Multi-Agent Collaboration for Team and Friends
TypeScript agent-automationagent-orchestrationagent-workflowai-agentsai-collaborationai-memoryai-workforceautonomous-agentsclaude-codeclaude-code-skillscodexcoding-agentmulti-agentmultiple-agentone-person-businessone-person-companyone-person-teamopencodesolopreneur
fix(email-worker): use UID tracking instead of SEARCH UNSEEN for IMAP polling (#19)
Gener599d9a3637a4
diff --git a/src/email-worker/src/imap-poller-do.test.ts b/src/email-worker/src/imap-poller-do.test.tsindex 853977388..f5dc1551b 100644--- a/src/email-worker/src/imap-poller-do.test.ts+++ b/src/email-worker/src/imap-poller-do.test.ts@@ -1,6 +1,5 @@import { describe, it, expect, vi, beforeEach } from "vitest"-// Mock cloudflare:workers — provide a real-enough DurableObject base classvi.mock("cloudflare:workers", () => ({DurableObject: class {ctx: any@@ -12,47 +11,45 @@ vi.mock("cloudflare:workers", () => ({},}))-// Mock cf-imap — using vi.hoisted so mock fns are available inside vi.mock factory-const { mockConnect, mockSelectFolder, mockSearchEmails, mockFetchEmails, mockLogout } = vi.hoisted(() => ({+const { mockConnect, mockSelectFolder, mockLogout } = vi.hoisted(() => ({mockConnect: vi.fn().mockResolvedValue(undefined),mockSelectFolder: vi.fn().mockResolvedValue({ exists: 5 }),- mockSearchEmails: vi.fn<() => Promise<number[]>>().mockResolvedValue([]),- mockFetchEmails: vi.fn<() => Promise<any[]>>().mockResolvedValue([]),mockLogout: vi.fn().mockResolvedValue(true),}))-const { mockStoreWriter } = vi.hoisted(() => ({- mockStoreWriter: vi.fn().mockResolvedValue(undefined),+const { mockWriterWrite, mockReaderRead } = vi.hoisted(() => ({+ mockWriterWrite: vi.fn().mockResolvedValue(undefined),+ mockReaderRead: vi.fn<() => Promise<{ value: Uint8Array; done: boolean }>>(),}))-vi.mock("cf-imap", () => {- return {- CFImap: class {- connect = mockConnect- selectFolder = mockSelectFolder- searchEmails = mockSearchEmails- fetchEmails = mockFetchEmails- logout = mockLogout- writer = { write: mockStoreWriter }- reader = { read: vi.fn().mockResolvedValue({ value: new Uint8Array() }) }- encoder = new TextEncoder()- },- }-})+vi.mock("cf-imap", () => ({+ CFImap: class {+ connect = mockConnect+ selectFolder = mockSelectFolder+ logout = mockLogout+ writer = { write: mockWriterWrite }+ reader = { read: mockReaderRead }+ },+}))++const { mockPostalParse } = vi.hoisted(() => ({+ mockPostalParse: vi.fn(),+}))++vi.mock("postal-mime", () => ({+ default: { parse: mockPostalParse },+}))-// Mock nanoidlet nanoidCounter = 0vi.mock("nanoid", () => ({nanoid: (len?: number) => `mock-${++nanoidCounter}`,}))-// Mock @alook/shared/cryptovi.mock("@alook/shared/crypto", () => ({encrypt: (val: string) => `encrypted:${val}`,decrypt: (val: string) => `decrypted:${val}`,}))-// Mock @alook/sharedconst mockGetEmailAccount = vi.fn()const mockUpdateEmailAccount = vi.fn()const mockIsWhitelisted = vi.fn().mockResolvedValue(true)@@ -147,28 +144,58 @@ function createDO() {return { durable, ctx, storage, env, putR2, webFetch, getAlarm }}+const encoder = new TextEncoder()++/** Queue raw IMAP response strings for the mock reader. */+function queueReaderResponses(...responses: string[]) {+ let idx = 0+ mockReaderRead.mockImplementation(async () => {+ if (idx < responses.length) {+ return { value: encoder.encode(responses[idx++]), done: false }+ }+ return { value: new Uint8Array(), done: true }+ })+}++function buildFetchResponse(uid: number, rawEmail: string): string {+ return `* 1 FETCH (UID ${uid} BODY[] {${rawEmail.length}}\r\n${rawEmail}\r\n)\r\nF${uid} OK UID FETCH completed\r\n`+}++const RAW_EMAIL_1 = "From: [email protected]\r\nSubject: Hi\r\nMessage-ID: <msg1>\r\n\r\nHello"+const RAW_EMAIL_2 = "From: [email protected]\r\nSubject: Hey\r\nMessage-ID: <msg2>\r\n\r\nWorld"+beforeEach(() => {nanoidCounter = 0vi.clearAllMocks()mockGetEmailAccount.mockResolvedValue({ ...ACCOUNT })mockUpdateEmailAccount.mockResolvedValue(undefined)mockIsWhitelisted.mockResolvedValue(true)- mockSearchEmails.mockResolvedValue([])- mockFetchEmails.mockResolvedValue([])+ mockPostalParse.mockResolvedValue({+ from: { name: "", address: "[email protected]" },+ subject: "Hi",+ messageId: "<msg1>",+ inReplyTo: "",+ references: "",+ })})-// ─── T3: alarm normal flow ───+// ─── Normal flow with UID tracking ───-describe("alarm — normal flow", () => {- it("fetches unseen emails, stores in R2, notifies web, and reschedules", async () => {+describe("alarm — normal UID-based flow", () => {+ it("searches by UID, fetches, stores in R2, notifies web, and updates lastSyncedUid", async () => {const { durable, ctx, putR2, webFetch } = createDO()- mockSearchEmails.mockResolvedValue([1, 2])- mockFetchEmails.mockResolvedValue([- { from: "[email protected]", to: "[email protected]", subject: "Hi", messageID: "<msg1>", raw: "raw1", body: "", contentType: "text/plain", date: new Date() },- { from: "[email protected]", to: "[email protected]", subject: "Hey", messageID: "<msg2>", raw: "raw2", body: "", contentType: "text/plain", date: new Date() },- ])+ mockPostalParse+ .mockResolvedValueOnce({ from: { name: "", address: "[email protected]" }, subject: "Hi", messageId: "<msg1>", inReplyTo: "", references: "" })+ .mockResolvedValueOnce({ from: { name: "", address: "[email protected]" }, subject: "Hey", messageId: "<msg2>", inReplyTo: "", references: "" })+ queueReaderResponses(+ "* SEARCH 101 102\r\nS1 OK UID SEARCH completed\r\n",+ buildFetchResponse(101, RAW_EMAIL_1),+ buildFetchResponse(102, RAW_EMAIL_2),+ )++ mockGetEmailAccount.mockResolvedValue({ ...ACCOUNT, lastSyncedUid: "100" })await ctx.storage.put("accountId", "aea_test1")await durable.alarm()@@ -180,21 +207,61 @@ describe("alarm — normal flow", () => {expect(notify1.from).toBe("[email protected]")expect(notify1.subject).toBe("Hi")expect(notify1.isWhitelisted).toBe(true)+ expect(notify1.messageId).toBe("<msg1>")++ const notify2 = JSON.parse(webFetch.mock.calls[1][1].body)+ expect(notify2.from).toBe("[email protected]")+ expect(notify2.subject).toBe("Hey")- expect(mockUpdateEmailAccount).toHaveBeenCalled()+ expect(mockUpdateEmailAccount).toHaveBeenCalledWith(+ expect.anything(), "aea_test1", "ws_test1",+ expect.objectContaining({ status: "active", lastSyncedUid: "102" })+ )expect(ctx.storage.setAlarm).toHaveBeenCalled()+ expect(mockLogout).toHaveBeenCalled()+ })++ it("uses SINCE filter for first sync (lastSyncedUid=0)", async () => {+ const { durable, ctx } = createDO()+ queueReaderResponses("* SEARCH\r\nS1 OK UID SEARCH completed\r\n")+ mockGetEmailAccount.mockResolvedValue({ ...ACCOUNT, lastSyncedUid: "0" })++ await ctx.storage.put("accountId", "aea_test1")+ await durable.alarm()++ const writeCall = mockWriterWrite.mock.calls[0][0]+ const cmd = new TextDecoder().decode(writeCall)+ expect(cmd).toMatch(/S1 UID SEARCH SINCE \d{1,2}-\w{3}-\d{4}/)+ })++ it("filters out UIDs <= lastSyncedUid from SEARCH results", async () => {+ const { durable, ctx, putR2 } = createDO()+ // UID SEARCH UID 101:* might return UID 100 on some servers+ queueReaderResponses(+ "* SEARCH 100 101 102\r\nS1 OK UID SEARCH completed\r\n",+ buildFetchResponse(101, RAW_EMAIL_1),+ buildFetchResponse(102, RAW_EMAIL_2),+ )+ mockGetEmailAccount.mockResolvedValue({ ...ACCOUNT, lastSyncedUid: "100" })++ await ctx.storage.put("accountId", "aea_test1")+ await durable.alarm()++ // Only 101 and 102 should be fetched, not 100+ expect(putR2).toHaveBeenCalledTimes(2)})})-// ─── T4: whitelist filtering ───+// ─── Whitelist filtering ───describe("alarm — whitelist filtering", () => {it("passes isWhitelisted=true for whitelisted sender", async () => {const { durable, ctx, webFetch } = createDO()- mockSearchEmails.mockResolvedValue([1])- mockFetchEmails.mockResolvedValue([- { from: "[email protected]", to: "[email protected]", subject: "Hi", messageID: "", raw: "raw", body: "", contentType: "text/plain", date: new Date() },- ])+ queueReaderResponses(+ "* SEARCH 50\r\nS1 OK UID SEARCH completed\r\n",+ buildFetchResponse(50, RAW_EMAIL_1),+ )+ mockGetEmailAccount.mockResolvedValue({ ...ACCOUNT, lastSyncedUid: "0" })mockIsWhitelisted.mockResolvedValue(true)await ctx.storage.put("accountId", "aea_test1")@@ -206,10 +273,11 @@ describe("alarm — whitelist filtering", () => {it("passes isWhitelisted=false for non-whitelisted sender", async () => {const { durable, ctx, webFetch } = createDO()- mockSearchEmails.mockResolvedValue([1])- mockFetchEmails.mockResolvedValue([- { from: "[email protected]", to: "[email protected]", subject: "Spam", messageID: "", raw: "raw", body: "", contentType: "text/plain", date: new Date() },- ])+ queueReaderResponses(+ "* SEARCH 50\r\nS1 OK UID SEARCH completed\r\n",+ buildFetchResponse(50, RAW_EMAIL_1),+ )+ mockGetEmailAccount.mockResolvedValue({ ...ACCOUNT, lastSyncedUid: "0" })mockIsWhitelisted.mockResolvedValue(false)await ctx.storage.put("accountId", "aea_test1")@@ -220,7 +288,7 @@ describe("alarm — whitelist filtering", () => {})})-// ─── T5: connection failure & backoff ───+// ─── Connection failure & backoff ───describe("alarm — connection failure & backoff", () => {it("sets error status and schedules with backoff on connection failure", async () => {@@ -238,7 +306,7 @@ describe("alarm — connection failure & backoff", () => {})})-// ─── T6: auth failure ───+// ─── Auth failure ───describe("alarm — auth failure", () => {it("stops polling on authentication error", async () => {@@ -257,17 +325,17 @@ describe("alarm — auth failure", () => {})})-// ─── T7: no new emails ───+// ─── No new emails ───describe("alarm — no new emails", () => {- it("reschedules without fetching when SEARCH returns empty", async () => {+ it("reschedules without fetching when UID SEARCH returns empty", async () => {const { durable, ctx, putR2, webFetch } = createDO()- mockSearchEmails.mockResolvedValue([])+ queueReaderResponses("* SEARCH\r\nS1 OK UID SEARCH completed\r\n")+ mockGetEmailAccount.mockResolvedValue({ ...ACCOUNT, lastSyncedUid: "50" })await ctx.storage.put("accountId", "aea_test1")await durable.alarm()- expect(mockFetchEmails).not.toHaveBeenCalled()expect(putR2).not.toHaveBeenCalled()expect(webFetch).not.toHaveBeenCalled()expect(ctx.storage.setAlarm).toHaveBeenCalled()@@ -278,7 +346,61 @@ describe("alarm — no new emails", () => {})})-// ─── T8: fetch() routing ───+// ─── Multi-chunk and error responses ───++describe("alarm — IMAP response edge cases", () => {+ it("handles FETCH response split across multiple TCP reads", async () => {+ const { durable, ctx, putR2 } = createDO()+ const rawEmail = RAW_EMAIL_1+ const fullFetch = buildFetchResponse(101, rawEmail)+ const mid = Math.floor(fullFetch.length / 2)++ queueReaderResponses(+ "* SEARCH 101\r\nS1 OK UID SEARCH completed\r\n",+ fullFetch.substring(0, mid),+ fullFetch.substring(mid),+ )+ mockGetEmailAccount.mockResolvedValue({ ...ACCOUNT, lastSyncedUid: "100" })++ await ctx.storage.put("accountId", "aea_test1")+ await durable.alarm()++ expect(putR2).toHaveBeenCalledTimes(1)+ })++ it("throws and triggers backoff when IMAP SEARCH returns NO", async () => {+ const { durable, ctx } = createDO()+ queueReaderResponses("S1 NO [NONEXISTENT] Mailbox not found\r\n")+ mockGetEmailAccount.mockResolvedValue({ ...ACCOUNT, lastSyncedUid: "50" })++ await ctx.storage.put("accountId", "aea_test1")+ await durable.alarm()++ expect(mockUpdateEmailAccount).toHaveBeenCalledWith(+ expect.anything(), "aea_test1", "ws_test1",+ expect.objectContaining({ status: "error", errorMessage: expect.stringContaining("S1") })+ )+ expect(ctx.storage.setAlarm).toHaveBeenCalled()+ })++ it("throws and triggers backoff when IMAP stream ends prematurely", async () => {+ const { durable, ctx } = createDO()+ // Stream ends without a tagged response+ mockReaderRead.mockResolvedValueOnce({ value: encoder.encode("* SEARCH 101\r\n"), done: false })+ mockReaderRead.mockResolvedValueOnce({ value: new Uint8Array(), done: true })+ mockGetEmailAccount.mockResolvedValue({ ...ACCOUNT, lastSyncedUid: "100" })++ await ctx.storage.put("accountId", "aea_test1")+ await durable.alarm()++ expect(mockUpdateEmailAccount).toHaveBeenCalledWith(+ expect.anything(), "aea_test1", "ws_test1",+ expect.objectContaining({ status: "error", errorMessage: expect.stringContaining("stream ended") })+ )+ })+})++// ─── fetch() routing ───describe("fetch() routing", () => {it("POST /start sets accountId and schedules alarm", async () => {@@ -303,10 +425,11 @@ describe("fetch() routing", () => {it("POST /sync triggers immediate poll", async () => {const { durable, ctx } = createDO()+ queueReaderResponses("* SEARCH\r\nS1 OK UID SEARCH completed\r\n")await ctx.storage.put("accountId", "aea_test1")const res = await durable.fetch(new Request("http://internal/sync", { method: "POST" }))expect(res.status).toBe(200)- expect(mockSearchEmails).toHaveBeenCalled()+ expect(mockConnect).toHaveBeenCalled()})it("GET /status returns account status", async () => {@@ -327,7 +450,7 @@ describe("fetch() routing", () => {})})-// ─── T9: lifecycle ───+// ─── Lifecycle ───describe("lifecycle", () => {it("stops polling when account is deleted from DB", async () => {@@ -339,6 +462,6 @@ describe("lifecycle", () => {expect(ctx.storage.deleteAlarm).toHaveBeenCalled()expect(ctx.storage.deleteAll).toHaveBeenCalled()- expect(mockSearchEmails).not.toHaveBeenCalled()+ expect(mockConnect).not.toHaveBeenCalled()})})diff --git a/src/email-worker/src/imap-poller-do.ts b/src/email-worker/src/imap-poller-do.tsindex 5c013e497..81540801e 100644--- a/src/email-worker/src/imap-poller-do.ts+++ b/src/email-worker/src/imap-poller-do.ts@@ -1,5 +1,6 @@import { DurableObject } from "cloudflare:workers"import { CFImap } from "cf-imap"+import PostalMime from "postal-mime"import { nanoid } from "nanoid"import { createDb, queries, createLogger } from "@alook/shared"import { decrypt } from "@alook/shared/crypto"@@ -9,6 +10,7 @@ const log = createLogger({ service: "imap-poller" })const MAX_BACKOFF_MS = 15 * 60 * 1000const MAX_EMAILS_PER_POLL = 50+const FIRST_SYNC_DAYS = 7export class ImapPollerDO extends DurableObject<EmailEnv> {private accountId: string | null = null@@ -88,10 +90,33 @@ export class ImapPollerDO extends DurableObject<EmailEnv> {await imap.connect()await imap.selectFolder("INBOX")- const unseenSeqs = await imap.searchEmails({ seen: false })- pollLog.info("search complete", { unseen: unseenSeqs.length })+ const encoder = new TextEncoder()+ const writer = (imap as any).writer as WritableStreamDefaultWriter+ const reader = (imap as any).reader as ReadableStreamDefaultReader++ // UID-based search instead of SEARCH UNSEEN — catches emails+ // regardless of \Seen flag (fixes Feishu and similar IMAP servers+ // that auto-mark emails as read).+ const lastUid = parseInt(account.lastSyncedUid, 10) || 0+ let searchCmd: string+ if (lastUid > 0) {+ searchCmd = `UID SEARCH UID ${lastUid + 1}:*`+ } else {+ const since = new Date()+ since.setDate(since.getDate() - FIRST_SYNC_DAYS)+ const months = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]+ searchCmd = `UID SEARCH SINCE ${since.getDate()}-${months[since.getMonth()]}-${since.getFullYear()}`+ }++ const searchResp = await this.imapReadUntilTag(writer, reader, encoder, "S1", searchCmd)+ const searchLine = searchResp.split("\r\n").find(l => l.startsWith("* SEARCH"))+ const uids: number[] = searchLine+ ? searchLine.replace("* SEARCH", "").trim().split(/\s+/).map(Number).filter(n => !isNaN(n) && n > lastUid)+ : []- if (unseenSeqs.length === 0) {+ pollLog.info("uid search complete", { found: uids.length, lastUid })++ if (uids.length === 0) {await queries.emailAccount.updateEmailAccount(db, accountId, account.workspaceId, {lastSyncedAt: new Date().toISOString(),status: "active",@@ -102,61 +127,59 @@ export class ImapPollerDO extends DurableObject<EmailEnv> {return}- const sorted = [...unseenSeqs].sort((a, b) => a - b)+ const sorted = uids.sort((a, b) => a - b)const batch = sorted.slice(0, MAX_EMAILS_PER_POLL)- const start = batch[0]!- const end = batch[batch.length - 1]!-- // peek:true to work around cf-imap bug where peek:false generates "BODYfalse" (invalid IMAP)- const fetched = await imap.fetchEmails({- folder: "INBOX",- limit: [start, end],- fetchBody: true,- peek: true,- })- pollLog.info("fetched emails", { count: fetched.length })+ let maxUid = lastUid+ for (const uid of batch) {+ const tag = `F${uid}`+ const fetchResp = await this.imapReadUntilTag(writer, reader, encoder, tag, `UID FETCH ${uid} (BODY.PEEK[])`)- // Mark fetched emails as \Seen since we used peek:true above- const encoder = new TextEncoder()- const decoder = new TextDecoder()- const writer = (imap as any).writer as WritableStreamDefaultWriter- const reader = (imap as any).reader as ReadableStreamDefaultReader- await writer.write(encoder.encode(`A6 STORE ${start}:${end} +FLAGS (\\Seen)\r\n`))- await decoder.decode((await reader.read()).value)+ const rawEmail = this.extractEmailFromFetch(fetchResp)+ if (!rawEmail) {+ pollLog.warn("failed to extract email content", { uid })+ maxUid = Math.max(maxUid, uid)+ continue+ }++ const parsed = await PostalMime.parse(rawEmail)- for (const email of fetched) {const r2Id = nanoid()const r2Key = `emails/${r2Id}/raw`- await this.env.EMAIL_BUCKET.put(r2Key, email.raw, {+ await this.env.EMAIL_BUCKET.put(r2Key, rawEmail, {httpMetadata: { contentType: "message/rfc822" },})- const fromEmail = email.from+ const fromAddr = parsed.from?.address || parsed.from?.name || ""const isWhitelisted = await queries.whitelist.isWhitelisted(- db, account.agentId, account.workspaceId, fromEmail+ db, account.agentId, account.workspaceId, fromAddr)await this.notifyWeb({agentId: account.agentId,workspaceId: account.workspaceId,r2Key,- from: fromEmail,+ from: fromAddr,to: account.emailAddress,- subject: email.subject || "",+ subject: parsed.subject || "",isWhitelisted,- messageId: email.messageID || "",- inReplyTo: "",- references: "",+ messageId: parsed.messageId || "",+ inReplyTo: parsed.inReplyTo || "",+ references: parsed.references || "",})++ maxUid = Math.max(maxUid, uid)}await queries.emailAccount.updateEmailAccount(db, accountId, account.workspaceId, {lastSyncedAt: new Date().toISOString(),+ lastSyncedUid: String(maxUid),status: "active",errorMessage: "",})+ pollLog.info("poll complete", { processed: batch.length, maxUid })+await this.ctx.storage.put("backoffMs", 0)await this.scheduleNext(account.pollIntervalSeconds * 1000)await imap.logout()@@ -189,6 +212,50 @@ export class ImapPollerDO extends DurableObject<EmailEnv> {}}+ private async imapReadUntilTag(+ writer: WritableStreamDefaultWriter,+ reader: ReadableStreamDefaultReader,+ encoder: TextEncoder,+ tag: string,+ cmd: string,+ ): Promise<string> {+ await writer.write(encoder.encode(`${tag} ${cmd}\r\n`))+ const decoder = new TextDecoder()+ let buf = ""+ const READ_TIMEOUT_MS = 15_000+ while (true) {+ const { value, done } = await Promise.race([+ reader.read(),+ new Promise<never>((_, reject) =>+ setTimeout(() => reject(new Error(`IMAP read timeout for ${tag}`)), READ_TIMEOUT_MS)+ ),+ ])+ if (done) break+ buf += decoder.decode(value, { stream: true })+ for (const line of buf.split("\r\n")) {+ if (line.startsWith(`${tag} OK`) || line.startsWith(`${tag} NO`) || line.startsWith(`${tag} BAD`)) {+ if (line.startsWith(`${tag} NO`) || line.startsWith(`${tag} BAD`)) {+ throw new Error(`IMAP ${tag} failed: ${line}`)+ }+ return buf+ }+ }+ }+ throw new Error(`IMAP stream ended without tagged response for ${tag}`)+ }++ private extractEmailFromFetch(response: string): string | null {+ const literalMatch = response.match(/\{(\d+)\}\r\n/)+ if (!literalMatch) return null++ const contentStart = response.indexOf(literalMatch[0]) + literalMatch[0].length+ const closingIdx = response.lastIndexOf("\r\n)")+ if (closingIdx > contentStart) {+ return response.substring(contentStart, closingIdx)+ }+ return null+ }+private async scheduleNext(delayMs: number): Promise<void> {await this.ctx.storage.setAlarm(Date.now() + delayMs)}