Skip to content

armgit / alook

Public Imported from GitHub
0

Multi-Agent Collaboration for Team and Friends

fix(email-worker): use UID tracking instead of SEARCH UNSEEN for IMAP polling (#19)

Gener599d9a3637a4
diff --git a/src/email-worker/src/imap-poller-do.test.ts b/src/email-worker/src/imap-poller-do.test.ts
index 853977388..f5dc1551b 100644
--- a/src/email-worker/src/imap-poller-do.test.ts
+++ b/src/email-worker/src/imap-poller-do.test.ts
@@ -1,6 +1,5 @@
import { describe, it, expect, vi, beforeEach } from "vitest"
-// Mock cloudflare:workers — provide a real-enough DurableObject base class
vi.mock("cloudflare:workers", () => ({
DurableObject: class {
ctx: any
@@ -12,47 +11,45 @@ vi.mock("cloudflare:workers", () => ({
},
}))
-// Mock cf-imap — using vi.hoisted so mock fns are available inside vi.mock factory
-const { mockConnect, mockSelectFolder, mockSearchEmails, mockFetchEmails, mockLogout } = vi.hoisted(() => ({
+const { mockConnect, mockSelectFolder, mockLogout } = vi.hoisted(() => ({
mockConnect: vi.fn().mockResolvedValue(undefined),
mockSelectFolder: vi.fn().mockResolvedValue({ exists: 5 }),
- mockSearchEmails: vi.fn<() => Promise<number[]>>().mockResolvedValue([]),
- mockFetchEmails: vi.fn<() => Promise<any[]>>().mockResolvedValue([]),
mockLogout: vi.fn().mockResolvedValue(true),
}))
-const { mockStoreWriter } = vi.hoisted(() => ({
- mockStoreWriter: vi.fn().mockResolvedValue(undefined),
+const { mockWriterWrite, mockReaderRead } = vi.hoisted(() => ({
+ mockWriterWrite: vi.fn().mockResolvedValue(undefined),
+ mockReaderRead: vi.fn<() => Promise<{ value: Uint8Array; done: boolean }>>(),
}))
-vi.mock("cf-imap", () => {
- return {
- CFImap: class {
- connect = mockConnect
- selectFolder = mockSelectFolder
- searchEmails = mockSearchEmails
- fetchEmails = mockFetchEmails
- logout = mockLogout
- writer = { write: mockStoreWriter }
- reader = { read: vi.fn().mockResolvedValue({ value: new Uint8Array() }) }
- encoder = new TextEncoder()
- },
- }
-})
+vi.mock("cf-imap", () => ({
+ CFImap: class {
+ connect = mockConnect
+ selectFolder = mockSelectFolder
+ logout = mockLogout
+ writer = { write: mockWriterWrite }
+ reader = { read: mockReaderRead }
+ },
+}))
+
+const { mockPostalParse } = vi.hoisted(() => ({
+ mockPostalParse: vi.fn(),
+}))
+
+vi.mock("postal-mime", () => ({
+ default: { parse: mockPostalParse },
+}))
-// Mock nanoid
let nanoidCounter = 0
vi.mock("nanoid", () => ({
nanoid: (len?: number) => `mock-${++nanoidCounter}`,
}))
-// Mock @alook/shared/crypto
vi.mock("@alook/shared/crypto", () => ({
encrypt: (val: string) => `encrypted:${val}`,
decrypt: (val: string) => `decrypted:${val}`,
}))
-// Mock @alook/shared
const mockGetEmailAccount = vi.fn()
const mockUpdateEmailAccount = vi.fn()
const mockIsWhitelisted = vi.fn().mockResolvedValue(true)
@@ -147,28 +144,58 @@ function createDO() {
return { durable, ctx, storage, env, putR2, webFetch, getAlarm }
}
+const encoder = new TextEncoder()
+
+/** Queue raw IMAP response strings for the mock reader. */
+function queueReaderResponses(...responses: string[]) {
+ let idx = 0
+ mockReaderRead.mockImplementation(async () => {
+ if (idx < responses.length) {
+ return { value: encoder.encode(responses[idx++]), done: false }
+ }
+ return { value: new Uint8Array(), done: true }
+ })
+}
+
+function buildFetchResponse(uid: number, rawEmail: string): string {
+ return `* 1 FETCH (UID ${uid} BODY[] {${rawEmail.length}}\r\n${rawEmail}\r\n)\r\nF${uid} OK UID FETCH completed\r\n`
+}
+
+const RAW_EMAIL_1 = "From: [email protected]\r\nSubject: Hi\r\nMessage-ID: <msg1>\r\n\r\nHello"
+const RAW_EMAIL_2 = "From: [email protected]\r\nSubject: Hey\r\nMessage-ID: <msg2>\r\n\r\nWorld"
+
beforeEach(() => {
nanoidCounter = 0
vi.clearAllMocks()
mockGetEmailAccount.mockResolvedValue({ ...ACCOUNT })
mockUpdateEmailAccount.mockResolvedValue(undefined)
mockIsWhitelisted.mockResolvedValue(true)
- mockSearchEmails.mockResolvedValue([])
- mockFetchEmails.mockResolvedValue([])
+ mockPostalParse.mockResolvedValue({
+ from: { name: "", address: "[email protected]" },
+ subject: "Hi",
+ messageId: "<msg1>",
+ inReplyTo: "",
+ references: "",
+ })
})
-// ─── T3: alarm normal flow ───
+// ─── Normal flow with UID tracking ───
-describe("alarm — normal flow", () => {
- it("fetches unseen emails, stores in R2, notifies web, and reschedules", async () => {
+describe("alarm — normal UID-based flow", () => {
+ it("searches by UID, fetches, stores in R2, notifies web, and updates lastSyncedUid", async () => {
const { durable, ctx, putR2, webFetch } = createDO()
- mockSearchEmails.mockResolvedValue([1, 2])
- mockFetchEmails.mockResolvedValue([
- { from: "[email protected]", to: "[email protected]", subject: "Hi", messageID: "<msg1>", raw: "raw1", body: "", contentType: "text/plain", date: new Date() },
- { from: "[email protected]", to: "[email protected]", subject: "Hey", messageID: "<msg2>", raw: "raw2", body: "", contentType: "text/plain", date: new Date() },
- ])
+ mockPostalParse
+ .mockResolvedValueOnce({ from: { name: "", address: "[email protected]" }, subject: "Hi", messageId: "<msg1>", inReplyTo: "", references: "" })
+ .mockResolvedValueOnce({ from: { name: "", address: "[email protected]" }, subject: "Hey", messageId: "<msg2>", inReplyTo: "", references: "" })
+ queueReaderResponses(
+ "* SEARCH 101 102\r\nS1 OK UID SEARCH completed\r\n",
+ buildFetchResponse(101, RAW_EMAIL_1),
+ buildFetchResponse(102, RAW_EMAIL_2),
+ )
+
+ mockGetEmailAccount.mockResolvedValue({ ...ACCOUNT, lastSyncedUid: "100" })
await ctx.storage.put("accountId", "aea_test1")
await durable.alarm()
@@ -180,21 +207,61 @@ describe("alarm — normal flow", () => {
expect(notify1.from).toBe("[email protected]")
expect(notify1.subject).toBe("Hi")
expect(notify1.isWhitelisted).toBe(true)
+ expect(notify1.messageId).toBe("<msg1>")
+
+ const notify2 = JSON.parse(webFetch.mock.calls[1][1].body)
+ expect(notify2.from).toBe("[email protected]")
+ expect(notify2.subject).toBe("Hey")
- expect(mockUpdateEmailAccount).toHaveBeenCalled()
+ expect(mockUpdateEmailAccount).toHaveBeenCalledWith(
+ expect.anything(), "aea_test1", "ws_test1",
+ expect.objectContaining({ status: "active", lastSyncedUid: "102" })
+ )
expect(ctx.storage.setAlarm).toHaveBeenCalled()
+ expect(mockLogout).toHaveBeenCalled()
+ })
+
+ it("uses SINCE filter for first sync (lastSyncedUid=0)", async () => {
+ const { durable, ctx } = createDO()
+ queueReaderResponses("* SEARCH\r\nS1 OK UID SEARCH completed\r\n")
+ mockGetEmailAccount.mockResolvedValue({ ...ACCOUNT, lastSyncedUid: "0" })
+
+ await ctx.storage.put("accountId", "aea_test1")
+ await durable.alarm()
+
+ const writeCall = mockWriterWrite.mock.calls[0][0]
+ const cmd = new TextDecoder().decode(writeCall)
+ expect(cmd).toMatch(/S1 UID SEARCH SINCE \d{1,2}-\w{3}-\d{4}/)
+ })
+
+ it("filters out UIDs <= lastSyncedUid from SEARCH results", async () => {
+ const { durable, ctx, putR2 } = createDO()
+ // UID SEARCH UID 101:* might return UID 100 on some servers
+ queueReaderResponses(
+ "* SEARCH 100 101 102\r\nS1 OK UID SEARCH completed\r\n",
+ buildFetchResponse(101, RAW_EMAIL_1),
+ buildFetchResponse(102, RAW_EMAIL_2),
+ )
+ mockGetEmailAccount.mockResolvedValue({ ...ACCOUNT, lastSyncedUid: "100" })
+
+ await ctx.storage.put("accountId", "aea_test1")
+ await durable.alarm()
+
+ // Only 101 and 102 should be fetched, not 100
+ expect(putR2).toHaveBeenCalledTimes(2)
})
})
-// ─── T4: whitelist filtering ───
+// ─── Whitelist filtering ───
describe("alarm — whitelist filtering", () => {
it("passes isWhitelisted=true for whitelisted sender", async () => {
const { durable, ctx, webFetch } = createDO()
- mockSearchEmails.mockResolvedValue([1])
- mockFetchEmails.mockResolvedValue([
- { from: "[email protected]", to: "[email protected]", subject: "Hi", messageID: "", raw: "raw", body: "", contentType: "text/plain", date: new Date() },
- ])
+ queueReaderResponses(
+ "* SEARCH 50\r\nS1 OK UID SEARCH completed\r\n",
+ buildFetchResponse(50, RAW_EMAIL_1),
+ )
+ mockGetEmailAccount.mockResolvedValue({ ...ACCOUNT, lastSyncedUid: "0" })
mockIsWhitelisted.mockResolvedValue(true)
await ctx.storage.put("accountId", "aea_test1")
@@ -206,10 +273,11 @@ describe("alarm — whitelist filtering", () => {
it("passes isWhitelisted=false for non-whitelisted sender", async () => {
const { durable, ctx, webFetch } = createDO()
- mockSearchEmails.mockResolvedValue([1])
- mockFetchEmails.mockResolvedValue([
- { from: "[email protected]", to: "[email protected]", subject: "Spam", messageID: "", raw: "raw", body: "", contentType: "text/plain", date: new Date() },
- ])
+ queueReaderResponses(
+ "* SEARCH 50\r\nS1 OK UID SEARCH completed\r\n",
+ buildFetchResponse(50, RAW_EMAIL_1),
+ )
+ mockGetEmailAccount.mockResolvedValue({ ...ACCOUNT, lastSyncedUid: "0" })
mockIsWhitelisted.mockResolvedValue(false)
await ctx.storage.put("accountId", "aea_test1")
@@ -220,7 +288,7 @@ describe("alarm — whitelist filtering", () => {
})
})
-// ─── T5: connection failure & backoff ───
+// ─── Connection failure & backoff ───
describe("alarm — connection failure & backoff", () => {
it("sets error status and schedules with backoff on connection failure", async () => {
@@ -238,7 +306,7 @@ describe("alarm — connection failure & backoff", () => {
})
})
-// ─── T6: auth failure ───
+// ─── Auth failure ───
describe("alarm — auth failure", () => {
it("stops polling on authentication error", async () => {
@@ -257,17 +325,17 @@ describe("alarm — auth failure", () => {
})
})
-// ─── T7: no new emails ───
+// ─── No new emails ───
describe("alarm — no new emails", () => {
- it("reschedules without fetching when SEARCH returns empty", async () => {
+ it("reschedules without fetching when UID SEARCH returns empty", async () => {
const { durable, ctx, putR2, webFetch } = createDO()
- mockSearchEmails.mockResolvedValue([])
+ queueReaderResponses("* SEARCH\r\nS1 OK UID SEARCH completed\r\n")
+ mockGetEmailAccount.mockResolvedValue({ ...ACCOUNT, lastSyncedUid: "50" })
await ctx.storage.put("accountId", "aea_test1")
await durable.alarm()
- expect(mockFetchEmails).not.toHaveBeenCalled()
expect(putR2).not.toHaveBeenCalled()
expect(webFetch).not.toHaveBeenCalled()
expect(ctx.storage.setAlarm).toHaveBeenCalled()
@@ -278,7 +346,61 @@ describe("alarm — no new emails", () => {
})
})
-// ─── T8: fetch() routing ───
+// ─── Multi-chunk and error responses ───
+
+describe("alarm — IMAP response edge cases", () => {
+ it("handles FETCH response split across multiple TCP reads", async () => {
+ const { durable, ctx, putR2 } = createDO()
+ const rawEmail = RAW_EMAIL_1
+ const fullFetch = buildFetchResponse(101, rawEmail)
+ const mid = Math.floor(fullFetch.length / 2)
+
+ queueReaderResponses(
+ "* SEARCH 101\r\nS1 OK UID SEARCH completed\r\n",
+ fullFetch.substring(0, mid),
+ fullFetch.substring(mid),
+ )
+ mockGetEmailAccount.mockResolvedValue({ ...ACCOUNT, lastSyncedUid: "100" })
+
+ await ctx.storage.put("accountId", "aea_test1")
+ await durable.alarm()
+
+ expect(putR2).toHaveBeenCalledTimes(1)
+ })
+
+ it("throws and triggers backoff when IMAP SEARCH returns NO", async () => {
+ const { durable, ctx } = createDO()
+ queueReaderResponses("S1 NO [NONEXISTENT] Mailbox not found\r\n")
+ mockGetEmailAccount.mockResolvedValue({ ...ACCOUNT, lastSyncedUid: "50" })
+
+ await ctx.storage.put("accountId", "aea_test1")
+ await durable.alarm()
+
+ expect(mockUpdateEmailAccount).toHaveBeenCalledWith(
+ expect.anything(), "aea_test1", "ws_test1",
+ expect.objectContaining({ status: "error", errorMessage: expect.stringContaining("S1") })
+ )
+ expect(ctx.storage.setAlarm).toHaveBeenCalled()
+ })
+
+ it("throws and triggers backoff when IMAP stream ends prematurely", async () => {
+ const { durable, ctx } = createDO()
+ // Stream ends without a tagged response
+ mockReaderRead.mockResolvedValueOnce({ value: encoder.encode("* SEARCH 101\r\n"), done: false })
+ mockReaderRead.mockResolvedValueOnce({ value: new Uint8Array(), done: true })
+ mockGetEmailAccount.mockResolvedValue({ ...ACCOUNT, lastSyncedUid: "100" })
+
+ await ctx.storage.put("accountId", "aea_test1")
+ await durable.alarm()
+
+ expect(mockUpdateEmailAccount).toHaveBeenCalledWith(
+ expect.anything(), "aea_test1", "ws_test1",
+ expect.objectContaining({ status: "error", errorMessage: expect.stringContaining("stream ended") })
+ )
+ })
+})
+
+// ─── fetch() routing ───
describe("fetch() routing", () => {
it("POST /start sets accountId and schedules alarm", async () => {
@@ -303,10 +425,11 @@ describe("fetch() routing", () => {
it("POST /sync triggers immediate poll", async () => {
const { durable, ctx } = createDO()
+ queueReaderResponses("* SEARCH\r\nS1 OK UID SEARCH completed\r\n")
await ctx.storage.put("accountId", "aea_test1")
const res = await durable.fetch(new Request("http://internal/sync", { method: "POST" }))
expect(res.status).toBe(200)
- expect(mockSearchEmails).toHaveBeenCalled()
+ expect(mockConnect).toHaveBeenCalled()
})
it("GET /status returns account status", async () => {
@@ -327,7 +450,7 @@ describe("fetch() routing", () => {
})
})
-// ─── T9: lifecycle ───
+// ─── Lifecycle ───
describe("lifecycle", () => {
it("stops polling when account is deleted from DB", async () => {
@@ -339,6 +462,6 @@ describe("lifecycle", () => {
expect(ctx.storage.deleteAlarm).toHaveBeenCalled()
expect(ctx.storage.deleteAll).toHaveBeenCalled()
- expect(mockSearchEmails).not.toHaveBeenCalled()
+ expect(mockConnect).not.toHaveBeenCalled()
})
})
diff --git a/src/email-worker/src/imap-poller-do.ts b/src/email-worker/src/imap-poller-do.ts
index 5c013e497..81540801e 100644
--- a/src/email-worker/src/imap-poller-do.ts
+++ b/src/email-worker/src/imap-poller-do.ts
@@ -1,5 +1,6 @@
import { DurableObject } from "cloudflare:workers"
import { CFImap } from "cf-imap"
+import PostalMime from "postal-mime"
import { nanoid } from "nanoid"
import { createDb, queries, createLogger } from "@alook/shared"
import { decrypt } from "@alook/shared/crypto"
@@ -9,6 +10,7 @@ const log = createLogger({ service: "imap-poller" })
const MAX_BACKOFF_MS = 15 * 60 * 1000
const MAX_EMAILS_PER_POLL = 50
+const FIRST_SYNC_DAYS = 7
export class ImapPollerDO extends DurableObject<EmailEnv> {
private accountId: string | null = null
@@ -88,10 +90,33 @@ export class ImapPollerDO extends DurableObject<EmailEnv> {
await imap.connect()
await imap.selectFolder("INBOX")
- const unseenSeqs = await imap.searchEmails({ seen: false })
- pollLog.info("search complete", { unseen: unseenSeqs.length })
+ const encoder = new TextEncoder()
+ const writer = (imap as any).writer as WritableStreamDefaultWriter
+ const reader = (imap as any).reader as ReadableStreamDefaultReader
+
+ // UID-based search instead of SEARCH UNSEEN — catches emails
+ // regardless of \Seen flag (fixes Feishu and similar IMAP servers
+ // that auto-mark emails as read).
+ const lastUid = parseInt(account.lastSyncedUid, 10) || 0
+ let searchCmd: string
+ if (lastUid > 0) {
+ searchCmd = `UID SEARCH UID ${lastUid + 1}:*`
+ } else {
+ const since = new Date()
+ since.setDate(since.getDate() - FIRST_SYNC_DAYS)
+ const months = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]
+ searchCmd = `UID SEARCH SINCE ${since.getDate()}-${months[since.getMonth()]}-${since.getFullYear()}`
+ }
+
+ const searchResp = await this.imapReadUntilTag(writer, reader, encoder, "S1", searchCmd)
+ const searchLine = searchResp.split("\r\n").find(l => l.startsWith("* SEARCH"))
+ const uids: number[] = searchLine
+ ? searchLine.replace("* SEARCH", "").trim().split(/\s+/).map(Number).filter(n => !isNaN(n) && n > lastUid)
+ : []
- if (unseenSeqs.length === 0) {
+ pollLog.info("uid search complete", { found: uids.length, lastUid })
+
+ if (uids.length === 0) {
await queries.emailAccount.updateEmailAccount(db, accountId, account.workspaceId, {
lastSyncedAt: new Date().toISOString(),
status: "active",
@@ -102,61 +127,59 @@ export class ImapPollerDO extends DurableObject<EmailEnv> {
return
}
- const sorted = [...unseenSeqs].sort((a, b) => a - b)
+ const sorted = uids.sort((a, b) => a - b)
const batch = sorted.slice(0, MAX_EMAILS_PER_POLL)
- const start = batch[0]!
- const end = batch[batch.length - 1]!
-
- // peek:true to work around cf-imap bug where peek:false generates "BODYfalse" (invalid IMAP)
- const fetched = await imap.fetchEmails({
- folder: "INBOX",
- limit: [start, end],
- fetchBody: true,
- peek: true,
- })
- pollLog.info("fetched emails", { count: fetched.length })
+ let maxUid = lastUid
+ for (const uid of batch) {
+ const tag = `F${uid}`
+ const fetchResp = await this.imapReadUntilTag(writer, reader, encoder, tag, `UID FETCH ${uid} (BODY.PEEK[])`)
- // Mark fetched emails as \Seen since we used peek:true above
- const encoder = new TextEncoder()
- const decoder = new TextDecoder()
- const writer = (imap as any).writer as WritableStreamDefaultWriter
- const reader = (imap as any).reader as ReadableStreamDefaultReader
- await writer.write(encoder.encode(`A6 STORE ${start}:${end} +FLAGS (\\Seen)\r\n`))
- await decoder.decode((await reader.read()).value)
+ const rawEmail = this.extractEmailFromFetch(fetchResp)
+ if (!rawEmail) {
+ pollLog.warn("failed to extract email content", { uid })
+ maxUid = Math.max(maxUid, uid)
+ continue
+ }
+
+ const parsed = await PostalMime.parse(rawEmail)
- for (const email of fetched) {
const r2Id = nanoid()
const r2Key = `emails/${r2Id}/raw`
- await this.env.EMAIL_BUCKET.put(r2Key, email.raw, {
+ await this.env.EMAIL_BUCKET.put(r2Key, rawEmail, {
httpMetadata: { contentType: "message/rfc822" },
})
- const fromEmail = email.from
+ const fromAddr = parsed.from?.address || parsed.from?.name || ""
const isWhitelisted = await queries.whitelist.isWhitelisted(
- db, account.agentId, account.workspaceId, fromEmail
+ db, account.agentId, account.workspaceId, fromAddr
)
await this.notifyWeb({
agentId: account.agentId,
workspaceId: account.workspaceId,
r2Key,
- from: fromEmail,
+ from: fromAddr,
to: account.emailAddress,
- subject: email.subject || "",
+ subject: parsed.subject || "",
isWhitelisted,
- messageId: email.messageID || "",
- inReplyTo: "",
- references: "",
+ messageId: parsed.messageId || "",
+ inReplyTo: parsed.inReplyTo || "",
+ references: parsed.references || "",
})
+
+ maxUid = Math.max(maxUid, uid)
}
await queries.emailAccount.updateEmailAccount(db, accountId, account.workspaceId, {
lastSyncedAt: new Date().toISOString(),
+ lastSyncedUid: String(maxUid),
status: "active",
errorMessage: "",
})
+ pollLog.info("poll complete", { processed: batch.length, maxUid })
+
await this.ctx.storage.put("backoffMs", 0)
await this.scheduleNext(account.pollIntervalSeconds * 1000)
await imap.logout()
@@ -189,6 +212,50 @@ export class ImapPollerDO extends DurableObject<EmailEnv> {
}
}
+ private async imapReadUntilTag(
+ writer: WritableStreamDefaultWriter,
+ reader: ReadableStreamDefaultReader,
+ encoder: TextEncoder,
+ tag: string,
+ cmd: string,
+ ): Promise<string> {
+ await writer.write(encoder.encode(`${tag} ${cmd}\r\n`))
+ const decoder = new TextDecoder()
+ let buf = ""
+ const READ_TIMEOUT_MS = 15_000
+ while (true) {
+ const { value, done } = await Promise.race([
+ reader.read(),
+ new Promise<never>((_, reject) =>
+ setTimeout(() => reject(new Error(`IMAP read timeout for ${tag}`)), READ_TIMEOUT_MS)
+ ),
+ ])
+ if (done) break
+ buf += decoder.decode(value, { stream: true })
+ for (const line of buf.split("\r\n")) {
+ if (line.startsWith(`${tag} OK`) || line.startsWith(`${tag} NO`) || line.startsWith(`${tag} BAD`)) {
+ if (line.startsWith(`${tag} NO`) || line.startsWith(`${tag} BAD`)) {
+ throw new Error(`IMAP ${tag} failed: ${line}`)
+ }
+ return buf
+ }
+ }
+ }
+ throw new Error(`IMAP stream ended without tagged response for ${tag}`)
+ }
+
+ private extractEmailFromFetch(response: string): string | null {
+ const literalMatch = response.match(/\{(\d+)\}\r\n/)
+ if (!literalMatch) return null
+
+ const contentStart = response.indexOf(literalMatch[0]) + literalMatch[0].length
+ const closingIdx = response.lastIndexOf("\r\n)")
+ if (closingIdx > contentStart) {
+ return response.substring(contentStart, closingIdx)
+ }
+ return null
+ }
+
private async scheduleNext(delayMs: number): Promise<void> {
await this.ctx.storage.setAlarm(Date.now() + delayMs)
}