Multi-Agent Collaboration for Team and Friends
TypeScript agent-automationagent-orchestrationagent-workflowai-agentsai-collaborationai-memoryai-workforceautonomous-agentsclaude-codeclaude-code-skillscodexcoding-agentmulti-agentmultiple-agentone-person-businessone-person-companyone-person-teamopencodesolopreneur
fix(wake): retry transient recipient failures
Gus622ba8dd3a2b
diff --git a/src/wake-worker/src/index.test.ts b/src/wake-worker/src/index.test.tsindex ffe680877..306718092 100644--- a/src/wake-worker/src/index.test.ts+++ b/src/wake-worker/src/index.test.ts@@ -150,21 +150,68 @@ describe("wake-worker dev-only HTTP entrypoint (fetch)", () => {expect(res.status).toBe(202)})- it("still returns 202 when one candidate rejects — siblings are unaffected, failure is only logged", async () => {- mockDispatchOneUnreadWake- .mockResolvedValueOnce({ outcome: "sent" })- .mockRejectedValueOnce(new Error("D1 exploded"))+ it("deduplicates candidates by stable messageId + botUserId within one request", async () => {+ mockDispatchOneUnreadWake.mockResolvedValue({ outcome: "sent" })+ const first = { messageId: "msg:one", botUserId: "bot" }+ const delimiterCollision = { messageId: "msg", botUserId: "one:bot" }++ const res = await handler.fetch!(makeRequest("POST", [first, delimiterCollision, first]), env)++ expect(res.status).toBe(202)+ expect(mockDispatchOneUnreadWake).toHaveBeenCalledTimes(2)+ expect(mockDispatchOneUnreadWake).toHaveBeenCalledWith({ __db: true }, env, first)+ expect(mockDispatchOneUnreadWake).toHaveBeenCalledWith({ __db: true }, env, delimiterCollision)+ })++ it("retries only a transiently failing candidate while a healthy sibling is dispatched once", async () => {+ const attempts = new Map<string, number>()+ mockDispatchOneUnreadWake.mockImplementation(async (_db: unknown, _env: unknown, item: { botUserId: string }) => {+ const attempt = (attempts.get(item.botUserId) ?? 0) + 1+ attempts.set(item.botUserId, attempt)+ if (item.botUserId === "bot_flaky" && attempt === 1) throw new Error("D1 exploded")+ return { outcome: "sent" }+ })const res = await handler.fetch!(makeRequest("POST", [- { messageId: "msg1", botUserId: "bot1" },- { messageId: "msg2", botUserId: "bot2" },+ { messageId: "msg1", botUserId: "bot_flaky" },+ { messageId: "msg1", botUserId: "bot_healthy" },]),env,)expect(res.status).toBe(202)- expect(mockDispatchOneUnreadWake).toHaveBeenCalledTimes(2)+ expect(attempts).toEqual(new Map([+ ["bot_flaky", 2],+ ["bot_healthy", 1],+ ]))+ })++ it("returns 207 with failed stable keys after bounded retries are exhausted without redispatching a healthy sibling", async () => {+ const attempts = new Map<string, number>()+ mockDispatchOneUnreadWake.mockImplementation(async (_db: unknown, _env: unknown, item: { botUserId: string }) => {+ const attempt = (attempts.get(item.botUserId) ?? 0) + 1+ attempts.set(item.botUserId, attempt)+ if (item.botUserId === "bot_flaky") throw new Error("D1 stayed down")+ return { outcome: "sent" }+ })++ const res = await handler.fetch!(+ makeRequest("POST", [+ { messageId: "msg1", botUserId: "bot_flaky" },+ { messageId: "msg1", botUserId: "bot_healthy" },+ ]),+ env,+ )++ expect(res.status).toBe(207)+ expect(await res.json()).toEqual({+ failed: [{ messageId: "msg1", botUserId: "bot_flaky" }],+ })+ expect(attempts).toEqual(new Map([+ ["bot_flaky", 3],+ ["bot_healthy", 1],+ ]))})it("returns 400 on invalid JSON body", async () => {@@ -199,12 +246,10 @@ describe("wake-worker dev-only HTTP entrypoint (fetch)", () => {* entrypoints so a future change that special-cases one of them (instead of* `resolveAndLog`) fails a test here, not just in production later.*- * What this table deliberately does NOT assert as equivalent — and never- * will, see plans/minimal-wake-queue-unread-notice.md — is- * retry-on-transient-failure: `queue()` retries via Cloudflare's- * backoff/DLQ; `fetch()` has no queue infra behind it, so a rejection is- * just logged. That's covered by the entrypoint-specific `describe` blocks- * above, not here.+ * Queue infrastructure and dev HTTP use different retry mechanisms, but+ * both must preserve sibling isolation and surface exhausted transient+ * failures instead of silently swallowing them. Entry-point-specific retry+ * details are covered by the `describe` blocks above.*/const RESOLUTION_SCENARIOS = [{ name: "sent", outcome: { outcome: "sent" as const } },@@ -235,7 +280,7 @@ describe("wake-worker queue() vs fetch() — same resolution for the same candidexpect(fetchCallArgs).toEqual(queueCallArgs)})- it("both entrypoints let a sibling's success/skip proceed when one candidate throws", async () => {+ it("both entrypoints let a sibling proceed when one candidate has a transient failure", async () => {const items = [{ messageId: "msg1", botUserId: "bot1" },{ messageId: "msg2", botUserId: "bot2" },@@ -248,9 +293,18 @@ describe("wake-worker queue() vs fetch() — same resolution for the same candidexpect(msgs[1]!.ack).toHaveBeenCalledTimes(1)vi.clearAllMocks()- mockDispatchOneUnreadWake.mockRejectedValueOnce(new Error("boom")).mockResolvedValueOnce({ outcome: "sent" })+ const attempts = new Map<string, number>()+ mockDispatchOneUnreadWake.mockImplementation(async (_db: unknown, _env: unknown, item: { botUserId: string }) => {+ const attempt = (attempts.get(item.botUserId) ?? 0) + 1+ attempts.set(item.botUserId, attempt)+ if (item.botUserId === "bot1" && attempt === 1) throw new Error("boom")+ return { outcome: "sent" }+ })const res = await handler.fetch!(new Request("http://internal/", { method: "POST", body: JSON.stringify(items) }), env)expect(res.status).toBe(202)- expect(mockDispatchOneUnreadWake).toHaveBeenCalledTimes(2)+ expect(attempts).toEqual(new Map([+ ["bot1", 2],+ ["bot2", 1],+ ]))})})diff --git a/src/wake-worker/src/index.ts b/src/wake-worker/src/index.tsindex f58541ab1..8606a3012 100644--- a/src/wake-worker/src/index.ts+++ b/src/wake-worker/src/index.ts@@ -2,6 +2,8 @@ import { createLogger, createDb, dispatchOneUnreadWake } from "@alook/shared"import type { WakePayload, Database } from "@alook/shared"const log = createLogger({ service: "wake-worker" })+const DEV_HTTP_MAX_ATTEMPTS = 3+const DEV_HTTP_RETRY_DELAYS_MS = [25, 100] as const/*** Resolve ONE wake candidate via `dispatchOneUnreadWake` and log its@@ -9,8 +11,8 @@ const log = createLogger({ service: "wake-worker" })* traffic) and `fetch()` (dev-only stand-in for the local Cloudflare Queue,* see its doc comment) — so the log lines and interpretation logic exist* exactly once regardless of which entrypoint received the item. Throws- * propagate untouched; each caller decides retry (queue) vs best-effort- * (dev HTTP shim) on its own.+ * propagate untouched; each caller applies its own retry contract (Queue+ * retry/DLQ versus bounded in-process retries for the dev HTTP shim).*/async function resolveAndLog(db: Database, env: Env, item: WakePayload) {const result = await dispatchOneUnreadWake(db, env, item)@@ -26,6 +28,55 @@ async function resolveAndLog(db: Database, env: Env, item: WakePayload) {return result}+function dedupeWakePayloads(payloads: WakePayload[]): WakePayload[] {+ const seen = new Map<string, Set<string>>()+ return payloads.filter((item) => {+ let botIds = seen.get(item.messageId)+ if (!botIds) {+ botIds = new Set<string>()+ seen.set(item.messageId, botIds)+ }+ if (botIds.has(item.botUserId)) return false+ botIds.add(item.botUserId)+ return true+ })+}++async function waitForDevHttpRetry(delayMs: number): Promise<void> {+ await new Promise<void>((resolve) => setTimeout(resolve, delayMs))+}++async function resolveDevHttpCandidate(db: Database, env: Env, item: WakePayload): Promise<boolean> {+ for (let attempt = 1; attempt <= DEV_HTTP_MAX_ATTEMPTS; attempt++) {+ try {+ await resolveAndLog(db, env, item)+ return true+ } catch (err) {+ if (attempt === DEV_HTTP_MAX_ATTEMPTS) {+ log.warn("dev_http_wake_dispatch_exhausted", {+ botUserId: item.botUserId,+ messageId: item.messageId,+ attempts: attempt,+ err: String(err),+ })+ return false+ }++ const delayMs = DEV_HTTP_RETRY_DELAYS_MS[attempt - 1]!+ log.warn("dev_http_wake_dispatch_retrying", {+ botUserId: item.botUserId,+ messageId: item.messageId,+ attempt,+ delayMs,+ err: String(err),+ })+ await waitForDevHttpRetry(delayMs)+ }+ }++ return false+}+export default {/*** `alook-wake-worker` — Cloudflare Queue consumer for `alook-wake`@@ -68,8 +119,11 @@ export default {* `resolveAndLog`/`dispatchOneUnreadWake` real orchestration `queue()`* does, including the real D1 read and the real forward to `alook-ws-do`* — this is the actual worker process handling actual wake candidates,- * not a simulation of it. Best-effort: no queue infra backs this, so one- * candidate's failure is logged but never blocks siblings or the response.+ * not a simulation of it. There is no durable queue behind this dev path:+ * candidates get bounded in-process retries and exhausted stable keys are+ * returned visibly as a 207 partial result. A partial result is deliberately+ * not a 5xx because the caller's binding/HTTP fallback would otherwise+ * replay the full batch and duplicate already-successful siblings.*/async fetch(request: Request, env: Env): Promise<Response> {const url = new URL(request.url)@@ -87,13 +141,13 @@ export default {}const db = createDb(env.DB)- const results = await Promise.allSettled(payloads.map((p) => resolveAndLog(db, env, p)))- for (let i = 0; i < results.length; i++) {- const r = results[i]!- if (r.status === "rejected") {- log.warn("dev_http_wake_dispatch_failed", { botUserId: payloads[i]!.botUserId, messageId: payloads[i]!.messageId, err: String(r.reason) })- }- }+ const uniquePayloads = dedupeWakePayloads(payloads)+ const results = await Promise.all(+ uniquePayloads.map(async (item) => ({ item, resolved: await resolveDevHttpCandidate(db, env, item) })),+ )+ const failed = results.filter((result) => !result.resolved).map((result) => result.item)+ if (failed.length > 0) return Response.json({ failed }, { status: 207 })+return new Response(null, { status: 202 })},}diff --git a/src/web/src/lib/community/wake-producer.test.ts b/src/web/src/lib/community/wake-producer.test.tsindex 1decbc1bc..7db6f7d0a 100644--- a/src/web/src/lib/community/wake-producer.test.ts+++ b/src/web/src/lib/community/wake-producer.test.ts@@ -206,9 +206,10 @@ describe("enqueueBotWakes", () => {expect(mockQueueSend).toHaveBeenCalledTimes(1)})- it("drops (does NOT throw or collapse the batch) when a single candidate's gate check rejects", async () => {+ it("retains a candidate when its producer prefilter rejects without collapsing the batch", async () => {// Regression guard: a transient D1 blip on ONE candidate's gate check- // must not wipe every wake for the message — `allSettled`, not `all`.+ // must not permanently lose that candidate. The consumer performs the+ // authoritative access check and will retry transient failures there.mockFindWakeCandidates.mockResolvedValue([{ botUserId: "bot_ok_1", name: "a", machineId: "m1", runtime: "claude" },{ botUserId: "bot_flaky", name: "b", machineId: "m2", runtime: "codex" },@@ -229,6 +230,7 @@ describe("enqueueBotWakes", () => {const [payloads] = mockQueueSend.mock.calls[0]!expect(payloads).toEqual([{ messageId: "msg_1", botUserId: "bot_ok_1" },+ { messageId: "msg_1", botUserId: "bot_flaky" },{ messageId: "msg_1", botUserId: "bot_ok_2" },])})diff --git a/src/web/src/lib/community/wake-producer.ts b/src/web/src/lib/community/wake-producer.tsindex f24878889..232507aaa 100644--- a/src/web/src/lib/community/wake-producer.ts+++ b/src/web/src/lib/community/wake-producer.ts@@ -113,9 +113,9 @@ async function doEnqueueBotWakes(env: Env, opts: EnqueueBotWakesOpts): Promise<v//// `allSettled` (not `all`): a transient D1 blip on ONE candidate's gate// check must not collapse the whole batch's enqueue. Rejected legs are- // treated as "gate indeterminate" — we drop just that candidate (the- // queue consumer re-runs the same gate at consume time anyway) rather- // than losing every wake for the message.+ // treated as "gate indeterminate" and retained because this check is only+ // a producer-side prefilter. The consumer re-runs the same gate+ // authoritatively and fails closed/retries if that check is unavailable.const scope: { channelId: string } = { channelId }const gateResults = await Promise.allSettled(candidates.map((c) => queries.communityMember.canBotReadWakeScope(db, c.botUserId, scope))@@ -124,7 +124,7 @@ async function doEnqueueBotWakes(env: Env, opts: EnqueueBotWakesOpts): Promise<vconst r = gateResults[i]!if (r.status === "rejected") {log.warn("wake_gate_check_failed", { botUserId: c.botUserId, err: String(r.reason) })- return false+ return true}return r.value})diff --git a/src/web/src/lib/community/wake-transport.test.ts b/src/web/src/lib/community/wake-transport.test.tsindex 641a04da3..5481e3a9d 100644--- a/src/web/src/lib/community/wake-transport.test.ts+++ b/src/web/src/lib/community/wake-transport.test.ts@@ -99,6 +99,49 @@ describe("createDevHttpWakeTransport", () => {expect(mockGlobalFetch).toHaveBeenCalledTimes(1)})+ it("does not replay a processed batch through HTTP fallback when the binding reports partial candidate exhaustion", async () => {+ const bindingFetch = vi.fn(async () => Response.json(+ { failed: [{ messageId: "msg_1", botUserId: "bot1" }] },+ { status: 207 },+ ))+ mockGlobalFetch.mockResolvedValue(new Response(null, { status: 202 }))+ const transport = createDevHttpWakeTransport(makeEnv(bindingFetch))++ await expect(transport.send(payloads)).rejects.toThrow(/partial.*1 candidate/i)++ expect(bindingFetch).toHaveBeenCalledTimes(1)+ expect(mockGlobalFetch).not.toHaveBeenCalled()+ })++ it.each([+ { name: "non-JSON body", body: "not json" },+ { name: "missing botUserId", body: JSON.stringify({ failed: [{ messageId: "msg_1" }] }) },+ { name: "unexpected top-level field", body: JSON.stringify({ failed: [], extra: true }) },+ { name: "empty failed list", body: JSON.stringify({ failed: [] }) },+ { name: "candidate outside the sent batch", body: JSON.stringify({ failed: [{ messageId: "other", botUserId: "bot1" }] }) },+ { name: "duplicate failed tuple", body: JSON.stringify({ failed: [payloads[0], payloads[0]] }) },+ ])("fails closed without fallback for an invalid 207 response: $name", async ({ body }) => {+ const bindingFetch = vi.fn(async () => new Response(body, { status: 207 }))+ mockGlobalFetch.mockResolvedValue(new Response(null, { status: 202 }))+ const transport = createDevHttpWakeTransport(makeEnv(bindingFetch))++ await expect(transport.send(payloads)).rejects.toThrow(/invalid partial response/i)++ expect(bindingFetch).toHaveBeenCalledTimes(1)+ expect(mockGlobalFetch).not.toHaveBeenCalled()+ })++ it("still falls back to HTTP when the binding returns a worker-level 5xx", async () => {+ const bindingFetch = vi.fn(async () => new Response("worker unavailable", { status: 503 }))+ mockGlobalFetch.mockResolvedValue(new Response(null, { status: 202 }))+ const transport = createDevHttpWakeTransport(makeEnv(bindingFetch))++ await expect(transport.send(payloads)).resolves.toBeUndefined()++ expect(bindingFetch).toHaveBeenCalledTimes(1)+ expect(mockGlobalFetch).toHaveBeenCalledTimes(1)+ })+it("throws when both the binding and the HTTP fallback respond non-OK (caller logs, dev-only best effort)", async () => {const bindingFetch = vi.fn(async () => new Response("boom", { status: 500 }))mockGlobalFetch.mockResolvedValue(new Response("still bad", { status: 500 }))diff --git a/src/web/src/lib/community/wake-transport.ts b/src/web/src/lib/community/wake-transport.tsindex 832f82a1d..85cac9938 100644--- a/src/web/src/lib/community/wake-transport.ts+++ b/src/web/src/lib/community/wake-transport.ts@@ -4,6 +4,50 @@ import { fetchViaBindingOrDevFallback } from "../dev-binding-fetch"const log = createLogger({ service: "wake-transport" })+function isExactFailedCandidate(value: unknown): value is WakePayload {+ if (typeof value !== "object" || value === null || Array.isArray(value)) return false+ const keys = Object.keys(value)+ if (keys.length !== 2 || !keys.includes("messageId") || !keys.includes("botUserId")) return false+ const candidate = value as Record<string, unknown>+ return typeof candidate.messageId === "string"+ && candidate.messageId.length > 0+ && typeof candidate.botUserId === "string"+ && candidate.botUserId.length > 0+}++function parsePartialFailure(value: unknown, payloads: WakePayload[]): WakePayload[] | null {+ if (typeof value !== "object" || value === null || Array.isArray(value)) return null+ const keys = Object.keys(value)+ if (keys.length !== 1 || keys[0] !== "failed") return null++ const failed = (value as Record<string, unknown>).failed+ if (!Array.isArray(failed) || failed.length === 0 || !failed.every(isExactFailedCandidate)) return null++ const allowed = new Map<string, Set<string>>()+ for (const item of payloads) {+ let botIds = allowed.get(item.messageId)+ if (!botIds) {+ botIds = new Set<string>()+ allowed.set(item.messageId, botIds)+ }+ botIds.add(item.botUserId)+ }++ const seen = new Map<string, Set<string>>()+ for (const item of failed) {+ if (!allowed.get(item.messageId)?.has(item.botUserId)) return null+ let botIds = seen.get(item.messageId)+ if (!botIds) {+ botIds = new Set<string>()+ seen.set(item.messageId, botIds)+ }+ if (botIds.has(item.botUserId)) return null+ botIds.add(item.botUserId)+ }++ return failed+}+/*** How `enqueueBotWakes` hands a batch of wake candidates off to whatever* will actually resolve them. Exactly two implementations below — neither@@ -54,6 +98,17 @@ export function createDevHttpWakeTransport(env: Env): WakeTransport {},{ logPrefix: "wake_transport", log, label: `${payloads.length}_candidates` },)+ if (res.status === 207) {+ let body: unknown+ try {+ body = await res.json()+ } catch {+ throw new Error("dev wake transport: invalid partial response")+ }+ const failed = parsePartialFailure(body, payloads)+ if (!failed) throw new Error("dev wake transport: invalid partial response")+ throw new Error(`dev wake transport partial failure: ${failed.length} candidate(s) exhausted`)+ }if (!res.ok) {throw new Error(`dev wake transport: alook-wake-worker responded ${res.status}`)}