Skip to content

armgit / alook

Public Imported from GitHub
0

Multi-Agent Collaboration for Team and Friends

fix(wake): retry transient recipient failures

Gus622ba8dd3a2b
diff --git a/src/wake-worker/src/index.test.ts b/src/wake-worker/src/index.test.ts
index ffe680877..306718092 100644
--- a/src/wake-worker/src/index.test.ts
+++ b/src/wake-worker/src/index.test.ts
@@ -150,21 +150,68 @@ describe("wake-worker dev-only HTTP entrypoint (fetch)", () => {
expect(res.status).toBe(202)
})
- it("still returns 202 when one candidate rejects — siblings are unaffected, failure is only logged", async () => {
- mockDispatchOneUnreadWake
- .mockResolvedValueOnce({ outcome: "sent" })
- .mockRejectedValueOnce(new Error("D1 exploded"))
+ it("deduplicates candidates by stable messageId + botUserId within one request", async () => {
+ mockDispatchOneUnreadWake.mockResolvedValue({ outcome: "sent" })
+ const first = { messageId: "msg:one", botUserId: "bot" }
+ const delimiterCollision = { messageId: "msg", botUserId: "one:bot" }
+
+ const res = await handler.fetch!(makeRequest("POST", [first, delimiterCollision, first]), env)
+
+ expect(res.status).toBe(202)
+ expect(mockDispatchOneUnreadWake).toHaveBeenCalledTimes(2)
+ expect(mockDispatchOneUnreadWake).toHaveBeenCalledWith({ __db: true }, env, first)
+ expect(mockDispatchOneUnreadWake).toHaveBeenCalledWith({ __db: true }, env, delimiterCollision)
+ })
+
+ it("retries only a transiently failing candidate while a healthy sibling is dispatched once", async () => {
+ const attempts = new Map<string, number>()
+ mockDispatchOneUnreadWake.mockImplementation(async (_db: unknown, _env: unknown, item: { botUserId: string }) => {
+ const attempt = (attempts.get(item.botUserId) ?? 0) + 1
+ attempts.set(item.botUserId, attempt)
+ if (item.botUserId === "bot_flaky" && attempt === 1) throw new Error("D1 exploded")
+ return { outcome: "sent" }
+ })
const res = await handler.fetch!(
makeRequest("POST", [
- { messageId: "msg1", botUserId: "bot1" },
- { messageId: "msg2", botUserId: "bot2" },
+ { messageId: "msg1", botUserId: "bot_flaky" },
+ { messageId: "msg1", botUserId: "bot_healthy" },
]),
env,
)
expect(res.status).toBe(202)
- expect(mockDispatchOneUnreadWake).toHaveBeenCalledTimes(2)
+ expect(attempts).toEqual(new Map([
+ ["bot_flaky", 2],
+ ["bot_healthy", 1],
+ ]))
+ })
+
+ it("returns 207 with failed stable keys after bounded retries are exhausted without redispatching a healthy sibling", async () => {
+ const attempts = new Map<string, number>()
+ mockDispatchOneUnreadWake.mockImplementation(async (_db: unknown, _env: unknown, item: { botUserId: string }) => {
+ const attempt = (attempts.get(item.botUserId) ?? 0) + 1
+ attempts.set(item.botUserId, attempt)
+ if (item.botUserId === "bot_flaky") throw new Error("D1 stayed down")
+ return { outcome: "sent" }
+ })
+
+ const res = await handler.fetch!(
+ makeRequest("POST", [
+ { messageId: "msg1", botUserId: "bot_flaky" },
+ { messageId: "msg1", botUserId: "bot_healthy" },
+ ]),
+ env,
+ )
+
+ expect(res.status).toBe(207)
+ expect(await res.json()).toEqual({
+ failed: [{ messageId: "msg1", botUserId: "bot_flaky" }],
+ })
+ expect(attempts).toEqual(new Map([
+ ["bot_flaky", 3],
+ ["bot_healthy", 1],
+ ]))
})
it("returns 400 on invalid JSON body", async () => {
@@ -199,12 +246,10 @@ describe("wake-worker dev-only HTTP entrypoint (fetch)", () => {
* entrypoints so a future change that special-cases one of them (instead of
* `resolveAndLog`) fails a test here, not just in production later.
*
- * What this table deliberately does NOT assert as equivalent — and never
- * will, see plans/minimal-wake-queue-unread-notice.md — is
- * retry-on-transient-failure: `queue()` retries via Cloudflare's
- * backoff/DLQ; `fetch()` has no queue infra behind it, so a rejection is
- * just logged. That's covered by the entrypoint-specific `describe` blocks
- * above, not here.
+ * Queue infrastructure and dev HTTP use different retry mechanisms, but
+ * both must preserve sibling isolation and surface exhausted transient
+ * failures instead of silently swallowing them. Entry-point-specific retry
+ * details are covered by the `describe` blocks above.
*/
const RESOLUTION_SCENARIOS = [
{ name: "sent", outcome: { outcome: "sent" as const } },
@@ -235,7 +280,7 @@ describe("wake-worker queue() vs fetch() — same resolution for the same candid
expect(fetchCallArgs).toEqual(queueCallArgs)
})
- it("both entrypoints let a sibling's success/skip proceed when one candidate throws", async () => {
+ it("both entrypoints let a sibling proceed when one candidate has a transient failure", async () => {
const items = [
{ messageId: "msg1", botUserId: "bot1" },
{ messageId: "msg2", botUserId: "bot2" },
@@ -248,9 +293,18 @@ describe("wake-worker queue() vs fetch() — same resolution for the same candid
expect(msgs[1]!.ack).toHaveBeenCalledTimes(1)
vi.clearAllMocks()
- mockDispatchOneUnreadWake.mockRejectedValueOnce(new Error("boom")).mockResolvedValueOnce({ outcome: "sent" })
+ const attempts = new Map<string, number>()
+ mockDispatchOneUnreadWake.mockImplementation(async (_db: unknown, _env: unknown, item: { botUserId: string }) => {
+ const attempt = (attempts.get(item.botUserId) ?? 0) + 1
+ attempts.set(item.botUserId, attempt)
+ if (item.botUserId === "bot1" && attempt === 1) throw new Error("boom")
+ return { outcome: "sent" }
+ })
const res = await handler.fetch!(new Request("http://internal/", { method: "POST", body: JSON.stringify(items) }), env)
expect(res.status).toBe(202)
- expect(mockDispatchOneUnreadWake).toHaveBeenCalledTimes(2)
+ expect(attempts).toEqual(new Map([
+ ["bot1", 2],
+ ["bot2", 1],
+ ]))
})
})
diff --git a/src/wake-worker/src/index.ts b/src/wake-worker/src/index.ts
index f58541ab1..8606a3012 100644
--- a/src/wake-worker/src/index.ts
+++ b/src/wake-worker/src/index.ts
@@ -2,6 +2,8 @@ import { createLogger, createDb, dispatchOneUnreadWake } from "@alook/shared"
import type { WakePayload, Database } from "@alook/shared"
const log = createLogger({ service: "wake-worker" })
+const DEV_HTTP_MAX_ATTEMPTS = 3
+const DEV_HTTP_RETRY_DELAYS_MS = [25, 100] as const
/**
* Resolve ONE wake candidate via `dispatchOneUnreadWake` and log its
@@ -9,8 +11,8 @@ const log = createLogger({ service: "wake-worker" })
* traffic) and `fetch()` (dev-only stand-in for the local Cloudflare Queue,
* see its doc comment) — so the log lines and interpretation logic exist
* exactly once regardless of which entrypoint received the item. Throws
- * propagate untouched; each caller decides retry (queue) vs best-effort
- * (dev HTTP shim) on its own.
+ * propagate untouched; each caller applies its own retry contract (Queue
+ * retry/DLQ versus bounded in-process retries for the dev HTTP shim).
*/
async function resolveAndLog(db: Database, env: Env, item: WakePayload) {
const result = await dispatchOneUnreadWake(db, env, item)
@@ -26,6 +28,55 @@ async function resolveAndLog(db: Database, env: Env, item: WakePayload) {
return result
}
+function dedupeWakePayloads(payloads: WakePayload[]): WakePayload[] {
+ const seen = new Map<string, Set<string>>()
+ return payloads.filter((item) => {
+ let botIds = seen.get(item.messageId)
+ if (!botIds) {
+ botIds = new Set<string>()
+ seen.set(item.messageId, botIds)
+ }
+ if (botIds.has(item.botUserId)) return false
+ botIds.add(item.botUserId)
+ return true
+ })
+}
+
+async function waitForDevHttpRetry(delayMs: number): Promise<void> {
+ await new Promise<void>((resolve) => setTimeout(resolve, delayMs))
+}
+
+async function resolveDevHttpCandidate(db: Database, env: Env, item: WakePayload): Promise<boolean> {
+ for (let attempt = 1; attempt <= DEV_HTTP_MAX_ATTEMPTS; attempt++) {
+ try {
+ await resolveAndLog(db, env, item)
+ return true
+ } catch (err) {
+ if (attempt === DEV_HTTP_MAX_ATTEMPTS) {
+ log.warn("dev_http_wake_dispatch_exhausted", {
+ botUserId: item.botUserId,
+ messageId: item.messageId,
+ attempts: attempt,
+ err: String(err),
+ })
+ return false
+ }
+
+ const delayMs = DEV_HTTP_RETRY_DELAYS_MS[attempt - 1]!
+ log.warn("dev_http_wake_dispatch_retrying", {
+ botUserId: item.botUserId,
+ messageId: item.messageId,
+ attempt,
+ delayMs,
+ err: String(err),
+ })
+ await waitForDevHttpRetry(delayMs)
+ }
+ }
+
+ return false
+}
+
export default {
/**
* `alook-wake-worker` — Cloudflare Queue consumer for `alook-wake`
@@ -68,8 +119,11 @@ export default {
* `resolveAndLog`/`dispatchOneUnreadWake` real orchestration `queue()`
* does, including the real D1 read and the real forward to `alook-ws-do`
* — this is the actual worker process handling actual wake candidates,
- * not a simulation of it. Best-effort: no queue infra backs this, so one
- * candidate's failure is logged but never blocks siblings or the response.
+ * not a simulation of it. There is no durable queue behind this dev path:
+ * candidates get bounded in-process retries and exhausted stable keys are
+ * returned visibly as a 207 partial result. A partial result is deliberately
+ * not a 5xx because the caller's binding/HTTP fallback would otherwise
+ * replay the full batch and duplicate already-successful siblings.
*/
async fetch(request: Request, env: Env): Promise<Response> {
const url = new URL(request.url)
@@ -87,13 +141,13 @@ export default {
}
const db = createDb(env.DB)
- const results = await Promise.allSettled(payloads.map((p) => resolveAndLog(db, env, p)))
- for (let i = 0; i < results.length; i++) {
- const r = results[i]!
- if (r.status === "rejected") {
- log.warn("dev_http_wake_dispatch_failed", { botUserId: payloads[i]!.botUserId, messageId: payloads[i]!.messageId, err: String(r.reason) })
- }
- }
+ const uniquePayloads = dedupeWakePayloads(payloads)
+ const results = await Promise.all(
+ uniquePayloads.map(async (item) => ({ item, resolved: await resolveDevHttpCandidate(db, env, item) })),
+ )
+ const failed = results.filter((result) => !result.resolved).map((result) => result.item)
+ if (failed.length > 0) return Response.json({ failed }, { status: 207 })
+
return new Response(null, { status: 202 })
},
}
diff --git a/src/web/src/lib/community/wake-producer.test.ts b/src/web/src/lib/community/wake-producer.test.ts
index 1decbc1bc..7db6f7d0a 100644
--- a/src/web/src/lib/community/wake-producer.test.ts
+++ b/src/web/src/lib/community/wake-producer.test.ts
@@ -206,9 +206,10 @@ describe("enqueueBotWakes", () => {
expect(mockQueueSend).toHaveBeenCalledTimes(1)
})
- it("drops (does NOT throw or collapse the batch) when a single candidate's gate check rejects", async () => {
+ it("retains a candidate when its producer prefilter rejects without collapsing the batch", async () => {
// Regression guard: a transient D1 blip on ONE candidate's gate check
- // must not wipe every wake for the message — `allSettled`, not `all`.
+ // must not permanently lose that candidate. The consumer performs the
+ // authoritative access check and will retry transient failures there.
mockFindWakeCandidates.mockResolvedValue([
{ botUserId: "bot_ok_1", name: "a", machineId: "m1", runtime: "claude" },
{ botUserId: "bot_flaky", name: "b", machineId: "m2", runtime: "codex" },
@@ -229,6 +230,7 @@ describe("enqueueBotWakes", () => {
const [payloads] = mockQueueSend.mock.calls[0]!
expect(payloads).toEqual([
{ messageId: "msg_1", botUserId: "bot_ok_1" },
+ { messageId: "msg_1", botUserId: "bot_flaky" },
{ messageId: "msg_1", botUserId: "bot_ok_2" },
])
})
diff --git a/src/web/src/lib/community/wake-producer.ts b/src/web/src/lib/community/wake-producer.ts
index f24878889..232507aaa 100644
--- a/src/web/src/lib/community/wake-producer.ts
+++ b/src/web/src/lib/community/wake-producer.ts
@@ -113,9 +113,9 @@ async function doEnqueueBotWakes(env: Env, opts: EnqueueBotWakesOpts): Promise<v
//
// `allSettled` (not `all`): a transient D1 blip on ONE candidate's gate
// check must not collapse the whole batch's enqueue. Rejected legs are
- // treated as "gate indeterminate" — we drop just that candidate (the
- // queue consumer re-runs the same gate at consume time anyway) rather
- // than losing every wake for the message.
+ // treated as "gate indeterminate" and retained because this check is only
+ // a producer-side prefilter. The consumer re-runs the same gate
+ // authoritatively and fails closed/retries if that check is unavailable.
const scope: { channelId: string } = { channelId }
const gateResults = await Promise.allSettled(
candidates.map((c) => queries.communityMember.canBotReadWakeScope(db, c.botUserId, scope))
@@ -124,7 +124,7 @@ async function doEnqueueBotWakes(env: Env, opts: EnqueueBotWakesOpts): Promise<v
const r = gateResults[i]!
if (r.status === "rejected") {
log.warn("wake_gate_check_failed", { botUserId: c.botUserId, err: String(r.reason) })
- return false
+ return true
}
return r.value
})
diff --git a/src/web/src/lib/community/wake-transport.test.ts b/src/web/src/lib/community/wake-transport.test.ts
index 641a04da3..5481e3a9d 100644
--- a/src/web/src/lib/community/wake-transport.test.ts
+++ b/src/web/src/lib/community/wake-transport.test.ts
@@ -99,6 +99,49 @@ describe("createDevHttpWakeTransport", () => {
expect(mockGlobalFetch).toHaveBeenCalledTimes(1)
})
+ it("does not replay a processed batch through HTTP fallback when the binding reports partial candidate exhaustion", async () => {
+ const bindingFetch = vi.fn(async () => Response.json(
+ { failed: [{ messageId: "msg_1", botUserId: "bot1" }] },
+ { status: 207 },
+ ))
+ mockGlobalFetch.mockResolvedValue(new Response(null, { status: 202 }))
+ const transport = createDevHttpWakeTransport(makeEnv(bindingFetch))
+
+ await expect(transport.send(payloads)).rejects.toThrow(/partial.*1 candidate/i)
+
+ expect(bindingFetch).toHaveBeenCalledTimes(1)
+ expect(mockGlobalFetch).not.toHaveBeenCalled()
+ })
+
+ it.each([
+ { name: "non-JSON body", body: "not json" },
+ { name: "missing botUserId", body: JSON.stringify({ failed: [{ messageId: "msg_1" }] }) },
+ { name: "unexpected top-level field", body: JSON.stringify({ failed: [], extra: true }) },
+ { name: "empty failed list", body: JSON.stringify({ failed: [] }) },
+ { name: "candidate outside the sent batch", body: JSON.stringify({ failed: [{ messageId: "other", botUserId: "bot1" }] }) },
+ { name: "duplicate failed tuple", body: JSON.stringify({ failed: [payloads[0], payloads[0]] }) },
+ ])("fails closed without fallback for an invalid 207 response: $name", async ({ body }) => {
+ const bindingFetch = vi.fn(async () => new Response(body, { status: 207 }))
+ mockGlobalFetch.mockResolvedValue(new Response(null, { status: 202 }))
+ const transport = createDevHttpWakeTransport(makeEnv(bindingFetch))
+
+ await expect(transport.send(payloads)).rejects.toThrow(/invalid partial response/i)
+
+ expect(bindingFetch).toHaveBeenCalledTimes(1)
+ expect(mockGlobalFetch).not.toHaveBeenCalled()
+ })
+
+ it("still falls back to HTTP when the binding returns a worker-level 5xx", async () => {
+ const bindingFetch = vi.fn(async () => new Response("worker unavailable", { status: 503 }))
+ mockGlobalFetch.mockResolvedValue(new Response(null, { status: 202 }))
+ const transport = createDevHttpWakeTransport(makeEnv(bindingFetch))
+
+ await expect(transport.send(payloads)).resolves.toBeUndefined()
+
+ expect(bindingFetch).toHaveBeenCalledTimes(1)
+ expect(mockGlobalFetch).toHaveBeenCalledTimes(1)
+ })
+
it("throws when both the binding and the HTTP fallback respond non-OK (caller logs, dev-only best effort)", async () => {
const bindingFetch = vi.fn(async () => new Response("boom", { status: 500 }))
mockGlobalFetch.mockResolvedValue(new Response("still bad", { status: 500 }))
diff --git a/src/web/src/lib/community/wake-transport.ts b/src/web/src/lib/community/wake-transport.ts
index 832f82a1d..85cac9938 100644
--- a/src/web/src/lib/community/wake-transport.ts
+++ b/src/web/src/lib/community/wake-transport.ts
@@ -4,6 +4,50 @@ import { fetchViaBindingOrDevFallback } from "../dev-binding-fetch"
const log = createLogger({ service: "wake-transport" })
+function isExactFailedCandidate(value: unknown): value is WakePayload {
+ if (typeof value !== "object" || value === null || Array.isArray(value)) return false
+ const keys = Object.keys(value)
+ if (keys.length !== 2 || !keys.includes("messageId") || !keys.includes("botUserId")) return false
+ const candidate = value as Record<string, unknown>
+ return typeof candidate.messageId === "string"
+ && candidate.messageId.length > 0
+ && typeof candidate.botUserId === "string"
+ && candidate.botUserId.length > 0
+}
+
+function parsePartialFailure(value: unknown, payloads: WakePayload[]): WakePayload[] | null {
+ if (typeof value !== "object" || value === null || Array.isArray(value)) return null
+ const keys = Object.keys(value)
+ if (keys.length !== 1 || keys[0] !== "failed") return null
+
+ const failed = (value as Record<string, unknown>).failed
+ if (!Array.isArray(failed) || failed.length === 0 || !failed.every(isExactFailedCandidate)) return null
+
+ const allowed = new Map<string, Set<string>>()
+ for (const item of payloads) {
+ let botIds = allowed.get(item.messageId)
+ if (!botIds) {
+ botIds = new Set<string>()
+ allowed.set(item.messageId, botIds)
+ }
+ botIds.add(item.botUserId)
+ }
+
+ const seen = new Map<string, Set<string>>()
+ for (const item of failed) {
+ if (!allowed.get(item.messageId)?.has(item.botUserId)) return null
+ let botIds = seen.get(item.messageId)
+ if (!botIds) {
+ botIds = new Set<string>()
+ seen.set(item.messageId, botIds)
+ }
+ if (botIds.has(item.botUserId)) return null
+ botIds.add(item.botUserId)
+ }
+
+ return failed
+}
+
/**
* How `enqueueBotWakes` hands a batch of wake candidates off to whatever
* will actually resolve them. Exactly two implementations below — neither
@@ -54,6 +98,17 @@ export function createDevHttpWakeTransport(env: Env): WakeTransport {
},
{ logPrefix: "wake_transport", log, label: `${payloads.length}_candidates` },
)
+ if (res.status === 207) {
+ let body: unknown
+ try {
+ body = await res.json()
+ } catch {
+ throw new Error("dev wake transport: invalid partial response")
+ }
+ const failed = parsePartialFailure(body, payloads)
+ if (!failed) throw new Error("dev wake transport: invalid partial response")
+ throw new Error(`dev wake transport partial failure: ${failed.length} candidate(s) exhausted`)
+ }
if (!res.ok) {
throw new Error(`dev wake transport: alook-wake-worker responded ${res.status}`)
}