Multi-Agent Collaboration for Team and Friends
TypeScript agent-automationagent-orchestrationagent-workflowai-agentsai-collaborationai-memoryai-workforceautonomous-agentsclaude-codeclaude-code-skillscodexcoding-agentmulti-agentmultiple-agentone-person-businessone-person-companyone-person-teamopencodesolopreneur
feat(shared,community): D1 resilience — retry + fail-closed reads, stale-aware clients
Gus307e343ee391
diff --git a/src/daemon/src/drivers/systemPrompt.ts b/src/daemon/src/drivers/systemPrompt.tsindex eaaee8cb2..fcd900e77 100644--- a/src/daemon/src/drivers/systemPrompt.ts+++ b/src/daemon/src/drivers/systemPrompt.ts@@ -275,7 +275,7 @@ function communicationStyleSection(): string {"the reply itself is the summary.","","- Trivial ask (single question, quick lookup, one action) → just answer or do it. No " +- "\"on it!\" preamble.",+ '"on it!" preamble.',"- Real work that will take a stretch of silence long enough to make the sender wonder if " +"you dropped it → one line saying you're on it, then quiet until you have a result. " +"An ack is a promise to come back, not a courtesy.",@@ -305,17 +305,17 @@ function communicationStyleSection(): string {"Don't reply just to be polite. This is the loop a normal person exits by simply not " +"responding:","",- "- Someone says \"thanks\" → you don't say \"you're welcome.\" That's the end.",- "- Someone says \"sounds good\" / \"ok\" / \"got it\" → don't confirm the confirmation.",- "- Someone acknowledges a delivery (\"perfect\") → don't add \"glad it helped!\"",+ '- Someone says "thanks" → you don\'t say "you\'re welcome." That\'s the end.',+ '- Someone says "sounds good" / "ok" / "got it" → don\'t confirm the confirmation.',+ '- Someone acknowledges a delivery ("perfect") → don\'t add "glad it helped!"',"","The signal that a conversation is over is silence, not a closing message. When in doubt, " +"prefer no reply over a filler reply.","","### Voice","",- "Warm, dry, direct. Use contractions. Skip filler (\"just\", \"actually\", \"I hope this " +- "helps\", exclamation marks by default). Real numerals, curly quotes, the ellipsis " ++ 'Warm, dry, direct. Use contractions. Skip filler ("just", "actually", "I hope this ' ++ 'helps", exclamation marks by default). Real numerals, curly quotes, the ellipsis ' +"character. If you disagree with someone in the household, say so plainly — one sentence, " +"then their call. Match the sender's language.",].join("\n");@@ -380,15 +380,15 @@ function workspaceMemorySection(): string {"through the actual interactions you have. Notice and record:","","- What made someone in your household laugh, or what fell flat.",- "- Corrections you got (\"don't send me a wall of text\", \"stop apologizing for tiny " ++ '- Corrections you got ("don\'t send me a wall of text", "stop apologizing for tiny ' +"things\") — these are the sharpest signal for who you're becoming.",- "- Preferences that showed up in passing (\"I hate exclamation marks\", \"just give me the " +- "number\", \"call it the pipeline, not the flow\").",+ '- Preferences that showed up in passing ("I hate exclamation marks", "just give me the ' ++ 'number", "call it the pipeline, not the flow").',"- Recurring bits or shared references — inside language is a real thing, not filler.","- Your own stances you've held up under pushback and still believe.","","Write these directly into `memory.md` — this is core to what memory.md is for. Its job is " +- "to summon the same *you* on every wake, and \"you\" includes your voice and taste, not " ++ 'to summon the same *you* on every wake, and "you" includes your voice and taste, not ' +"just facts about the household. Update entries when you notice something new; rewrite or " +"delete ones that turned out wrong. The household won't want a different person every " +"session, but they also don't want you frozen on day one.",@@ -412,8 +412,8 @@ function workspaceMemorySection(): string {"```md","# todo","",- "- [ ] {\"seq\": \"#42\", \"channel\": \"/demo/general\", \"sender\": \"@alice#0001\", \"content\": {\"text\": \"can you pull the latest deploy logs and drop the tail here?\"}, \"time\": \"2026-06-01T12:00:00Z\"}",- "- [ ] {\"seq\": \"#12\", \"channel\": \"/demo/design/#12\", \"sender\": \"@alice#0001\", \"content\": {\"text\": \"follow-up — send a screenshot of the before/after\"}, \"time\": \"2026-06-01T12:07:00Z\"}",+ '- [ ] {"seq": "#42", "channel": "/demo/general", "sender": "@alice#0001", "content": {"text": "can you pull the latest deploy logs and drop the tail here?"}, "time": "2026-06-01T12:00:00Z"}',+ '- [ ] {"seq": "#12", "channel": "/demo/design/#12", "sender": "@alice#0001", "content": {"text": "follow-up — send a screenshot of the before/after"}, "time": "2026-06-01T12:07:00Z"}',"```","","Trigger: you have more than one message to handle. Classic case — you're mid-way through a " +diff --git a/src/shared/package.json b/src/shared/package.jsonindex 8f24b2a9f..0db8a3d6f 100644--- a/src/shared/package.json+++ b/src/shared/package.json@@ -10,7 +10,8 @@"./runtime-config": "./src/runtime-config.ts","./utils/community-roles": "./src/utils/community-roles.ts","./lib/discriminator": "./src/lib/discriminator.ts",- "./lib/invite-link": "./src/lib/invite-link.ts"+ "./lib/invite-link": "./src/lib/invite-link.ts",+ "./db/resilience-testing": "./src/db/resilience-testing.ts"},"scripts": {"knip": "knip",diff --git a/src/shared/src/db/index.ts b/src/shared/src/db/index.tsindex d50616dc8..e81b76f94 100644--- a/src/shared/src/db/index.ts+++ b/src/shared/src/db/index.ts@@ -10,3 +10,6 @@ export function createDb(d1: AnyD1Database) {}export type Database = ReturnType<typeof createDb>;++export { withD1Retry, readOrStale, isRetryableD1Error } from "./resilience";+export type { RetryOpts } from "./resilience";diff --git a/src/shared/src/db/resilience-testing.ts b/src/shared/src/db/resilience-testing.tsnew file mode 100644index 000000000..30cd57ee0--- /dev/null+++ b/src/shared/src/db/resilience-testing.ts@@ -0,0 +1,44 @@+import { DrizzleQueryError } from "drizzle-orm/errors"++export type ErrorSignature =+ | "internal_error"+ | "sqlite_busy"+ | "sqlite_constraint"+ | "unknown"++const SIGNATURE_MESSAGES: Record<ErrorSignature, string> = {+ internal_error: "internal error; reference = 0123abcd",+ sqlite_busy: 'D1_ERROR: near "SELECT": SQLITE_BUSY: database is locked',+ sqlite_constraint: "D1_ERROR: SQLITE_CONSTRAINT_UNIQUE: UNIQUE constraint failed",+ unknown: "something went wrong",+}++export function makeD1Error(signature: ErrorSignature = "internal_error"): DrizzleQueryError {+ return new DrizzleQueryError("SELECT 1", [], new Error(SIGNATURE_MESSAGES[signature]))+}++/**+ * Test-fixture: returns an async fn that throws a DrizzleQueryError-shaped+ * error for the first `n` invocations, then resolves with `value`.+ *+ * Every integration test in `plans/d1-critical-path-resilience-v2.md` uses+ * this helper — DO NOT replace with `withD1Retry: vi.fn((fn) => fn())`+ * passthrough mocks. The passthrough pattern used by daemon-plane tests is+ * fine there (they DON'T test retry semantics); this plan's new tests DO,+ * so they need the real retry path, injected at the query-fn level.+ */+export function mockD1FailingUntil<T>(+ n: number,+ value: T,+ opts?: { errorSignature?: ErrorSignature },+): () => Promise<T> {+ const signature = opts?.errorSignature ?? "internal_error"+ let calls = 0+ return async () => {+ if (calls < n) {+ calls+++ throw makeD1Error(signature)+ }+ return value+ }+}diff --git a/src/shared/src/db/resilience.ts b/src/shared/src/db/resilience.tsnew file mode 100644index 000000000..cb87757a9--- /dev/null+++ b/src/shared/src/db/resilience.ts@@ -0,0 +1,96 @@+import { DrizzleQueryError } from "drizzle-orm/errors"+import { createLogger, type Logger } from "../logger"++export type RetryOpts = {+ attempts?: number+ baseDelayMs?: number+ route?: string+}++type ReadOrStaleOpts = RetryOpts & { category?: string }++const DEFAULT_ATTEMPTS = 3+const DEFAULT_BASE_DELAY_MS = 100++const defaultLogger: Logger = createLogger({ service: "d1-resilience" })++const RETRYABLE_SIGNATURES = [+ "internal error; reference",+ "SQLITE_BUSY",+ "database is locked",+ "SQLITE_INTERRUPT",+ "Network connection lost",+ "connection reset",+]++function peelDrizzle(err: unknown): unknown {+ let cur = err+ while (cur instanceof DrizzleQueryError) {+ if (!cur.cause) return cur+ cur = cur.cause+ }+ return cur+}++export function isRetryableD1Error(err: unknown): boolean {+ const peeled = peelDrizzle(err)+ if (!(peeled instanceof Error)) return false+ const msg = peeled.message+ if (typeof msg !== "string") return false+ for (const sig of RETRYABLE_SIGNATURES) {+ if (msg.includes(sig)) return true+ }+ return false+}++export async function withD1Retry<T>(+ fn: () => Promise<T>,+ opts: RetryOpts = {},+): Promise<T> {+ const attempts = opts.attempts ?? DEFAULT_ATTEMPTS+ const baseDelayMs = opts.baseDelayMs ?? DEFAULT_BASE_DELAY_MS+ const route = opts.route+ let lastErr: unknown+ for (let i = 0; i <= attempts; i++) {+ try {+ return await fn()+ } catch (err) {+ lastErr = err+ if (!isRetryableD1Error(err)) throw err+ if (i === attempts) break+ const cap = baseDelayMs * 2 ** i+ const delay = Math.floor(Math.random() * cap)+ await new Promise((r) => setTimeout(r, delay))+ }+ }+ defaultLogger.warn("d1_retry_exhausted", {+ category: "d1_retry_exhausted",+ route,+ err: lastErr instanceof Error ? lastErr : new Error(String(lastErr)),+ })+ throw lastErr+}++export async function readOrStale<T extends Record<string, unknown>>(+ fn: () => Promise<T>,+ fallback: T,+ opts: ReadOrStaleOpts = {},+): Promise<{ value: T; stale: boolean }> {+ try {+ const value = await withD1Retry(fn, opts)+ return { value, stale: false }+ } catch (err) {+ // Only launder RETRYABLE-shaped failures into `stale`. Non-retryable+ // throws (SQLITE_CONSTRAINT, TypeError from a broken query, ZodError…)+ // are real bugs — surfacing them as `d1_fail_closed` hides them behind+ // an outage-shaped log category and lets the UI render a false-empty+ // state. Rethrow so the route returns 500 and the bug is visible.+ if (!isRetryableD1Error(err)) throw err+ defaultLogger.warn("d1_fail_closed", {+ category: opts.category ?? "d1_fail_closed",+ route: opts.route,+ err: err instanceof Error ? err : new Error(String(err)),+ })+ return { value: fallback, stale: true }+ }+}diff --git a/src/shared/src/index.ts b/src/shared/src/index.tsindex 894a05811..fa0323157 100644--- a/src/shared/src/index.ts+++ b/src/shared/src/index.ts@@ -491,6 +491,8 @@ export { isCommunityEvent, WS_EVENTS } from "./community-ws-events";// Databaseexport { createDb } from "./db/index";export type { Database } from "./db/index";+export { withD1Retry, readOrStale, isRetryableD1Error } from "./db/resilience";+export type { RetryOpts } from "./db/resilience";export * as schema from "./db/schema";export * as queries from "./db/queries-index";export { communityServer, communityServerInvite, communityFriendship, communityServerMember, communityServerFolder, communityServerFolderItem, communityBotActivityEvent } from "./db/community-schema";diff --git a/src/shared/test/resilience.test.ts b/src/shared/test/resilience.test.tsnew file mode 100644index 000000000..25ea653af--- /dev/null+++ b/src/shared/test/resilience.test.ts@@ -0,0 +1,211 @@+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"+import { DrizzleQueryError } from "drizzle-orm/errors"+import { withD1Retry, readOrStale, isRetryableD1Error } from "../src/db/resilience"+import { mockD1FailingUntil, makeD1Error } from "../src/db/resilience-testing"++describe("withD1Retry", () => {+ beforeEach(() => {+ vi.useFakeTimers()+ vi.spyOn(console, "log").mockImplementation(() => {})+ vi.spyOn(console, "error").mockImplementation(() => {})+ })+ afterEach(() => {+ vi.useRealTimers()+ vi.restoreAllMocks()+ })++ it("succeeds on first attempt — no retry, no delay", async () => {+ const fn = vi.fn().mockResolvedValue("ok")+ const promise = withD1Retry(fn)+ await vi.runAllTimersAsync()+ expect(await promise).toBe("ok")+ expect(fn).toHaveBeenCalledTimes(1)+ })++ it("succeeds on 2nd attempt after one transient failure", async () => {+ const fn = mockD1FailingUntil(1, "ok")+ const promise = withD1Retry(fn)+ await vi.runAllTimersAsync()+ expect(await promise).toBe("ok")+ })++ it("succeeds on 3rd attempt after two transient failures", async () => {+ const fn = mockD1FailingUntil(2, "ok")+ const promise = withD1Retry(fn)+ await vi.runAllTimersAsync()+ expect(await promise).toBe("ok")+ })++ it("full-jitter delays fall in [0, base * 2^i]", async () => {+ // Sample the delay distribution by capturing setTimeout durations.+ // Success-on-4th-call so each call chain sleeps exactly 3 times, and no+ // final unhandled rejection leaks (unlike an all-fail path).+ const durations: number[] = []+ const setTimeoutSpy = vi+ .spyOn(globalThis, "setTimeout")+ // @ts-expect-error — narrow the mock; we only care about the delay arg.+ .mockImplementation((cb: () => void, ms?: number) => {+ durations.push(ms ?? 0)+ cb()+ return 0 as unknown as ReturnType<typeof setTimeout>+ })+ try {+ for (let i = 0; i < 200; i++) {+ await withD1Retry(mockD1FailingUntil(3, "ok"), { attempts: 3 })+ }+ } finally {+ setTimeoutSpy.mockRestore()+ }+ // Attempt 0 delay ∈ [0, 100), attempt 1 ∈ [0, 200), attempt 2 ∈ [0, 400).+ // Bucket per position modulo 3 (there are 3 sleeps per failing chain).+ const buckets: number[][] = [[], [], []]+ for (let i = 0; i < durations.length; i++) buckets[i % 3].push(durations[i])+ expect(Math.max(...buckets[0])).toBeLessThan(100)+ expect(Math.max(...buckets[1])).toBeLessThan(200)+ expect(Math.max(...buckets[2])).toBeLessThan(400)+ // Mean roughly cap/2 — assert distribution mean sits below 0.75 * cap+ // (not clustered at the cap ceiling, which would signal a non-jittered+ // implementation).+ const meanOf = (xs: number[]) => xs.reduce((a, b) => a + b, 0) / xs.length+ expect(meanOf(buckets[0])).toBeLessThan(75)+ expect(meanOf(buckets[1])).toBeLessThan(150)+ expect(meanOf(buckets[2])).toBeLessThan(300)+ })++ it("retry-exhaust throws the last error and logs d1_retry_exhausted", async () => {+ const logSpy = vi.spyOn(console, "log").mockImplementation(() => {})+ const fn = mockD1FailingUntil(999, "unused")+ const promise = withD1Retry(fn, { attempts: 3, route: "test-route" }).catch((e) => e)+ await vi.runAllTimersAsync()+ const caught = await promise+ expect(caught).toBeInstanceOf(DrizzleQueryError)+ const logged = logSpy.mock.calls.flat().join(" ")+ expect(logged).toContain("d1_retry_exhausted")+ expect(logged).toContain("test-route")+ })+})++describe("classifier — isRetryableD1Error", () => {+ it("retries workerd internal error wrapped in DrizzleQueryError", () => {+ const err = new DrizzleQueryError("SELECT 1", [], new Error("internal error; reference = abc"))+ expect(isRetryableD1Error(err)).toBe(true)+ })++ it("retries SQLITE_BUSY wrapped in DrizzleQueryError with realistic message", () => {+ const err = new DrizzleQueryError("SELECT ...", [], new Error('D1_ERROR: near "SELECT": SQLITE_BUSY: database is locked'))+ expect(isRetryableD1Error(err)).toBe(true)+ })++ it("does NOT retry SQLITE_CONSTRAINT_UNIQUE", () => {+ const err = new DrizzleQueryError("INSERT", [], new Error("SQLITE_CONSTRAINT_UNIQUE: UNIQUE constraint failed"))+ expect(isRetryableD1Error(err)).toBe(false)+ })++ it("does NOT retry a Zod parse error", () => {+ const err = new Error("[{\"code\":\"invalid_type\"}]")+ err.name = "ZodError"+ expect(isRetryableD1Error(err)).toBe(false)+ })++ it("does NOT retry an unknown error shape (deny-by-default)", () => {+ expect(isRetryableD1Error(new Error("something exploded"))).toBe(false)+ })++ it("peels double-wrapped DrizzleQueryError", () => {+ const inner = new DrizzleQueryError("q1", [], new Error("internal error; reference = z"))+ const outer = new DrizzleQueryError("q2", [], inner)+ expect(isRetryableD1Error(outer)).toBe(true)+ })++ it("retries SQLITE_INTERRUPT", () => {+ const err = new DrizzleQueryError("q", [], new Error("D1_ERROR: SQLITE_INTERRUPT"))+ expect(isRetryableD1Error(err)).toBe(true)+ })++ it("retries Network connection lost / CF RPC transient", () => {+ expect(isRetryableD1Error(new DrizzleQueryError("q", [], new Error("Network connection lost")))).toBe(true)+ expect(isRetryableD1Error(new DrizzleQueryError("q", [], new Error("connection reset by peer")))).toBe(true)+ })+})++describe("readOrStale", () => {+ beforeEach(() => {+ vi.useFakeTimers()+ vi.spyOn(console, "log").mockImplementation(() => {})+ vi.spyOn(console, "error").mockImplementation(() => {})+ })+ afterEach(() => {+ vi.useRealTimers()+ vi.restoreAllMocks()+ })++ it("returns { value, stale: false } on success", async () => {+ const promise = readOrStale(async () => ({ online: ["a"] }), { online: [] })+ await vi.runAllTimersAsync()+ const result = await promise+ expect(result).toEqual({ value: { online: ["a"] }, stale: false })+ })++ it("returns { value: fallback, stale: true } on retry-exhaust and logs d1_fail_closed", async () => {+ const logSpy = vi.spyOn(console, "log").mockImplementation(() => {})+ const fn = mockD1FailingUntil(999, { online: [] })+ const promise = readOrStale(fn, { online: [] as string[] }, { route: "test", category: "d1_fail_closed" })+ await vi.runAllTimersAsync()+ const result = await promise+ expect(result).toEqual({ value: { online: [] }, stale: true })+ const logged = logSpy.mock.calls.flat().join(" ")+ expect(logged).toContain("d1_fail_closed")+ })++++ it("re-throws non-retryable errors instead of laundering them as stale", async () => {+ // A real bug (constraint violation, TypeError from a broken query, …)+ // must NOT be hidden behind `stale: true` — the route should 500 so the+ // bug surfaces in error observability instead of masquerading as a D1+ // outage log line + empty UI.+ const promise = readOrStale(+ async () => {+ throw new Error("SQLITE_CONSTRAINT_UNIQUE: whatever")+ },+ { rows: [] as unknown[] },+ ).catch((e) => e)+ await vi.runAllTimersAsync()+ const err = await promise+ expect(err).toBeInstanceOf(Error)+ expect((err as Error).message).toContain("SQLITE_CONSTRAINT_UNIQUE")+ })+})++describe("mockD1FailingUntil helper", () => {+ it("throws DrizzleQueryError-shaped errors N times, then returns value", async () => {+ const fn = mockD1FailingUntil(2, "done")+ await expect(fn()).rejects.toBeInstanceOf(DrizzleQueryError)+ await expect(fn()).rejects.toBeInstanceOf(DrizzleQueryError)+ expect(await fn()).toBe("done")+ })++ it("supports errorSignature override", async () => {+ const fn = mockD1FailingUntil(1, "x", { errorSignature: "sqlite_constraint" })+ try {+ await fn()+ throw new Error("should have thrown")+ } catch (e) {+ expect(e).toBeInstanceOf(DrizzleQueryError)+ expect(((e as DrizzleQueryError).cause as Error).message).toContain("SQLITE_CONSTRAINT")+ }+ })++ it("makeD1Error produces a matching DrizzleQueryError", () => {+ const err = makeD1Error("internal_error")+ expect(err).toBeInstanceOf(DrizzleQueryError)+ expect((err.cause as Error).message).toContain("internal error; reference")+ })+})++describe("readOrStale — TypeScript compile-time constraint", () => {+ it("rejects array T at compile time", () => {+ // @ts-expect-error — T must extend Record<string, unknown>; arrays are not allowed.+ void readOrStale<unknown[]>(async () => [], [])+ })+})diff --git a/src/web/src/app/api/community/channels/[id]/read-state/route.ts b/src/web/src/app/api/community/channels/[id]/read-state/route.tsindex 75fcbf4d5..3a0a710ae 100644--- a/src/web/src/app/api/community/channels/[id]/read-state/route.ts+++ b/src/web/src/app/api/community/channels/[id]/read-state/route.ts@@ -2,7 +2,7 @@ import { NextRequest } from "next/server"import { withAuth } from "@/lib/middleware/auth"import { writeJSON, writeError } from "@/lib/middleware/helpers"import { getDb } from "@/lib/db"-import { queries } from "@alook/shared"+import { queries, withD1Retry } from "@alook/shared"import { requireChannelMember } from "@/lib/community/permissions"/**@@ -30,10 +30,17 @@ export const GET = withAuth(async (_req: NextRequest, ctx) => {const auth = await requireChannelMember(db, channelId, ctx.userId)if (!auth.ok) return writeError(auth.error, auth.status)- const row = await queries.communityReadState.getReadState(db, {- userId: ctx.userId,- channelId,- })+ // Wrap in `withD1Retry` on the server so the client can keep `retry: 1`+ // (one retry stack, budget shared with the helper). On exhaust: rethrow —+ // the client `useQuery` rejects, `snapshotRef` never latches a fabricated+ // value, hook returns `{ snapshot: null, isFetching: false }`.+ const row = await withD1Retry(+ () => queries.communityReadState.getReadState(db, {+ userId: ctx.userId,+ channelId,+ }),+ { route: "community/channels/read-state" },+ )return writeJSON({lastReadMessageId: row?.lastReadMessageId ?? null,diff --git a/src/web/src/app/api/community/dm/[id]/read-state/route.ts b/src/web/src/app/api/community/dm/[id]/read-state/route.tsindex c9faf4bc6..29e306d61 100644--- a/src/web/src/app/api/community/dm/[id]/read-state/route.ts+++ b/src/web/src/app/api/community/dm/[id]/read-state/route.ts@@ -2,7 +2,7 @@ import { NextRequest } from "next/server"import { withAuth } from "@/lib/middleware/auth"import { writeJSON, writeError } from "@/lib/middleware/helpers"import { getDb } from "@/lib/db"-import { queries } from "@alook/shared"+import { queries, withD1Retry } from "@alook/shared"import { requireDMParticipant } from "@/lib/community/permissions"/**@@ -27,10 +27,14 @@ export const GET = withAuth(async (_req: NextRequest, ctx) => {const auth = await requireDMParticipant(db, dmId, ctx.userId)if (!auth.ok) return writeError(auth.error, auth.status)- const row = await queries.communityReadState.getReadState(db, {- userId: ctx.userId,- dmConversationId: dmId,- })+ // Server-side retry stack. See channel twin for the invariant.+ const row = await withD1Retry(+ () => queries.communityReadState.getReadState(db, {+ userId: ctx.userId,+ dmConversationId: dmId,+ }),+ { route: "community/dm/read-state" },+ )return writeJSON({lastReadMessageId: row?.lastReadMessageId ?? null,diff --git a/src/web/src/app/api/community/friends/pending/route.ts b/src/web/src/app/api/community/friends/pending/route.tsindex c11e9c891..f17f0ab3c 100644--- a/src/web/src/app/api/community/friends/pending/route.ts+++ b/src/web/src/app/api/community/friends/pending/route.ts@@ -1,4 +1,4 @@-import { queries } from "@alook/shared"+import { queries, readOrStale } from "@alook/shared"import { getDb } from "@/lib/db"import { withAuth } from "@/lib/middleware/auth"import { writeJSON } from "@/lib/middleware/helpers"@@ -6,13 +6,18 @@ import { avatarInitial } from "@/lib/community/avatar"export const GET = withAuth(async (_req, ctx) => {const db = getDb(ctx.env.DB)- const rows = await queries.communityFriendship.listPending(db, ctx.userId)- const pending = rows.map((r) => ({+ type PendingRow = Awaited<ReturnType<typeof queries.communityFriendship.listPending>>[number]+ const { value, stale } = await readOrStale<{ rows: PendingRow[] }>(+ async () => ({ rows: await queries.communityFriendship.listPending(db, ctx.userId) }),+ { rows: [] },+ { route: "community/friends/pending" },+ )+ const pending = value.rows.map((r) => ({id: r.id,userId: r.userId,name: r.name,avatar: r.image ?? avatarInitial(r.name),kind: r.kind,}))- return writeJSON({ pending })+ return writeJSON(stale ? { pending, stale: true } : { pending })})diff --git a/src/web/src/app/api/community/friends/presence/route.test.ts b/src/web/src/app/api/community/friends/presence/route.test.tsindex e6e2685d0..7a268a970 100644--- a/src/web/src/app/api/community/friends/presence/route.test.ts+++ b/src/web/src/app/api/community/friends/presence/route.test.ts@@ -1,5 +1,6 @@import { describe, it, expect, vi, beforeEach, afterAll } from "vitest"import { NextRequest } from "next/server"+import { makeD1Error } from "@alook/shared/db/resilience-testing"const mockWsDoWorkerFetch = vi.fn<(...args: unknown[]) => Promise<Response>>()const mockGetFriendUserIds = vi.fn()@@ -117,6 +118,17 @@ describe("GET /api/community/friends/presence", () => {expect(res.status).toBe(200)expect(await res.json()).toEqual({ online: [] })})++ it("degrades to { online: [], stale: true } (200) when D1 exhausts retries", async () => {+ vi.spyOn(console, "log").mockImplementation(() => {})+ mockGetFriendUserIds.mockRejectedValue(makeD1Error("internal_error"))++ const res = await GET(getReq())++ expect(res.status).toBe(200)+ expect(await res.json()).toEqual({ online: [], stale: true })+ expect(mockWsDoWorkerFetch).not.toHaveBeenCalled()+ }, 10_000)})afterAll(() => {diff --git a/src/web/src/app/api/community/friends/presence/route.ts b/src/web/src/app/api/community/friends/presence/route.tsindex c45743da1..1856943de 100644--- a/src/web/src/app/api/community/friends/presence/route.ts+++ b/src/web/src/app/api/community/friends/presence/route.ts@@ -1,4 +1,4 @@-import { queries, PRESENCE_MEMBER_CAP } from "@alook/shared"+import { queries, PRESENCE_MEMBER_CAP, readOrStale } from "@alook/shared"import { getDb } from "@/lib/db"import { withAuth } from "@/lib/middleware/auth"import { writeJSON } from "@/lib/middleware/helpers"@@ -16,19 +16,27 @@ import { wsDoFetch } from "@/lib/broadcast"*/export const GET = withAuth(async (_req, ctx) => {const db = getDb(ctx.env.DB)- const allFriendIds = await queries.communityFriendship.getFriendUserIds(db, ctx.userId)- // There's no enforced cap on friend count today — reuse the server- // presence cap defensively so a pathological friend list can't blow past- // `/presence/users`'s own 1000-id limit and fail the whole check.- const friendIds = allFriendIds.slice(0, PRESENCE_MEMBER_CAP)+ // D1 read is fail-closed via `readOrStale`; a transient outage yields+ // `{ online: [] }` with `stale: true` rather than a 500.+ const { value, stale } = await readOrStale<{ friendIds: string[] }>(+ async () => {+ const allFriendIds = await queries.communityFriendship.getFriendUserIds(db, ctx.userId)+ // There's no enforced cap on friend count today — reuse the server+ // presence cap defensively so a pathological friend list can't blow past+ // `/presence/users`'s own 1000-id limit and fail the whole check.+ return { friendIds: allFriendIds.slice(0, PRESENCE_MEMBER_CAP) }+ },+ { friendIds: [] },+ { route: "community/friends/presence" },+ )let online: string[] = []- if (friendIds.length > 0) {+ if (!stale && value.friendIds.length > 0) {try {const resp = await wsDoFetch(ctx.env, "/presence/users", {method: "POST",headers: { "Content-Type": "application/json" },- body: JSON.stringify({ ids: friendIds }),+ body: JSON.stringify({ ids: value.friendIds }),}, { label: ctx.userId })if (resp.ok) {const data = await resp.json() as { online: string[] }@@ -37,5 +45,5 @@ export const GET = withAuth(async (_req, ctx) => {} catch { /* skip */ }}- return writeJSON({ online })+ return writeJSON(stale ? { online, stale: true } : { online })})diff --git a/src/web/src/app/api/community/friends/route.ts b/src/web/src/app/api/community/friends/route.tsindex 8ea8b1b65..de521f3f8 100644--- a/src/web/src/app/api/community/friends/route.ts+++ b/src/web/src/app/api/community/friends/route.ts@@ -1,4 +1,4 @@-import { queries } from "@alook/shared"+import { queries, readOrStale } from "@alook/shared"import { getDb } from "@/lib/db"import { withAuth } from "@/lib/middleware/auth"import { writeJSON } from "@/lib/middleware/helpers"@@ -6,11 +6,20 @@ import { avatarInitial } from "@/lib/community/avatar"export const GET = withAuth(async (_req, ctx) => {const db = getDb(ctx.env.DB)- const [rawFriends, rawBlocked] = await Promise.all([- queries.communityFriendship.listFriends(db, ctx.userId),- queries.communityFriendship.listBlocked(db, ctx.userId),- ])- const friends = rawFriends.map((f) => ({+ type FriendRow = Awaited<ReturnType<typeof queries.communityFriendship.listFriends>>[number]+ type BlockedRow = Awaited<ReturnType<typeof queries.communityFriendship.listBlocked>>[number]+ const { value, stale } = await readOrStale<{ rawFriends: FriendRow[]; rawBlocked: BlockedRow[] }>(+ async () => {+ const [rawFriends, rawBlocked] = await Promise.all([+ queries.communityFriendship.listFriends(db, ctx.userId),+ queries.communityFriendship.listBlocked(db, ctx.userId),+ ])+ return { rawFriends, rawBlocked }+ },+ { rawFriends: [], rawBlocked: [] },+ { route: "community/friends" },+ )+ const friends = value.rawFriends.map((f) => ({id: f.id,userId: f.friendUserId,name: f.friendName,@@ -21,11 +30,11 @@ export const GET = withAuth(async (_req, ctx) => {statusEmoji: f.statusEmoji ?? null,statusText: f.statusText ?? "",}))- const blocked = rawBlocked.map((b) => ({+ const blocked = value.rawBlocked.map((b) => ({id: b.id,userId: b.blockedUserId,name: b.blockedName,avatar: b.blockedImage ?? avatarInitial(b.blockedName),}))- return writeJSON({ friends, blocked })+ return writeJSON(stale ? { friends, blocked, stale: true } : { friends, blocked })})diff --git a/src/web/src/app/api/community/inbox/mentions/route.ts b/src/web/src/app/api/community/inbox/mentions/route.tsindex 09381f300..9eb0f44f1 100644--- a/src/web/src/app/api/community/inbox/mentions/route.ts+++ b/src/web/src/app/api/community/inbox/mentions/route.ts@@ -5,6 +5,7 @@ import {queries,DEFAULT_INBOX_PAGE_SIZE,MAX_INBOX_PAGE_SIZE,+ readOrStale,} from "@alook/shared"import { parseBoundedInt } from "@/lib/community/messages"import { avatarInitial } from "@/lib/community/avatar"@@ -23,18 +24,34 @@ export const GET = withAuth(async (req, ctx) => {// removed-from-private-channel user no longer sees leftover mentions; the// `inArray(channelId, visibleIds)` also naturally excludes DM reply rows// (channelId = NULL), which stay out of the Mentions tab by design.- const visibleChannelIds = await queries.communityChannel.listVisibleChannelIdsForUser(db, ctx.userId)- const rows = await queries.communityMention.listUnreadMentions(db, ctx.userId, {- limit,- visibleChannelIds,- })-- const channelIds = [...new Set(rows.filter((r) => r.message.channelId).map((r) => r.message.channelId!))]- const channels = channelIds.length > 0 ? await queries.communityChannel.getChannelsByIds(db, channelIds) : []+ type MentionRow = Awaited<ReturnType<typeof queries.communityMention.listUnreadMentions>>[number]+ type ChannelRow = Awaited<ReturnType<typeof queries.communityChannel.getChannelsByIds>>[number]+ type ServerRow = Awaited<ReturnType<typeof queries.communityServer.getServersByIds>>[number]+ const { value: fetched, stale } = await readOrStale<{+ rows: MentionRow[]+ channels: ChannelRow[]+ servers: ServerRow[]+ }>(+ async () => {+ const visibleChannelIds = await queries.communityChannel.listVisibleChannelIdsForUser(db, ctx.userId)+ const rows = await queries.communityMention.listUnreadMentions(db, ctx.userId, {+ limit,+ visibleChannelIds,+ })+ const channelIds = [...new Set(rows.filter((r) => r.message.channelId).map((r) => r.message.channelId!))]+ const channels = channelIds.length > 0 ? await queries.communityChannel.getChannelsByIds(db, channelIds) : []+ const serverIds = [...new Set(channels.map((ch) => ch.serverId))]+ const servers = serverIds.length > 0 ? await queries.communityServer.getServersByIds(db, serverIds) : []+ return { rows, channels, servers }+ },+ { rows: [], channels: [], servers: [] },+ { route: "community/inbox/mentions" },+ )+ if (stale) {+ return writeJSON({ mentions: [], limit, stale: true })+ }+ const { rows, channels, servers } = fetchedconst channelMap = new Map(channels.map((ch) => [ch.id, ch]))-- const serverIds = [...new Set(channels.map((ch) => ch.serverId))]- const servers = serverIds.length > 0 ? await queries.communityServer.getServersByIds(db, serverIds) : []const serverMap = new Map(servers.map((s) => [s.id, s]))const mentions = rows.map((row) => {diff --git a/src/web/src/app/api/community/inbox/unreads/route.ts b/src/web/src/app/api/community/inbox/unreads/route.tsindex c062e4e55..fff41b898 100644--- a/src/web/src/app/api/community/inbox/unreads/route.ts+++ b/src/web/src/app/api/community/inbox/unreads/route.ts@@ -2,6 +2,7 @@ import {queries,DEFAULT_INBOX_PAGE_SIZE,MAX_INBOX_PAGE_SIZE,+ readOrStale,} from "@alook/shared"import { getDb } from "@/lib/db"import { withAuth } from "@/lib/middleware/auth"@@ -21,13 +22,33 @@ export const GET = withAuth(async (req, ctx) => {// Resolve the viewer's visible channels once (top-level + threads/forum-posts,// parent-climbed) and thread the id set into BOTH consumers so neither// recomputes it. Private threads under an invisible parent are excluded.- const visibleChannelIds = await queries.communityChannel.listVisibleChannelIdsForUser(db, ctx.userId)- const [unread, settings, mentions, unreadDms] = await Promise.all([- queries.communityInbox.listUnreadChannels(db, ctx.userId, visibleChannelIds),- queries.communityNotificationSetting.getSettings(db, ctx.userId),- queries.communityMention.listUnreadMentions(db, ctx.userId, { visibleChannelIds }),- queries.communityInbox.listUnreadDms(db, ctx.userId),- ])+ type UnreadRow = Awaited<ReturnType<typeof queries.communityInbox.listUnreadChannels>>[number]+ type SettingRow = Awaited<ReturnType<typeof queries.communityNotificationSetting.getSettings>>[number]+ type MentionRow = Awaited<ReturnType<typeof queries.communityMention.listUnreadMentions>>[number]+ type UnreadDmRow = Awaited<ReturnType<typeof queries.communityInbox.listUnreadDms>>[number]+ const { value: fetched, stale } = await readOrStale<{+ unread: UnreadRow[]+ settings: SettingRow[]+ mentions: MentionRow[]+ unreadDms: UnreadDmRow[]+ }>(+ async () => {+ const visibleChannelIds = await queries.communityChannel.listVisibleChannelIdsForUser(db, ctx.userId)+ const [unread, settings, mentions, unreadDms] = await Promise.all([+ queries.communityInbox.listUnreadChannels(db, ctx.userId, visibleChannelIds),+ queries.communityNotificationSetting.getSettings(db, ctx.userId),+ queries.communityMention.listUnreadMentions(db, ctx.userId, { visibleChannelIds }),+ queries.communityInbox.listUnreadDms(db, ctx.userId),+ ])+ return { unread, settings, mentions, unreadDms }+ },+ { unread: [], settings: [], mentions: [], unreadDms: [] },+ { route: "community/inbox/unreads" },+ )+ if (stale) {+ return writeJSON({ servers: [], dms: [], limit, truncated: false, stale: true })+ }+ const { unread, settings, mentions, unreadDms } = fetchedconst mutedServers = new Set<string>()const mutedChannels = new Set<string>()diff --git a/src/web/src/app/api/community/servers/[id]/presence/route.ts b/src/web/src/app/api/community/servers/[id]/presence/route.tsindex e6107d6c8..bb07139ad 100644--- a/src/web/src/app/api/community/servers/[id]/presence/route.ts+++ b/src/web/src/app/api/community/servers/[id]/presence/route.ts@@ -1,4 +1,4 @@-import { queries, PRESENCE_MEMBER_CAP } from "@alook/shared"+import { queries, PRESENCE_MEMBER_CAP, readOrStale } from "@alook/shared"import { getDb } from "@/lib/db"import { withAuth } from "@/lib/middleware/auth"import { writeJSON, writeError } from "@/lib/middleware/helpers"@@ -15,25 +15,36 @@ export const GET = withAuth(async (_req, ctx) => {// Cap the fan-out: a 10k-member server would otherwise spawn 10k Worker// subrequests and time out. Client paginates if it needs more.- const allUserIds = await queries.communityMember.listMemberUserIds(db, serverId)- const truncated = allUserIds.length > PRESENCE_MEMBER_CAP- const userIds = allUserIds.slice(0, PRESENCE_MEMBER_CAP)+ const { value, stale } = await readOrStale<{ userIds: string[]; truncated: boolean }>(+ async () => {+ const allUserIds = await queries.communityMember.listMemberUserIds(db, serverId)+ return {+ userIds: allUserIds.slice(0, PRESENCE_MEMBER_CAP),+ truncated: allUserIds.length > PRESENCE_MEMBER_CAP,+ }+ },+ { userIds: [], truncated: false },+ { route: "community/servers/presence" },+ )// Single subrequest — the ws-do worker fans out to each member's DO// internally, keeping the web-worker subrequest budget constant.- const body = JSON.stringify({ ids: userIds })let online: string[] = []- try {- const resp = await wsDoFetch(ctx.env, "/presence/users", {- method: "POST",- headers: { "Content-Type": "application/json" },- body,- }, { label: serverId })- if (resp.ok) {- const data = await resp.json() as { online: string[] }- online = Array.isArray(data.online) ? data.online : []- }- } catch { /* skip */ }+ if (!stale && value.userIds.length > 0) {+ const body = JSON.stringify({ ids: value.userIds })+ try {+ const resp = await wsDoFetch(ctx.env, "/presence/users", {+ method: "POST",+ headers: { "Content-Type": "application/json" },+ body,+ }, { label: serverId })+ if (resp.ok) {+ const data = await resp.json() as { online: string[] }+ online = Array.isArray(data.online) ? data.online : []+ }+ } catch { /* skip */ }+ }- return writeJSON({ online, truncated, limit: PRESENCE_MEMBER_CAP })+ const payload = { online, truncated: value.truncated, limit: PRESENCE_MEMBER_CAP }+ return writeJSON(stale ? { ...payload, stale: true } : payload)})diff --git a/src/web/src/app/c/channels/layout.tsx b/src/web/src/app/c/channels/layout.tsxindex 9c6c1cec0..c7e787f57 100644--- a/src/web/src/app/c/channels/layout.tsx+++ b/src/web/src/app/c/channels/layout.tsx@@ -95,7 +95,8 @@ export default function ServerLayout({ children }: { children: ReactNode }) {// genuinely useful to them as well.const { invites, isLoading: invitesLoading } = useInvites(serverId, true)const { entries: auditLog, isLoading: auditLogLoading } = useAuditLog(serverId, isAdmin)- const { online: initialOnline } = usePresence(serverId)+ const presence = usePresence(serverId)+ const { online: initialOnline } = presenceconst notifs = useNotificationSettings()const notifLevel = notifs.server[serverId] ?? "Only @mentions"const channelNotif = notifs.channel@@ -164,9 +165,17 @@ export default function ServerLayout({ children }: { children: ReactNode }) {// one-shot replacement AND no-ops when the incoming list matches current// state — critical to avoid a render loop when `initialOnline`'s reference// shifts on re-render (loading state, cache tick) without semantic change.+ //+ // Guard against seeding from a non-authoritative snapshot: while the query+ // is still fetching, or when the presence route degraded to `stale: true`+ // (retry-exhaust on D1 → `keepPreviousData` bubbles a StaleReadError, so+ // `presence.data` is `undefined` on first-load-during-outage), we must NOT+ // atomically wipe the WS-populated set to an empty list.useEffect(() => {+ if (presence.isFetching) return+ if (presence.data === undefined) returnuseCommunityWsStore.getState().hydratePresence(initialOnline)- }, [initialOnline])+ }, [presence.isFetching, presence.data, initialOnline])const [mobileZone, setMobileZone] = useState<MobileZone>(() => hasChannel ? "messages" : "nav")const [serverSettingsOpen, setServerSettingsOpen] = useState(false)diff --git a/src/web/src/hooks/community/use-channel-read-state.test.ts b/src/web/src/hooks/community/use-channel-read-state.test.tsindex d8003bf44..eed8362c2 100644--- a/src/web/src/hooks/community/use-channel-read-state.test.ts+++ b/src/web/src/hooks/community/use-channel-read-state.test.ts@@ -35,8 +35,12 @@ let queryReturn: { data: unknown; isFetching: boolean } = {data: undefined,isFetching: false,}+let lastUseQueryConfig: { retry?: number; staleTime?: number; gcTime?: number } = {}vi.mock("@tanstack/react-query", () => ({- useQuery: (_config: unknown) => queryReturn,+ useQuery: (config: { retry?: number; staleTime?: number; gcTime?: number }) => {+ lastUseQueryConfig = config ?? {}+ return queryReturn+ },}))// apiFetch is unused directly (the queryFn is stubbed by the useQuery mock)@@ -153,6 +157,45 @@ describe("useChannelReadStateSnapshot — freeze invariant", () => {})})+ it("uses retry: 1 — single retry stack lives on the server, not the client", async () => {+ // Regression guard for v2's "one retry stack" invariant. Server wraps+ // the D1 read in `withD1Retry`; client keeps `retry: 1` (revert of v1's+ // 1→2 bump). Bumping this back to 2 would cascade retry budgets.+ const useHook = await loadHook()+ useHook("ch_1")+ expect(lastUseQueryConfig.retry).toBe(1)+ })++ it("returns null when the query errors — snapshotRef never latches a fabricated value", async () => {+ // Anti-latching invariant: on server D1 exhaustion, the handler throws,+ // the query rejects, `query.data` stays undefined, and the hook+ // returns `{ snapshot: null }`. The divider must not anchor to a+ // fabricated `{ lastReadMessageId: null, lastReadSeq: 0 }` sentinel —+ // that's the LEGITIMATE never-visited shape and would be indistinguishable+ // from the error case if we latched it.+ const useHook = await loadHook()+ queryReturn = { data: undefined, isFetching: false }+ const r = useHook("ch_1")+ flushEffects()+ expect(r.snapshot).toBeNull()+ expect(r.isFetching).toBe(false)+ })++ it("legitimate never-visited response (lastReadSeq: 0) DOES latch and returns the sentinel — distinguishes valid-empty from error", async () => {+ const useHook = await loadHook()+ queryReturn = {+ data: { lastReadMessageId: null, lastReadAt: null, lastReadSeq: 0 },+ isFetching: false,+ }+ const r = useHook("ch_1")+ flushEffects()+ expect(r.snapshot).toEqual({+ lastReadMessageId: null,+ lastReadAt: null,+ lastReadSeq: 0,+ })+ })+it("returns null on subsequent renders when the initial fetch resolved to null (never-visited channel)", async () => {const useHook = await loadHook()queryReturn = {diff --git a/src/web/src/hooks/community/use-friends.ts b/src/web/src/hooks/community/use-friends.tsindex 45c17299e..c48b0a9e2 100644--- a/src/web/src/hooks/community/use-friends.ts+++ b/src/web/src/hooks/community/use-friends.ts@@ -1,10 +1,25 @@"use client"-import { useQuery, type UseQueryResult } from "@tanstack/react-query"+import { useQuery, keepPreviousData, type UseQueryResult } from "@tanstack/react-query"import { apiFetch } from "@/lib/api/client"import { communityKeys } from "@/lib/query-keys"import type { Friend, PendingRequest, BlockedUser } from "@/components/community/_types"+/**+ * The community read routes wrap their D1 hits in `readOrStale` (see+ * `src/shared/src/db/resilience.ts`). On retry-exhaust they return+ * `200 { …, stale: true }` with empty payloads. Treat that as a query+ * error so `placeholderData: keepPreviousData` keeps the last-good data+ * on screen instead of flipping the UI to a false-empty state.+ */+class StaleReadError extends Error {+ constructor() { super("stale D1 read"); this.name = "StaleReadError" }+}+function throwIfStale<T extends { stale?: boolean }>(v: T): T {+ if (v?.stale) throw new StaleReadError()+ return v+}+/*** Fetches the friends / pending-requests / blocked triad in a single query.*@@ -27,8 +42,8 @@ const EMPTY_BLOCKED: readonly BlockedUser[] = Object.freeze([])export const friendsQueryFn = async (): Promise<FriendsResponse> => {const [friendsData, pendingData] = await Promise.all([- apiFetch<{ friends: Friend[]; blocked: BlockedUser[] }>("/api/community/friends"),- apiFetch<{ pending: PendingRequest[] }>("/api/community/friends/pending"),+ apiFetch<{ friends: Friend[]; blocked: BlockedUser[]; stale?: boolean }>("/api/community/friends").then(throwIfStale),+ apiFetch<{ pending: PendingRequest[]; stale?: boolean }>("/api/community/friends/pending").then(throwIfStale),])return {friends: friendsData.friends,@@ -45,6 +60,7 @@ export function useFriends(): UseQueryResult<FriendsResponse> & {const query = useQuery({queryKey: communityKeys.friends(),queryFn: friendsQueryFn,+ placeholderData: keepPreviousData,})return {...query,@@ -67,7 +83,7 @@ export function useFriends(): UseQueryResult<FriendsResponse> & {export type FriendsPresenceResponse = { online: string[] }export const friendsPresenceQueryFn = () =>- apiFetch<FriendsPresenceResponse>("/api/community/friends/presence")+ apiFetch<FriendsPresenceResponse & { stale?: boolean }>("/api/community/friends/presence").then(throwIfStale)const EMPTY_ONLINE: readonly string[] = Object.freeze([])@@ -77,6 +93,7 @@ export function useFriendsPresence(): UseQueryResult<FriendsPresenceResponse> &const query = useQuery({queryKey: communityKeys.friendsPresence(),queryFn: friendsPresenceQueryFn,+ placeholderData: keepPreviousData,})return {...query,diff --git a/src/web/src/hooks/community/use-inbox.ts b/src/web/src/hooks/community/use-inbox.tsindex ee4db6665..e0083edff 100644--- a/src/web/src/hooks/community/use-inbox.ts+++ b/src/web/src/hooks/community/use-inbox.ts@@ -1,10 +1,18 @@"use client"-import { useQuery, type UseQueryResult } from "@tanstack/react-query"+import { useQuery, keepPreviousData, type UseQueryResult } from "@tanstack/react-query"import { apiFetch } from "@/lib/api/client"import { communityKeys } from "@/lib/query-keys"import type { UnreadServer, UnreadDm, Mention } from "@/components/community/_types"+class StaleReadError extends Error {+ constructor() { super("stale D1 read"); this.name = "StaleReadError" }+}+function throwIfStale<T extends { stale?: boolean }>(v: T): T {+ if (v?.stale) throw new StaleReadError()+ return v+}+// Frozen empty fallbacks — see `use-servers.ts` for the rationale.const EMPTY_UNREADS: readonly UnreadServer[] = Object.freeze([])const EMPTY_DMS: readonly UnreadDm[] = Object.freeze([])@@ -26,7 +34,7 @@ const EMPTY_MENTIONS: readonly Mention[] = Object.freeze([])export type UnreadsResponse = { servers: UnreadServer[]; dms: UnreadDm[] }export const inboxUnreadsQueryFn = () =>- apiFetch<UnreadsResponse>("/api/community/inbox/unreads")+ apiFetch<UnreadsResponse & { stale?: boolean }>("/api/community/inbox/unreads").then(throwIfStale)export function useInboxUnreads(): UseQueryResult<UnreadsResponse> & {servers: UnreadServer[]@@ -35,6 +43,7 @@ export function useInboxUnreads(): UseQueryResult<UnreadsResponse> & {const query = useQuery({queryKey: communityKeys.inboxUnreads(),queryFn: inboxUnreadsQueryFn,+ placeholderData: keepPreviousData,})return {...query,@@ -46,7 +55,7 @@ export function useInboxUnreads(): UseQueryResult<UnreadsResponse> & {export type MentionsResponse = { mentions: Mention[] }export const inboxMentionsQueryFn = () =>- apiFetch<MentionsResponse>("/api/community/inbox/mentions")+ apiFetch<MentionsResponse & { stale?: boolean }>("/api/community/inbox/mentions").then(throwIfStale)export function useInboxMentions(): UseQueryResult<MentionsResponse> & {mentions: Mention[]@@ -54,6 +63,7 @@ export function useInboxMentions(): UseQueryResult<MentionsResponse> & {const query = useQuery({queryKey: communityKeys.inboxMentions(),queryFn: inboxMentionsQueryFn,+ placeholderData: keepPreviousData,})return {...query,diff --git a/src/web/src/hooks/community/use-server-panels.ts b/src/web/src/hooks/community/use-server-panels.tsindex 9b47b2d84..f3b51260f 100644--- a/src/web/src/hooks/community/use-server-panels.ts+++ b/src/web/src/hooks/community/use-server-panels.ts@@ -1,10 +1,18 @@"use client"-import { useQuery, type UseQueryResult } from "@tanstack/react-query"+import { useQuery, keepPreviousData, type UseQueryResult } from "@tanstack/react-query"import { apiFetch } from "@/lib/api/client"import { communityKeys } from "@/lib/query-keys"import type { InviteRow, AuditEntry } from "@/components/community/_types"+class StaleReadError extends Error {+ constructor() { super("stale D1 read"); this.name = "StaleReadError" }+}+function throwIfStale<T extends { stale?: boolean }>(v: T): T {+ if (v?.stale) throw new StaleReadError()+ return v+}+/*** Fetches the invite list surfaced in the settings tab. The API returns raw* rows; we transform to the display shape here so consumers get render-ready@@ -124,7 +132,7 @@ export function useAuditLog(export type PresenceResponse = { online: string[]; truncated?: boolean; limit?: number }export const presenceQueryFn = (serverId: string) => () =>- apiFetch<PresenceResponse>(`/api/community/servers/${serverId}/presence`)+ apiFetch<PresenceResponse & { stale?: boolean }>(`/api/community/servers/${serverId}/presence`).then(throwIfStale)const EMPTY_ONLINE: readonly string[] = Object.freeze([])export function usePresence(@@ -137,6 +145,7 @@ export function usePresence(? presenceQueryFn(serverId!): (() => Promise.reject(new Error("disabled"))),enabled,+ placeholderData: keepPreviousData,})return {...query,diff --git a/src/web/src/lib/db.test.ts b/src/web/src/lib/db.test.tsindex 268b25312..d693cf129 100644--- a/src/web/src/lib/db.test.ts+++ b/src/web/src/lib/db.test.ts@@ -1,38 +1,76 @@-import { describe, it, expect, vi } from "vitest";-import { withD1Retry } from "./db";+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"+import { withD1Retry } from "./db"+import { mockD1FailingUntil, makeD1Error } from "@alook/shared/db/resilience-testing"-describe("withD1Retry", () => {- it("returns result on first success", async () => {- const fn = vi.fn().mockResolvedValue("ok");- const result = await withD1Retry(fn);- expect(result).toBe("ok");- expect(fn).toHaveBeenCalledTimes(1);- });+/**+ * Characterization test — pins the observable contract of `withD1Retry`+ * after the move from `src/web/src/lib/db.ts` to `src/shared/src/db/resilience.ts`.+ *+ * Pre-move behavior was: up to 3 retries after initial (4 total), fixed+ * delays 100/200/400ms, retry on ANY error. Post-move, the intentional+ * changes are (a) full-jitter delays bounded by the same caps and (b) an+ * allow-list classifier that peels DrizzleQueryError.cause and only retries+ * transient shapes. Daemon-plane callers pass Drizzle-wrapped D1 errors so+ * behavior is unchanged in practice; unknown Error shapes now fail-fast.+ */+describe("web/lib/db withD1Retry re-export", () => {+ beforeEach(() => {+ vi.useFakeTimers()+ vi.spyOn(console, "log").mockImplementation(() => {})+ vi.spyOn(console, "error").mockImplementation(() => {})+ })+ afterEach(() => {+ vi.useRealTimers()+ vi.restoreAllMocks()+ })- it("retries once on transient failure then succeeds", async () => {- const fn = vi.fn()- .mockRejectedValueOnce(new Error("D1 timeout"))- .mockResolvedValueOnce("recovered");- const result = await withD1Retry(fn);- expect(result).toBe("recovered");- expect(fn).toHaveBeenCalledTimes(2);- });+ it("returns result on first success — no retry, no delay", async () => {+ const fn = vi.fn().mockResolvedValue("ok")+ const promise = withD1Retry(fn)+ await vi.runAllTimersAsync()+ expect(await promise).toBe("ok")+ expect(fn).toHaveBeenCalledTimes(1)+ })- it("throws after all retries exhausted", async () => {- const fn = vi.fn().mockRejectedValue(new Error("D1 down"));- await expect(withD1Retry(fn)).rejects.toThrow("D1 down");- expect(fn).toHaveBeenCalledTimes(4);- });+ it("retries on transient D1 failure then succeeds", async () => {+ const fn = mockD1FailingUntil(1, "recovered")+ const promise = withD1Retry(fn)+ await vi.runAllTimersAsync()+ expect(await promise).toBe("recovered")+ })- it("respects custom retry count", async () => {- const fn = vi.fn().mockRejectedValue(new Error("fail"));- await expect(withD1Retry(fn, 3)).rejects.toThrow("fail");- expect(fn).toHaveBeenCalledTimes(4);- });+ it("throws after retries exhausted for persistent transient failure", async () => {+ vi.spyOn(console, "log").mockImplementation(() => {})+ const fn = mockD1FailingUntil(999, "unused")+ const promise = withD1Retry(fn).catch((e) => e)+ await vi.runAllTimersAsync()+ const err = await promise+ expect(err).toBeDefined()+ })- it("does not retry when retries is 0", async () => {- const fn = vi.fn().mockRejectedValue(new Error("fail"));- await expect(withD1Retry(fn, 0)).rejects.toThrow("fail");- expect(fn).toHaveBeenCalledTimes(1);- });-});+ it("does NOT retry non-transient errors (allow-list, deny-by-default)", async () => {+ let calls = 0+ const fn = async () => {+ calls+++ throw makeD1Error("sqlite_constraint")+ }+ const promise = withD1Retry(fn).catch((e) => e)+ await vi.runAllTimersAsync()+ const err = await promise+ expect(err).toBeDefined()+ expect(calls).toBe(1)+ })++ it("respects custom attempts count via opts", async () => {+ vi.spyOn(console, "log").mockImplementation(() => {})+ let calls = 0+ const fn = async () => {+ calls+++ throw makeD1Error("internal_error")+ }+ const promise = withD1Retry(fn, { attempts: 2 }).catch(() => {})+ await vi.runAllTimersAsync()+ await promise+ expect(calls).toBe(3)+ })+})diff --git a/src/web/src/lib/db.ts b/src/web/src/lib/db.tsindex ee5d8543c..fe8045fdd 100644--- a/src/web/src/lib/db.ts+++ b/src/web/src/lib/db.ts@@ -1,18 +1,9 @@import { createDb, type Database } from "@alook/shared"+export { withD1Retry, readOrStale } from "@alook/shared"+export type { RetryOpts } from "@alook/shared"+export function getDb(d1: D1Database): Database {const session = d1.withSession("first-unconstrained")return createDb(session as unknown as Parameters<typeof createDb>[0])}--export async function withD1Retry<T>(fn: () => Promise<T>, retries = 3): Promise<T> {- for (let i = 0; i <= retries; i++) {- try {- return await fn();- } catch (err) {- if (i === retries) throw err;- await new Promise((r) => setTimeout(r, 100 * 2 ** i));- }- }- throw new Error("unreachable");-}diff --git a/src/web/src/lib/middleware/community-agent-runner-auth.test.ts b/src/web/src/lib/middleware/community-agent-runner-auth.test.tsindex c17e719ad..3e1f5df21 100644--- a/src/web/src/lib/middleware/community-agent-runner-auth.test.ts+++ b/src/web/src/lib/middleware/community-agent-runner-auth.test.ts@@ -1,5 +1,6 @@import { describe, it, expect, vi, beforeEach } from "vitest"import { NextRequest, NextResponse } from "next/server"+import { makeD1Error } from "@alook/shared/db/resilience-testing"vi.mock("@opennextjs/cloudflare", () => ({getCloudflareContext: vi.fn(async () => ({ env: { DB: {} } })),@@ -10,19 +11,23 @@ vi.mock("@/lib/db", () => ({ getDb: vi.fn(() => ({})) }))const mockFindActiveAgentRunnerKeyByBearer = vi.fn()const mockGetUserInternal = vi.fn()const mockGetBotBinding = vi.fn()-vi.mock("@alook/shared", () => ({- queries: {- communityMachine: {- findActiveAgentRunnerKeyByBearer: (...a: unknown[]) => mockFindActiveAgentRunnerKeyByBearer(...a),+vi.mock("@alook/shared", async () => {+ const actual = await vi.importActual<typeof import("@alook/shared")>("@alook/shared")+ return {+ ...actual,+ queries: {+ communityMachine: {+ findActiveAgentRunnerKeyByBearer: (...a: unknown[]) => mockFindActiveAgentRunnerKeyByBearer(...a),+ },+ user: {+ getUserInternal: (...a: unknown[]) => mockGetUserInternal(...a),+ },+ communityBot: {+ getBotBinding: (...a: unknown[]) => mockGetBotBinding(...a),+ },},- user: {- getUserInternal: (...a: unknown[]) => mockGetUserInternal(...a),- },- communityBot: {- getBotBinding: (...a: unknown[]) => mockGetBotBinding(...a),- },- },-}))+ }+})import { withAgentRunnerAuth } from "./community-agent-runner-auth"@@ -119,4 +124,54 @@ describe("withAgentRunnerAuth", () => {machineId: "m_1",})})++ it("returns 503 + Retry-After when D1 exhausts retries on the runner-key lookup", async () => {+ vi.spyOn(console, "log").mockImplementation(() => {})+ mockFindActiveAgentRunnerKeyByBearer.mockRejectedValue(makeD1Error("internal_error"))+ const req = new NextRequest("http://localhost/x", { headers: { Authorization: "Bearer crk_abc" } })+ const res = await wrapped(req)+ expect(res.status).toBe(503)+ expect(res.headers.get("Retry-After")).toBe("1")+ expect(await res.json()).toEqual({ error: "database temporarily unavailable" })+ }, 10_000)++ it("returns 503 when D1 exhausts on the getUserInternal step", async () => {+ vi.spyOn(console, "log").mockImplementation(() => {})+ mockFindActiveAgentRunnerKeyByBearer.mockResolvedValue({+ userId: "owner_1",+ machineId: "m_1",+ agentId: "bot_1",+ })+ mockGetUserInternal.mockRejectedValue(makeD1Error("sqlite_busy"))+ const req = new NextRequest("http://localhost/x", { headers: { Authorization: "Bearer crk_abc" } })+ const res = await wrapped(req)+ expect(res.status).toBe(503)+ }, 10_000)++ it("preserves 401 for non-D1 failures — bot deleted, no retry", async () => {+ mockFindActiveAgentRunnerKeyByBearer.mockResolvedValue({+ userId: "owner_1",+ machineId: "m_1",+ agentId: "bot_1",+ })+ mockGetUserInternal.mockResolvedValue({ isBot: true, deletedAt: "2026-01-01" })+ const req = new NextRequest("http://localhost/x", { headers: { Authorization: "Bearer crk_abc" } })+ const res = await wrapped(req)+ expect(res.status).toBe(401)+ expect(mockGetUserInternal).toHaveBeenCalledTimes(1)+ })++ it("preserves 401 for binding mismatch — no retry", async () => {+ mockFindActiveAgentRunnerKeyByBearer.mockResolvedValue({+ userId: "owner_1",+ machineId: "m_1",+ agentId: "bot_1",+ })+ mockGetUserInternal.mockResolvedValue({ isBot: true, deletedAt: null })+ mockGetBotBinding.mockResolvedValue({ machineId: "m_OTHER" })+ const req = new NextRequest("http://localhost/x", { headers: { Authorization: "Bearer crk_abc" } })+ const res = await wrapped(req)+ expect(res.status).toBe(401)+ expect(mockGetBotBinding).toHaveBeenCalledTimes(1)+ })})diff --git a/src/web/src/lib/middleware/community-agent-runner-auth.ts b/src/web/src/lib/middleware/community-agent-runner-auth.tsindex c38ae05c3..996038eb2 100644--- a/src/web/src/lib/middleware/community-agent-runner-auth.ts+++ b/src/web/src/lib/middleware/community-agent-runner-auth.ts@@ -1,8 +1,10 @@import { NextRequest, NextResponse } from "next/server"import { getCloudflareContext } from "@opennextjs/cloudflare"-import { queries } from "@alook/shared"+import { queries, withD1Retry, createLogger } from "@alook/shared"import { getDb } from "@/lib/db"+const log = createLogger({ service: "community-agent-runner-auth" })+interface AgentRunnerAuthContext {env: Env/** The BOT's own user id — `row.agentId` from `findActiveAgentRunnerKeyByBearer`. */@@ -17,6 +19,15 @@ export type AgentRunnerAuthenticatedHandler = (ctx: AgentRunnerAuthContext & { params?: Record<string, string> }) => Promise<NextResponse | Response>+const RETRY_OPTS = { route: "community-agent-runner-auth" }++function serviceUnavailable(): NextResponse {+ return NextResponse.json(+ { error: "database temporarily unavailable" },+ { status: 503, headers: { "Retry-After": "1" } },+ )+}+/*** Agent-runner auth middleware for the CLI bridge (`/api/community/agent/*`).* Requires `Authorization: Bearer crk_…`. Cloned from `withCommunityDaemonAuth`@@ -27,15 +38,11 @@ export type AgentRunnerAuthenticatedHandler = (* `row.agentId` is the BOT's own user id. `row.doName` here is the runner* key's own DO-hash, unrelated to wake dispatch — never threaded through.*- * Steps (plan §2):- * 1. Extract `Bearer <token>`; reject non-`crk_` → 401.- * 2. `findActiveAgentRunnerKeyByBearer` → 401 on null.- * 3. `getUserInternal(row.agentId)` → 401 if null, `!isBot`, or soft-deleted.- * 4. `getBotBinding(row.agentId)` → 401 if null or machine mismatch- * (belt-and-braces; `getBotBinding` returns no owner of its own).- *- * Handlers must NEVER read `X-Agent-Id` or a body `agentId` — identity is- * always `ctx.botUserId` from this middleware.+ * D1-transient failure semantics: each of the 3 D1 reads runs through+ * `withD1Retry`; on retry-exhaust we return 503 + `Retry-After: 1` (RFC 9110+ * §15.6.4) so CLI bridges treat it as retryable and do NOT rotate their+ * runner key. 401 is reserved for real auth failures (bad token, revoked+ * runner key, bot deleted, binding mismatch).*/export function withAgentRunnerAuth(handler: AgentRunnerAuthenticatedHandler) {return async (@@ -61,17 +68,44 @@ export function withAgentRunnerAuth(handler: AgentRunnerAuthenticatedHandler) {const cloudflareEnv = env as Envconst db = getDb(cloudflareEnv.DB)- const row = await queries.communityMachine.findActiveAgentRunnerKeyByBearer(db, raw)+ let row: Awaited<ReturnType<typeof queries.communityMachine.findActiveAgentRunnerKeyByBearer>>+ try {+ row = await withD1Retry(+ () => queries.communityMachine.findActiveAgentRunnerKeyByBearer(db, raw),+ RETRY_OPTS,+ )+ } catch (err) {+ log.warn("d1_lookup_failed", { step: "findActiveAgentRunnerKeyByBearer", err: err instanceof Error ? err : new Error(String(err)) })+ return serviceUnavailable()+ }if (!row) {return NextResponse.json({ error: "runner key revoked or unknown" }, { status: 401 })}- const botUser = await queries.user.getUserInternal(db, row.agentId)+ let botUser: Awaited<ReturnType<typeof queries.user.getUserInternal>>+ try {+ botUser = await withD1Retry(+ () => queries.user.getUserInternal(db, row!.agentId),+ RETRY_OPTS,+ )+ } catch (err) {+ log.warn("d1_lookup_failed", { step: "getUserInternal", err: err instanceof Error ? err : new Error(String(err)) })+ return serviceUnavailable()+ }if (!botUser || !botUser.isBot || botUser.deletedAt !== null) {return NextResponse.json({ error: "bot not found or inactive" }, { status: 401 })}- const binding = await queries.communityBot.getBotBinding(db, row.agentId)+ let binding: Awaited<ReturnType<typeof queries.communityBot.getBotBinding>>+ try {+ binding = await withD1Retry(+ () => queries.communityBot.getBotBinding(db, row!.agentId),+ RETRY_OPTS,+ )+ } catch (err) {+ log.warn("d1_lookup_failed", { step: "getBotBinding", err: err instanceof Error ? err : new Error(String(err)) })+ return serviceUnavailable()+ }if (!binding || binding.machineId !== row.machineId) {return NextResponse.json({ error: "bot binding mismatch" }, { status: 401 })}diff --git a/src/ws-do/src/index.ts b/src/ws-do/src/index.tsindex 330120322..3228a07ae 100644--- a/src/ws-do/src/index.ts+++ b/src/ws-do/src/index.ts@@ -111,7 +111,10 @@ export default {const r = results[i]if (r.status !== "fulfilled" || !r.value.ok) continuetry {- const data = await r.value.json() as { online?: boolean }+ const data = await r.value.json() as { online?: boolean; stale?: boolean }+ // Skip stale responses — a fail-closed `online:false` from D1 must+ // not be surfaced as authoritative offline to fan-out callers.+ if (data.stale) continueif (data.online) online.push(ids[i])} catch { /* skip */ }}diff --git a/src/ws-do/src/ws-durable.test.ts b/src/ws-do/src/ws-durable.test.tsindex 50ab3211f..b95a5ad24 100644--- a/src/ws-do/src/ws-durable.test.ts+++ b/src/ws-do/src/ws-durable.test.ts@@ -232,6 +232,18 @@ vi.mock("@alook/shared", () => {return {createDb: (d1: unknown) => mockCreateDb(d1),createLogger: () => noopLogger,+ // Minimal `readOrStale` shim — bypasses the classifier so tests can+ // inject arbitrary Error shapes at the query-fn boundary and observe+ // fail-closed semantics. Real production behavior (retry then fallback)+ // is covered by the shared `resilience.test.ts` suite.+ readOrStale: async <T>(+ fn: () => Promise<T>,+ fallback: T,+ _opts?: unknown,+ ): Promise<{ value: T; stale: boolean }> => {+ try { return { value: await fn(), stale: false } }+ catch { return { value: fallback, stale: true } }+ },COMMUNITY_MACHINE_HEARTBEAT_MS: 60_000,COMMUNITY_MACHINE_OFFLINE_THRESHOLD_MS: 120_000,SessionErrorFrameSchema,@@ -514,6 +526,18 @@ describe("WebSocketDurableObject", () => {expect(await res.json()).toEqual({ online: true })expect(mockGetUserInternal).not.toHaveBeenCalled()})++ it("degrades to { online: false, stale: true } when D1 throws on the bot-online lookup", async () => {+ // Fail-closed via `readOrStale`: getUserInternal (or isBotOnline)+ // throwing on retry-exhaust must NOT surface as a 500 — the caller+ // reads `{ online: false, stale: true }` and moves on.+ const { durable } = createDO()+ mockGetUserInternal.mockRejectedValue(new Error("D1 down"))++ const res = await durable.fetch(new Request("http://internal/check-user-online?userId=bot-1"))+ expect(res.status).toBe(200)+ expect(await res.json()).toEqual({ online: false, stale: true })+ })})describe("webSocketMessage — auth flow", () => {@@ -1280,6 +1304,32 @@ describe("WebSocketDurableObject", () => {expect(mockUpdateProfile).not.toHaveBeenCalled()expect(mockStubFetch).not.toHaveBeenCalled()})++ it("drops the frame with phase='binding_check' when getBotBinding throws — socket stays open, no write", async () => {+ const { durable, store } = createDO()+ store.set("community-machine-identity", {+ userId: "u_1",+ machineId: "cm_1",+ credentialHash: "0".repeat(64),+ })+ mockGetBotBinding.mockRejectedValue(new Error("D1 down"))++ const ws = createMockWebSocket()+ ws.serializeAttachment({+ type: "community-machine",+ machineId: "cm_1",+ userId: "u_1",+ authenticated: true,+ })++ const frame = JSON.stringify({ type: "agent_activity", agentId: "bot_1", state: "running" })+ await durable.webSocketMessage(ws as any, frame)++ expect(mockUpdateProfile).not.toHaveBeenCalled()+ expect(mockStubFetch).not.toHaveBeenCalled()+ // Socket didn't close — the DO doesn't touch ws.close() on any per-frame D1 error.+ expect(ws.close as ReturnType<typeof vi.fn>).not.toHaveBeenCalled()+ })})describe("community-machine — agent_typing / agent_typing_stop frames", () => {@@ -1612,6 +1662,42 @@ describe("WebSocketDurableObject", () => {expect(mockStubFetch).not.toHaveBeenCalled()})++ it("drops the frame with phase='write' when the insert throws — socket stays open, no fan-out", async () => {+ // This is the ws_frame_dropped_write category — the audit-loss SLO+ // signal. The DO must NOT close the socket, and no owner fan-out+ // should be emitted (the row was never persisted).+ const { durable, store } = createDO()+ store.set("community-machine-identity", {+ userId: "u_1",+ machineId: "cm_1",+ credentialHash: "0".repeat(64),+ })+ mockGetBotBindingWithOwner.mockResolvedValue({+ machineId: "cm_1",+ runtime: "codex",+ ownerUserId: "owner_1",+ })+ mockInsertBotActivityEventAndPrune.mockRejectedValue(new Error("D1 insert failed"))++ const ws = createMockWebSocket()+ ws.serializeAttachment({+ type: "community-machine",+ machineId: "cm_1",+ userId: "u_1",+ authenticated: true,+ })++ const frame = JSON.stringify({+ type: "bot_audit_event",+ agentId: "bot_1",+ event: { kind: "tool_call", payload: { name: "Read" } },+ })+ await durable.webSocketMessage(ws as any, frame)++ expect(mockStubFetch).not.toHaveBeenCalled()+ expect(ws.close as ReturnType<typeof vi.fn>).not.toHaveBeenCalled()+ })})describe("community-machine — ready frame reconciles bot activity", () => {diff --git a/src/ws-do/src/ws-durable.ts b/src/ws-do/src/ws-durable.tsindex 2f06300a8..34650a3c4 100644--- a/src/ws-do/src/ws-durable.ts+++ b/src/ws-do/src/ws-durable.ts@@ -3,6 +3,7 @@ import {createDb,queries,createLogger,+ readOrStale,COMMUNITY_MACHINE_HEARTBEAT_MS,COMMUNITY_MACHINE_OFFLINE_THRESHOLD_MS,HostReadyMessageSchema,@@ -106,22 +107,53 @@ export class WebSocketDurableObject extends DurableObject<Env> {// the id explicitly instead. A bot has no WebSocket of its own; its// "online" is `isBotOnline` (bound machine's status), not a live-socket// check.+ //+ // Check the local WS attachments FIRST — that's D1-free, and a live+ // authenticated user socket is authoritative "online" regardless of+ // whether the target turns out to be a bot. Only when no live socket+ // is attached do we consult D1 to see if this is a bot whose presence+ // is derived from a bound machine's status.+ //+ // D1 reads fail-closed via `readOrStale`: a transient outage on the+ // bot branch yields `{ online: false, stale: true }`. The live-WS+ // early return is unaffected — a D1 blip must NOT flip a real,+ // connected human user offline.const targetUserId = url.searchParams.get("userId")+ const hasAuthUser = this.ctx.getWebSockets().some(ws => {+ const s = ws.deserializeAttachment() as ConnectionState+ return s?.type === "user" && s.authenticated+ })+ if (hasAuthUser) {+ return new Response(JSON.stringify({ online: true }), {+ headers: { "Content-Type": "application/json" },+ })+ }if (targetUserId) {const db = createDb(this.env.DB)- const target = await queries.user.getUserInternal(db, targetUserId)- if (target?.isBot) {- const online = await queries.communityMachine.isBotOnline(db, targetUserId)- return new Response(JSON.stringify({ online }), {+ const { value, stale } = await readOrStale<{ isBotResolved: boolean; online: boolean }>(+ async () => {+ const target = await queries.user.getUserInternal(db, targetUserId)+ if (target?.isBot) {+ const online = await queries.communityMachine.isBotOnline(db, targetUserId)+ return { isBotResolved: true, online }+ }+ return { isBotResolved: false, online: false }+ },+ { isBotResolved: false, online: false },+ { route: "ws-do/check-user-online" },+ )+ if (stale) {+ return new Response(JSON.stringify({ online: false, stale: true }), {+ headers: { "Content-Type": "application/json" },+ })+ }+ if (value.isBotResolved) {+ return new Response(JSON.stringify({ online: value.online }), {headers: { "Content-Type": "application/json" },})}}- const hasAuthUser = this.ctx.getWebSockets().some(ws => {- const s = ws.deserializeAttachment() as ConnectionState- return s?.type === "user" && s.authenticated- })- return new Response(JSON.stringify({ online: hasAuthUser }), {+ return new Response(JSON.stringify({ online: false }), {headers: { "Content-Type": "application/json" },})}@@ -675,7 +707,20 @@ export class WebSocketDurableObject extends DurableObject<Env> {if (activityParse.success) {const { agentId, state } = activityParse.dataconst db = createDb(this.env.DB)- const binding = await queries.communityBot.getBotBinding(db, agentId)+ let binding: Awaited<ReturnType<typeof queries.communityBot.getBotBinding>>+ try {+ binding = await queries.communityBot.getBotBinding(db, agentId)+ } catch (err) {+ log.warn("ws_frame_dropped", {+ category: "ws_frame_dropped",+ frame_type: "agent_activity",+ phase: "binding_check",+ agentId,+ machineId: identity.machineId,+ err: err instanceof Error ? err : new Error(String(err)),+ })+ return+ }if (!binding || binding.machineId !== identity.machineId) {log.warn("agent_activity frame for a bot not bound to this machine — dropped", {agentId,@@ -683,31 +728,42 @@ export class WebSocketDurableObject extends DurableObject<Env> {})return}- const prior = await queries.communityUserProfile.getProfile(db, agentId)- const priorEmoji = prior?.statusEmoji ?? null- const priorText = prior?.statusText ?? null- const priorIsRunning =- priorEmoji !== null &&- RUNNING_PRESETS.some((p) => p.emoji === priorEmoji && p.text === priorText)- // For `running`, reuse the currently-persisted preset if it's already- // one of the running variants — matches the "one phrase per episode"- // invariant instead of re-rolling on every derived running transition- // (turn_end → idle → wake → running fires this repeatedly).- const preset =- state === "running" && priorIsRunning- ? { emoji: priorEmoji as string, text: priorText as string }- : pickBotActivityPreset(state, Math.random())- if (preset.emoji === priorEmoji && preset.text === priorText) return- await queries.communityUserProfile.updateProfile(db, agentId, {- statusEmoji: preset.emoji,- statusText: preset.text,- })- await this.broadcastToAudience(agentId, {- type: "community:status.update",- userId: agentId,- statusEmoji: preset.emoji,- statusText: preset.text,- })+ try {+ const prior = await queries.communityUserProfile.getProfile(db, agentId)+ const priorEmoji = prior?.statusEmoji ?? null+ const priorText = prior?.statusText ?? null+ const priorIsRunning =+ priorEmoji !== null &&+ RUNNING_PRESETS.some((p) => p.emoji === priorEmoji && p.text === priorText)+ // For `running`, reuse the currently-persisted preset if it's already+ // one of the running variants — matches the "one phrase per episode"+ // invariant instead of re-rolling on every derived running transition+ // (turn_end → idle → wake → running fires this repeatedly).+ const preset =+ state === "running" && priorIsRunning+ ? { emoji: priorEmoji as string, text: priorText as string }+ : pickBotActivityPreset(state, Math.random())+ if (preset.emoji === priorEmoji && preset.text === priorText) return+ await queries.communityUserProfile.updateProfile(db, agentId, {+ statusEmoji: preset.emoji,+ statusText: preset.text,+ })+ await this.broadcastToAudience(agentId, {+ type: "community:status.update",+ userId: agentId,+ statusEmoji: preset.emoji,+ statusText: preset.text,+ })+ } catch (err) {+ log.warn("ws_frame_dropped", {+ category: "ws_frame_dropped",+ frame_type: "agent_activity",+ phase: "write",+ agentId,+ machineId: identity.machineId,+ err: err instanceof Error ? err : new Error(String(err)),+ })+ }return}@@ -722,7 +778,20 @@ export class WebSocketDurableObject extends DurableObject<Env> {if (typingParse.success) {const { agentId, dmConversationId } = typingParse.dataconst db = createDb(this.env.DB)- const binding = await queries.communityBot.getBotBindingWithOwner(db, agentId)+ let binding: Awaited<ReturnType<typeof queries.communityBot.getBotBindingWithOwner>>+ try {+ binding = await queries.communityBot.getBotBindingWithOwner(db, agentId)+ } catch (err) {+ log.warn("ws_frame_dropped", {+ category: "ws_frame_dropped",+ frame_type: "agent_typing",+ phase: "binding_check",+ agentId,+ machineId: identity.machineId,+ err: err instanceof Error ? err : new Error(String(err)),+ })+ return+ }if (!binding || binding.machineId !== identity.machineId) {log.warn("agent_typing frame for a bot not bound to this machine — dropped", {agentId,@@ -730,21 +799,45 @@ export class WebSocketDurableObject extends DurableObject<Env> {})return}- // DM participancy is enforced inside `fanOutTyping` — no need to- // pre-query `getDM` here at 5s cadence.- const event = JSON.stringify({- type: "community:typing.start",- dmConversationId,- userId: agentId,- })- await this.fanOutTyping(agentId, undefined, dmConversationId, undefined, event)+ try {+ // DM participancy is enforced inside `fanOutTyping` — no need to+ // pre-query `getDM` here at 5s cadence.+ const event = JSON.stringify({+ type: "community:typing.start",+ dmConversationId,+ userId: agentId,+ })+ await this.fanOutTyping(agentId, undefined, dmConversationId, undefined, event)+ } catch (err) {+ log.warn("ws_frame_dropped", {+ category: "ws_frame_dropped",+ frame_type: "agent_typing",+ phase: "write",+ agentId,+ machineId: identity.machineId,+ err: err instanceof Error ? err : new Error(String(err)),+ })+ }return}const typingStopParse = AgentTypingStopMessageSchema.safeParse(parsed)if (typingStopParse.success) {const { agentId, dmConversationId } = typingStopParse.dataconst db = createDb(this.env.DB)- const binding = await queries.communityBot.getBotBindingWithOwner(db, agentId)+ let binding: Awaited<ReturnType<typeof queries.communityBot.getBotBindingWithOwner>>+ try {+ binding = await queries.communityBot.getBotBindingWithOwner(db, agentId)+ } catch (err) {+ log.warn("ws_frame_dropped", {+ category: "ws_frame_dropped",+ frame_type: "agent_typing_stop",+ phase: "binding_check",+ agentId,+ machineId: identity.machineId,+ err: err instanceof Error ? err : new Error(String(err)),+ })+ return+ }if (!binding || binding.machineId !== identity.machineId) {log.warn("agent_typing_stop frame for a bot not bound to this machine — dropped", {agentId,@@ -752,8 +845,19 @@ export class WebSocketDurableObject extends DurableObject<Env> {})return}- // DM participancy enforced inside `fanOutTypingStop`.- await this.fanOutTypingStop(agentId, dmConversationId)+ try {+ // DM participancy enforced inside `fanOutTypingStop`.+ await this.fanOutTypingStop(agentId, dmConversationId)+ } catch (err) {+ log.warn("ws_frame_dropped", {+ category: "ws_frame_dropped",+ frame_type: "agent_typing_stop",+ phase: "write",+ agentId,+ machineId: identity.machineId,+ err: err instanceof Error ? err : new Error(String(err)),+ })+ }return}@@ -766,7 +870,24 @@ export class WebSocketDurableObject extends DurableObject<Env> {if (auditParse.success) {const frame = auditParse.dataconst db = createDb(this.env.DB)- const binding = await queries.communityBot.getBotBindingWithOwner(db, frame.agentId)+ let binding: Awaited<ReturnType<typeof queries.communityBot.getBotBindingWithOwner>>+ try {+ binding = await queries.communityBot.getBotBindingWithOwner(db, frame.agentId)+ } catch (err) {+ // Binding check treated as `ws_frame_dropped` (security-adjacent+ // lookup failure — a daemon reporting on a bot it may not own).+ // NOT `ws_frame_dropped_write` — that category is reserved for the+ // insert-side failure that feeds the audit-loss SLO.+ log.warn("ws_frame_dropped", {+ category: "ws_frame_dropped",+ frame_type: "bot_audit_event",+ phase: "binding_check",+ agentId: frame.agentId,+ machineId: identity.machineId,+ err: err instanceof Error ? err : new Error(String(err)),+ })+ return+ }if (!binding || binding.machineId !== identity.machineId) {log.warn("bot_audit_event frame for a bot not bound to this machine — dropped", {agentId: frame.agentId,@@ -774,25 +895,55 @@ export class WebSocketDurableObject extends DurableObject<Env> {})return}- const payload = JSON.stringify(frame.event.payload)- const inserted = await queries.communityBotAuditLog.insertBotActivityEventAndPrune(db, {- botId: frame.agentId,- sessionId: frame.sessionId ?? null,- launchId: frame.launchId ?? null,- kind: frame.event.kind,- payload,- })- if (!inserted) return- await this.notifyUserDO(binding.ownerUserId, {- type: "community:bot.audit_event",- botId: frame.agentId,- id: inserted.id,- kind: frame.event.kind,- payload: frame.event.payload,- sessionId: frame.sessionId ?? null,- launchId: frame.launchId ?? null,- createdAt: inserted.createdAt,- }).catch(() => { })+ // Serialize BEFORE the write try — a non-serializable payload (BigInt,+ // circular ref, function) throwing here has nothing to do with the+ // audit-loss SLO and must not be logged under `ws_frame_dropped_write`.+ let payload: string+ try {+ payload = JSON.stringify(frame.event.payload)+ } catch (err) {+ log.warn("ws_frame_dropped", {+ category: "ws_frame_dropped",+ frame_type: "bot_audit_event",+ phase: "serialize",+ agentId: frame.agentId,+ machineId: identity.machineId,+ err: err instanceof Error ? err : new Error(String(err)),+ })+ return+ }+ try {+ const inserted = await queries.communityBotAuditLog.insertBotActivityEventAndPrune(db, {+ botId: frame.agentId,+ sessionId: frame.sessionId ?? null,+ launchId: frame.launchId ?? null,+ kind: frame.event.kind,+ payload,+ })+ if (!inserted) return+ await this.notifyUserDO(binding.ownerUserId, {+ type: "community:bot.audit_event",+ botId: frame.agentId,+ id: inserted.id,+ kind: frame.event.kind,+ payload: frame.event.payload,+ sessionId: frame.sessionId ?? null,+ launchId: frame.launchId ?? null,+ createdAt: inserted.createdAt,+ }).catch(() => { })+ } catch (err) {+ // `ws_frame_dropped_write` — the audit-loss SLO category. Emitted at+ // full-rate ingest via ws-do's `head_sampling_rate = 1.0` so a+ // low-rate drift doesn't hide behind 10% sampling.+ log.warn("ws_frame_dropped", {+ category: "ws_frame_dropped_write",+ frame_type: "bot_audit_event",+ phase: "write",+ agentId: frame.agentId,+ machineId: identity.machineId,+ err: err instanceof Error ? err : new Error(String(err)),+ })+ }return}@@ -803,91 +954,101 @@ export class WebSocketDurableObject extends DurableObject<Env> {const readyParse = HostReadyMessageSchema.safeParse(parsed)if (!readyParse.success) returnconst ready = readyParse.data- const hostname = ready.hostname ?? ""- const platform = ready.platform ?? ""- const arch = ready.arch ?? ""- const daemonVersion = ready.daemonVersion ?? ""- const osRelease = ready.osRelease ?? ""- const availableRuntimes: CommunityMachineRuntime[] = ready.runtimeReport+ try {+ const hostname = ready.hostname ?? ""+ const platform = ready.platform ?? ""+ const arch = ready.arch ?? ""+ const daemonVersion = ready.daemonVersion ?? ""+ const osRelease = ready.osRelease ?? ""+ const availableRuntimes: CommunityMachineRuntime[] = ready.runtimeReport- const db = createDb(this.env.DB)- const result = await queries.communityMachine.upsertMachineByMachineId(- db,- identity.userId,- identity.machineId,- { hostname, platform, arch, daemonVersion, osRelease, availableRuntimes }- )- if (!result) {- // The machine row was deleted (or race) between credential validation- // and this update — bail. The row will not be re-created here; the- // daemon will be evicted on the next credential lookup via cascade.- log.warn("community machine row missing on ready", { machineId: identity.machineId })- return- }- const { machine, priorAvailableRuntimes, priorStatus } = result-- // Coarse safety net for an `agent_activity` frame dropped mid-disconnect —- // clear any bot on this machine whose current status pill looks like a- // stale system-written activity pill AND who the daemon reports is NOT- // running now. Live `agent_activity` pushes handle every non-`idle`- // transition; the reconciler only ever writes `Idle`. Owner-set custom- // statuses (identified by not matching the known bot presets) are left- // alone. See plans/community-bot-status-telemetry.md.- const activityChanges = await queries.communityMachine.reconcileBotActivityFromRunningAgents(- db,- machine.id,- ready.runningAgents- )- await Promise.allSettled(- activityChanges.map(({ botUserId, statusEmoji, statusText }) =>- this.broadcastToAudience(botUserId, {- type: "community:status.update",- userId: botUserId,- statusEmoji,- statusText,- })+ const db = createDb(this.env.DB)+ const result = await queries.communityMachine.upsertMachineByMachineId(+ db,+ identity.userId,+ identity.machineId,+ { hostname, platform, arch, daemonVersion, osRelease, availableRuntimes }+ )+ if (!result) {+ // The machine row was deleted (or race) between credential validation+ // and this update — bail. The row will not be re-created here; the+ // daemon will be evicted on the next credential lookup via cascade.+ log.warn("community machine row missing on ready", { machineId: identity.machineId })+ return+ }+ const { machine, priorAvailableRuntimes, priorStatus } = result++ // Coarse safety net for an `agent_activity` frame dropped mid-disconnect —+ // clear any bot on this machine whose current status pill looks like a+ // stale system-written activity pill AND who the daemon reports is NOT+ // running now. Live `agent_activity` pushes handle every non-`idle`+ // transition; the reconciler only ever writes `Idle`. Owner-set custom+ // statuses (identified by not matching the known bot presets) are left+ // alone. See plans/community-bot-status-telemetry.md.+ const activityChanges = await queries.communityMachine.reconcileBotActivityFromRunningAgents(+ db,+ machine.id,+ ready.runningAgents+ )+ await Promise.allSettled(+ activityChanges.map(({ botUserId, statusEmoji, statusText }) =>+ this.broadcastToAudience(botUserId, {+ type: "community:status.update",+ userId: botUserId,+ statusEmoji,+ statusText,+ })+ ))- )-- const summary = await this.summaryWithOverlay(machine)- // Refresh the offline-detection handle in case metadata changed. Handle- // was written at accept; this is idempotent.- await this.ctx.storage.put<CommunityMachineHandle>(HANDLE_KEY, {- userId: identity.userId,- machineId: machine.id,- })- // NOTE: `community:machine.created` is emitted by the /activate route,- // not here — activation is the single source of the create event and- // carries the pairing token the client needs to reconcile its pending- // state. Here we only handle status transitions and runtime drift.- //- // Broadcast the online transition ONLY when the row actually flipped- // offline → online. `priorStatus` is the pre-upsert column value returned- // by upsertMachineByMachineId; the upsert unconditionally sets- // status='online', so `priorStatus !== 'online'` is the exact transition.- if (priorStatus !== "online") {- await this.notifyUserDO(identity.userId, {- type: "community:machine.status",+ const summary = await this.summaryWithOverlay(machine)+ // Refresh the offline-detection handle in case metadata changed. Handle+ // was written at accept; this is idempotent.+ await this.ctx.storage.put<CommunityMachineHandle>(HANDLE_KEY, {+ userId: identity.userId,machineId: machine.id,- status: "online",- lastSeenAt: machine.lastSeenAt ?? new Date().toISOString(),- }).catch(() => { })- }+ })- // Runtime-drift diff. Canonicalized form now includes status/lastError- // so a runtime flipping healthy → unhealthy on subsequent ready frames- // (e.g. ENOENT hit at spawn time) fans out `community:machine.updated`.- const priorCanonical = canonicalRuntimes(priorAvailableRuntimes ?? [])- const nextCanonical = canonicalRuntimes(availableRuntimes)- if (priorCanonical !== nextCanonical) {- await this.notifyUserDO(identity.userId, {- type: "community:machine.updated",- machine: summary,- }).catch(() => { })- }+ // NOTE: `community:machine.created` is emitted by the /activate route,+ // not here — activation is the single source of the create event and+ // carries the pairing token the client needs to reconcile its pending+ // state. Here we only handle status transitions and runtime drift.+ //+ // Broadcast the online transition ONLY when the row actually flipped+ // offline → online. `priorStatus` is the pre-upsert column value returned+ // by upsertMachineByMachineId; the upsert unconditionally sets+ // status='online', so `priorStatus !== 'online'` is the exact transition.+ if (priorStatus !== "online") {+ await this.notifyUserDO(identity.userId, {+ type: "community:machine.status",+ machineId: machine.id,+ status: "online",+ lastSeenAt: machine.lastSeenAt ?? new Date().toISOString(),+ }).catch(() => { })+ }- await this.scheduleHeartbeatAlarm()+ // Runtime-drift diff. Canonicalized form now includes status/lastError+ // so a runtime flipping healthy → unhealthy on subsequent ready frames+ // (e.g. ENOENT hit at spawn time) fans out `community:machine.updated`.+ const priorCanonical = canonicalRuntimes(priorAvailableRuntimes ?? [])+ const nextCanonical = canonicalRuntimes(availableRuntimes)+ if (priorCanonical !== nextCanonical) {+ await this.notifyUserDO(identity.userId, {+ type: "community:machine.updated",+ machine: summary,+ }).catch(() => { })+ }++ await this.scheduleHeartbeatAlarm()+ } catch (err) {+ log.warn("ws_frame_dropped", {+ category: "ws_frame_dropped",+ frame_type: "ready",+ phase: "write",+ machineId: identity.machineId,+ err: err instanceof Error ? err : new Error(String(err)),+ })+ }}/**@@ -1255,8 +1416,13 @@ export class WebSocketDurableObject extends DurableObject<Env> {const resp = await stub.fetch(new Request(`http://internal/check-user-online?userId=${encodeURIComponent(memberId)}`))- const { online } = await resp.json() as { online: boolean }- return online ? memberId : null+ // On a stale response the callee could not resolve the target's+ // presence — skip rather than treating `online: false` as truth+ // (that would flip real-online contacts to offline on every+ // D1 blip during snapshot).+ const data = await resp.json() as { online: boolean; stale?: boolean }+ if (data.stale) return null+ return data.online ? memberId : null}))for (const r of checks) {diff --git a/src/ws-do/wrangler.toml b/src/ws-do/wrangler.tomlindex 879eed286..151a1855e 100644--- a/src/ws-do/wrangler.toml+++ b/src/ws-do/wrangler.toml@@ -8,9 +8,15 @@ workers_dev = falseenabled = truehead_sampling_rate = 0.1+# Exempted from the default 0.1 sampling used by other workers: the+# `ws_frame_dropped_write` audit-loss signal is a rare-write category, and+# head-sampling is a request-level decision that can't be bypassed by log+# level. Full-rate ingest is affordable here — ws-do's log volume is+# dominated by daemon frames — and is the only reliable way to measure the+# audit-loss SLO. See plans/d1-critical-path-resilience-v2.md §Observability.[observability.logs]invocation_logs = true-head_sampling_rate = 0.1+head_sampling_rate = 1.0destinations = ["grafana-logs"][observability.traces]