Multi-Agent Collaboration for Team and Friends
TypeScript agent-automationagent-orchestrationagent-workflowai-agentsai-collaborationai-memoryai-workforceautonomous-agentsclaude-codeclaude-code-skillscodexcoding-agentmulti-agentmultiple-agentone-person-businessone-person-companyone-person-teamopencodesolopreneur
refactor(shared,community): expand retry classifier + collapse duplicated D1-guard scaffolds
Gusbf52c563a36a
diff --git a/src/shared/src/db/resilience.ts b/src/shared/src/db/resilience.tsindex cb87757a9..80e4c4a9d 100644--- a/src/shared/src/db/resilience.ts+++ b/src/shared/src/db/resilience.ts@@ -15,25 +15,54 @@ const DEFAULT_BASE_DELAY_MS = 100const defaultLogger: Logger = createLogger({ service: "d1-resilience" })const RETRYABLE_SIGNATURES = [+ // workerd / D1 transient runtime errors."internal error; reference","SQLITE_BUSY","database is locked","SQLITE_INTERRUPT",+ // CF RPC / fetch transient shapes. `fetch failed` covers Node's+ // fetch-rejection wrapper, `ETIMEDOUT` / `ECONNRESET` / `EAI_AGAIN`+ // catch DNS + socket transients seen from daemon-plane routes."Network connection lost","connection reset",+ "fetch failed",+ "ETIMEDOUT",+ "ECONNRESET",+ "ECONNREFUSED",+ "EAI_AGAIN",+ // `timeout after` catches Node's / undici's phrase; leading space on the+ // others avoids matching column names like `timeout_at` inside a+ // SQLITE_CONSTRAINT message.+ "timeout after",+ " timed out",+ "network timeout",+ "socket hang up",]-function peelDrizzle(err: unknown): unknown {- let cur = err+/**+ * Peel a DrizzleQueryError chain to its underlying cause. If Drizzle ever+ * wraps a transient RPC error WITHOUT preserving `.cause` (older versions+ * of the ORM did this, and some codepaths still do), the bare wrapper's+ * message is `Failed query: …` — no signature matches, so classification+ * would silently return "not retryable" and every retry across the fleet+ * stops working. Return `null` in that case so the caller can conservatively+ * treat a bare DrizzleQueryError as retryable.+ */+function peelDrizzle(err: unknown): { peeled: unknown; bareWrapper: boolean } {+ if (!(err instanceof DrizzleQueryError)) return { peeled: err, bareWrapper: false }+ let cur: unknown = errwhile (cur instanceof DrizzleQueryError) {- if (!cur.cause) return cur+ if (!cur.cause) return { peeled: cur, bareWrapper: true }cur = cur.cause}- return cur+ return { peeled: cur, bareWrapper: false }}export function isRetryableD1Error(err: unknown): boolean {- const peeled = peelDrizzle(err)+ const { peeled, bareWrapper } = peelDrizzle(err)+ // A DrizzleQueryError with no `.cause` is a database error whose transient+ // shape we can't inspect — retry conservatively rather than fail-fast.+ if (bareWrapper) return trueif (!(peeled instanceof Error)) return falseconst msg = peeled.messageif (typeof msg !== "string") return falsediff --git a/src/shared/test/resilience.test.ts b/src/shared/test/resilience.test.tsindex 25ea653af..8b0e7b145 100644--- a/src/shared/test/resilience.test.ts+++ b/src/shared/test/resilience.test.ts@@ -126,6 +126,46 @@ describe("classifier — isRetryableD1Error", () => {expect(isRetryableD1Error(new DrizzleQueryError("q", [], new Error("Network connection lost")))).toBe(true)expect(isRetryableD1Error(new DrizzleQueryError("q", [], new Error("connection reset by peer")))).toBe(true)})++ it("retries fetch/socket-level transients daemon-plane routes see", () => {+ for (const sig of [+ "fetch failed",+ "ETIMEDOUT",+ "ECONNRESET",+ "ECONNREFUSED",+ "EAI_AGAIN",+ "request timeout after 5000ms",+ "socket hang up",+ ]) {+ expect(isRetryableD1Error(new DrizzleQueryError("q", [], new Error(sig)))).toBe(true)+ }+ })++ it("does NOT match column names containing 'timeout' in a SQLITE_CONSTRAINT message", () => {+ // Guard the tightened matcher — bare `timeout` used to match here.+ const err = new DrizzleQueryError(+ "INSERT",+ [],+ new Error("SQLITE_CONSTRAINT_NOTNULL: NOT NULL constraint failed: session.timeout_at"),+ )+ expect(isRetryableD1Error(err)).toBe(false)+ })++ it("conservatively retries a bare DrizzleQueryError with no .cause", () => {+ // Older Drizzle versions (and some codepaths in current ones) wrap a+ // transient RPC error without preserving `.cause` — the wrapper's+ // message is `Failed query: …` with no signature. Treat as retryable+ // rather than fail-fast.+ class BareDQE extends DrizzleQueryError {+ constructor() {+ super("SELECT 1", [], undefined as unknown as Error)+ // @ts-expect-error — deliberately null out cause to simulate the+ // bare-wrapper shape.+ this.cause = undefined+ }+ }+ expect(isRetryableD1Error(new BareDQE())).toBe(true)+ })})describe("readOrStale", () => {diff --git a/src/web/src/lib/middleware/community-agent-runner-auth.ts b/src/web/src/lib/middleware/community-agent-runner-auth.tsindex 996038eb2..4f7a84da7 100644--- a/src/web/src/lib/middleware/community-agent-runner-auth.ts+++ b/src/web/src/lib/middleware/community-agent-runner-auth.ts@@ -28,6 +28,25 @@ function serviceUnavailable(): NextResponse {)}+/**+ * Run a D1 lookup through `withD1Retry`; on retry-exhaust log the failing+ * step and return a `NextResponse` sentinel the caller can early-return.+ * Every lookup MUST route through this helper so a new step can't+ * accidentally ship with a bare `try/catch` that either swallows the log+ * or converts the 503 into a 401 (which would rotate CLI runner keys).+ */+async function lookupOr503<T>(+ step: string,+ fn: () => Promise<T>,+): Promise<{ ok: true; value: T } | { ok: false; response: NextResponse }> {+ try {+ return { ok: true, value: await withD1Retry(fn, RETRY_OPTS) }+ } catch (err) {+ log.warn("d1_lookup_failed", { step, err: err instanceof Error ? err : new Error(String(err)) })+ return { ok: false, response: serviceUnavailable() }+ }+}+/*** Agent-runner auth middleware for the CLI bridge (`/api/community/agent/*`).* Requires `Authorization: Bearer crk_…`. Cloned from `withCommunityDaemonAuth`@@ -40,9 +59,12 @@ function serviceUnavailable(): NextResponse {** D1-transient failure semantics: each of the 3 D1 reads runs through* `withD1Retry`; on retry-exhaust we return 503 + `Retry-After: 1` (RFC 9110- * §15.6.4) so CLI bridges treat it as retryable and do NOT rotate their- * runner key. 401 is reserved for real auth failures (bad token, revoked- * runner key, bot deleted, binding mismatch).+ * §15.6.4). What that buys us: 401 is reserved for real auth failures (bad+ * token, revoked runner key, bot deleted, binding mismatch) — so a+ * transient D1 blip surfaces as "temporarily unavailable" and does NOT+ * trip the CLI's runner-key rotation path (which is only ever driven by+ * 401). The CLI itself does not auto-retry on 503 today; the bot's next+ * wake naturally re-issues the command against a healthy D1.*/export function withAgentRunnerAuth(handler: AgentRunnerAuthenticatedHandler) {return async (@@ -68,44 +90,29 @@ export function withAgentRunnerAuth(handler: AgentRunnerAuthenticatedHandler) {const cloudflareEnv = env as Envconst db = getDb(cloudflareEnv.DB)- let row: Awaited<ReturnType<typeof queries.communityMachine.findActiveAgentRunnerKeyByBearer>>- try {- row = await withD1Retry(- () => queries.communityMachine.findActiveAgentRunnerKeyByBearer(db, raw),- RETRY_OPTS,- )- } catch (err) {- log.warn("d1_lookup_failed", { step: "findActiveAgentRunnerKeyByBearer", err: err instanceof Error ? err : new Error(String(err)) })- return serviceUnavailable()- }+ const rowLookup = await lookupOr503("findActiveAgentRunnerKeyByBearer", () =>+ queries.communityMachine.findActiveAgentRunnerKeyByBearer(db, raw),+ )+ if (!rowLookup.ok) return rowLookup.response+ const row = rowLookup.valueif (!row) {return NextResponse.json({ error: "runner key revoked or unknown" }, { status: 401 })}- let botUser: Awaited<ReturnType<typeof queries.user.getUserInternal>>- try {- botUser = await withD1Retry(- () => queries.user.getUserInternal(db, row!.agentId),- RETRY_OPTS,- )- } catch (err) {- log.warn("d1_lookup_failed", { step: "getUserInternal", err: err instanceof Error ? err : new Error(String(err)) })- return serviceUnavailable()- }+ const botLookup = await lookupOr503("getUserInternal", () =>+ queries.user.getUserInternal(db, row.agentId),+ )+ if (!botLookup.ok) return botLookup.response+ const botUser = botLookup.valueif (!botUser || !botUser.isBot || botUser.deletedAt !== null) {return NextResponse.json({ error: "bot not found or inactive" }, { status: 401 })}- let binding: Awaited<ReturnType<typeof queries.communityBot.getBotBinding>>- try {- binding = await withD1Retry(- () => queries.communityBot.getBotBinding(db, row!.agentId),- RETRY_OPTS,- )- } catch (err) {- log.warn("d1_lookup_failed", { step: "getBotBinding", err: err instanceof Error ? err : new Error(String(err)) })- return serviceUnavailable()- }+ const bindingLookup = await lookupOr503("getBotBinding", () =>+ queries.communityBot.getBotBinding(db, row.agentId),+ )+ if (!bindingLookup.ok) return bindingLookup.response+ const binding = bindingLookup.valueif (!binding || binding.machineId !== row.machineId) {return NextResponse.json({ error: "bot binding mismatch" }, { status: 401 })}diff --git a/src/ws-do/src/ws-durable.ts b/src/ws-do/src/ws-durable.tsindex 34650a3c4..7684b985d 100644--- a/src/ws-do/src/ws-durable.ts+++ b/src/ws-do/src/ws-durable.ts@@ -625,6 +625,71 @@ export class WebSocketDurableObject extends DurableObject<Env> {await this.ctx.storage.setAlarm(Date.now() + (COMMUNITY_MACHINE_OFFLINE_THRESHOLD_MS - elapsed))}+ /**+ * Shared scaffold for every daemon-reported bot frame (agent_activity,+ * agent_typing, agent_typing_stop, bot_audit_event):+ *+ * 1. resolve binding → if the query throws, log `phase=binding_check`+ * under `ws_frame_dropped` and drop (never `_write` — the write+ * never ran and this is a security-adjacent lookup failure).+ * 2. verify ownership → drop with a plain warn if `binding.machineId`+ * doesn't match this DO's identity (frame-supplied agentId can't+ * be trusted).+ * 3. run the frame-specific write → if it throws, log `phase=write`+ * under `writeCategory` (`ws_frame_dropped_write` for audit events+ * feeding the SLO, `ws_frame_dropped` for everything else).+ *+ * Centralized here so a new frame type CAN'T ship with the wrong log+ * category or a missing phase tag — the whole reason the discipline+ * exists.+ */+ private async handleFrameForBoundBot<B>(args: {+ frameType: string+ agentId: string+ machineId: string+ writeCategory?: "ws_frame_dropped" | "ws_frame_dropped_write"+ resolveBinding: () => Promise<B | null | undefined>+ isMatch: (binding: B) => boolean+ write: (binding: B) => Promise<void>+ }): Promise<void> {+ const { frameType, agentId, machineId, resolveBinding, isMatch, write } = args+ const writeCategory = args.writeCategory ?? "ws_frame_dropped"++ let binding: B | null | undefined+ try {+ binding = await resolveBinding()+ } catch (err) {+ log.warn("ws_frame_dropped", {+ category: "ws_frame_dropped",+ frame_type: frameType,+ phase: "binding_check",+ agentId,+ machineId,+ err: err instanceof Error ? err : new Error(String(err)),+ })+ return+ }+ if (!binding || !isMatch(binding)) {+ log.warn(`${frameType} frame for a bot not bound to this machine — dropped`, {+ agentId,+ machineId,+ })+ return+ }+ try {+ await write(binding)+ } catch (err) {+ log.warn("ws_frame_dropped", {+ category: writeCategory,+ frame_type: frameType,+ phase: "write",+ agentId,+ machineId,+ err: err instanceof Error ? err : new Error(String(err)),+ })+ }+ }+private async handleCommunityMachineMessage(parsed: unknown): Promise<void> {// Identity lives in ctx.storage — one D1 lookup at accept, zero here.const identity = await this.ctx.storage.get<CommunityMachineIdentity>(IDENTITY_KEY)@@ -707,63 +772,40 @@ export class WebSocketDurableObject extends DurableObject<Env> {if (activityParse.success) {const { agentId, state } = activityParse.dataconst db = createDb(this.env.DB)- let binding: Awaited<ReturnType<typeof queries.communityBot.getBotBinding>>- try {- binding = await queries.communityBot.getBotBinding(db, agentId)- } catch (err) {- log.warn("ws_frame_dropped", {- category: "ws_frame_dropped",- frame_type: "agent_activity",- phase: "binding_check",- agentId,- machineId: identity.machineId,- err: err instanceof Error ? err : new Error(String(err)),- })- return- }- if (!binding || binding.machineId !== identity.machineId) {- log.warn("agent_activity frame for a bot not bound to this machine — dropped", {- agentId,- machineId: identity.machineId,- })- return- }- try {- const prior = await queries.communityUserProfile.getProfile(db, agentId)- const priorEmoji = prior?.statusEmoji ?? null- const priorText = prior?.statusText ?? null- const priorIsRunning =- priorEmoji !== null &&- RUNNING_PRESETS.some((p) => p.emoji === priorEmoji && p.text === priorText)- // For `running`, reuse the currently-persisted preset if it's already- // one of the running variants — matches the "one phrase per episode"- // invariant instead of re-rolling on every derived running transition- // (turn_end → idle → wake → running fires this repeatedly).- const preset =- state === "running" && priorIsRunning- ? { emoji: priorEmoji as string, text: priorText as string }- : pickBotActivityPreset(state, Math.random())- if (preset.emoji === priorEmoji && preset.text === priorText) return- await queries.communityUserProfile.updateProfile(db, agentId, {- statusEmoji: preset.emoji,- statusText: preset.text,- })- await this.broadcastToAudience(agentId, {- type: "community:status.update",- userId: agentId,- statusEmoji: preset.emoji,- statusText: preset.text,- })- } catch (err) {- log.warn("ws_frame_dropped", {- category: "ws_frame_dropped",- frame_type: "agent_activity",- phase: "write",- agentId,- machineId: identity.machineId,- err: err instanceof Error ? err : new Error(String(err)),- })- }+ await this.handleFrameForBoundBot({+ frameType: "agent_activity",+ agentId,+ machineId: identity.machineId,+ resolveBinding: () => queries.communityBot.getBotBinding(db, agentId),+ isMatch: (binding) => binding.machineId === identity.machineId,+ write: async () => {+ const prior = await queries.communityUserProfile.getProfile(db, agentId)+ const priorEmoji = prior?.statusEmoji ?? null+ const priorText = prior?.statusText ?? null+ const priorIsRunning =+ priorEmoji !== null &&+ RUNNING_PRESETS.some((p) => p.emoji === priorEmoji && p.text === priorText)+ // For `running`, reuse the currently-persisted preset if it's already+ // one of the running variants — matches the "one phrase per episode"+ // invariant instead of re-rolling on every derived running transition+ // (turn_end → idle → wake → running fires this repeatedly).+ const preset =+ state === "running" && priorIsRunning+ ? { emoji: priorEmoji as string, text: priorText as string }+ : pickBotActivityPreset(state, Math.random())+ if (preset.emoji === priorEmoji && preset.text === priorText) return+ await queries.communityUserProfile.updateProfile(db, agentId, {+ statusEmoji: preset.emoji,+ statusText: preset.text,+ })+ await this.broadcastToAudience(agentId, {+ type: "community:status.update",+ userId: agentId,+ statusEmoji: preset.emoji,+ statusText: preset.text,+ })+ },+ })return}@@ -778,86 +820,38 @@ export class WebSocketDurableObject extends DurableObject<Env> {if (typingParse.success) {const { agentId, dmConversationId } = typingParse.dataconst db = createDb(this.env.DB)- let binding: Awaited<ReturnType<typeof queries.communityBot.getBotBindingWithOwner>>- try {- binding = await queries.communityBot.getBotBindingWithOwner(db, agentId)- } catch (err) {- log.warn("ws_frame_dropped", {- category: "ws_frame_dropped",- frame_type: "agent_typing",- phase: "binding_check",- agentId,- machineId: identity.machineId,- err: err instanceof Error ? err : new Error(String(err)),- })- return- }- if (!binding || binding.machineId !== identity.machineId) {- log.warn("agent_typing frame for a bot not bound to this machine — dropped", {- agentId,- machineId: identity.machineId,- })- return- }- try {- // DM participancy is enforced inside `fanOutTyping` — no need to- // pre-query `getDM` here at 5s cadence.- const event = JSON.stringify({- type: "community:typing.start",- dmConversationId,- userId: agentId,- })- await this.fanOutTyping(agentId, undefined, dmConversationId, undefined, event)- } catch (err) {- log.warn("ws_frame_dropped", {- category: "ws_frame_dropped",- frame_type: "agent_typing",- phase: "write",- agentId,- machineId: identity.machineId,- err: err instanceof Error ? err : new Error(String(err)),- })- }+ await this.handleFrameForBoundBot({+ frameType: "agent_typing",+ agentId,+ machineId: identity.machineId,+ resolveBinding: () => queries.communityBot.getBotBindingWithOwner(db, agentId),+ isMatch: (binding) => binding.machineId === identity.machineId,+ write: async () => {+ // DM participancy is enforced inside `fanOutTyping` — no need to+ // pre-query `getDM` here at 5s cadence.+ const event = JSON.stringify({+ type: "community:typing.start",+ dmConversationId,+ userId: agentId,+ })+ await this.fanOutTyping(agentId, undefined, dmConversationId, undefined, event)+ },+ })return}const typingStopParse = AgentTypingStopMessageSchema.safeParse(parsed)if (typingStopParse.success) {const { agentId, dmConversationId } = typingStopParse.dataconst db = createDb(this.env.DB)- let binding: Awaited<ReturnType<typeof queries.communityBot.getBotBindingWithOwner>>- try {- binding = await queries.communityBot.getBotBindingWithOwner(db, agentId)- } catch (err) {- log.warn("ws_frame_dropped", {- category: "ws_frame_dropped",- frame_type: "agent_typing_stop",- phase: "binding_check",- agentId,- machineId: identity.machineId,- err: err instanceof Error ? err : new Error(String(err)),- })- return- }- if (!binding || binding.machineId !== identity.machineId) {- log.warn("agent_typing_stop frame for a bot not bound to this machine — dropped", {- agentId,- machineId: identity.machineId,- })- return- }- try {+ await this.handleFrameForBoundBot({+ frameType: "agent_typing_stop",+ agentId,+ machineId: identity.machineId,+ resolveBinding: () => queries.communityBot.getBotBindingWithOwner(db, agentId),+ isMatch: (binding) => binding.machineId === identity.machineId,// DM participancy enforced inside `fanOutTypingStop`.- await this.fanOutTypingStop(agentId, dmConversationId)- } catch (err) {- log.warn("ws_frame_dropped", {- category: "ws_frame_dropped",- frame_type: "agent_typing_stop",- phase: "write",- agentId,- machineId: identity.machineId,- err: err instanceof Error ? err : new Error(String(err)),- })- }+ write: () => this.fanOutTypingStop(agentId, dmConversationId),+ })return}@@ -869,35 +863,10 @@ export class WebSocketDurableObject extends DurableObject<Env> {const auditParse = HostBotAuditEventFrameSchema.safeParse(parsed)if (auditParse.success) {const frame = auditParse.data- const db = createDb(this.env.DB)- let binding: Awaited<ReturnType<typeof queries.communityBot.getBotBindingWithOwner>>- try {- binding = await queries.communityBot.getBotBindingWithOwner(db, frame.agentId)- } catch (err) {- // Binding check treated as `ws_frame_dropped` (security-adjacent- // lookup failure — a daemon reporting on a bot it may not own).- // NOT `ws_frame_dropped_write` — that category is reserved for the- // insert-side failure that feeds the audit-loss SLO.- log.warn("ws_frame_dropped", {- category: "ws_frame_dropped",- frame_type: "bot_audit_event",- phase: "binding_check",- agentId: frame.agentId,- machineId: identity.machineId,- err: err instanceof Error ? err : new Error(String(err)),- })- return- }- if (!binding || binding.machineId !== identity.machineId) {- log.warn("bot_audit_event frame for a bot not bound to this machine — dropped", {- agentId: frame.agentId,- machineId: identity.machineId,- })- return- }- // Serialize BEFORE the write try — a non-serializable payload (BigInt,- // circular ref, function) throwing here has nothing to do with the- // audit-loss SLO and must not be logged under `ws_frame_dropped_write`.+ // Serialize BEFORE the shared scaffold — a non-serializable payload+ // (BigInt, circular ref, function) throwing here has nothing to do+ // with the audit-loss SLO and must not be logged under+ // `ws_frame_dropped_write`.let payload: stringtry {payload = JSON.stringify(frame.event.payload)@@ -912,38 +881,40 @@ export class WebSocketDurableObject extends DurableObject<Env> {})return}- try {- const inserted = await queries.communityBotAuditLog.insertBotActivityEventAndPrune(db, {- botId: frame.agentId,- sessionId: frame.sessionId ?? null,- launchId: frame.launchId ?? null,- kind: frame.event.kind,- payload,- })- if (!inserted) return- await this.notifyUserDO(binding.ownerUserId, {- type: "community:bot.audit_event",- botId: frame.agentId,- id: inserted.id,- kind: frame.event.kind,- payload: frame.event.payload,- sessionId: frame.sessionId ?? null,- launchId: frame.launchId ?? null,- createdAt: inserted.createdAt,- }).catch(() => { })- } catch (err) {+ const db = createDb(this.env.DB)+ await this.handleFrameForBoundBot({+ frameType: "bot_audit_event",+ agentId: frame.agentId,+ machineId: identity.machineId,// `ws_frame_dropped_write` — the audit-loss SLO category. Emitted at// full-rate ingest via ws-do's `head_sampling_rate = 1.0` so a- // low-rate drift doesn't hide behind 10% sampling.- log.warn("ws_frame_dropped", {- category: "ws_frame_dropped_write",- frame_type: "bot_audit_event",- phase: "write",- agentId: frame.agentId,- machineId: identity.machineId,- err: err instanceof Error ? err : new Error(String(err)),- })- }+ // low-rate drift doesn't hide behind 10% sampling. Binding-check+ // failures stay on the plain `ws_frame_dropped` category via the+ // helper's default.+ writeCategory: "ws_frame_dropped_write",+ resolveBinding: () => queries.communityBot.getBotBindingWithOwner(db, frame.agentId),+ isMatch: (binding) => binding.machineId === identity.machineId,+ write: async (binding) => {+ const inserted = await queries.communityBotAuditLog.insertBotActivityEventAndPrune(db, {+ botId: frame.agentId,+ sessionId: frame.sessionId ?? null,+ launchId: frame.launchId ?? null,+ kind: frame.event.kind,+ payload,+ })+ if (!inserted) return+ await this.notifyUserDO(binding.ownerUserId, {+ type: "community:bot.audit_event",+ botId: frame.agentId,+ id: inserted.id,+ kind: frame.event.kind,+ payload: frame.event.payload,+ sessionId: frame.sessionId ?? null,+ launchId: frame.launchId ?? null,+ createdAt: inserted.createdAt,+ }).catch(() => { })+ },+ })return}