Multi-Agent Collaboration for Team and Friends
TypeScript agent-automationagent-orchestrationagent-workflowai-agentsai-collaborationai-memoryai-workforceautonomous-agentsclaude-codeclaude-code-skillscodexcoding-agentmulti-agentmultiple-agentone-person-businessone-person-companyone-person-teamopencodesolopreneur
Harden control-plane receipts and sleep recovery (#489)
Gustavo Ye6dd9f9fda1cb
diff --git a/src/daemon/src/cli/daemonSelfUpdate.real.test.ts b/src/daemon/src/cli/daemonSelfUpdate.real.test.tsindex 5442b8c6a..fb873847a 100644--- a/src/daemon/src/cli/daemonSelfUpdate.real.test.ts+++ b/src/daemon/src/cli/daemonSelfUpdate.real.test.ts@@ -94,7 +94,7 @@ async function createControlServer(): Promise<ControlServer> {return;}if (request.url?.includes("/api/community/daemon/wakes/resync")) {- response.end(JSON.stringify({ woken: 0 }));+ response.end(JSON.stringify({ attempted: 0 }));return;}response.end(JSON.stringify({}));diff --git a/src/daemon/src/daemon/createDaemon.test.ts b/src/daemon/src/daemon/createDaemon.test.tsindex 79b78e4dc..d51e5f0bb 100644--- a/src/daemon/src/daemon/createDaemon.test.ts+++ b/src/daemon/src/daemon/createDaemon.test.ts@@ -164,7 +164,7 @@ describe("createDaemon", () => {if (url.includes("/daemon/bots")) {return Response.json({ bots: [{ id: "bot_1", name: "Bot", discriminator: "0001" }] });}- return Response.json({ woken: 0 });+ return Response.json({ attempted: 0 });}));const daemon = await createDaemon({@@ -347,7 +347,7 @@ describe("createDaemon", () => {bots: [{ id: "bot_1", name: "Bot", discriminator: "0001" }],});}- return Response.json({ woken: 0 });+ return Response.json({ attempted: 0 });}));const daemon = await createDaemon({machineKey: "cmk_diagnostics",@@ -418,7 +418,7 @@ describe("createDaemon", () => {const url = String(input);return url.includes("/api/community/daemon/bots")? Response.json({ bots: [{ id: "bot_1", name: "Bot", discriminator: "0001" }] })- : Response.json({ woken: 0 });+ : Response.json({ attempted: 0 });}));const daemon = await createDaemon({machineKey: "cmk_diagnostics_failure",@@ -1074,12 +1074,12 @@ describe("createDaemon — logging", () => {await daemon.stop();});- it("calls resync-wakes with the machine key bearer on open and logs the woken count", async () => {+ it("calls resync-wakes with the machine key bearer on open and logs the attempted count", async () => {global.fetch = vi.fn(async (url: string | URL, init?: RequestInit) => {const href = String(url);if (href.includes("/resync-wakes")) {expect((init?.headers as Record<string, string>).authorization).toBe("Bearer cmk_resync");- return new Response(JSON.stringify({ woken: 2 }), { status: 200 });+ return new Response(JSON.stringify({ attempted: 2 }), { status: 200 });}return new Response(JSON.stringify({ bots: [] }), { status: 200 });}) as unknown as typeof fetch;@@ -1100,7 +1100,7 @@ describe("createDaemon — logging", () => {await new Promise((r) => setTimeout(r, 20));expect(- logger.calls.info.some(([, m, d]) => m === "wake resync completed" && (d[0] as any).woken === 2),+ logger.calls.info.some(([, m, d]) => m === "wake resync completed" && (d[0] as any).attempted === 2),).toBe(true);await daemon.stop();});diff --git a/src/daemon/src/daemon/createDaemon.ts b/src/daemon/src/daemon/createDaemon.tsindex f7d732567..9fb035b0e 100644--- a/src/daemon/src/daemon/createDaemon.ts+++ b/src/daemon/src/daemon/createDaemon.ts@@ -581,7 +581,7 @@ export async function createDaemon(opts: CreateDaemonOptions): Promise<RunningDa* Ask the server to re-check each of this machine's bots for unread work* and re-wake any that have some. Recovers a message that arrived while* this daemon was offline: the wake-queue consumer acks (never retries) a- * `delivered_nowhere` outcome, so without this call that wake is gone for+ * `attempted_nowhere` outcome, so without this call that wake is gone for* good once the daemon reconnects. The daemon drives WHEN this runs; the* server alone decides WHAT an `agent:wake` looks like (same* `dispatchOneUnreadWake` rebuild the queue consumer uses) — this call@@ -595,13 +595,31 @@ export async function createDaemon(opts: CreateDaemonOptions): Promise<RunningDaheaders: { authorization: `Bearer ${opts.machineKey}` },});if (!res.ok) throw new Error(`resync-wakes ${res.status}`);- const json = (await res.json()) as { woken?: number };- log.info("wake resync completed", { woken: json.woken ?? 0 });+ const json = (await res.json()) as { attempted?: number };+ log.info("wake resync completed", { attempted: json.attempted ?? 0 });} catch (err) {log.warn("wake resync failed", { err: err instanceof Error ? err.message : String(err) });}}+ async function resyncPendingDiagnostics(): Promise<void> {+ try {+ const res = await fetch(`${opts.serverUrl}/api/community/daemon/resync-diagnostics`, {+ method: "POST",+ headers: { authorization: `Bearer ${opts.machineKey}` },+ });+ if (!res.ok) throw new Error(`resync-diagnostics ${res.status}`);+ const json = (await res.json()) as { pending?: number; attempted?: number; ambiguous?: number };+ log.info("diagnostics resync completed", {+ pending: json.pending ?? 0,+ attempted: json.attempted ?? 0,+ ambiguous: json.ambiguous ?? 0,+ });+ } catch (err) {+ log.warn("diagnostics resync failed", { err: err instanceof Error ? err.message : String(err) });+ }+ }+const enrollAgent = async (agentId: string): Promise<string> => {const existing = enrolledKeys.get(agentId);if (existing) return existing;@@ -941,6 +959,7 @@ export async function createDaemon(opts: CreateDaemonOptions): Promise<RunningDachannel.onOpen(() => {void coldStartWarmup();void resyncPendingWakes();+ void resyncPendingDiagnostics();});channel.connect();diff --git a/src/daemon/src/manager/agentRouter.ts b/src/daemon/src/manager/agentRouter.tsindex 523705e49..1af2e3237 100644--- a/src/daemon/src/manager/agentRouter.ts+++ b/src/daemon/src/manager/agentRouter.ts@@ -19,6 +19,7 @@* server recovers this host's state after a dropped control connection.*/import type { HostCommand, HostControlChannel, HostReady, HostReadyRuntime, UnreadNotice, AgentSessionReport, SessionErrorFrame, AgentId } from "../server/contract.js";+import { CONTROL_HEARTBEAT_CAPABILITY } from "../server/contract.js";import type { AgentProcessManager } from "./managerRuntime.js";import type { TypingScopeTracker } from "./typingScopeTracker.js";import { createLogger, type Logger } from "../logger.js";@@ -217,6 +218,7 @@ export class AgentRouter {buildReady(): HostReady {return {runtimeReport: [...this.runtimes.values()],+ capabilities: [CONTROL_HEARTBEAT_CAPABILITY],runningAgents: [...this.running],hostname: this.opts.hostname,platform: this.opts.platform,diff --git a/src/daemon/src/server/contract.ts b/src/daemon/src/server/contract.tsindex 950bb7ebd..2d679028e 100644--- a/src/daemon/src/server/contract.ts+++ b/src/daemon/src/server/contract.ts@@ -82,6 +82,7 @@ export type {} from "@alook/shared/community-cli-contract";export {+ CONTROL_HEARTBEAT_CAPABILITY,DM_SERVER,parseRef,formatRef,diff --git a/src/daemon/src/server/wsControlChannel.test.ts b/src/daemon/src/server/wsControlChannel.test.tsindex 7815df8cb..ef02d0d39 100644--- a/src/daemon/src/server/wsControlChannel.test.ts+++ b/src/daemon/src/server/wsControlChannel.test.ts@@ -10,6 +10,7 @@ import type { Logger } from "../logger";*/class FakeSocket implements WebSocketLike {sent: string[] = [];+ terminated = false;private handlers: Record<string, ((...a: any[]) => void)[]> = {};on(event: string, cb: (...a: any[]) => void): void {(this.handlers[event] ??= []).push(cb);@@ -20,6 +21,10 @@ class FakeSocket implements WebSocketLike {close(): void {this.emit("close");}+ terminate(): void {+ this.terminated = true;+ this.emit("close");+ }ping(): void {}emit(event: string, ...args: unknown[]): void {(this.handlers[event] ?? []).forEach((h) => h(...args));@@ -616,6 +621,39 @@ describe("WsControlChannel — downlink HostCommand validation (convergence #6)"expect(received).toEqual([command]);});+ it("acks an app heartbeat at ingress without dispatching it to lifecycle listeners", () => {+ const { sockets, received } = driven();++ sockets[0].emit("message", JSON.stringify({ type: "machine:heartbeat", nonce: "nonce_1" }));++ expect(received).toEqual([]);+ expect(sockets[0].frames()).toContainEqual({+ type: "machine_heartbeat_ack",+ nonce: "nonce_1",+ });+ });++ it("acks every diagnostics frame at ingress, including duplicates, before dispatch", () => {+ const command = {+ type: "diagnostics:collect",+ reportId: "dbr_0123456789abcdef",+ agentId: "bot_1",+ fromMs: 1_700_000_000_000,+ deadlineAt: 1_700_087_000_000,+ } as const;+ const { sockets, received } = driven();++ sockets[0].emit("message", JSON.stringify(command));+ sockets[0].emit("message", JSON.stringify(command));++ expect(received).toEqual([command, command]);+ expect(sockets[0].frames().filter((frame) => frame.type === "diagnostics_ack"))+ .toEqual([+ { type: "diagnostics_ack", reportId: command.reportId },+ { type: "diagnostics_ack", reportId: command.reportId },+ ]);+ });+it("drops diagnostics commands with missing or unknown top-level fields", () => {for (const command of [{@@ -727,5 +765,6 @@ describe("WsControlChannel — logging", () => {await new Promise((r) => setTimeout(r, 30));expect(logger.calls.warn.some(([m]) => m === "heartbeat pong timeout — forcing reconnect")).toBe(true);+ expect(sockets[0].terminated).toBe(true);});});diff --git a/src/daemon/src/server/wsControlChannel.ts b/src/daemon/src/server/wsControlChannel.tsindex 9bbbc545e..6a109f6bb 100644--- a/src/daemon/src/server/wsControlChannel.ts+++ b/src/daemon/src/server/wsControlChannel.ts@@ -134,6 +134,8 @@ type OutboundFrame =status: AgentCommandAckStatus;error?: AgentCommandAckError;}+ | { type: "machine_heartbeat_ack"; nonce: string }+ | { type: "diagnostics_ack"; reportId: string }| HostBotAuditEventFrame| SessionErrorFrame;@@ -462,6 +464,18 @@ export class WsControlChannel implements HostControlChannel {}private async dispatchIngressCommand(cmd: HostCommand): Promise<void> {+ if (cmd.type === "machine:heartbeat") {+ this.sendFrame({ type: "machine_heartbeat_ack", nonce: cmd.nonce });+ return;+ }+ if (cmd.type === "diagnostics:collect") {+ // Receipt means only that the daemon websocket ingress parsed this frame.+ // Collection acceptance/completion remains authoritative in the durable+ // diagnostic-report row and must never be inferred from this ack.+ this.sendFrame({ type: "diagnostics_ack", reportId: cmd.reportId });+ await this.dispatchListeners(cmd);+ return;+ }if (cmd.type === "agent:wake") {const result = await this.wakeCoordinator.run(cmd, (accepted) =>this.dispatchListeners(accepted), (advanced) => {@@ -546,7 +560,8 @@ export class WsControlChannel implements HostControlChannel {if (this.now() > this.pongDeadline) {// Watchdog: no pong in time → treat as dead, force reconnect.this.log.warn("heartbeat pong timeout — forcing reconnect");- this.ws?.close();+ if (this.ws?.terminate) this.ws.terminate();+ else this.ws?.close();return;}this.log.debug("heartbeat ping");diff --git a/src/shared/src/community-cli-contract.ts b/src/shared/src/community-cli-contract.tsindex 32aaeb70d..5705f60e5 100644--- a/src/shared/src/community-cli-contract.ts+++ b/src/shared/src/community-cli-contract.ts@@ -709,6 +709,8 @@ export interface UnreadNotice {/* Control plane — server → host commands *//* ------------------------------------------------------------------ */+export const CONTROL_HEARTBEAT_CAPABILITY = "control-heartbeat-v1";+/*** Commands the SERVER pushes DOWN to a host (daemon). This is the control plane —* distinct from the agent-initiated data plane (`ServerApi`). The server owns@@ -722,6 +724,7 @@ export interface UnreadNotice {* coalesce the notice for the next turn (see `AgentProcessManager`).*/export type HostCommand =+ | { type: "machine:heartbeat"; nonce: string }| {type: "agent:wake";agentId: AgentId;@@ -842,6 +845,8 @@ export interface HostReady {* reader-side concern (server-side bot-create validator, client picker).*/runtimeReport: HostReadyRuntime[];+ /** Capability gates for wire behavior that is unsafe to assume on legacy daemons. */+ capabilities?: string[];/** Agents currently running on this host. */runningAgents: AgentId[];hostname?: string;@@ -1027,6 +1032,8 @@ export interface WebSocketLike {send(data: string): void;close(): void;ping?(): void;+ /** Hard-close a half-open client socket when the implementation supports it (`ws`). */+ terminate?(): void;}/** Builds a client `WebSocketLike` for a url + headers (injected; no hard `ws` dep). */@@ -1379,6 +1386,10 @@ export function formatSeq(seq: Seq): string {// downstream with its own defaulting, so deep-validating it here would only turn// a forward-compatible server field into a hard drop on an older daemon.export const HostCommandSchema = z.discriminatedUnion("type", [+ z.strictObject({+ type: z.literal("machine:heartbeat"),+ nonce: z.string().min(1).max(128),+ }),z.object({type: z.literal("agent:wake"),agentId: z.string().min(1),diff --git a/src/shared/src/community/wake-dispatch.ts b/src/shared/src/community/wake-dispatch.tsindex 6f7819826..538482b4c 100644--- a/src/shared/src/community/wake-dispatch.ts+++ b/src/shared/src/community/wake-dispatch.ts@@ -1,4 +1,3 @@-import { nanoid } from "nanoid";import type { HostCommand, UnreadNotice } from "../community-cli-contract";import { makeRuntimeConfig } from "../runtime-config";import { resolveModelConfig } from "./bot-model";@@ -14,6 +13,7 @@ import * as notificationEligibility from "../db/queries/community/notification-eimport { policyAllows } from "../db/queries/community/notification-setting";import { getUsersByIds } from "../db/queries/user";import type { Database } from "../db/index";+import { parseAttemptedCountReceipt } from "../transport-receipt";/*** Deliberately NOT `@cloudflare/workers-types`' `Fetcher` — this module is@@ -31,13 +31,40 @@ interface WakeDispatchEnv {WS_DO_WORKER: FetcherLike;}+/**+ * Stable, opaque identity for one semantic unread wake. Queue retries and+ * daemon reconnect resync rebuild the command independently from D1, so a+ * random id here would turn one unread message into multiple commands. The+ * domain-separated digest avoids adding a command ledger without exposing the+ * internal message id through daemon logs or the agent launch environment.+ */+async function unreadWakeLaunchId(input: {+ botUserId: string;+ messageId: string;+}): Promise<string> {+ const identity = JSON.stringify([+ "alook:unread-wake:v1",+ input.botUserId,+ input.messageId,+ ]);+ const digest = new Uint8Array(+ await crypto.subtle.digest(+ "SHA-256",+ new TextEncoder().encode(identity),+ ),+ );+ return `wake_${Array.from(digest, (byte) => byte.toString(16).padStart(2, "0")).join("")}`;+}+/*** One `WAKE_QUEUE` message — deliberately minimal (plan* minimal-wake-queue-unread-notice §1): just enough to rebuild the wake* command from CURRENT D1 state at consume time. No `machineId`, `runtime`,* `launchId`, message text, sender, or preview — all of that is re-derived* by `buildUnreadWakeCommand` in `src/wake-worker` so a stale queue item- * never wakes an old machine or carries stale content.+ * never wakes an old machine or carries stale content. `launchId` is derived+ * deterministically from the bot/message tuple so retry and reconnect replay+ * retain one semantic command identity.*/export interface WakePayload {messageId: string;@@ -55,14 +82,14 @@ export interface WakePayload {* surface (never a raw DO namespace — `src/web`/`src/wake-worker` cannot* fetch a DO stub directly). This function POSTs an already-fully-built* `HostCommand` to that worker's `/community-machine/by-id/<machineId>/forward-agent-wake`- * route and normalizes the response to a boolean — it never inspects,+ * route and normalizes the attempted socket-write count to a boolean — it never inspects,* validates, or constructs any part of `command`, and it never exposes the* DO-naming mechanics (no public `getMachineDoName` here or anywhere else).** Error contract (load-bearing for the queue consumer's retry semantics):- * - `{ sent: true }` — at least one live doName's DO forwarded the command- * to an authenticated daemon WebSocket.- * - `{ sent: false }` — the ws-do route responded 200 with `{ sent: 0 }`:+ * - `{ attempted: true }` — at least one live doName's DO attempted the command+ * on an authenticated daemon WebSocket. This is not a daemon receipt.+ * - `{ attempted: false }` — the ws-do route responded 200 with `{ attempted: 0 }`:* no active credential for this machine, or a live credential but no open* WS (daemon offline). This is a known-permanent state for this attempt —* the consumer must `ack()`, not `retry()`. Daemon reconnect warmup@@ -70,14 +97,14 @@ export interface WakePayload {* - throws — the ws-do route (or the service-binding fetch itself) returned* non-2xx, or the fetch itself threw (network error/timeout). This is* transient — the consumer must `retry()`. Never swallowed into- * `{ sent: false }`, or a real outage would look identical to "daemon is+ * `{ attempted: false }`, or a real outage would look identical to "daemon is* just offline" and stop retrying.*/export async function sendWakeToMachine(env: { WS_DO_WORKER: FetcherLike },machineId: string,command: HostCommand-): Promise<{ sent: boolean }> {+): Promise<{ attempted: boolean }> {const path = `/community-machine/by-id/${encodeURIComponent(machineId)}/forward-agent-wake`;const res = await env.WS_DO_WORKER.fetch(`http://internal${path}`, {method: "POST",@@ -89,8 +116,7 @@ export async function sendWakeToMachine(throw new Error(`sendWakeToMachine: ws-do route returned ${res.status} for machine ${machineId}`);}- const data = (await res.json()) as { sent?: number };- return { sent: (data.sent ?? 0) > 0 };+ return { attempted: parseAttemptedCountReceipt(await res.json()) > 0 };}/** Why `buildUnreadWakeCommand` decided NOT to wake — every reason is a permanent, current-state miss the consumer must `ack()`, never `retry()`. */@@ -194,7 +220,10 @@ export async function buildUnreadWakeCommand(type: "agent:wake",agentId: botCtx.botUserId,config,- launchId: nanoid(),+ launchId: await unreadWakeLaunchId({+ botUserId: botCtx.botUserId,+ messageId: msg.id,+ }),unreadNotice,};@@ -306,8 +335,8 @@ async function writeWakeTriggerAudit(/** Outcome of resolving ONE wake candidate — what every caller (the real queue consumer, and the dev-only inline stand-in) needs to decide what to log. */export type DispatchOneWakeResult =| { outcome: "skip"; reason: SkipReason }- | { outcome: "sent" }- | { outcome: "delivered_nowhere"; machineId: string };+ | { outcome: "attempted" }+ | { outcome: "attempted_nowhere"; machineId: string };/*** The ONE place that decides what happens for a single `{ messageId,@@ -328,6 +357,6 @@ export async function dispatchOneUnreadWake(): Promise<DispatchOneWakeResult> {const result = await buildUnreadWakeCommand(db, input, env);if (result.state === "skip") return { outcome: "skip", reason: result.reason };- const { sent } = await sendWakeToMachine(env, result.machineId, result.command);- return sent ? { outcome: "sent" } : { outcome: "delivered_nowhere", machineId: result.machineId };+ const { attempted } = await sendWakeToMachine(env, result.machineId, result.command);+ return attempted ? { outcome: "attempted" } : { outcome: "attempted_nowhere", machineId: result.machineId };}diff --git a/src/shared/src/db/queries/community/diagnostic-report.ts b/src/shared/src/db/queries/community/diagnostic-report.tsindex dd8c64fdb..2756e67a6 100644--- a/src/shared/src/db/queries/community/diagnostic-report.ts+++ b/src/shared/src/db/queries/community/diagnostic-report.ts@@ -1,4 +1,4 @@-import { and, eq, gt, isNull, lte, sql } from "drizzle-orm";+import { and, asc, eq, gt, isNull, lte, sql } from "drizzle-orm";import { alias } from "drizzle-orm/sqlite-core";import { nanoid } from "nanoid";import {@@ -280,6 +280,42 @@ export async function getPendingDiagnosticReportForMachine();}+export async function listPendingDiagnosticReportsForMachine(+ db: Database,+ input: { machineId: string; nowMs: number }+): Promise<DiagnosticReportRow[]> {+ assertSafeEpoch(input.nowMs);+ return db+ .select()+ .from(communityDiagnosticReport)+ .where(+ and(+ eq(communityDiagnosticReport.machineId, input.machineId),+ eq(communityDiagnosticReport.status, "pending"),+ gt(communityDiagnosticReport.deadlineAt, input.nowMs)+ )+ )+ .orderBy(asc(communityDiagnosticReport.deadlineAt));+}++export async function getNextPendingDiagnosticDeadlineForMachine(+ db: Database,+ input: { machineId: string }+): Promise<number | null> {+ const rows = await db+ .select({ deadlineAt: communityDiagnosticReport.deadlineAt })+ .from(communityDiagnosticReport)+ .where(+ and(+ eq(communityDiagnosticReport.machineId, input.machineId),+ eq(communityDiagnosticReport.status, "pending")+ )+ )+ .orderBy(asc(communityDiagnosticReport.deadlineAt))+ .limit(1);+ return rows[0]?.deadlineAt ?? null;+}+export async function getDiagnosticReportForMachine(db: Database,input: { reportId: string; machineId: string }@@ -318,6 +354,24 @@ export async function timeoutPendingDiagnosticReport(return rows[0] ?? null;}+export async function timeoutPendingDiagnosticReportsForMachine(+ db: Database,+ input: { machineId: string; nowMs: number }+): Promise<DiagnosticReportRow[]> {+ assertSafeEpoch(input.nowMs);+ return db+ .update(communityDiagnosticReport)+ .set({ status: "failed", failureCode: "timeout", completedAt: input.nowMs })+ .where(+ and(+ eq(communityDiagnosticReport.machineId, input.machineId),+ eq(communityDiagnosticReport.status, "pending"),+ lte(communityDiagnosticReport.deadlineAt, input.nowMs)+ )+ )+ .returning();+}+export async function failPendingDiagnosticReport(db: Database,input: {diff --git a/src/shared/src/index.ts b/src/shared/src/index.tsindex c120e3292..70aa9816e 100644--- a/src/shared/src/index.ts+++ b/src/shared/src/index.ts@@ -270,6 +270,8 @@ export {AgentTypingStopMessageSchema,AgentSessionMessageSchema,AgentWakeAckMessageSchema,+ MachineHeartbeatAckMessageSchema,+ DiagnosticCommandAckMessageSchema,COMMUNITY_RUNTIME_ID_MAX,COMMUNITY_RUNTIME_VERSION_MAX,COMMUNITY_RUNTIME_LIST_MAX,@@ -369,6 +371,8 @@ export type {AgentTypingStopMessage,AgentSessionMessage,AgentWakeAckMessage,+ MachineHeartbeatAckMessage,+ DiagnosticCommandAckMessage,CommunityBotCreateRequest,CommunityBotPatchRequest,CommunityBotAddToServerRequest,@@ -454,7 +458,15 @@ export type {HostBotAuditEventFrame,BotAuditEventPayload,} from "./community-cli-contract";-export { DM_SERVER, parseRef, formatRef, formatCanonicalRef, parseSeq, formatSeq } from "./community-cli-contract";+export {+ CONTROL_HEARTBEAT_CAPABILITY,+ DM_SERVER,+ parseRef,+ formatRef,+ formatCanonicalRef,+ parseSeq,+ formatSeq,+} from "./community-cli-contract";export type { CanonicalRefScope } from "./community-cli-contract";export type {@@ -482,7 +494,12 @@ export {modelNameFromSelect,} from "./community/bot-model";-export { sendWakeToMachine, buildUnreadWakeCommand, dispatchOneUnreadWake } from "./community/wake-dispatch";+export {+ sendWakeToMachine,+ buildUnreadWakeCommand,+ dispatchOneUnreadWake,+} from "./community/wake-dispatch";+export { parseAttemptedCountReceipt } from "./transport-receipt";export type { DispatchOneWakeResult } from "./community/wake-dispatch";export type { WakePayload, BuildUnreadWakeResult } from "./community/wake-dispatch";diff --git a/src/shared/src/schemas.ts b/src/shared/src/schemas.tsindex 4afd6baf1..eb6f73cf2 100644--- a/src/shared/src/schemas.ts+++ b/src/shared/src/schemas.ts@@ -1,6 +1,7 @@import { z } from "zod";import { IssueStatus, TASK_TYPES } from "./constants";import { sanitizeSlug } from "./utils/slug";+import { DiagnosticReportIdSchema } from "./diagnostics-contract";import {ALLOWED_ICON_MIME_TYPES,MAX_ATTACHMENTS_PER_MESSAGE,@@ -899,6 +900,7 @@ export const CommunityMachineSummarySchema = z.object({export const HostReadyMessageSchema = z.object({type: z.literal("ready"),runtimeReport: CommunityMachineRuntimeListSchema,+ capabilities: z.array(z.string().min(1).max(64)).max(16).optional().default([]),runningAgents: z.array(z.string()).default([]),hostname: z.string().optional(),platform: z.string().optional(),@@ -1002,6 +1004,18 @@ export const AgentWakeAckMessageSchema = z.object({});export type AgentWakeAckMessage = z.infer<typeof AgentWakeAckMessageSchema>;+export const MachineHeartbeatAckMessageSchema = z.strictObject({+ type: z.literal("machine_heartbeat_ack"),+ nonce: z.string().min(1).max(128),+});+export type MachineHeartbeatAckMessage = z.infer<typeof MachineHeartbeatAckMessageSchema>;++export const DiagnosticCommandAckMessageSchema = z.strictObject({+ type: z.literal("diagnostics_ack"),+ reportId: DiagnosticReportIdSchema,+});+export type DiagnosticCommandAckMessage = z.infer<typeof DiagnosticCommandAckMessageSchema>;+export const CommunityPairTokenResponseSchema = z.object({tokenId: z.string(),diff --git a/src/shared/src/transport-receipt.ts b/src/shared/src/transport-receipt.tsnew file mode 100644index 000000000..c453c6d58--- /dev/null+++ b/src/shared/src/transport-receipt.ts@@ -0,0 +1,29 @@+function isAttemptCount(value: unknown): value is number {+ return Number.isSafeInteger(value) && (value as number) >= 0;+}++/**+ * Parse a rolling transport-attempt receipt. `sent` is a temporary wire+ * alias for independently deployed legacy consumers; it never means that+ * the command was received or completed.+ */+export function parseAttemptedCountReceipt(+ value: unknown,+ options: { allowLegacySentOnly?: boolean } = {},+): number {+ if (!value || typeof value !== "object" || Array.isArray(value)) {+ throw new Error("invalid attempted-count receipt");+ }+ const data = value as Record<string, unknown>;+ const hasAttempted = Object.prototype.hasOwnProperty.call(data, "attempted");+ const hasSent = Object.prototype.hasOwnProperty.call(data, "sent");+ if (hasAttempted) {+ if (!isAttemptCount(data.attempted)) throw new Error("invalid attempted count");+ if (hasSent && (!isAttemptCount(data.sent) || data.sent !== data.attempted)) {+ throw new Error("inconsistent attempted-count receipt");+ }+ return data.attempted;+ }+ if (hasSent && isAttemptCount(data.sent) && options.allowLegacySentOnly !== false) return data.sent;+ throw new Error("missing attempted count");+}diff --git a/src/shared/test/community-cli-contract.test.ts b/src/shared/test/community-cli-contract.test.tsindex a47a6fc59..023320d4e 100644--- a/src/shared/test/community-cli-contract.test.ts+++ b/src/shared/test/community-cli-contract.test.ts@@ -313,6 +313,18 @@ describe("HostCommand diagnostics contract", () => {});});+describe("HostCommand control heartbeat contract", () => {+ it("keeps a strict nonce-only arm", () => {+ expect(HostCommandSchema.parse({ type: "machine:heartbeat", nonce: "nonce_1" }))+ .toEqual({ type: "machine:heartbeat", nonce: "nonce_1" });+ expect(HostCommandSchema.safeParse({+ type: "machine:heartbeat",+ nonce: "nonce_1",+ machineId: "injected",+ }).success).toBe(false);+ });+});+describe("HostCommand machine update contract", () => {it("keeps the no-payload arm in the type and strict runtime schema", () => {type UpdateCommand = { type: "machine:update" };diff --git a/src/shared/test/community/wake-dispatch-build.test.ts b/src/shared/test/community/wake-dispatch-build.test.tsindex 3b6ab1883..5f8afb921 100644--- a/src/shared/test/community/wake-dispatch-build.test.ts+++ b/src/shared/test/community/wake-dispatch-build.test.ts@@ -135,6 +135,59 @@ describe("buildUnreadWakeCommand", () => {expect(mockResolveUnreadNoticeChannel).toHaveBeenCalledWith(fakeDb, { channelId: "ch_1" }, "bot_1");});+ it("reuses one launchId across queue retry, reconnect resync, and duplicate concurrent rebuilds", async () => {+ seedHappyPath();++ const initial = await buildUnreadWakeCommand(fakeDb, {+ messageId: "msg_1",+ botUserId: "bot_1",+ });+ const queueRetry = await buildUnreadWakeCommand(fakeDb, {+ messageId: "msg_1",+ botUserId: "bot_1",+ });+ const [reconnectResync, duplicate] = await Promise.all([+ buildUnreadWakeCommand(fakeDb, {+ messageId: "msg_1",+ botUserId: "bot_1",+ }),+ buildUnreadWakeCommand(fakeDb, {+ messageId: "msg_1",+ botUserId: "bot_1",+ }),+ ]);++ if (+ initial.state !== "ready" ||+ queueRetry.state !== "ready" ||+ reconnectResync.state !== "ready" ||+ duplicate.state !== "ready"+ ) {+ throw new Error("expected ready wakes");+ }+ expect(initial.command.launchId).toBe(+ "wake_59f46f28b814c6f6b597cc0db2a5c9166d2fee5d3148a3e536a629cc227a8531",+ );+ expect(initial.command.launchId).not.toContain("bot_1");+ expect(initial.command.launchId).not.toContain("msg_1");+ expect(queueRetry.command.launchId).toBe(initial.command.launchId);+ expect(reconnectResync.command.launchId).toBe(initial.command.launchId);+ expect(duplicate.command.launchId).toBe(initial.command.launchId);+ expect(+ mockInsertBotAuditWakeTrigger.mock.calls+ .slice(0, 4)+ .map(([, audit]) => audit.launchId),+ ).toEqual(Array(4).fill(initial.command.launchId));++ seedHappyPath({ message: { id: "msg_2" } });+ const nextMessage = await buildUnreadWakeCommand(fakeDb, {+ messageId: "msg_2",+ botUserId: "bot_1",+ });+ if (nextMessage.state !== "ready") throw new Error("expected ready wake");+ expect(nextMessage.command.launchId).not.toBe(initial.command.launchId);+ });+it("ready: resolves a DM scope — a DM is a type=dm channel, so it carries a channelId like any other", async () => {seedHappyPath({message: { channelId: "dm_ch_1" },diff --git a/src/shared/test/community/wake-dispatch-one.test.ts b/src/shared/test/community/wake-dispatch-one.test.tsindex ad2a2a3d6..ac49ae44b 100644--- a/src/shared/test/community/wake-dispatch-one.test.ts+++ b/src/shared/test/community/wake-dispatch-one.test.ts@@ -94,22 +94,22 @@ describe("dispatchOneUnreadWake", () => {vi.clearAllMocks();});- it("resolves to { outcome: 'sent' } when the rebuild is ready and the daemon is online", async () => {+ it("resolves to { outcome: 'attempted' } when a socket write is attempted", async () => {seedReady();- const env = makeEnv(async () => new Response(JSON.stringify({ sent: 1 }), { status: 200 }));+ const env = makeEnv(async () => new Response(JSON.stringify({ attempted: 1 }), { status: 200 }));const result = await dispatchOneUnreadWake(fakeDb, env, { messageId: "msg_1", botUserId: "bot_1" });- expect(result).toEqual({ outcome: "sent" });+ expect(result).toEqual({ outcome: "attempted" });});- it("resolves to { outcome: 'delivered_nowhere', machineId } when the daemon is offline", async () => {+ it("resolves to { outcome: 'attempted_nowhere', machineId } when no socket write is attempted", async () => {seedReady();- const env = makeEnv(async () => new Response(JSON.stringify({ sent: 0 }), { status: 200 }));+ const env = makeEnv(async () => new Response(JSON.stringify({ attempted: 0 }), { status: 200 }));const result = await dispatchOneUnreadWake(fakeDb, env, { messageId: "msg_1", botUserId: "bot_1" });- expect(result).toEqual({ outcome: "delivered_nowhere", machineId: "machine_1" });+ expect(result).toEqual({ outcome: "attempted_nowhere", machineId: "machine_1" });});it("resolves to { outcome: 'skip', reason } without ever calling fetch when the rebuild is a skip", async () => {diff --git a/src/shared/test/community/wake-dispatch.test.ts b/src/shared/test/community/wake-dispatch.test.tsindex 53da6667f..05c5e9952 100644--- a/src/shared/test/community/wake-dispatch.test.ts+++ b/src/shared/test/community/wake-dispatch.test.ts@@ -20,30 +20,51 @@ describe("sendWakeToMachine", () => {expect(url).toBe("http://internal/community-machine/by-id/machine-1/forward-agent-wake");expect(init?.method).toBe("POST");expect(JSON.parse(init!.body as string)).toEqual(dummyCommand);- return new Response(JSON.stringify({ sent: 1 }), { status: 200 });+ return new Response(JSON.stringify({ attempted: 1, sent: 1 }), { status: 200 });});const env = makeEnv(fetchMock);const result = await sendWakeToMachine(env, "machine-1", dummyCommand);- expect(result).toEqual({ sent: true });+ expect(result).toEqual({ attempted: true });expect(fetchMock).toHaveBeenCalledTimes(1);});- it("normalizes { sent: 0 } (daemon offline) to { sent: false }", async () => {- const env = makeEnv(async () => new Response(JSON.stringify({ sent: 0 }), { status: 200 }));+ it("normalizes { attempted: 0 } to { attempted: false }", async () => {+ const env = makeEnv(async () => new Response(JSON.stringify({ attempted: 0 }), { status: 200 }));const result = await sendWakeToMachine(env, "machine-1", dummyCommand);- expect(result).toEqual({ sent: false });+ expect(result).toEqual({ attempted: false });});- it("normalizes any positive sent count to { sent: true }", async () => {- const env = makeEnv(async () => new Response(JSON.stringify({ sent: 3 }), { status: 200 }));+ it("normalizes any positive attempted count to { attempted: true }", async () => {+ const env = makeEnv(async () => new Response(JSON.stringify({ attempted: 3 }), { status: 200 }));const result = await sendWakeToMachine(env, "machine-1", dummyCommand);- expect(result).toEqual({ sent: true });+ expect(result).toEqual({ attempted: true });+ });++ it("accepts a valid legacy sent-only receipt during rolling deployment", async () => {+ const env = makeEnv(async () => new Response(JSON.stringify({ sent: 1 }), { status: 200 }));++ await expect(sendWakeToMachine(env, "machine-1", dummyCommand)).resolves.toEqual({ attempted: true });+ });++ it.each([+ ["missing fields", {}],+ ["wrong attempted type", { attempted: "1" }],+ ["negative attempted", { attempted: -1 }],+ ["fractional attempted", { attempted: 0.5 }],+ ["unsafe attempted", { attempted: Number.MAX_SAFE_INTEGER + 1 }],+ ["invalid legacy sent", { sent: -1 }],+ ["mismatched dual receipt", { attempted: 1, sent: 0 }],+ ["invalid sent alias", { attempted: 1, sent: "1" }],+ ])("throws on %s so the queue retries", async (_name, receipt) => {+ const env = makeEnv(async () => new Response(JSON.stringify(receipt), { status: 200 }));++ await expect(sendWakeToMachine(env, "machine-1", dummyCommand)).rejects.toThrow();});it("throws on non-2xx response (transient — consumer must retry)", async () => {@@ -63,7 +84,7 @@ describe("sendWakeToMachine", () => {it("encodes the machineId in the URL path", async () => {const fetchMock = vi.fn(async (url: string) => {expect(url).toBe("http://internal/community-machine/by-id/machine%2Fwith%2Fslash/forward-agent-wake");- return new Response(JSON.stringify({ sent: 0 }), { status: 200 });+ return new Response(JSON.stringify({ attempted: 0 }), { status: 200 });});const env = makeEnv(fetchMock);diff --git a/src/shared/test/queries/community-diagnostic-report.test.ts b/src/shared/test/queries/community-diagnostic-report.test.tsindex 3a597859e..bd2733ce5 100644--- a/src/shared/test/queries/community-diagnostic-report.test.ts+++ b/src/shared/test/queries/community-diagnostic-report.test.ts@@ -264,10 +264,45 @@ describe("diagnostic report query contract", () => {expect(typeof q.getPendingDiagnosticReportForMachine).toBe("function");expect(typeof q.getDiagnosticReportForMachine).toBe("function");expect(typeof q.timeoutPendingDiagnosticReport).toBe("function");+ expect(typeof q.listPendingDiagnosticReportsForMachine).toBe("function");+ expect(typeof q.getNextPendingDiagnosticDeadlineForMachine).toBe("function");+ expect(typeof q.timeoutPendingDiagnosticReportsForMachine).toBe("function");expect(typeof q.failPendingDiagnosticReport).toBe("function");expect(typeof q.finalizeDiagnosticReportUpload).toBe("function");});+ it("bulk timeout and reconnect reads stay scoped to one machine and preserve future rows", async () => {+ const { db, sqlite } = realSnapshotDb();+ const insert = sqlite.prepare(`+ INSERT INTO community_diagnostic_report (+ id, owner_user_id, agent_id, machine_id, client_nonce, rate_bucket,+ status, failure_code, from_ms, created_at, deadline_at, completed_at,+ r2_key, sha256, size_bytes, uploaded_at, object_expires_at+ ) VALUES (?, ?, ?, ?, ?, ?, 'pending', NULL, ?, ?, ?, NULL, NULL, NULL, NULL, NULL, NULL)+ `);+ insert.run("dbr_expired", "owner_a", "bot_a", MACHINE_ID, "nonce_a_123456789", 1, FROM_MS, NOW_MS - 20_000, NOW_MS - 1);+ insert.run("dbr_future", "owner_b", "bot_b", MACHINE_ID, "nonce_b_123456789", 2, FROM_MS, NOW_MS, NOW_MS + 20_000);+ insert.run("dbr_other", "owner_c", "bot_c", "cm_other", "nonce_c_123456789", 3, FROM_MS, NOW_MS, NOW_MS + 10_000);++ try {+ await expect(q.timeoutPendingDiagnosticReportsForMachine(db, {+ machineId: MACHINE_ID,+ nowMs: NOW_MS,+ })).resolves.toMatchObject([{ id: "dbr_expired", status: "failed", failureCode: "timeout" }]);+ await expect(q.listPendingDiagnosticReportsForMachine(db, {+ machineId: MACHINE_ID,+ nowMs: NOW_MS,+ })).resolves.toMatchObject([{ id: "dbr_future" }]);+ await expect(q.getNextPendingDiagnosticDeadlineForMachine(db, {+ machineId: MACHINE_ID,+ })).resolves.toBe(NOW_MS + 20_000);+ expect(sqlite.prepare(`SELECT status FROM community_diagnostic_report WHERE id='dbr_other'`).get())+ .toEqual({ status: "pending" });+ } finally {+ sqlite.close();+ }+ });+it("first create is one atomic INSERT…SELECT from the live owner/bot/binding snapshot", async () => {const created = { ...pendingRow };const { db, chain } = queryDb([[created]]);diff --git a/src/shared/test/schemas/community-runtime.test.ts b/src/shared/test/schemas/community-runtime.test.tsindex 7a543ac43..c5e932e75 100644--- a/src/shared/test/schemas/community-runtime.test.ts+++ b/src/shared/test/schemas/community-runtime.test.ts@@ -3,6 +3,8 @@ import {CommunityMachineRuntimeSchema,CommunityMachineRuntimeListSchema,HostReadyMessageSchema,+ MachineHeartbeatAckMessageSchema,+ DiagnosticCommandAckMessageSchema,SessionErrorFrameSchema,COMMUNITY_RUNTIME_ID_MAX,COMMUNITY_RUNTIME_LIST_MAX,@@ -125,6 +127,7 @@ describe("HostReadyMessageSchema", () => {{ id: "claude", version: "1.0.0", status: "healthy" },]);expect(parsed.runningAgents).toEqual(["agent_a"]);+ expect(parsed.capabilities).toEqual([]);});it("defaults runningAgents to an empty array", () => {@@ -133,6 +136,20 @@ describe("HostReadyMessageSchema", () => {runtimeReport: [],});expect(parsed.runningAgents).toEqual([]);+ expect(parsed.capabilities).toEqual([]);+ });++ it("accepts bounded capability names and rejects an empty capability", () => {+ expect(HostReadyMessageSchema.parse({+ type: "ready",+ runtimeReport: [],+ capabilities: ["control-heartbeat-v1"],+ }).capabilities).toEqual(["control-heartbeat-v1"]);+ expect(HostReadyMessageSchema.safeParse({+ type: "ready",+ runtimeReport: [],+ capabilities: [""],+ }).success).toBe(false);});it("rejects the legacy string-only `runtimes` field (no runtimeReport)", () => {@@ -164,6 +181,24 @@ describe("HostReadyMessageSchema", () => {});});+describe("control-plane receipt schemas", () => {+ it("strictly parses heartbeat and diagnostic ingress receipts", () => {+ expect(MachineHeartbeatAckMessageSchema.parse({+ type: "machine_heartbeat_ack",+ nonce: "nonce_1",+ })).toEqual({ type: "machine_heartbeat_ack", nonce: "nonce_1" });+ expect(DiagnosticCommandAckMessageSchema.parse({+ type: "diagnostics_ack",+ reportId: "dbr_0123456789abcdef",+ })).toEqual({ type: "diagnostics_ack", reportId: "dbr_0123456789abcdef" });+ expect(MachineHeartbeatAckMessageSchema.safeParse({+ type: "machine_heartbeat_ack",+ nonce: "nonce_1",+ extra: true,+ }).success).toBe(false);+ });+});+describe("SessionErrorFrameSchema", () => {it("accepts a runtime_not_available frame", () => {const parsed = SessionErrorFrameSchema.parse({diff --git a/src/shared/test/transport-receipt.test.ts b/src/shared/test/transport-receipt.test.tsnew file mode 100644index 000000000..9a94db8ad--- /dev/null+++ b/src/shared/test/transport-receipt.test.ts@@ -0,0 +1,23 @@+import { describe, expect, it } from "vitest";+import { parseAttemptedCountReceipt } from "../src/transport-receipt";++describe("parseAttemptedCountReceipt", () => {+ it("accepts attempted-only, equal dual-write, and optionally legacy sent-only receipts", () => {+ expect(parseAttemptedCountReceipt({ attempted: 2 })).toBe(2);+ expect(parseAttemptedCountReceipt({ attempted: 2, sent: 2 })).toBe(2);+ expect(parseAttemptedCountReceipt({ sent: 2 })).toBe(2);+ expect(() => parseAttemptedCountReceipt({ sent: 2 }, { allowLegacySentOnly: false })).toThrow();+ });++ it.each([+ null,+ {},+ { attempted: -1 },+ { attempted: 0.5 },+ { attempted: Number.MAX_SAFE_INTEGER + 1 },+ { attempted: 1, sent: 0 },+ { attempted: 1, sent: "1" },+ ])("rejects malformed or inconsistent receipt %#", (receipt) => {+ expect(() => parseAttemptedCountReceipt(receipt)).toThrow();+ });+});diff --git a/src/wake-worker/src/index.test.ts b/src/wake-worker/src/index.test.tsindex 96183d7be..0f141052a 100644--- a/src/wake-worker/src/index.test.ts+++ b/src/wake-worker/src/index.test.ts@@ -2,8 +2,9 @@ import { describe, it, expect, vi, beforeEach } from "vitest"const mockDispatchOneUnreadWake = vi.fn()const mockCreateDb = vi.fn((..._a: unknown[]) => ({ __db: true }))+const { mockLogInfo } = vi.hoisted(() => ({ mockLogInfo: vi.fn() }))vi.mock("@alook/shared", () => {- const noopLogger = { debug: () => { }, info: () => { }, warn: () => { }, error: () => { }, child() { return this } }+ const noopLogger = { debug: () => { }, info: mockLogInfo, warn: () => { }, error: () => { }, child() { return this } }return {createLogger: () => noopLogger,createDb: (...a: unknown[]) => mockCreateDb(...a),@@ -24,8 +25,8 @@ describe("wake-worker queue consumer", () => {vi.clearAllMocks()})- it("resolves via dispatchOneUnreadWake and acks on successful delivery", async () => {- mockDispatchOneUnreadWake.mockResolvedValue({ outcome: "sent" })+ it("resolves via dispatchOneUnreadWake and acks after an attempted write", async () => {+ mockDispatchOneUnreadWake.mockResolvedValue({ outcome: "attempted" })const msg = makeMsg({ messageId: "msg1", botUserId: "bot1" })const batch = { messages: [msg] } as unknown as MessageBatch<unknown>@@ -37,8 +38,8 @@ describe("wake-worker queue consumer", () => {expect(msg.retry).not.toHaveBeenCalled()})- it("acks (does not retry) when daemon is offline (delivered_nowhere) — known-permanent state", async () => {- mockDispatchOneUnreadWake.mockResolvedValue({ outcome: "delivered_nowhere", machineId: "m1" })+ it("acks and logs when no socket write was attempted — reconnect resync owns recovery", async () => {+ mockDispatchOneUnreadWake.mockResolvedValue({ outcome: "attempted_nowhere", machineId: "m1" })const msg = makeMsg({ messageId: "msg1", botUserId: "bot1" })const batch = { messages: [msg] } as unknown as MessageBatch<unknown>@@ -46,6 +47,10 @@ describe("wake-worker queue consumer", () => {expect(msg.ack).toHaveBeenCalledTimes(1)expect(msg.retry).not.toHaveBeenCalled()+ expect(mockLogInfo).toHaveBeenCalledWith("wake_attempted_nowhere", {+ botUserId: "bot1",+ machineId: "m1",+ })})it("acks when the resolution is a skip", async () => {@@ -93,7 +98,7 @@ describe("wake-worker queue consumer", () => {it("processes every message in the batch independently — one failure doesn't block others", async () => {mockDispatchOneUnreadWake- .mockResolvedValueOnce({ outcome: "sent" })+ .mockResolvedValueOnce({ outcome: "attempted" }).mockRejectedValueOnce(new Error("boom")).mockResolvedValueOnce({ outcome: "skip", reason: "already_read" })@@ -127,7 +132,7 @@ describe("wake-worker dev-only HTTP entrypoint (fetch)", () => {}it("resolves every candidate in the body via dispatchOneUnreadWake — the SAME function queue() calls", async () => {- mockDispatchOneUnreadWake.mockResolvedValue({ outcome: "sent" })+ mockDispatchOneUnreadWake.mockResolvedValue({ outcome: "attempted" })const payloads = [{ messageId: "msg1", botUserId: "bot1" },{ messageId: "msg1", botUserId: "bot2" },@@ -142,7 +147,7 @@ describe("wake-worker dev-only HTTP entrypoint (fetch)", () => {expect(mockDispatchOneUnreadWake).toHaveBeenCalledWith({ __db: true }, env, payloads[1])})- it("returns 202 even when a skip/delivered_nowhere outcome resolves (no queue infra to ack/retry against)", async () => {+ it("returns 202 even when a skip/attempted_nowhere outcome resolves (no queue infra to ack/retry against)", async () => {mockDispatchOneUnreadWake.mockResolvedValue({ outcome: "skip", reason: "already_read" })const res = await handler.fetch!(makeRequest("POST", [{ messageId: "msg1", botUserId: "bot1" }]), env)@@ -151,7 +156,7 @@ describe("wake-worker dev-only HTTP entrypoint (fetch)", () => {})it("deduplicates candidates by stable messageId + botUserId within one request", async () => {- mockDispatchOneUnreadWake.mockResolvedValue({ outcome: "sent" })+ mockDispatchOneUnreadWake.mockResolvedValue({ outcome: "attempted" })const first = { messageId: "msg:one", botUserId: "bot" }const delimiterCollision = { messageId: "msg", botUserId: "one:bot" }@@ -169,7 +174,7 @@ describe("wake-worker dev-only HTTP entrypoint (fetch)", () => {const attempt = (attempts.get(item.botUserId) ?? 0) + 1attempts.set(item.botUserId, attempt)if (item.botUserId === "bot_flaky" && attempt === 1) throw new Error("D1 exploded")- return { outcome: "sent" }+ return { outcome: "attempted" }})const res = await handler.fetch!(@@ -193,7 +198,7 @@ describe("wake-worker dev-only HTTP entrypoint (fetch)", () => {const attempt = (attempts.get(item.botUserId) ?? 0) + 1attempts.set(item.botUserId, attempt)if (item.botUserId === "bot_flaky") throw new Error("D1 stayed down")- return { outcome: "sent" }+ return { outcome: "attempted" }})const res = await handler.fetch!(@@ -252,8 +257,8 @@ describe("wake-worker dev-only HTTP entrypoint (fetch)", () => {* details are covered by the `describe` blocks above.*/const RESOLUTION_SCENARIOS = [- { name: "sent", outcome: { outcome: "sent" as const } },- { name: "delivered_nowhere", outcome: { outcome: "delivered_nowhere" as const, machineId: "m1" } },+ { name: "attempted", outcome: { outcome: "attempted" as const } },+ { name: "attempted_nowhere", outcome: { outcome: "attempted_nowhere" as const, machineId: "m1" } },{ name: "skip: already_read", outcome: { outcome: "skip" as const, reason: "already_read" as const } },{ name: "skip: forbidden", outcome: { outcome: "skip" as const, reason: "forbidden" as const } },{ name: "skip: muted", outcome: { outcome: "skip" as const, reason: "muted" as const } },@@ -288,7 +293,7 @@ describe("wake-worker queue() vs fetch() — same resolution for the same candid{ messageId: "msg2", botUserId: "bot2" },]- mockDispatchOneUnreadWake.mockRejectedValueOnce(new Error("boom")).mockResolvedValueOnce({ outcome: "sent" })+ mockDispatchOneUnreadWake.mockRejectedValueOnce(new Error("boom")).mockResolvedValueOnce({ outcome: "attempted" })const msgs = items.map((body) => ({ body, ack: vi.fn(), retry: vi.fn() }))await handler.queue({ messages: msgs } as never, env)expect(msgs[0]!.retry).toHaveBeenCalledTimes(1)@@ -300,7 +305,7 @@ describe("wake-worker queue() vs fetch() — same resolution for the same candidconst attempt = (attempts.get(item.botUserId) ?? 0) + 1attempts.set(item.botUserId, attempt)if (item.botUserId === "bot1" && attempt === 1) throw new Error("boom")- return { outcome: "sent" }+ return { outcome: "attempted" }})const res = await handler.fetch!(new Request("http://internal/", { method: "POST", body: JSON.stringify(items) }), env)expect(res.status).toBe(202)diff --git a/src/wake-worker/src/index.ts b/src/wake-worker/src/index.tsindex 8606a3012..8e5c58fe0 100644--- a/src/wake-worker/src/index.ts+++ b/src/wake-worker/src/index.ts@@ -19,11 +19,11 @@ async function resolveAndLog(db: Database, env: Env, item: WakePayload) {if (result.outcome === "skip") {// Every skip reason is a permanent current-state miss — caller must ack, never retry.log.info("wake_skipped", { botUserId: item.botUserId, messageId: item.messageId, reason: result.reason })- } else if (result.outcome === "delivered_nowhere") {+ } else if (result.outcome === "attempted_nowhere") {// ws-do resolved cleanly but the daemon is offline — a known-permanent// state for this attempt (plan §3's error contract). Daemon reconnect// warmup recovers; retrying would just spin, so this also acks.- log.info("wake_delivered_nowhere", { botUserId: item.botUserId, machineId: result.machineId })+ log.info("wake_attempted_nowhere", { botUserId: item.botUserId, machineId: result.machineId })}return result}diff --git a/src/web/AGENTS.md b/src/web/AGENTS.mdnew file mode 100644index 000000000..643577dfa--- /dev/null+++ b/src/web/AGENTS.md@@ -0,0 +1,9 @@+<!-- BEGIN:nextjs-agent-rules -->++# This is NOT the Next.js you know++This version has breaking changes — APIs, conventions, and file structure may all differ from your training data. Read the relevant guide in `node_modules/next/dist/docs/` (resolved from this file's directory; in monorepos the `next` package may not be visible from the repo root) before writing any code. Heed deprecation notices.++This block is written and re-added by `next dev` — verify at `node_modules/next/dist/server/lib/generate-agent-files.js`. Removing it from a diff only re-creates the uncommitted change; committing it with your work keeps the tree clean.++<!-- END:nextjs-agent-rules -->diff --git a/src/web/CLAUDE.md b/src/web/CLAUDE.mdnew file mode 100644index 000000000..43c994c2d--- /dev/null+++ b/src/web/CLAUDE.md@@ -0,0 +1 @@diff --git a/src/web/migrations/0088_timeout_expired_diagnostic_reports.sql b/src/web/migrations/0088_timeout_expired_diagnostic_reports.sqlnew file mode 100644index 000000000..902e0f609--- /dev/null+++ b/src/web/migrations/0088_timeout_expired_diagnostic_reports.sql@@ -0,0 +1,10 @@+-- Close diagnostic rows that expired before receipt-driven alarms shipped.+-- New reports are expired by the ws-do alarm and daemon reconnect resync path;+-- this idempotent backfill is only the deploy-time bridge for legacy pending+-- rows (including the two reports stranded by the 2026-08-17 incident).+UPDATE community_diagnostic_report+SET status = 'failed',+ failure_code = 'timeout',+ completed_at = CAST(strftime('%s', 'now') AS INTEGER) * 1000+WHERE status = 'pending'+ AND deadline_at <= CAST(strftime('%s', 'now') AS INTEGER) * 1000;diff --git a/src/web/src/app/api/community/bots/[id]/diagnostics/route.test.ts b/src/web/src/app/api/community/bots/[id]/diagnostics/route.test.tsindex 8dacc9384..fad35e341 100644--- a/src/web/src/app/api/community/bots/[id]/diagnostics/route.test.ts+++ b/src/web/src/app/api/community/bots/[id]/diagnostics/route.test.ts@@ -89,7 +89,7 @@ describe("POST /api/community/bots/:id/diagnostics", () => {vi.clearAllMocks();vi.spyOn(Date, "now").mockReturnValue(NOW);createOrGet.mockResolvedValue({ kind: "created", report: row() });- pushDiagnostic.mockResolvedValue({ kind: "delivered", sent: 1 });+ pushDiagnostic.mockResolvedValue({ kind: "attempted", attempted: 1 });});it.each([@@ -136,7 +136,7 @@ describe("POST /api/community/bots/:id/diagnostics", () => {expect(getForOwner).not.toHaveBeenCalled();});- it("creates through the atomic owner gate and returns 202 accepted pending", async () => {+ it("creates through the atomic owner gate and returns 202 unknown after a socket attempt", async () => {const response = await POST(request(), context);expect(createOrGet).toHaveBeenCalledWith({}, {@@ -158,7 +158,7 @@ describe("POST /api/community/bots/:id/diagnostics", () => {);expect(response.status).toBe(202);await expect(response.json()).resolves.toEqual({- delivery: "accepted",+ delivery: "unknown",report: {reportId: "dbr_0123456789abcdef",status: "pending",@@ -170,7 +170,7 @@ describe("POST /api/community/bots/:id/diagnostics", () => {});});- it("turns definitive sent=0 into guarded offline and returns a 200 terminal envelope", async () => {+ it("turns definitive attempted=0 into guarded offline and returns a 200 terminal envelope", async () => {pushDiagnostic.mockResolvedValue({ kind: "offline" });const failed = row({ status: "failed", failureCode: "offline", completedAt: NOW + 1 });failPending.mockResolvedValue(failed);@@ -197,7 +197,7 @@ describe("POST /api/community/bots/:id/diagnostics", () => {});});- it("reads the authoritative owner row when the sent=0 offline CAS loses", async () => {+ it("reads the authoritative owner row when the attempted=0 offline CAS loses", async () => {pushDiagnostic.mockResolvedValue({ kind: "offline" });failPending.mockResolvedValue(null);getForOwner.mockResolvedValue(row({@@ -219,7 +219,7 @@ describe("POST /api/community/bots/:id/diagnostics", () => {});});- it("keeps delivery unknown when the sent=0 offline CAS loses to a still-pending row", async () => {+ it("keeps delivery unknown when the attempted=0 offline CAS loses to a still-pending row", async () => {pushDiagnostic.mockResolvedValue({ kind: "offline" });failPending.mockResolvedValue(null);getForOwner.mockResolvedValue(row());diff --git a/src/web/src/app/api/community/bots/[id]/diagnostics/route.ts b/src/web/src/app/api/community/bots/[id]/diagnostics/route.tsindex e09c82b3d..d2c49fe3b 100644--- a/src/web/src/app/api/community/bots/[id]/diagnostics/route.ts+++ b/src/web/src/app/api/community/bots/[id]/diagnostics/route.ts@@ -68,8 +68,8 @@ export const POST = withAuth(async (req: NextRequest, ctx) => {fromMs: report.fromMs,deadlineAt: report.deadlineAt,});- if (delivery.kind === "delivered") {- return reportEnvelope("accepted", report, nowMs, 202);+ if (delivery.kind === "attempted") {+ return reportEnvelope("unknown", report, nowMs, 202);}if (delivery.kind === "ambiguous") {return reportEnvelope("unknown", report, nowMs, 202);diff --git a/src/web/src/app/api/community/daemon/resync-diagnostics/route.test.ts b/src/web/src/app/api/community/daemon/resync-diagnostics/route.test.tsnew file mode 100644index 000000000..c78c5f23b--- /dev/null+++ b/src/web/src/app/api/community/daemon/resync-diagnostics/route.test.ts@@ -0,0 +1,94 @@+import { beforeEach, describe, expect, it, vi } from "vitest";+import { NextRequest } from "next/server";++const findCredential = vi.fn();+const timeoutPending = vi.fn();+const listPending = vi.fn();+const pushDiagnostic = vi.fn();++vi.mock("@opennextjs/cloudflare", () => ({+ getCloudflareContext: vi.fn(async () => ({ env: { DB: {}, WS_DO_WORKER: {} } })),+}));+vi.mock("@/lib/db", () => ({ getDb: vi.fn(() => ({})) }));+vi.mock("@/lib/community/diagnostic-report-push", () => ({+ pushDiagnosticReportToMachine: (...args: unknown[]) => pushDiagnostic(...args),+}));+vi.mock("@alook/shared", async () => {+ const actual = await vi.importActual<typeof import("@alook/shared")>("@alook/shared");+ return {+ ...actual,+ queries: {+ ...actual.queries,+ communityMachine: {+ ...actual.queries.communityMachine,+ findActiveCredentialByBearer: (...args: unknown[]) => findCredential(...args),+ },+ communityDiagnosticReport: {+ ...actual.queries.communityDiagnosticReport,+ timeoutPendingDiagnosticReportsForMachine: (...args: unknown[]) => timeoutPending(...args),+ listPendingDiagnosticReportsForMachine: (...args: unknown[]) => listPending(...args),+ },+ },+ };+});++import { POST } from "./route";++const NOW = 1_800_000_000_000;+const pending = (id: string, deadlineAt: number) => ({+ id,+ agentId: `bot_${id}`,+ fromMs: NOW - 86_400_000,+ deadlineAt,+});++function request(): NextRequest {+ return new NextRequest("http://localhost/api/community/daemon/resync-diagnostics", {+ method: "POST",+ headers: { Authorization: "Bearer cmk_ok" },+ });+}++describe("POST /api/community/daemon/resync-diagnostics", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ vi.spyOn(Date, "now").mockReturnValue(NOW);+ findCredential.mockResolvedValue({+ credentialId: "cmk_ok",+ userId: "owner_1",+ machineId: "cm_1",+ });+ timeoutPending.mockResolvedValue([]);+ listPending.mockResolvedValue([]);+ });++ it("expires overdue rows, then re-derives and re-attempts every pending report from D1", async () => {+ listPending.mockResolvedValue([+ pending("dbr_one", NOW + 10_000),+ pending("dbr_two", NOW + 20_000),+ ]);+ pushDiagnostic+ .mockResolvedValueOnce({ kind: "attempted", attempted: 1 })+ .mockResolvedValueOnce({ kind: "ambiguous" });++ const response = await POST(request());++ expect(timeoutPending).toHaveBeenCalledWith({}, { machineId: "cm_1", nowMs: NOW });+ expect(listPending).toHaveBeenCalledWith({}, { machineId: "cm_1", nowMs: NOW });+ expect(pushDiagnostic).toHaveBeenNthCalledWith(1, expect.anything(), "cm_1", {+ reportId: "dbr_one",+ agentId: "bot_dbr_one",+ fromMs: NOW - 86_400_000,+ deadlineAt: NOW + 10_000,+ });+ expect(response.status).toBe(200);+ await expect(response.json()).resolves.toEqual({ pending: 2, attempted: 1, ambiguous: 1 });+ });++ it("returns an empty recovery set without attempting delivery", async () => {+ const response = await POST(request());++ await expect(response.json()).resolves.toEqual({ pending: 0, attempted: 0, ambiguous: 0 });+ expect(pushDiagnostic).not.toHaveBeenCalled();+ });+});diff --git a/src/web/src/app/api/community/daemon/resync-diagnostics/route.ts b/src/web/src/app/api/community/daemon/resync-diagnostics/route.tsnew file mode 100644index 000000000..c81de75f9--- /dev/null+++ b/src/web/src/app/api/community/daemon/resync-diagnostics/route.ts@@ -0,0 +1,33 @@+import { NextResponse } from "next/server";+import { queries } from "@alook/shared";+import { getDb } from "@/lib/db";+import { pushDiagnosticReportToMachine } from "@/lib/community/diagnostic-report-push";+import { withCommunityDaemonAuth } from "@/lib/middleware/community-daemon-auth";++export const POST = withCommunityDaemonAuth(async (_request, ctx) => {+ const db = getDb(ctx.env.DB);+ const nowMs = Date.now();+ await queries.communityDiagnosticReport.timeoutPendingDiagnosticReportsForMachine(db, {+ machineId: ctx.machineId,+ nowMs,+ });+ const reports = await queries.communityDiagnosticReport.listPendingDiagnosticReportsForMachine(db, {+ machineId: ctx.machineId,+ nowMs,+ });++ let attempted = 0;+ let ambiguous = 0;+ for (const report of reports) {+ const outcome = await pushDiagnosticReportToMachine(ctx.env, ctx.machineId, {+ reportId: report.id,+ agentId: report.agentId,+ fromMs: report.fromMs,+ deadlineAt: report.deadlineAt,+ });+ if (outcome.kind === "attempted") attempted++;+ else if (outcome.kind === "ambiguous") ambiguous++;+ }++ return NextResponse.json({ pending: reports.length, attempted, ambiguous });+});diff --git a/src/web/src/app/api/community/daemon/resync-wakes/route.test.ts b/src/web/src/app/api/community/daemon/resync-wakes/route.test.tsindex 064acd484..76fffdb95 100644--- a/src/web/src/app/api/community/daemon/resync-wakes/route.test.ts+++ b/src/web/src/app/api/community/daemon/resync-wakes/route.test.ts@@ -51,11 +51,11 @@ describe("POST /api/community/daemon/resync-wakes", () => {});});- it("returns { woken: 0 } and does nothing when the machine has no bots", async () => {+ it("returns { attempted: 0 } and does nothing when the machine has no bots", async () => {mockListBotsForMachine.mockResolvedValue([]);const res = await POST(req({ Authorization: "Bearer cmk_ok" }));expect(res.status).toBe(200);- expect(await res.json()).toEqual({ woken: 0 });+ expect(await res.json()).toEqual({ attempted: 0 });expect(mockGetLatestUnreadMessageForAgent).not.toHaveBeenCalled();expect(mockDispatchOneUnreadWake).not.toHaveBeenCalled();});@@ -67,7 +67,7 @@ describe("POST /api/community/daemon/resync-wakes", () => {mockGetLatestUnreadMessageForAgent.mockResolvedValue({messageId: "msg_1",});- mockDispatchOneUnreadWake.mockResolvedValue({ outcome: "sent" });+ mockDispatchOneUnreadWake.mockResolvedValue({ outcome: "attempted" });const res = await POST(req({ Authorization: "Bearer cmk_ok" }));@@ -76,7 +76,7 @@ describe("POST /api/community/daemon/resync-wakes", () => {expect.anything(),{ messageId: "msg_1", botUserId: "bot_1" },);- expect(await res.json()).toEqual({ woken: 1 });+ expect(await res.json()).toEqual({ attempted: 1 });});it("skips bots with no pending unread — not counted, dispatch never called for them", async () => {@@ -88,10 +88,10 @@ describe("POST /api/community/daemon/resync-wakes", () => {const res = await POST(req({ Authorization: "Bearer cmk_ok" }));expect(mockDispatchOneUnreadWake).not.toHaveBeenCalled();- expect(await res.json()).toEqual({ woken: 0 });+ expect(await res.json()).toEqual({ attempted: 0 });});- it("does not count a skip/delivered_nowhere outcome as woken", async () => {+ it("does not count a skip/attempted_nowhere outcome as attempted", async () => {mockListBotsForMachine.mockResolvedValue([{ id: "bot_1", name: "a", description: "" },{ id: "bot_2", name: "b", description: "" },@@ -102,13 +102,13 @@ describe("POST /api/community/daemon/resync-wakes", () => {mockDispatchOneUnreadWake.mockResolvedValueOnce({ outcome: "skip", reason: "already_read" }).mockResolvedValueOnce({- outcome: "delivered_nowhere",+ outcome: "attempted_nowhere",machineId: "cm_1",});const res = await POST(req({ Authorization: "Bearer cmk_ok" }));- expect(await res.json()).toEqual({ woken: 0 });+ expect(await res.json()).toEqual({ attempted: 0 });});it("401 without Authorization", async () => {@@ -123,11 +123,11 @@ describe("POST /api/community/daemon/resync-wakes", () => {expect(res.status).toBe(401);});- it("Mellicent regression: bot on a server whose only unread lives in a non-participated forum_post — feeder returns null → woken: 0", async () => {+ it("Mellicent regression: bot on a server whose only unread lives in a non-participated forum_post — feeder returns null → attempted: 0", async () => {// Post-fix behaviour: `getLatestUnreadMessageForAgent` applies the// thread-participation post-filter (`plans/agent-unread-visibility-unify.md`),// so a message in a forum_post the bot isn't a participant of no longer- // surfaces. The feeder returns null → resync-wakes reports woken: 0 and+ // surfaces. The feeder returns null → resync-wakes reports attempted: 0 and// never touches the wake dispatcher.mockListBotsForMachine.mockResolvedValue([{ id: "bot_Y", name: "Mellicent", description: "" },@@ -136,7 +136,7 @@ describe("POST /api/community/daemon/resync-wakes", () => {const res = await POST(req({ Authorization: "Bearer cmk_ok" }));- expect(await res.json()).toEqual({ woken: 0 });+ expect(await res.json()).toEqual({ attempted: 0 });expect(mockDispatchOneUnreadWake).not.toHaveBeenCalled();});});diff --git a/src/web/src/app/api/community/daemon/resync-wakes/route.ts b/src/web/src/app/api/community/daemon/resync-wakes/route.tsindex 3b6c59b81..299745357 100644--- a/src/web/src/app/api/community/daemon/resync-wakes/route.ts+++ b/src/web/src/app/api/community/daemon/resync-wakes/route.ts@@ -9,7 +9,7 @@ import { withCommunityDaemonAuth } from "@/lib/middleware/community-daemon-auth"* Daemon-initiated recovery for the "message sent while the daemon was* offline" gap: `WAKE_QUEUE`'s consumer acks (never retries) a wake whose* daemon was unreachable at delivery time (`dispatchOneUnreadWake`'s- * `delivered_nowhere` outcome) — that queue item is gone for good. Rather+ * `attempted_nowhere` outcome) — that queue item is gone for good. Rather* than the server pushing a catch-up wake on its own when the daemon's WS* reconnects, the DAEMON proactively calls this route right after it opens* its control-plane connection (`channel.onOpen()`), asking "do any of my@@ -29,7 +29,7 @@ export const POST = withCommunityDaemonAuth(async (_req, ctx) => {{ route: "community/daemon/resync-wakes:list-bots" },)- let woken = 0+ let attempted = 0for (const bot of bots) {const latest = await withD1Retry(() => queries.communityAgentInbox.getLatestUnreadMessageForAgent(db, bot.id),@@ -37,8 +37,8 @@ export const POST = withCommunityDaemonAuth(async (_req, ctx) => {)if (!latest) continueconst result = await dispatchOneUnreadWake(db, ctx.env, { messageId: latest.messageId, botUserId: bot.id })- if (result.outcome === "sent") woken+++ if (result.outcome === "attempted") attempted++}- return NextResponse.json({ woken })+ return NextResponse.json({ attempted })})diff --git a/src/web/src/lib/community/diagnostic-report-push.test.ts b/src/web/src/lib/community/diagnostic-report-push.test.tsindex dfee51260..90d224501 100644--- a/src/web/src/lib/community/diagnostic-report-push.test.ts+++ b/src/web/src/lib/community/diagnostic-report-push.test.ts@@ -19,11 +19,11 @@ describe("pushDiagnosticReportToMachine", () => {beforeEach(() => wsDoFetch.mockReset());it("uses only the purpose-built diagnostics route and narrow body", async () => {- wsDoFetch.mockResolvedValue(Response.json({ sent: 1 }));+ wsDoFetch.mockResolvedValue(Response.json({ attempted: 1, sent: 1 }));const outcome = await pushDiagnosticReportToMachine(env, "cm one", command);- expect(outcome).toEqual({ kind: "delivered", sent: 1 });+ expect(outcome).toEqual({ kind: "attempted", attempted: 1 });const [, path, init, audit] = wsDoFetch.mock.calls[0]!;expect(path).toBe("/community-machine/by-id/cm%20one/forward-diagnostics-collect");expect(init.method).toBe("POST");@@ -32,14 +32,14 @@ describe("pushDiagnosticReportToMachine", () => {expect(audit).toEqual({ label: "cm one", type: "diagnostics:collect" });});- it("distinguishes delivered, definite offline, and ambiguous transport", async () => {- wsDoFetch.mockResolvedValueOnce(Response.json({ sent: 2 }));+ it("distinguishes attempted, definite offline, and ambiguous transport", async () => {+ wsDoFetch.mockResolvedValueOnce(Response.json({ attempted: 2 }));await expect(pushDiagnosticReportToMachine(env, "cm_1", command)).resolves.toEqual({- kind: "delivered",- sent: 2,+ kind: "attempted",+ attempted: 2,});- wsDoFetch.mockResolvedValueOnce(Response.json({ sent: 0 }));+ wsDoFetch.mockResolvedValueOnce(Response.json({ attempted: 0 }));await expect(pushDiagnosticReportToMachine(env, "cm_1", command)).resolves.toEqual({kind: "offline",});@@ -56,10 +56,12 @@ describe("pushDiagnosticReportToMachine", () => {});it.each([- ["missing sent", {}],- ["negative sent", { sent: -1 }],- ["fractional sent", { sent: 0.5 }],- ["string sent", { sent: "1" }],+ ["missing attempted", {}],+ ["legacy sent-only without deadline proof", { sent: 1 }],+ ["negative attempted", { attempted: -1 }],+ ["fractional attempted", { attempted: 0.5 }],+ ["string attempted", { attempted: "1" }],+ ["mismatched sent alias", { attempted: 1, sent: 0 }],])("fails closed on a 2xx response with %s", async (_label, response) => {wsDoFetch.mockResolvedValue(Response.json(response));diff --git a/src/web/src/lib/community/diagnostic-report-push.ts b/src/web/src/lib/community/diagnostic-report-push.tsindex 63dd6779a..235d96f71 100644--- a/src/web/src/lib/community/diagnostic-report-push.ts+++ b/src/web/src/lib/community/diagnostic-report-push.ts@@ -1,8 +1,9 @@+import { parseAttemptedCountReceipt } from "@alook/shared";import type { DiagnosticCollectPayload } from "@alook/shared";import { wsDoFetch } from "@/lib/broadcast";export type DiagnosticDeliveryOutcome =- | { kind: "delivered"; sent: number }+ | { kind: "attempted"; attempted: number }| { kind: "offline" }| { kind: "ambiguous" };@@ -24,13 +25,14 @@ export async function pushDiagnosticReportToMachine({ label: machineId, type: "diagnostics:collect" },);if (!response.ok) return { kind: "ambiguous" };- const result = await response.json() as { sent?: unknown };- if (!Number.isSafeInteger(result.sent) || (result.sent as number) < 0) {- return { kind: "ambiguous" };- }- return (result.sent as number) === 0+ const attempted = parseAttemptedCountReceipt(await response.json(), {+ // A legacy sent-only ws-do predates deterministic deadline ownership,+ // so it cannot establish a safe offline/attempted result for new web.+ allowLegacySentOnly: false,+ });+ return attempted === 0? { kind: "offline" }- : { kind: "delivered", sent: result.sent as number };+ : { kind: "attempted", attempted };} catch {return { kind: "ambiguous" };}diff --git a/src/ws-do/src/routes/community-machine-delivery.test.ts b/src/ws-do/src/routes/community-machine-delivery.test.tsindex 1e0c173d4..bd598a275 100644--- a/src/ws-do/src/routes/community-machine-delivery.test.ts+++ b/src/ws-do/src/routes/community-machine-delivery.test.ts@@ -111,7 +111,7 @@ describe("ws-do router", () => {mockWithD1Retry.mockImplementation(async (fn) => fn())})- it("zero active doNames → { sent: 0 } without touching any DO", async () => {+ it("zero active doNames → dual-write zero without touching any DO", async () => {const req = new Request("http://localhost/community-machine/by-id/machine-1/forward-agent-wake", {method: "POST",body: JSON.stringify({ type: "agent:wake" }),@@ -120,13 +120,13 @@ describe("ws-do router", () => {expect(mockGetActiveDoNamesForMachine).toHaveBeenCalledWith({}, "machine-1")expect(res.status).toBe(200)- expect(await res.json()).toEqual({ sent: 0 })+ expect(await res.json()).toEqual({ attempted: 0, sent: 0 })expect(doMock.stubFetch).not.toHaveBeenCalled()})- it("single active doName, daemon connected → forwards and aggregates sent count", async () => {+ it("single active doName, daemon connected → forwards and aggregates attempted count", async () => {mockGetActiveDoNamesForMachine.mockResolvedValue(["do-abc"])- doMock.stubFetch.mockResolvedValue(new Response(JSON.stringify({ sent: 1 }), { status: 200 }))+ doMock.stubFetch.mockResolvedValue(new Response(JSON.stringify({ attempted: 1, sent: 1 }), { status: 200 }))const req = new Request("http://localhost/community-machine/by-id/machine-1/forward-agent-wake", {method: "POST",@@ -139,12 +139,12 @@ describe("ws-do router", () => {const stubReq = doMock.stubFetch.mock.calls[0][0] as Requestexpect(stubReq.url).toBe("http://internal/forward-agent-wake")expect(res.status).toBe(200)- expect(await res.json()).toEqual({ sent: 1 })+ expect(await res.json()).toEqual({ attempted: 1, sent: 1 })})- it("daemon offline (DO reports { sent: 0 }) → does not count as delivered", async () => {+ it("daemon offline (DO reports { attempted: 0 }) → does not count a write", async () => {mockGetActiveDoNamesForMachine.mockResolvedValue(["do-abc"])- doMock.stubFetch.mockResolvedValue(new Response(JSON.stringify({ sent: 0 }), { status: 200 }))+ doMock.stubFetch.mockResolvedValue(new Response(JSON.stringify({ attempted: 0 }), { status: 200 }))const req = new Request("http://localhost/community-machine/by-id/machine-1/forward-agent-wake", {method: "POST",@@ -153,16 +153,16 @@ describe("ws-do router", () => {const res = await handler.fetch(req, env as any)expect(res.status).toBe(200)- expect(await res.json()).toEqual({ sent: 0 })+ expect(await res.json()).toEqual({ attempted: 0, sent: 0 })})- it("multi-doName fan-out: both DOs hit, aggregate sums delivered counts", async () => {+ it("multi-doName fan-out: both DOs hit, aggregate sums attempted counts", async () => {mockGetActiveDoNamesForMachine.mockResolvedValue(["do-a", "do-b"])let call = 0doMock.stubFetch.mockImplementation(() => {call++- const sent = call === 1 ? 0 : 1- return Promise.resolve(new Response(JSON.stringify({ sent }), { status: 200 }))+ const attempted = call === 1 ? 0 : 1+ return Promise.resolve(new Response(JSON.stringify({ attempted }), { status: 200 }))})const req = new Request("http://localhost/community-machine/by-id/machine-1/forward-agent-wake", {@@ -174,16 +174,16 @@ describe("ws-do router", () => {expect(doMock.idFromName).toHaveBeenCalledWith("community-machine:do-a")expect(doMock.idFromName).toHaveBeenCalledWith("community-machine:do-b")expect(res.status).toBe(200)- expect(await res.json()).toEqual({ sent: 1 })+ expect(await res.json()).toEqual({ attempted: 1, sent: 1 })})- it("DO fetch throws — tolerated, other doNames still evaluated", async () => {+ it("DO fetch throws — other doNames are evaluated but the aggregate remains retryable", async () => {mockGetActiveDoNamesForMachine.mockResolvedValue(["do-a", "do-b"])let call = 0doMock.stubFetch.mockImplementation(() => {call++if (call === 1) return Promise.reject(new Error("network error"))- return Promise.resolve(new Response(JSON.stringify({ sent: 1 }), { status: 200 }))+ return Promise.resolve(new Response(JSON.stringify({ attempted: 1 }), { status: 200 }))})const req = new Request("http://localhost/community-machine/by-id/machine-1/forward-agent-wake", {@@ -192,8 +192,9 @@ describe("ws-do router", () => {})const res = await handler.fetch(req, env as any)- expect(res.status).toBe(200)- expect(await res.json()).toEqual({ sent: 1 })+ expect(doMock.stubFetch).toHaveBeenCalledTimes(2)+ expect(res.status).toBe(503)+ expect(await res.json()).toEqual({ error: "failed to forward agent wake" })})it("DO fetch throws with no delivery → retryable 503", async () => {@@ -210,13 +211,13 @@ describe("ws-do router", () => {expect(await res.json()).toEqual({ error: "failed to forward agent wake" })})- it("non-2xx or malformed DO responses with no delivery → retryable 503", async () => {+ it("non-2xx or malformed DO responses → retryable 503", async () => {mockGetActiveDoNamesForMachine.mockResolvedValue(["do-a", "do-b"])let call = 0doMock.stubFetch.mockImplementation(() => {call++if (call === 1) return Promise.resolve(new Response("oops", { status: 502 }))- return Promise.resolve(new Response(JSON.stringify({ sent: "bad" }), { status: 200 }))+ return Promise.resolve(new Response(JSON.stringify({ attempted: "bad" }), { status: 200 }))})const req = new Request("http://localhost/community-machine/by-id/machine-1/forward-agent-wake", {@@ -229,7 +230,38 @@ describe("ws-do router", () => {expect(await res.json()).toEqual({ error: "failed to forward agent wake" })})- it("DB lookup failure → retryable 503, not offline { sent: 0 }", async () => {+ it("accepts a legacy sent-only inner DO and dual-writes the outer receipt", async () => {+ mockGetActiveDoNamesForMachine.mockResolvedValue(["do-legacy"])+ doMock.stubFetch.mockResolvedValue(new Response(JSON.stringify({ sent: 1 }), { status: 200 }))++ const res = await handler.fetch(new Request(+ "http://localhost/community-machine/by-id/machine-1/forward-agent-wake",+ { method: "POST", body: JSON.stringify({ type: "agent:wake" }) },+ ), env as any)++ expect(res.status).toBe(200)+ expect(await res.json()).toEqual({ attempted: 1, sent: 1 })+ })++ it.each([+ ["missing", {}],+ ["negative", { attempted: -1 }],+ ["fractional", { attempted: 0.5 }],+ ["mismatched", { attempted: 1, sent: 0 }],+ ])("maps a %s inner receipt to retryable 503", async (_name, receipt) => {+ mockGetActiveDoNamesForMachine.mockResolvedValue(["do-bad"])+ doMock.stubFetch.mockResolvedValue(new Response(JSON.stringify(receipt), { status: 200 }))++ const res = await handler.fetch(new Request(+ "http://localhost/community-machine/by-id/machine-1/forward-agent-wake",+ { method: "POST", body: JSON.stringify({ type: "agent:wake" }) },+ ), env as any)++ expect(res.status).toBe(503)+ expect(await res.json()).toEqual({ error: "failed to forward agent wake" })+ })++ it("DB lookup failure → retryable 503, not { attempted: 0 }", async () => {mockGetActiveDoNamesForMachine.mockRejectedValue(new Error("d1 unreachable"))const req = new Request("http://localhost/community-machine/by-id/machine-1/forward-agent-wake", {@@ -254,7 +286,7 @@ describe("ws-do router", () => {return fn()}})- doMock.stubFetch.mockResolvedValue(new Response(JSON.stringify({ sent: 1 }), { status: 200 }))+ doMock.stubFetch.mockResolvedValue(new Response(JSON.stringify({ attempted: 1 }), { status: 200 }))const res = await handler.fetch(new Request("http://localhost/community-machine/by-id/machine-1/forward-agent-wake",@@ -267,7 +299,7 @@ describe("ws-do router", () => {})expect(doMock.stubFetch).toHaveBeenCalledTimes(1)expect(res.status).toBe(200)- expect(await res.json()).toEqual({ sent: 1 })+ expect(await res.json()).toEqual({ attempted: 1, sent: 1 })})it("maps exhausted SQLITE_BUSY retries to the existing 503 response", async () => {diff --git a/src/ws-do/src/routes/community-machine-delivery.ts b/src/ws-do/src/routes/community-machine-delivery.tsindex b75cd046a..b4684eecf 100644--- a/src/ws-do/src/routes/community-machine-delivery.ts+++ b/src/ws-do/src/routes/community-machine-delivery.ts@@ -1,4 +1,4 @@-import { queries, withD1Retry } from "@alook/shared"+import { parseAttemptedCountReceipt, queries, withD1Retry } from "@alook/shared"import type { RouterContext } from "../router-context"export async function handleMachinePush({ request, env, url, traceId, log }: RouterContext): Promise<Response | null> {@@ -56,10 +56,9 @@ export async function handleMachineWake({ request, env, url, traceId, log }: Rou// of the `/push` route above, for the minimal-wake-queue-unread-notice// wake path. Forwards an already-built `HostCommand` (`agent:wake`)// verbatim to every live DO for this machine's active credential(s),- // then aggregates by parsing each DO's own `{ sent: N }` response —- // unlike `/push`, we must NOT just count `res.ok`: a 200 with- // `{ sent: 0 }` means that DO has no authenticated daemon socket at all,- // which must not be reported as delivered.+ // then aggregates each DO's rolling `{ attempted: N, sent: N }` receipt.+ // A successful socket write is only an attempt; daemon `agent_wake_ack`+ // is the receipt and unread D1 state remains the reconnect truth.const forwardAgentWake = url.pathname.match(/^\/community-machine\/by-id\/([^/]+)\/forward-agent-wake$/)if (!forwardAgentWake || request.method !== "POST") return null@@ -80,10 +79,10 @@ export async function handleMachineWake({ request, env, url, traceId, log }: Roureturn Response.json({ error: "failed to resolve machine" }, { status: 503 })}if (doNames.length === 0) {- return Response.json({ sent: 0 })+ return Response.json({ attempted: 0, sent: 0 })}const bodyText = await request.text()- let delivered = 0+ let attempted = 0let transientFailure = falsefor (const dn of doNames) {const doId = env.WS_DO.idFromName("community-machine:" + dn)@@ -100,18 +99,13 @@ export async function handleMachineWake({ request, env, url, traceId, log }: RoutransientFailure = truecontinue}- const data = (await res.json()) as { sent?: unknown }- if (typeof data.sent !== "number" || !Number.isFinite(data.sent) || data.sent < 0) {- transientFailure = true- continue- }- delivered += data.sent+ attempted += parseAttemptedCountReceipt(await res.json())} catch {transientFailure = true}}- if (delivered === 0 && transientFailure) {+ if (transientFailure) {return Response.json({ error: "failed to forward agent wake" }, { status: 503 })}- return Response.json({ sent: delivered })+ return Response.json({ attempted, sent: attempted })}diff --git a/src/ws-do/src/routes/community-machine-diagnostics.test.ts b/src/ws-do/src/routes/community-machine-diagnostics.test.tsindex 97f0fff8c..7896362bf 100644--- a/src/ws-do/src/routes/community-machine-diagnostics.test.ts+++ b/src/ws-do/src/routes/community-machine-diagnostics.test.ts@@ -50,19 +50,23 @@ describe("POST /community-machine/by-id/:machineId/forward-diagnostics-collect",expect(context.doMock.stubFetch).not.toHaveBeenCalled();});- it("constructs the exact HostCommand and attempts every active credential", async () => {+ it("constructs the exact HostCommand, accepts a legacy inner receipt, and dual-writes for old web", async () => {getActiveDoNamesForMachine.mockResolvedValue(["do-a", "do-b", "do-c"]);context.doMock.stubFetch+ .mockResolvedValueOnce(Response.json({ registered: true }))+ .mockResolvedValueOnce(Response.json({ attempted: 0, sent: 0 })).mockResolvedValueOnce(Response.json({ sent: 0 }))- .mockRejectedValueOnce(new Error("credential transport failed"))- .mockResolvedValueOnce(Response.json({ sent: 1 }));+ .mockResolvedValueOnce(Response.json({ attempted: 1, sent: 1 }));const response = await handler.fetch(request(payload), context.env as never);expect(response.status).toBe(200);- await expect(response.json()).resolves.toEqual({ sent: 1 });- expect(context.doMock.stubFetch).toHaveBeenCalledTimes(3);- const forwarded = context.doMock.stubFetch.mock.calls.map(([value]) => value as Request);+ await expect(response.json()).resolves.toEqual({ attempted: 1, sent: 1 });+ expect(context.doMock.stubFetch).toHaveBeenCalledTimes(4);+ const deadline = context.doMock.stubFetch.mock.calls[0]![0] as Request;+ expect(deadline.url).toBe("http://internal/register-diagnostic-deadline?machineId=cm_1");+ await expect(deadline.clone().json()).resolves.toEqual({ deadlineAt: payload.deadlineAt });+ const forwarded = context.doMock.stubFetch.mock.calls.slice(1).map(([value]) => value as Request);expect(forwarded.every((value) => value.url === "http://internal/forward-diagnostics-collect")).toBe(true);expect(await forwarded[0].clone().json()).toEqual({ type: "diagnostics:collect", ...payload });expect(forwarded.some((value) => value.url.endsWith("/push"))).toBe(false);@@ -71,26 +75,118 @@ describe("POST /community-machine/by-id/:machineId/forward-diagnostics-collect",it("returns definite offline only when every credential reports zero", async () => {getActiveDoNamesForMachine.mockResolvedValue(["do-a", "do-b"]);context.doMock.stubFetch- .mockResolvedValueOnce(Response.json({ sent: 0 }))- .mockResolvedValueOnce(Response.json({ sent: 0 }));+ .mockResolvedValueOnce(Response.json({ registered: true }))+ .mockResolvedValueOnce(Response.json({ attempted: 0, sent: 0 }))+ .mockResolvedValueOnce(Response.json({ attempted: 0, sent: 0 }));const response = await handler.fetch(request(payload), context.env as never);expect(response.status).toBe(200);- await expect(response.json()).resolves.toEqual({ sent: 0 });- expect(context.doMock.stubFetch).toHaveBeenCalledTimes(2);+ await expect(response.json()).resolves.toEqual({ attempted: 0, sent: 0 });+ expect(context.doMock.stubFetch).toHaveBeenCalledTimes(3);});it("returns ambiguous when no delivery succeeds and any credential is non-definitive", async () => {getActiveDoNamesForMachine.mockResolvedValue(["do-a", "do-b", "do-c"]);context.doMock.stubFetch- .mockResolvedValueOnce(Response.json({ sent: 0 }))+ .mockResolvedValueOnce(Response.json({ registered: true }))+ .mockResolvedValueOnce(Response.json({ attempted: 0 })).mockResolvedValueOnce(new Response("unavailable", { status: 503 }))- .mockResolvedValueOnce(Response.json({ sent: "one" }));+ .mockResolvedValueOnce(Response.json({ attempted: "one" }));++ const response = await handler.fetch(request(payload), context.env as never);++ expect(response.status).toBe(503);+ expect(context.doMock.stubFetch).toHaveBeenCalledTimes(4);+ });++ it("returns ambiguous when one credential succeeds but another forward fails", async () => {+ getActiveDoNamesForMachine.mockResolvedValue(["do-a", "do-b"]);+ context.doMock.stubFetch+ .mockResolvedValueOnce(Response.json({ registered: true }))+ .mockResolvedValueOnce(Response.json({ attempted: 1, sent: 1 }))+ .mockRejectedValueOnce(new Error("credential transport failed"));++ const response = await handler.fetch(request(payload), context.env as never);++ expect(response.status).toBe(503);+ await expect(response.json()).resolves.toEqual({ error: "diagnostic delivery ambiguous" });+ });++ it.each([+ ["missing", {}],+ ["negative", { attempted: -1 }],+ ["fractional", { attempted: 0.5 }],+ ["mismatched", { attempted: 1, sent: 0 }],+ ])("returns ambiguous for a %s inner receipt", async (_label, receipt) => {+ getActiveDoNamesForMachine.mockResolvedValue(["do-a"]);+ context.doMock.stubFetch+ .mockResolvedValueOnce(Response.json({ registered: true }))+ .mockResolvedValueOnce(Response.json(receipt));++ const response = await handler.fetch(request(payload), context.env as never);++ expect(response.status).toBe(503);+ await expect(response.json()).resolves.toEqual({ error: "diagnostic delivery ambiguous" });+ });++ it("dual-writes zero after deadline registration when there are no active credentials", async () => {+ context.doMock.stubFetch.mockResolvedValueOnce(Response.json({ registered: true }));++ const response = await handler.fetch(request(payload), context.env as never);++ expect(response.status).toBe(200);+ await expect(response.json()).resolves.toEqual({ attempted: 0, sent: 0 });+ });++ it("registers the deadline before an active-credential lookup failure", async () => {+ context.doMock.stubFetch.mockResolvedValueOnce(Response.json({ registered: true }));+ getActiveDoNamesForMachine.mockRejectedValueOnce(new Error("D1 unavailable"));++ const response = await handler.fetch(request(payload), context.env as never);++ expect(response.status).toBe(503);+ expect(context.doMock.stubFetch).toHaveBeenCalledOnce();+ const deadline = context.doMock.stubFetch.mock.calls[0]![0] as Request;+ expect(deadline.url).toBe("http://internal/register-diagnostic-deadline?machineId=cm_1");+ });++ it("keeps the registered deadline when every active credential forward throws", async () => {+ getActiveDoNamesForMachine.mockResolvedValue(["do-a", "do-b"]);+ context.doMock.stubFetch+ .mockResolvedValueOnce(Response.json({ registered: true }))+ .mockRejectedValueOnce(new Error("do-a unavailable"))+ .mockRejectedValueOnce(new Error("do-b unavailable"));const response = await handler.fetch(request(payload), context.env as never);expect(response.status).toBe(503);expect(context.doMock.stubFetch).toHaveBeenCalledTimes(3);+ expect((context.doMock.stubFetch.mock.calls[0]![0] as Request).url)+ .toContain("/register-diagnostic-deadline");+ });++ it("fails explicitly before lookup or fanout when deadline registration fails", async () => {+ context.doMock.stubFetch.mockResolvedValueOnce(new Response("unavailable", { status: 503 }));++ const response = await handler.fetch(request(payload), context.env as never);++ expect(response.status).toBe(503);+ await expect(response.json()).resolves.toEqual({ error: "diagnostic deadline registration failed" });+ expect(getActiveDoNamesForMachine).not.toHaveBeenCalled();+ });++ it.each([+ ["registered false", { registered: false }],+ ["missing receipt", {}],+ ["extra field", { registered: true, attempted: 1 }],+ ])("rejects a 200 deadline response with %s", async (_label, receipt) => {+ context.doMock.stubFetch.mockResolvedValueOnce(Response.json(receipt));++ const response = await handler.fetch(request(payload), context.env as never);++ expect(response.status).toBe(503);+ await expect(response.json()).resolves.toEqual({ error: "diagnostic deadline registration failed" });+ expect(getActiveDoNamesForMachine).not.toHaveBeenCalled();});});diff --git a/src/ws-do/src/routes/community-machine-diagnostics.ts b/src/ws-do/src/routes/community-machine-diagnostics.tsindex 8320af876..9bda837a5 100644--- a/src/ws-do/src/routes/community-machine-diagnostics.ts+++ b/src/ws-do/src/routes/community-machine-diagnostics.ts@@ -2,6 +2,7 @@ import {createDb,DiagnosticCollectCommandSchema,DiagnosticCollectPayloadSchema,+ parseAttemptedCountReceipt,queries,} from "@alook/shared";import type { RouterContext } from "../router-context";@@ -35,6 +36,38 @@ export async function handleMachineDiagnostics(return Response.json({ error: "invalid payload" }, { status: 400 });}+ // Deadline ownership is deterministic by machineId, independent of the+ // currently active credential doName. Register before D1 lookup/fanout so+ // lookup failures, credential rotation, and ambiguous socket writes cannot+ // strand the durable report row forever.+ try {+ const deadlineId = env.WS_DO.idFromName(`community-machine-deadline:${machineId}`);+ const deadlineStub = env.WS_DO.get(deadlineId);+ const registration = await deadlineStub.fetch(new Request(+ `http://internal/register-diagnostic-deadline?machineId=${encodeURIComponent(machineId)}`,+ {+ method: "POST",+ headers: { "content-type": "application/json" },+ body: JSON.stringify({ deadlineAt: command.data.deadlineAt }),+ },+ ));+ if (!registration.ok) {+ return Response.json({ error: "diagnostic deadline registration failed" }, { status: 503 });+ }+ const receipt = await registration.json() as unknown;+ if (+ !receipt ||+ typeof receipt !== "object" ||+ Object.keys(receipt).length !== 1 ||+ (receipt as Record<string, unknown>).registered !== true+ ) {+ return Response.json({ error: "diagnostic deadline registration failed" }, { status: 503 });+ }+ } catch (err) {+ reqLog.error("failed to register diagnostic deadline", { err });+ return Response.json({ error: "diagnostic deadline registration failed" }, { status: 503 });+ }+let doNames: string[];try {const db = createDb((env as unknown as { DB: D1Database }).DB);@@ -43,10 +76,10 @@ export async function handleMachineDiagnostics(reqLog.error("failed to resolve machine doNames for diagnostics", { err });return Response.json({ error: "failed to resolve machine" }, { status: 503 });}- if (doNames.length === 0) return Response.json({ sent: 0 });+ if (doNames.length === 0) return Response.json({ attempted: 0, sent: 0 });const body = JSON.stringify(command.data);- let delivered = 0;+ let attempted = 0;let ambiguous = false;for (const doName of doNames) {const id = env.WS_DO.idFromName(`community-machine:${doName}`);@@ -64,19 +97,14 @@ export async function handleMachineDiagnostics(ambiguous = true;continue;}- const result = await response.json() as { sent?: unknown };- if (!Number.isSafeInteger(result.sent) || (result.sent as number) < 0) {- ambiguous = true;- continue;- }- delivered += result.sent as number;+ attempted += parseAttemptedCountReceipt(await response.json());} catch {ambiguous = true;}}- if (delivered === 0 && ambiguous) {+ if (ambiguous) {return Response.json({ error: "diagnostic delivery ambiguous" }, { status: 503 });}- return Response.json({ sent: delivered });+ return Response.json({ attempted, sent: attempted });}diff --git a/src/ws-do/src/ws-durable.ts b/src/ws-do/src/ws-durable.tsindex dd22b7fe5..99e66bd04 100644--- a/src/ws-do/src/ws-durable.ts+++ b/src/ws-do/src/ws-durable.ts@@ -103,6 +103,7 @@ export class WebSocketDurableObject extends DurableObject<Env> {message,(parsed) => handleCommunityMachineMessage(this.domainContext(),+ ws,parsed,{fanOutMachineUpdated: (userId, machineId) =>diff --git a/src/ws-do/src/ws-durable/internal.ts b/src/ws-do/src/ws-durable/internal.tsindex 8b4d87fdd..bd861ba1e 100644--- a/src/ws-do/src/ws-durable/internal.ts+++ b/src/ws-do/src/ws-durable/internal.ts@@ -15,6 +15,9 @@ export type ConnectionState =machineId: stringuserId: stringauthenticated: boolean+ controlHeartbeat?: boolean+ lastHeartbeatAckAt?: number+ pendingHeartbeatNonce?: string}export type UserConnectionState = Extract<ConnectionState, { type: "user" }>@@ -34,6 +37,7 @@ export interface CommunityMachineHandle {export const IDENTITY_KEY = "community-machine-identity"export const HANDLE_KEY = "community-machine-handle"export const RUNTIME_ERROR_KEY = "community-machine-runtime-error"+export const DIAGNOSTIC_ALARM_HINT_KEY = "community-machine-diagnostic-alarm-hint"const RESTART_PENDING_PREFIX = "reset-pending:"export const restartPendingKey = (launchId: string) => RESTART_PENDING_PREFIX + launchIddiff --git a/src/ws-do/src/ws-durable/machine-control.test.ts b/src/ws-do/src/ws-durable/machine-control.test.tsindex afd555800..44e32e8e8 100644--- a/src/ws-do/src/ws-durable/machine-control.test.ts+++ b/src/ws-do/src/ws-durable/machine-control.test.ts@@ -291,7 +291,7 @@ describe("WebSocketDurableObject", () => {body,}))- await expect(response.json()).resolves.toEqual({ sent: 1 })+ await expect(response.json()).resolves.toEqual({ attempted: 1, sent: 1 })expect(ws.send).toHaveBeenCalledWith(body)})@@ -320,11 +320,34 @@ describe("WebSocketDurableObject", () => {}))expect(response.status).toBe(200)- await expect(response.json()).resolves.toEqual({ sent: 1 })+ await expect(response.json()).resolves.toEqual({ attempted: 1, sent: 1 })expect(ws.send).toHaveBeenCalledOnce()expect(JSON.parse(ws.send.mock.calls[0]![0] as string)).toEqual(payload)})+ it("registers a strict machine-keyed diagnostic deadline without forwarding a command", async () => {+ vi.spyOn(Date, "now").mockReturnValue(1_700_000_000_000)+ const { durable, getWebSockets, store, storage } = createDO()+ getWebSockets.mockReturnValue([])++ const response = await durable.fetch(new Request(+ "http://internal/register-diagnostic-deadline?machineId=cm_1",+ {+ method: "POST",+ headers: { "content-type": "application/json" },+ body: JSON.stringify({ deadlineAt: 1_700_087_000_000 }),+ },+ ))++ expect(response.status).toBe(200)+ await expect(response.json()).resolves.toEqual({ registered: true })+ expect(store.get("community-machine-diagnostic-alarm-hint")).toEqual({+ machineId: "cm_1",+ deadlineAt: 1_700_087_000_000,+ })+ expect(storage.setAlarm).toHaveBeenCalledWith(1_700_087_000_000)+ })+it.each([["malformed JSON", "{"],["missing field", JSON.stringify({ type: "diagnostics:collect", agentId: "bot_1" })],diff --git a/src/ws-do/src/ws-durable/machine-control.ts b/src/ws-do/src/ws-durable/machine-control.tsindex 72535cb04..3054f30bd 100644--- a/src/ws-do/src/ws-durable/machine-control.ts+++ b/src/ws-do/src/ws-durable/machine-control.ts@@ -12,7 +12,7 @@ import type {RestartAttribution,WsDurableContext,} from "./internal"-import { scheduleHeartbeatAlarm } from "./machine-lifecycle"+import { registerDiagnosticDeadline, scheduleHeartbeatAlarm } from "./machine-lifecycle"export type MachineControlHooks = Readonly<{recordPendingRestarts: (body: string) => Promise<void>@@ -92,9 +92,33 @@ export async function handleMachineControlFetch(if (url.pathname === "/forward-agent-wake" && request.method === "POST") {const body = await request.text()- const sent = forwardToCommunityMachine(context, body)+ const attempted = forwardToCommunityMachine(context, body)await hooks.clearRuntimeErrorOverlay().catch(() => { })- return new Response(JSON.stringify({ sent }), {+ return new Response(JSON.stringify({ attempted, sent: attempted }), {+ headers: { "Content-Type": "application/json" },+ })+ }++ if (url.pathname === "/register-diagnostic-deadline" && request.method === "POST") {+ let raw: unknown+ try {+ raw = await request.json()+ } catch {+ return new Response(JSON.stringify({ registered: false }), { status: 400 })+ }+ const machineId = url.searchParams.get("machineId")+ const record = raw && typeof raw === "object" ? raw as Record<string, unknown> : null+ if (+ !machineId ||+ !record ||+ Object.keys(record).length !== 1 ||+ !Number.isSafeInteger(record.deadlineAt) ||+ (record.deadlineAt as number) < 0+ ) {+ return new Response(JSON.stringify({ registered: false }), { status: 400 })+ }+ await registerDiagnosticDeadline(context, machineId, record.deadlineAt as number)+ return new Response(JSON.stringify({ registered: true }), {headers: { "Content-Type": "application/json" },})}@@ -104,14 +128,14 @@ export async function handleMachineControlFetch(try {raw = await request.json()} catch {- return new Response(JSON.stringify({ sent: 0 }), { status: 400 })+ return new Response(JSON.stringify({ attempted: 0 }), { status: 400 })}const command = DiagnosticCollectCommandSchema.safeParse(raw)if (!command.success) {- return new Response(JSON.stringify({ sent: 0 }), { status: 400 })+ return new Response(JSON.stringify({ attempted: 0 }), { status: 400 })}- const sent = forwardToCommunityMachine(context, JSON.stringify(command.data))- return new Response(JSON.stringify({ sent }), {+ const attempted = forwardToCommunityMachine(context, JSON.stringify(command.data))+ return new Response(JSON.stringify({ attempted, sent: attempted }), {headers: { "Content-Type": "application/json" },})}diff --git a/src/ws-do/src/ws-durable/machine-frames.test.ts b/src/ws-do/src/ws-durable/machine-frames.test.tsindex e76554864..91715e40a 100644--- a/src/ws-do/src/ws-durable/machine-frames.test.ts+++ b/src/ws-do/src/ws-durable/machine-frames.test.ts@@ -183,6 +183,37 @@ describe("WebSocketDurableObject", () => {})})+ it.each([+ ["legacy ready", undefined, false],+ ["heartbeat-capable ready", ["control-heartbeat-v1"], true],+ ] as const)("capability-gates the app heartbeat lease for %s", async (_name, capabilities, expected) => {+ vi.spyOn(Date, "now").mockReturnValue(9_000_000)+ const { durable, store } = createDO()+ store.set("community-machine-identity", {+ userId: "u_1",+ machineId: "cm_1",+ credentialHash: "hash",+ })+ const ws = createMockWebSocket()+ ws.serializeAttachment({+ type: "community-machine",+ machineId: "cm_1",+ userId: "u_1",+ authenticated: true,+ })++ await durable.webSocketMessage(ws as any, JSON.stringify({+ type: "ready",+ runtimeReport: [],+ runningAgents: [],+ ...(capabilities ? { capabilities } : {}),+ }))++ expect(ws._attachment).toMatchObject({ controlHeartbeat: expected })+ if (expected) expect((ws._attachment as any).lastHeartbeatAckAt).toBe(9_000_000)+ else expect((ws._attachment as any).lastHeartbeatAckAt).toBeUndefined()+ })+it("silently drops a wrapped `{ready:{...}}` frame (legacy shape — regression guard)", async () => {const { durable, store } = createDO()store.set("community-machine-identity", {@@ -214,6 +245,35 @@ describe("WebSocketDurableObject", () => {})})+ describe("community-machine — control-plane receipt scope", () => {+ it("consumes but does not accept a diagnostics receipt from a mismatched machine socket", async () => {+ const { durable, store } = createDO()+ store.set("community-machine-identity", {+ userId: "u_1",+ machineId: "cm_1",+ credentialHash: "hash",+ })+ const ws = createMockWebSocket()+ ws.serializeAttachment({+ type: "community-machine",+ machineId: "cm_other",+ userId: "u_1",+ authenticated: true,+ })+ mockLogDebug.mockClear()++ await durable.webSocketMessage(ws as any, JSON.stringify({+ type: "diagnostics_ack",+ reportId: "dbr_0123456789abcdef",+ }))++ expect(mockLogDebug).not.toHaveBeenCalledWith(+ "diagnostics command receipted",+ expect.anything(),+ )+ })+ })+describe("community-machine — ready frame reconciles bot activity", () => {beforeEach(() => {mockUpsertMachineByMachineId.mockReset()diff --git a/src/ws-do/src/ws-durable/machine-frames.ts b/src/ws-do/src/ws-durable/machine-frames.tsindex 05538b8dc..0461fe053 100644--- a/src/ws-do/src/ws-durable/machine-frames.ts+++ b/src/ws-do/src/ws-durable/machine-frames.ts@@ -1,6 +1,10 @@import { IDENTITY_KEY } from "./internal"import type { CommunityMachineIdentity, WsDurableContext } from "./internal"import { handleReadyFrame } from "./machine-ready"+import {+ handleDiagnosticCommandAck,+ handleMachineHeartbeatAck,+} from "./machine-lifecycle"import {handleAgentCommandAck,handleAgentSessionFrame,@@ -16,6 +20,7 @@ import {export async function handleCommunityMachineMessage(context: WsDurableContext,+ ws: WebSocket,parsed: unknown,hooks: MachineRestartHooks,): Promise<void> {@@ -25,6 +30,9 @@ export async function handleCommunityMachineMessage(return}+ if (await handleMachineHeartbeatAck(context, ws, parsed, identity)) return+ if (handleDiagnosticCommandAck(context, ws, parsed, identity)) return+// The router is intentionally first-match. Keep identity before every// frame and preserve ack → session error → activity → typing → typing stop// → agent session → audit → strict ready. A recognized frame is consumed@@ -70,5 +78,6 @@ export async function handleCommunityMachineMessage(context,parsed,identity,+ ws,)}diff --git a/src/ws-do/src/ws-durable/machine-lifecycle.test.ts b/src/ws-do/src/ws-durable/machine-lifecycle.test.tsindex fac7a3866..896b78268 100644--- a/src/ws-do/src/ws-durable/machine-lifecycle.test.ts+++ b/src/ws-do/src/ws-durable/machine-lifecycle.test.ts@@ -45,6 +45,8 @@ import {mockToSummary,mockTouchBotRefreshContext,mockTouchMachineHeartbeat,+ mockTimeoutPendingDiagnosticReportsForMachine,+ mockGetNextPendingDiagnosticDeadlineForMachine,mockUpdateProfile,mockUpsertMachineByMachineId,resetHarness@@ -98,8 +100,9 @@ describe("WebSocketDurableObject", () => {expect(store.has("community-machine-handle")).toBe(false)})- it("null return (credential revoked or already offline) does NOT broadcast and leaves the alarm armed", async () => {+ it("revoked credential on normal close is terminal local cleanup without a broadcast", async () => {const { durable, store, ctx } = createDO()+ store.set("community-machine-handle", { userId: "u_1", machineId: "cm_1" })store.set("community-machine-identity", {userId: "u_1",machineId: "cm_1",@@ -116,16 +119,17 @@ describe("WebSocketDurableObject", () => {})mockStubFetch.mockClear()+ ;(ctx.storage.setAlarm as any).mockClear?.()+ ;(ctx.storage.deleteAlarm as any).mockClear?.()await durable.webSocketClose(ws as any)expect(mockMarkMachineOffline).toHaveBeenCalledTimes(1)// No broadcast fired — the guarded UPDATE returned zero rows.expect(mockStubFetch).not.toHaveBeenCalled()- // Alarm armed as the safety-net fallback (setAlarm was called; not deleted).- expect(ctx.storage.setAlarm).toHaveBeenCalled()+ expect(ctx.storage.setAlarm).not.toHaveBeenCalled()expect(ctx.storage.deleteAlarm).not.toHaveBeenCalled()- // Storage keys retained — a different DO instance may own the row now.- expect(store.has("community-machine-identity")).toBe(true)+ expect(store.has("community-machine-handle")).toBe(false)+ expect(store.has("community-machine-identity")).toBe(false)})it("missing IDENTITY_KEY (never fully accepted) is a clean no-op — no markMachineOffline, no alarm", async () => {@@ -289,6 +293,143 @@ describe("WebSocketDurableObject", () => {})describe("community-machine — alarm presence + backfill", () => {+ it("capability-gated heartbeat requires the matching nonce and ignores duplicate/stale acks", async () => {+ vi.spyOn(Date, "now").mockReturnValue(10_000_000)+ const { durable, store, getWebSockets } = createDO()+ store.set("community-machine-handle", { userId: "u_1", machineId: "cm_1" })+ store.set("community-machine-identity", {+ userId: "u_1",+ machineId: "cm_1",+ credentialHash: "hash",+ })+ const ws = createMockWebSocket()+ ws.serializeAttachment({+ type: "community-machine",+ machineId: "cm_1",+ userId: "u_1",+ authenticated: true,+ controlHeartbeat: true,+ lastHeartbeatAckAt: 9_940_000,+ })+ getWebSockets.mockReturnValue([ws])++ await durable.alarm()++ expect(ws.send).toHaveBeenCalledOnce()+ const heartbeat = JSON.parse(ws.send.mock.calls[0]![0] as string) as {+ type: string+ nonce: string+ }+ expect(heartbeat.type).toBe("machine:heartbeat")+ expect((ws._attachment as any).pendingHeartbeatNonce).toBe(heartbeat.nonce)++ await durable.webSocketMessage(ws as any, JSON.stringify({+ type: "machine_heartbeat_ack",+ nonce: "stale-nonce",+ }))+ expect((ws._attachment as any).pendingHeartbeatNonce).toBe(heartbeat.nonce)++ await durable.webSocketMessage(ws as any, JSON.stringify({+ type: "machine_heartbeat_ack",+ nonce: heartbeat.nonce,+ }))+ expect(ws._attachment).toMatchObject({+ lastHeartbeatAckAt: 10_000_000,+ pendingHeartbeatNonce: undefined,+ })++ await durable.webSocketMessage(ws as any, JSON.stringify({+ type: "machine_heartbeat_ack",+ nonce: heartbeat.nonce,+ }))+ expect((ws._attachment as any).lastHeartbeatAckAt).toBe(10_000_000)+ })++ it("expires a retained capability socket after the app-level lease and flips D1 once", async () => {+ vi.spyOn(Date, "now").mockReturnValue(20_000_000)+ const { durable, store, getWebSockets } = createDO()+ store.set("community-machine-handle", { userId: "u_1", machineId: "cm_1" })+ store.set("community-machine-identity", {+ userId: "u_1",+ machineId: "cm_1",+ credentialHash: "hash",+ })+ const ws = createMockWebSocket()+ ws.serializeAttachment({+ type: "community-machine",+ machineId: "cm_1",+ userId: "u_1",+ authenticated: true,+ controlHeartbeat: true,+ lastHeartbeatAckAt: 19_800_000,+ })+ getWebSockets.mockReturnValue([ws])+ mockMarkMachineOffline.mockResolvedValueOnce({+ id: "cm_1",+ userId: "u_1",+ status: "offline",+ lastSeenAt: "now",+ })++ await durable.alarm()++ expect(ws.close).toHaveBeenCalledWith(1011, "Heartbeat lease expired")+ expect(mockTouchMachineHeartbeat).not.toHaveBeenCalled()+ expect(mockMarkMachineOffline).toHaveBeenCalledOnce()+ expect(store.has("community-machine-handle")).toBe(false)+ expect(store.has("community-machine-identity")).toBe(false)+ })++ it("forgets an expired retained socket whose credential no longer owns the D1 row", async () => {+ vi.spyOn(Date, "now").mockReturnValue(20_000_000)+ const { durable, store, getWebSockets } = createDO()+ store.set("community-machine-handle", { userId: "u_1", machineId: "cm_1" })+ store.set("community-machine-identity", {+ userId: "u_1",+ machineId: "cm_1",+ credentialHash: "revoked-credential",+ })+ const ws = createMockWebSocket()+ ws.serializeAttachment({+ type: "community-machine",+ machineId: "cm_1",+ userId: "u_1",+ authenticated: true,+ controlHeartbeat: true,+ lastHeartbeatAckAt: 19_800_000,+ })+ getWebSockets.mockReturnValue([ws])+ mockMarkMachineOffline.mockResolvedValueOnce(null)+ mockStubFetch.mockClear()++ await durable.alarm()++ expect(ws.close).toHaveBeenCalledWith(1011, "Heartbeat lease expired")+ expect(mockMarkMachineOffline).toHaveBeenCalledOnce()+ expect(mockStubFetch).not.toHaveBeenCalled()+ expect(store.has("community-machine-handle")).toBe(false)+ expect(store.has("community-machine-identity")).toBe(false)+ })++ it("machine-keyed deadline owner expires D1 without a socket, poll, or daemon reconnect", async () => {+ vi.spyOn(Date, "now").mockReturnValue(30_000_000)+ const { durable, store, getWebSockets } = createDO()+ getWebSockets.mockReturnValue([])+ store.set("community-machine-diagnostic-alarm-hint", {+ machineId: "cm_1",+ deadlineAt: 29_999_999,+ })+ mockGetNextPendingDiagnosticDeadlineForMachine.mockResolvedValueOnce(null)++ await durable.alarm()++ expect(mockTimeoutPendingDiagnosticReportsForMachine).toHaveBeenCalledWith({}, {+ machineId: "cm_1",+ nowMs: 30_000_000,+ })+ expect(store.has("community-machine-diagnostic-alarm-hint")).toBe(false)+ })+it("live-WS + status=offline row: markMachineOnlineIfOffline flips it back online and broadcasts (post-deploy backfill)", async () => {const { durable, store, getWebSockets, ctx } = createDO()store.set("community-machine-identity", {@@ -395,11 +536,9 @@ describe("WebSocketDurableObject", () => {expect(mockStubFetch).toHaveBeenCalled()expect(store.has("community-machine-handle")).toBe(false)expect(store.has("community-machine-identity")).toBe(false)- // No further alarm reschedule after the terminal offline flip.- // (setAlarm may have been called on the earlier setup path; we assert- // deleteAlarm was NOT called since alarm() doesn't need to explicitly- // delete — it just doesn't reschedule.)- expect(ctx.storage.deleteAlarm).not.toHaveBeenCalled()+ // Combined machine/diagnostics scheduling explicitly clears the alarm+ // after the terminal offline flip leaves no remaining work.+ expect(ctx.storage.deleteAlarm).toHaveBeenCalled()})it("no-live-WS + stale row + no identity (mid-lifecycle wipe): still broadcasts offline using stored handle so UI sees the transition", async () => {@@ -430,7 +569,7 @@ describe("WebSocketDurableObject", () => {expect(mockStubFetch).toHaveBeenCalled()// Storage keys dropped — this DO's presence lifecycle is done.expect(store.has("community-machine-handle")).toBe(false)- expect(ctx.storage.deleteAlarm).not.toHaveBeenCalled()+ expect(ctx.storage.deleteAlarm).toHaveBeenCalled()})it("no-live-WS + fresh row: reschedules alarm to exact stale moment, no broadcast, no DB flip", async () => {@@ -557,7 +696,7 @@ describe("WebSocketDurableObject", () => {expect(storage.setAlarm).toHaveBeenCalledWith(5_060_000)})- it("cleans lifecycle storage when a stale offline flip rejects", async () => {+ it("retains lifecycle storage and retries when a stale offline flip rejects", async () => {vi.spyOn(Date, "now").mockReturnValue(6_000_000)const { durable, getWebSockets, store, storage } = createDO()getWebSockets.mockReturnValue([])@@ -577,11 +716,10 @@ describe("WebSocketDurableObject", () => {mockMarkMachineOffline.mockRejectedValueOnce(new Error("d1 unavailable"))await expect(durable.alarm()).resolves.toBeUndefined()- expect(storage.delete.mock.calls).toEqual([- ["community-machine-handle"],- ["community-machine-identity"],- ])- expect(storage.setAlarm).not.toHaveBeenCalled()+ expect(storage.delete).not.toHaveBeenCalled()+ expect(store.has("community-machine-handle")).toBe(true)+ expect(store.has("community-machine-identity")).toBe(true)+ expect(storage.setAlarm).toHaveBeenCalledWith(6_060_000)})it("does not broadcast a stale null flip and still cleans lifecycle storage", async () => {diff --git a/src/ws-do/src/ws-durable/machine-lifecycle.ts b/src/ws-do/src/ws-durable/machine-lifecycle.tsindex 0adde6d26..cdfef1167 100644--- a/src/ws-do/src/ws-durable/machine-lifecycle.ts+++ b/src/ws-do/src/ws-durable/machine-lifecycle.ts@@ -1,11 +1,17 @@import {COMMUNITY_MACHINE_HEARTBEAT_MS,COMMUNITY_MACHINE_OFFLINE_THRESHOLD_MS,+ DiagnosticCommandAckMessageSchema,+ MachineHeartbeatAckMessageSchema,WS_EVENTS,createDb,queries,} from "@alook/shared"-import { HANDLE_KEY, IDENTITY_KEY } from "./internal"+import {+ DIAGNOSTIC_ALARM_HINT_KEY,+ HANDLE_KEY,+ IDENTITY_KEY,+} from "./internal"import type {CommunityMachineConnectionState,CommunityMachineHandle,@@ -15,11 +21,143 @@ import type {} from "./internal"import { notifyUserDO } from "./presence-typing"+type DiagnosticAlarmHint = Readonly<{ machineId: string; deadlineAt: number }>++function authenticatedMachineSockets(context: WsDurableContext): WebSocket[] {+ return context.ctx.getWebSockets().filter((socket) => {+ const state = socket.deserializeAttachment() as ConnectionState+ return state?.type === "community-machine" && state.authenticated+ })+}+export async function scheduleHeartbeatAlarm(context: WsDurableContext): Promise<void> {+ const now = Date.now()+ const hint = await context.ctx.storage.get<DiagnosticAlarmHint>(DIAGNOSTIC_ALARM_HINT_KEY)+ const handle = await context.ctx.storage.get<CommunityMachineHandle>(HANDLE_KEY)+ const identity = await context.ctx.storage.get<CommunityMachineIdentity>(IDENTITY_KEY)+ const hasMachineSocket = authenticatedMachineSockets(context).length > 0+ const candidates: number[] = []+ if (hasMachineSocket || handle || identity) candidates.push(now + COMMUNITY_MACHINE_HEARTBEAT_MS)+ if (hint) candidates.push(Math.max(now, hint.deadlineAt))+ if (candidates.length === 0) {+ await context.ctx.storage.deleteAlarm()+ return+ }+ const want = Math.min(...candidates)const current = await context.ctx.storage.getAlarm()- const want = Date.now() + COMMUNITY_MACHINE_HEARTBEAT_MS- if (current == null || current > want) {- await context.ctx.storage.setAlarm(want)+ if (current == null || current > want) await context.ctx.storage.setAlarm(want)+}++export async function registerDiagnosticDeadline(+ context: WsDurableContext,+ machineId: string,+ deadlineAt: number,+): Promise<void> {+ const current = await context.ctx.storage.get<DiagnosticAlarmHint>(DIAGNOSTIC_ALARM_HINT_KEY)+ if (!current || current.machineId !== machineId || deadlineAt < current.deadlineAt) {+ await context.ctx.storage.put<DiagnosticAlarmHint>(DIAGNOSTIC_ALARM_HINT_KEY, {+ machineId,+ deadlineAt,+ })+ }+ await scheduleHeartbeatAlarm(context)+}++export async function handleMachineHeartbeatAck(+ _context: WsDurableContext,+ ws: WebSocket,+ parsed: unknown,+ identity: CommunityMachineIdentity,+): Promise<boolean> {+ const result = MachineHeartbeatAckMessageSchema.safeParse(parsed)+ if (!result.success) return false+ const state = ws.deserializeAttachment() as ConnectionState+ if (+ state?.type !== "community-machine" ||+ !state.authenticated ||+ state.userId !== identity.userId ||+ state.machineId !== identity.machineId ||+ state.controlHeartbeat !== true ||+ state.pendingHeartbeatNonce !== result.data.nonce+ ) return true+ ws.serializeAttachment({+ ...state,+ lastHeartbeatAckAt: Date.now(),+ pendingHeartbeatNonce: undefined,+ } satisfies CommunityMachineConnectionState)+ return true+}++export function handleDiagnosticCommandAck(+ context: WsDurableContext,+ ws: WebSocket,+ parsed: unknown,+ identity: CommunityMachineIdentity,+): boolean {+ const result = DiagnosticCommandAckMessageSchema.safeParse(parsed)+ if (!result.success) return false+ const state = ws.deserializeAttachment() as ConnectionState+ if (+ state?.type !== "community-machine" ||+ !state.authenticated ||+ state.userId !== identity.userId ||+ state.machineId !== identity.machineId+ ) return true+ context.log.debug("diagnostics command receipted", {+ machineId: identity.machineId,+ reportId: result.data.reportId,+ })+ return true+}++async function markIdentityOffline(+ context: WsDurableContext,+ identity: CommunityMachineIdentity,+): Promise<boolean> {+ const db = createDb(context.env.DB)+ const flipped = await queries.communityMachine.markMachineOffline(db, {+ userId: identity.userId,+ machineId: identity.machineId,+ credentialHash: identity.credentialHash,+ })+ if (!flipped) return false+ await notifyUserDO(context, identity.userId, {+ type: WS_EVENTS.MACHINE_STATUS,+ machineId: identity.machineId,+ status: "offline",+ lastSeenAt: flipped.lastSeenAt ?? new Date().toISOString(),+ }).catch(() => { })+ await context.ctx.storage.delete(HANDLE_KEY)+ await context.ctx.storage.delete(IDENTITY_KEY)+ return true+}++async function sweepDiagnosticDeadlines(context: WsDurableContext, nowMs: number): Promise<void> {+ const hint = await context.ctx.storage.get<DiagnosticAlarmHint>(DIAGNOSTIC_ALARM_HINT_KEY)+ if (!hint || hint.deadlineAt > nowMs) return+ try {+ const db = createDb(context.env.DB)+ await queries.communityDiagnosticReport.timeoutPendingDiagnosticReportsForMachine(db, {+ machineId: hint.machineId,+ nowMs,+ })+ const next = await queries.communityDiagnosticReport.getNextPendingDiagnosticDeadlineForMachine(db, {+ machineId: hint.machineId,+ })+ if (next == null) await context.ctx.storage.delete(DIAGNOSTIC_ALARM_HINT_KEY)+ else await context.ctx.storage.put<DiagnosticAlarmHint>(DIAGNOSTIC_ALARM_HINT_KEY, {+ machineId: hint.machineId,+ deadlineAt: next,+ })+ } catch (err) {+ context.log.warn("diagnostics deadline sweep failed", {+ machineId: hint.machineId,+ err: String(err),+ })+ await context.ctx.storage.put<DiagnosticAlarmHint>(DIAGNOSTIC_ALARM_HINT_KEY, {+ machineId: hint.machineId,+ deadlineAt: nowMs + COMMUNITY_MACHINE_HEARTBEAT_MS,+ })}}@@ -30,60 +168,62 @@ export async function handleCommunityMachineClose(): Promise<void> {context.log.info("community machine websocket closed", { machineId: state.machineId, userId: state.userId })const identity = await context.ctx.storage.get<CommunityMachineIdentity>(IDENTITY_KEY)- if (!identity) return- const replacementIsLive = context.ctx.getWebSockets().some((socket) => {+ if (!identity) {+ await scheduleHeartbeatAlarm(context)+ return+ }+ const replacementIsLive = authenticatedMachineSockets(context).some((socket) => {if (socket === closingSocket) return false- const candidate = socket.deserializeAttachment() as ConnectionState- return candidate?.type === "community-machine"- && candidate.authenticated- && candidate.userId === state.userId- && candidate.machineId === state.machineId+ const candidate = socket.deserializeAttachment() as CommunityMachineConnectionState+ return candidate.userId === state.userId && candidate.machineId === state.machineId})if (replacementIsLive) returntry {- const db = createDb(context.env.DB)- const flipped = await queries.communityMachine.markMachineOffline(db, {- userId: identity.userId,- machineId: identity.machineId,- credentialHash: identity.credentialHash,- })- if (flipped) {- await notifyUserDO(context, identity.userId, {- type: WS_EVENTS.MACHINE_STATUS,- machineId: identity.machineId,- status: "offline",- lastSeenAt: flipped.lastSeenAt ?? new Date().toISOString(),- }).catch(() => { })- await context.ctx.storage.deleteAlarm()+ const flipped = await markIdentityOffline(context, identity)+ if (!flipped) {+ // The guarded row belongs to a newer credential (or is already+ // offline), so this close is terminal for the old DO identity.await context.ctx.storage.delete(HANDLE_KEY)await context.ctx.storage.delete(IDENTITY_KEY)- } else {- await context.ctx.storage.setAlarm(Date.now() + COMMUNITY_MACHINE_OFFLINE_THRESHOLD_MS)+ return}} catch (err) {context.log.warn("markMachineOffline failed on webSocketClose", { err: String(err) })await context.ctx.storage.setAlarm(Date.now() + COMMUNITY_MACHINE_OFFLINE_THRESHOLD_MS)}+ await scheduleHeartbeatAlarm(context)}export async function handleMachineAlarm(context: WsDurableContext): Promise<void> {- const sockets = context.ctx.getWebSockets()+ const now = Date.now()+ await sweepDiagnosticDeadlines(context, now)+const liveMachines: Array<{ userId: string; machineId: string }> = []- for (const ws of sockets) {- const s = ws.deserializeAttachment() as ConnectionState- if (s?.type === "community-machine" && s.authenticated) {- liveMachines.push({ userId: s.userId, machineId: s.machineId })+ let expiredCapableSocket = false+ for (const ws of authenticatedMachineSockets(context)) {+ const state = ws.deserializeAttachment() as CommunityMachineConnectionState+ if (state.controlHeartbeat === true) {+ const lastAckAt = state.lastHeartbeatAckAt ?? 0+ if (now - lastAckAt >= COMMUNITY_MACHINE_OFFLINE_THRESHOLD_MS) {+ expiredCapableSocket = true+ try { ws.close(1011, "Heartbeat lease expired") } catch { }+ continue+ }+ const nonce = state.pendingHeartbeatNonce ?? crypto.randomUUID()+ ws.serializeAttachment({ ...state, pendingHeartbeatNonce: nonce })+ try { ws.send(JSON.stringify({ type: "machine:heartbeat", nonce })) } catch { }}+ liveMachines.push({ userId: state.userId, machineId: state.machineId })}if (liveMachines.length > 0) {const db = createDb(context.env.DB)const identity = await context.ctx.storage.get<CommunityMachineIdentity>(IDENTITY_KEY)- for (const m of liveMachines) {+ for (const machine of liveMachines) {try {- await queries.communityMachine.touchMachineHeartbeat(db, m.userId, m.machineId)- } catch { /* ok */ }- if (identity && identity.userId === m.userId && identity.machineId === m.machineId) {+ await queries.communityMachine.touchMachineHeartbeat(db, machine.userId, machine.machineId)+ } catch { }+ if (identity && identity.userId === machine.userId && identity.machineId === machine.machineId) {try {const backfilled = await queries.communityMachine.markMachineOnlineIfOffline(db, {userId: identity.userId,@@ -99,48 +239,67 @@ export async function handleMachineAlarm(context: WsDurableContext): Promise<voi}).catch(() => { })}} catch (err) {- context.log.warn("markMachineOnlineIfOffline (alarm live-WS) failed", { err: String(err) })+ context.log.warn("markMachineOnlineIfOffline (alarm fresh-WS) failed", { err: String(err) })}}}- await context.ctx.storage.setAlarm(Date.now() + COMMUNITY_MACHINE_HEARTBEAT_MS)+ await scheduleHeartbeatAlarm(context)return}const stored = await context.ctx.storage.get<CommunityMachineHandle>(HANDLE_KEY)- if (!stored) returnconst identity = await context.ctx.storage.get<CommunityMachineIdentity>(IDENTITY_KEY)+ if (expiredCapableSocket && identity) {+ try {+ const flipped = await markIdentityOffline(context, identity)+ if (!flipped) {+ // A guarded no-op means this credential no longer owns the D1 row+ // (or another actor already won the offline transition). Forget only+ // this DO's stale lifecycle state; do not broadcast over the current+ // credential's presence.+ await context.ctx.storage.delete(HANDLE_KEY)+ await context.ctx.storage.delete(IDENTITY_KEY)+ }+ } catch (err) {+ context.log.warn("markMachineOffline (heartbeat lease) failed", { err: String(err) })+ await context.ctx.storage.setAlarm(now + COMMUNITY_MACHINE_HEARTBEAT_MS)+ return+ }+ await scheduleHeartbeatAlarm(context)+ return+ }+ if (!stored) {+ await scheduleHeartbeatAlarm(context)+ return+ }+const db = createDb(context.env.DB)- const machine = await queries.communityMachine.getMachineByIdForUser(- db,- stored.userId,- stored.machineId- )+ const machine = await queries.communityMachine.getMachineByIdForUser(db, stored.userId, stored.machineId)if (!machine) {await context.ctx.storage.delete(HANDLE_KEY)await context.ctx.storage.delete(IDENTITY_KEY)+ await scheduleHeartbeatAlarm(context)return}const lastSeen = machine.lastSeenAt ? Date.parse(machine.lastSeenAt) : 0- const elapsed = Date.now() - lastSeen+ const elapsed = now - lastSeenif (elapsed >= COMMUNITY_MACHINE_OFFLINE_THRESHOLD_MS) {if (identity) {+ let flipped: booleantry {- const flipped = await queries.communityMachine.markMachineOffline(db, {- userId: identity.userId,- machineId: identity.machineId,- credentialHash: identity.credentialHash,- })- if (flipped) {- await notifyUserDO(context, stored.userId, {- type: WS_EVENTS.MACHINE_STATUS,- machineId: stored.machineId,- status: "offline",- lastSeenAt: flipped.lastSeenAt ?? new Date().toISOString(),- }).catch(() => { })- }+ flipped = await markIdentityOffline(context, identity)} catch (err) {context.log.warn("markMachineOffline (alarm stale-flip) failed", { err: String(err) })+ await context.ctx.storage.setAlarm(now + COMMUNITY_MACHINE_HEARTBEAT_MS)+ return+ }+ // This is the legacy no-socket stale-row cleanup path. Once its D1+ // observation is stale, drop the local lifecycle record even when the+ // guarded flip was already won elsewhere. A thrown update is transient+ // and retains the identity for the retry path above.+ if (!flipped) {+ await context.ctx.storage.delete(HANDLE_KEY)+ await context.ctx.storage.delete(IDENTITY_KEY)}} else {await notifyUserDO(context, stored.userId, {@@ -149,10 +308,10 @@ export async function handleMachineAlarm(context: WsDurableContext): Promise<voistatus: "offline",lastSeenAt: machine.lastSeenAt ?? new Date().toISOString(),}).catch(() => { })+ await context.ctx.storage.delete(HANDLE_KEY)}- await context.ctx.storage.delete(HANDLE_KEY)- await context.ctx.storage.delete(IDENTITY_KEY)+ await scheduleHeartbeatAlarm(context)return}- await context.ctx.storage.setAlarm(Date.now() + (COMMUNITY_MACHINE_OFFLINE_THRESHOLD_MS - elapsed))+ await context.ctx.storage.setAlarm(now + (COMMUNITY_MACHINE_OFFLINE_THRESHOLD_MS - elapsed))}diff --git a/src/ws-do/src/ws-durable/machine-ready.ts b/src/ws-do/src/ws-durable/machine-ready.tsindex b17d90a04..8cb3067d0 100644--- a/src/ws-do/src/ws-durable/machine-ready.ts+++ b/src/ws-do/src/ws-durable/machine-ready.ts@@ -1,4 +1,5 @@import {+ CONTROL_HEARTBEAT_CAPABILITY,createDb,HostReadyMessageSchema,queries,@@ -59,10 +60,26 @@ export async function handleReadyFrame(context: WsDurableContext,parsed: unknown,identity: CommunityMachineIdentity,+ ws: WebSocket,): Promise<boolean> {const readyParse = HostReadyMessageSchema.safeParse(parsed)if (!readyParse.success) return falseconst ready = readyParse.data+ const state = ws.deserializeAttachment() as import("./internal").ConnectionState+ if (+ state?.type === "community-machine" &&+ state.authenticated &&+ state.userId === identity.userId &&+ state.machineId === identity.machineId+ ) {+ const controlHeartbeat = (ready.capabilities ?? []).includes(CONTROL_HEARTBEAT_CAPABILITY)+ ws.serializeAttachment({+ ...state,+ controlHeartbeat,+ ...(controlHeartbeat ? { lastHeartbeatAckAt: Date.now() } : {}),+ pendingHeartbeatNonce: undefined,+ })+ }try {const hostname = ready.hostname ?? ""const platform = ready.platform ?? ""diff --git a/src/ws-do/src/ws-durable/test-harness.ts b/src/ws-do/src/ws-durable/test-harness.tsindex 7536a448c..f448fb183 100644--- a/src/ws-do/src/ws-durable/test-harness.ts+++ b/src/ws-do/src/ws-durable/test-harness.ts@@ -87,6 +87,8 @@ export const mockUpsertMachineByMachineId = vi.fn()export const mockTouchMachineHeartbeat = vi.fn()export const mockMarkMachineOffline = vi.fn()export const mockMarkMachineOnlineIfOffline = vi.fn()+export const mockTimeoutPendingDiagnosticReportsForMachine = vi.fn().mockResolvedValue([])+export const mockGetNextPendingDiagnosticDeadlineForMachine = vi.fn().mockResolvedValue(null)export const mockGetCoMemberUserIds = vi.fn<(db: unknown, userId: string) => Promise<string[]>>().mockResolvedValue([])export const mockGetFriendUserIds = vi.fn<(db: unknown, userId: string) => Promise<string[]>>().mockResolvedValue([])export const mockGetChannelForMember = vi.fn()@@ -184,12 +186,13 @@ vi.mock("@alook/shared", async () => {// flat top-level fields.const HostReadyMessageSchema = {safeParse(v: unknown) {- const m = v as { type?: unknown; runtimeReport?: unknown; runningAgents?: unknown }+ const m = v as { type?: unknown; runtimeReport?: unknown; capabilities?: unknown; runningAgents?: unknown }if (m?.type !== "ready") return { success: false } as constif (!Array.isArray(m?.runtimeReport)) return { success: false } as constconst data: Record<string, unknown> = {type: "ready",runtimeReport: m.runtimeReport,+ capabilities: Array.isArray(m.capabilities) ? m.capabilities : [],runningAgents: Array.isArray(m.runningAgents) ? m.runningAgents : [],}for (const k of ["hostname", "platform", "arch", "osRelease", "daemonVersion"]) {@@ -199,6 +202,22 @@ vi.mock("@alook/shared", async () => {return { success: true as const, data }},}+ const MachineHeartbeatAckMessageSchema = {+ safeParse(v: unknown) {+ const m = v as { type?: unknown; nonce?: unknown }+ return m?.type === "machine_heartbeat_ack" && typeof m.nonce === "string" && m.nonce.length > 0+ ? { success: true as const, data: { type: "machine_heartbeat_ack" as const, nonce: m.nonce } }+ : { success: false as const }+ },+ }+ const DiagnosticCommandAckMessageSchema = {+ safeParse(v: unknown) {+ const m = v as { type?: unknown; reportId?: unknown }+ return m?.type === "diagnostics_ack" && typeof m.reportId === "string" && /^dbr_[A-Za-z0-9_-]+$/.test(m.reportId)+ ? { success: true as const, data: { type: "diagnostics_ack" as const, reportId: m.reportId } }+ : { success: false as const }+ },+ }const AgentActivityMessageSchema = {safeParse(v: unknown) {const m = v as { type?: unknown; agentId?: unknown; state?: unknown }@@ -374,6 +393,10 @@ vi.mock("@alook/shared", async () => {reconcileBotActivityFromRunningAgents: (...a: any[]) =>mockReconcileBotActivityFromRunningAgents(...(a as [unknown, string, string[]])),},+ communityDiagnosticReport: {+ timeoutPendingDiagnosticReportsForMachine: (...a: any[]) => mockTimeoutPendingDiagnosticReportsForMachine(...a),+ getNextPendingDiagnosticDeadlineForMachine: (...a: any[]) => mockGetNextPendingDiagnosticDeadlineForMachine(...a),+ },communityUserProfile: {updateProfile: (...a: any[]) =>mockUpdateProfile(...(a as [unknown, string, { statusEmoji?: string | null; statusText?: string | null }])),@@ -421,6 +444,9 @@ vi.mock("@alook/shared", async () => {getUserInternal: (...a: [unknown, string]) => mockGetUserInternal(...a),},},+ CONTROL_HEARTBEAT_CAPABILITY: "control-heartbeat-v1",+ MachineHeartbeatAckMessageSchema,+ DiagnosticCommandAckMessageSchema,}})@@ -473,6 +499,8 @@ export function resetHarness() {mockUpdateProfile.mockResolvedValue({})mockGetProfile.mockResolvedValue(null)mockReconcileBotActivityFromRunningAgents.mockResolvedValue([])+ mockTimeoutPendingDiagnosticReportsForMachine.mockResolvedValue([])+ mockGetNextPendingDiagnosticDeadlineForMachine.mockResolvedValue(null)// Daemon-auth binds the token to a machine row for daemonId+workspace;// default to a present row so auth-flow success tests pass. Tests that// exercise a missing machine override this.