Skip to content

armgit / alook

Public Imported from GitHub
0

Multi-Agent Collaboration for Team and Friends

Harden control-plane receipts and sleep recovery (#489)

Gustavo Ye6dd9f9fda1cb
diff --git a/src/daemon/src/cli/daemonSelfUpdate.real.test.ts b/src/daemon/src/cli/daemonSelfUpdate.real.test.ts
index 5442b8c6a..fb873847a 100644
--- a/src/daemon/src/cli/daemonSelfUpdate.real.test.ts
+++ b/src/daemon/src/cli/daemonSelfUpdate.real.test.ts
@@ -94,7 +94,7 @@ async function createControlServer(): Promise<ControlServer> {
return;
}
if (request.url?.includes("/api/community/daemon/wakes/resync")) {
- response.end(JSON.stringify({ woken: 0 }));
+ response.end(JSON.stringify({ attempted: 0 }));
return;
}
response.end(JSON.stringify({}));
diff --git a/src/daemon/src/daemon/createDaemon.test.ts b/src/daemon/src/daemon/createDaemon.test.ts
index 79b78e4dc..d51e5f0bb 100644
--- a/src/daemon/src/daemon/createDaemon.test.ts
+++ b/src/daemon/src/daemon/createDaemon.test.ts
@@ -164,7 +164,7 @@ describe("createDaemon", () => {
if (url.includes("/daemon/bots")) {
return Response.json({ bots: [{ id: "bot_1", name: "Bot", discriminator: "0001" }] });
}
- return Response.json({ woken: 0 });
+ return Response.json({ attempted: 0 });
}));
const daemon = await createDaemon({
@@ -347,7 +347,7 @@ describe("createDaemon", () => {
bots: [{ id: "bot_1", name: "Bot", discriminator: "0001" }],
});
}
- return Response.json({ woken: 0 });
+ return Response.json({ attempted: 0 });
}));
const daemon = await createDaemon({
machineKey: "cmk_diagnostics",
@@ -418,7 +418,7 @@ describe("createDaemon", () => {
const url = String(input);
return url.includes("/api/community/daemon/bots")
? Response.json({ bots: [{ id: "bot_1", name: "Bot", discriminator: "0001" }] })
- : Response.json({ woken: 0 });
+ : Response.json({ attempted: 0 });
}));
const daemon = await createDaemon({
machineKey: "cmk_diagnostics_failure",
@@ -1074,12 +1074,12 @@ describe("createDaemon — logging", () => {
await daemon.stop();
});
- it("calls resync-wakes with the machine key bearer on open and logs the woken count", async () => {
+ it("calls resync-wakes with the machine key bearer on open and logs the attempted count", async () => {
global.fetch = vi.fn(async (url: string | URL, init?: RequestInit) => {
const href = String(url);
if (href.includes("/resync-wakes")) {
expect((init?.headers as Record<string, string>).authorization).toBe("Bearer cmk_resync");
- return new Response(JSON.stringify({ woken: 2 }), { status: 200 });
+ return new Response(JSON.stringify({ attempted: 2 }), { status: 200 });
}
return new Response(JSON.stringify({ bots: [] }), { status: 200 });
}) as unknown as typeof fetch;
@@ -1100,7 +1100,7 @@ describe("createDaemon — logging", () => {
await new Promise((r) => setTimeout(r, 20));
expect(
- logger.calls.info.some(([, m, d]) => m === "wake resync completed" && (d[0] as any).woken === 2),
+ logger.calls.info.some(([, m, d]) => m === "wake resync completed" && (d[0] as any).attempted === 2),
).toBe(true);
await daemon.stop();
});
diff --git a/src/daemon/src/daemon/createDaemon.ts b/src/daemon/src/daemon/createDaemon.ts
index f7d732567..9fb035b0e 100644
--- a/src/daemon/src/daemon/createDaemon.ts
+++ b/src/daemon/src/daemon/createDaemon.ts
@@ -581,7 +581,7 @@ export async function createDaemon(opts: CreateDaemonOptions): Promise<RunningDa
* Ask the server to re-check each of this machine's bots for unread work
* and re-wake any that have some. Recovers a message that arrived while
* this daemon was offline: the wake-queue consumer acks (never retries) a
- * `delivered_nowhere` outcome, so without this call that wake is gone for
+ * `attempted_nowhere` outcome, so without this call that wake is gone for
* good once the daemon reconnects. The daemon drives WHEN this runs; the
* server alone decides WHAT an `agent:wake` looks like (same
* `dispatchOneUnreadWake` rebuild the queue consumer uses) — this call
@@ -595,13 +595,31 @@ export async function createDaemon(opts: CreateDaemonOptions): Promise<RunningDa
headers: { authorization: `Bearer ${opts.machineKey}` },
});
if (!res.ok) throw new Error(`resync-wakes ${res.status}`);
- const json = (await res.json()) as { woken?: number };
- log.info("wake resync completed", { woken: json.woken ?? 0 });
+ const json = (await res.json()) as { attempted?: number };
+ log.info("wake resync completed", { attempted: json.attempted ?? 0 });
} catch (err) {
log.warn("wake resync failed", { err: err instanceof Error ? err.message : String(err) });
}
}
+ async function resyncPendingDiagnostics(): Promise<void> {
+ try {
+ const res = await fetch(`${opts.serverUrl}/api/community/daemon/resync-diagnostics`, {
+ method: "POST",
+ headers: { authorization: `Bearer ${opts.machineKey}` },
+ });
+ if (!res.ok) throw new Error(`resync-diagnostics ${res.status}`);
+ const json = (await res.json()) as { pending?: number; attempted?: number; ambiguous?: number };
+ log.info("diagnostics resync completed", {
+ pending: json.pending ?? 0,
+ attempted: json.attempted ?? 0,
+ ambiguous: json.ambiguous ?? 0,
+ });
+ } catch (err) {
+ log.warn("diagnostics resync failed", { err: err instanceof Error ? err.message : String(err) });
+ }
+ }
+
const enrollAgent = async (agentId: string): Promise<string> => {
const existing = enrolledKeys.get(agentId);
if (existing) return existing;
@@ -941,6 +959,7 @@ export async function createDaemon(opts: CreateDaemonOptions): Promise<RunningDa
channel.onOpen(() => {
void coldStartWarmup();
void resyncPendingWakes();
+ void resyncPendingDiagnostics();
});
channel.connect();
diff --git a/src/daemon/src/manager/agentRouter.ts b/src/daemon/src/manager/agentRouter.ts
index 523705e49..1af2e3237 100644
--- a/src/daemon/src/manager/agentRouter.ts
+++ b/src/daemon/src/manager/agentRouter.ts
@@ -19,6 +19,7 @@
* server recovers this host's state after a dropped control connection.
*/
import type { HostCommand, HostControlChannel, HostReady, HostReadyRuntime, UnreadNotice, AgentSessionReport, SessionErrorFrame, AgentId } from "../server/contract.js";
+import { CONTROL_HEARTBEAT_CAPABILITY } from "../server/contract.js";
import type { AgentProcessManager } from "./managerRuntime.js";
import type { TypingScopeTracker } from "./typingScopeTracker.js";
import { createLogger, type Logger } from "../logger.js";
@@ -217,6 +218,7 @@ export class AgentRouter {
buildReady(): HostReady {
return {
runtimeReport: [...this.runtimes.values()],
+ capabilities: [CONTROL_HEARTBEAT_CAPABILITY],
runningAgents: [...this.running],
hostname: this.opts.hostname,
platform: this.opts.platform,
diff --git a/src/daemon/src/server/contract.ts b/src/daemon/src/server/contract.ts
index 950bb7ebd..2d679028e 100644
--- a/src/daemon/src/server/contract.ts
+++ b/src/daemon/src/server/contract.ts
@@ -82,6 +82,7 @@ export type {
} from "@alook/shared/community-cli-contract";
export {
+ CONTROL_HEARTBEAT_CAPABILITY,
DM_SERVER,
parseRef,
formatRef,
diff --git a/src/daemon/src/server/wsControlChannel.test.ts b/src/daemon/src/server/wsControlChannel.test.ts
index 7815df8cb..ef02d0d39 100644
--- a/src/daemon/src/server/wsControlChannel.test.ts
+++ b/src/daemon/src/server/wsControlChannel.test.ts
@@ -10,6 +10,7 @@ import type { Logger } from "../logger";
*/
class FakeSocket implements WebSocketLike {
sent: string[] = [];
+ terminated = false;
private handlers: Record<string, ((...a: any[]) => void)[]> = {};
on(event: string, cb: (...a: any[]) => void): void {
(this.handlers[event] ??= []).push(cb);
@@ -20,6 +21,10 @@ class FakeSocket implements WebSocketLike {
close(): void {
this.emit("close");
}
+ terminate(): void {
+ this.terminated = true;
+ this.emit("close");
+ }
ping(): void {}
emit(event: string, ...args: unknown[]): void {
(this.handlers[event] ?? []).forEach((h) => h(...args));
@@ -616,6 +621,39 @@ describe("WsControlChannel — downlink HostCommand validation (convergence #6)"
expect(received).toEqual([command]);
});
+ it("acks an app heartbeat at ingress without dispatching it to lifecycle listeners", () => {
+ const { sockets, received } = driven();
+
+ sockets[0].emit("message", JSON.stringify({ type: "machine:heartbeat", nonce: "nonce_1" }));
+
+ expect(received).toEqual([]);
+ expect(sockets[0].frames()).toContainEqual({
+ type: "machine_heartbeat_ack",
+ nonce: "nonce_1",
+ });
+ });
+
+ it("acks every diagnostics frame at ingress, including duplicates, before dispatch", () => {
+ const command = {
+ type: "diagnostics:collect",
+ reportId: "dbr_0123456789abcdef",
+ agentId: "bot_1",
+ fromMs: 1_700_000_000_000,
+ deadlineAt: 1_700_087_000_000,
+ } as const;
+ const { sockets, received } = driven();
+
+ sockets[0].emit("message", JSON.stringify(command));
+ sockets[0].emit("message", JSON.stringify(command));
+
+ expect(received).toEqual([command, command]);
+ expect(sockets[0].frames().filter((frame) => frame.type === "diagnostics_ack"))
+ .toEqual([
+ { type: "diagnostics_ack", reportId: command.reportId },
+ { type: "diagnostics_ack", reportId: command.reportId },
+ ]);
+ });
+
it("drops diagnostics commands with missing or unknown top-level fields", () => {
for (const command of [
{
@@ -727,5 +765,6 @@ describe("WsControlChannel — logging", () => {
await new Promise((r) => setTimeout(r, 30));
expect(logger.calls.warn.some(([m]) => m === "heartbeat pong timeout — forcing reconnect")).toBe(true);
+ expect(sockets[0].terminated).toBe(true);
});
});
diff --git a/src/daemon/src/server/wsControlChannel.ts b/src/daemon/src/server/wsControlChannel.ts
index 9bbbc545e..6a109f6bb 100644
--- a/src/daemon/src/server/wsControlChannel.ts
+++ b/src/daemon/src/server/wsControlChannel.ts
@@ -134,6 +134,8 @@ type OutboundFrame =
status: AgentCommandAckStatus;
error?: AgentCommandAckError;
}
+ | { type: "machine_heartbeat_ack"; nonce: string }
+ | { type: "diagnostics_ack"; reportId: string }
| HostBotAuditEventFrame
| SessionErrorFrame;
@@ -462,6 +464,18 @@ export class WsControlChannel implements HostControlChannel {
}
private async dispatchIngressCommand(cmd: HostCommand): Promise<void> {
+ if (cmd.type === "machine:heartbeat") {
+ this.sendFrame({ type: "machine_heartbeat_ack", nonce: cmd.nonce });
+ return;
+ }
+ if (cmd.type === "diagnostics:collect") {
+ // Receipt means only that the daemon websocket ingress parsed this frame.
+ // Collection acceptance/completion remains authoritative in the durable
+ // diagnostic-report row and must never be inferred from this ack.
+ this.sendFrame({ type: "diagnostics_ack", reportId: cmd.reportId });
+ await this.dispatchListeners(cmd);
+ return;
+ }
if (cmd.type === "agent:wake") {
const result = await this.wakeCoordinator.run(cmd, (accepted) =>
this.dispatchListeners(accepted), (advanced) => {
@@ -546,7 +560,8 @@ export class WsControlChannel implements HostControlChannel {
if (this.now() > this.pongDeadline) {
// Watchdog: no pong in time → treat as dead, force reconnect.
this.log.warn("heartbeat pong timeout — forcing reconnect");
- this.ws?.close();
+ if (this.ws?.terminate) this.ws.terminate();
+ else this.ws?.close();
return;
}
this.log.debug("heartbeat ping");
diff --git a/src/shared/src/community-cli-contract.ts b/src/shared/src/community-cli-contract.ts
index 32aaeb70d..5705f60e5 100644
--- a/src/shared/src/community-cli-contract.ts
+++ b/src/shared/src/community-cli-contract.ts
@@ -709,6 +709,8 @@ export interface UnreadNotice {
/* Control plane — server → host commands */
/* ------------------------------------------------------------------ */
+export const CONTROL_HEARTBEAT_CAPABILITY = "control-heartbeat-v1";
+
/**
* Commands the SERVER pushes DOWN to a host (daemon). This is the control plane —
* distinct from the agent-initiated data plane (`ServerApi`). The server owns
@@ -722,6 +724,7 @@ export interface UnreadNotice {
* coalesce the notice for the next turn (see `AgentProcessManager`).
*/
export type HostCommand =
+ | { type: "machine:heartbeat"; nonce: string }
| {
type: "agent:wake";
agentId: AgentId;
@@ -842,6 +845,8 @@ export interface HostReady {
* reader-side concern (server-side bot-create validator, client picker).
*/
runtimeReport: HostReadyRuntime[];
+ /** Capability gates for wire behavior that is unsafe to assume on legacy daemons. */
+ capabilities?: string[];
/** Agents currently running on this host. */
runningAgents: AgentId[];
hostname?: string;
@@ -1027,6 +1032,8 @@ export interface WebSocketLike {
send(data: string): void;
close(): void;
ping?(): void;
+ /** Hard-close a half-open client socket when the implementation supports it (`ws`). */
+ terminate?(): void;
}
/** Builds a client `WebSocketLike` for a url + headers (injected; no hard `ws` dep). */
@@ -1379,6 +1386,10 @@ export function formatSeq(seq: Seq): string {
// downstream with its own defaulting, so deep-validating it here would only turn
// a forward-compatible server field into a hard drop on an older daemon.
export const HostCommandSchema = z.discriminatedUnion("type", [
+ z.strictObject({
+ type: z.literal("machine:heartbeat"),
+ nonce: z.string().min(1).max(128),
+ }),
z.object({
type: z.literal("agent:wake"),
agentId: z.string().min(1),
diff --git a/src/shared/src/community/wake-dispatch.ts b/src/shared/src/community/wake-dispatch.ts
index 6f7819826..538482b4c 100644
--- a/src/shared/src/community/wake-dispatch.ts
+++ b/src/shared/src/community/wake-dispatch.ts
@@ -1,4 +1,3 @@
-import { nanoid } from "nanoid";
import type { HostCommand, UnreadNotice } from "../community-cli-contract";
import { makeRuntimeConfig } from "../runtime-config";
import { resolveModelConfig } from "./bot-model";
@@ -14,6 +13,7 @@ import * as notificationEligibility from "../db/queries/community/notification-e
import { policyAllows } from "../db/queries/community/notification-setting";
import { getUsersByIds } from "../db/queries/user";
import type { Database } from "../db/index";
+import { parseAttemptedCountReceipt } from "../transport-receipt";
/**
* Deliberately NOT `@cloudflare/workers-types`' `Fetcher` — this module is
@@ -31,13 +31,40 @@ interface WakeDispatchEnv {
WS_DO_WORKER: FetcherLike;
}
+/**
+ * Stable, opaque identity for one semantic unread wake. Queue retries and
+ * daemon reconnect resync rebuild the command independently from D1, so a
+ * random id here would turn one unread message into multiple commands. The
+ * domain-separated digest avoids adding a command ledger without exposing the
+ * internal message id through daemon logs or the agent launch environment.
+ */
+async function unreadWakeLaunchId(input: {
+ botUserId: string;
+ messageId: string;
+}): Promise<string> {
+ const identity = JSON.stringify([
+ "alook:unread-wake:v1",
+ input.botUserId,
+ input.messageId,
+ ]);
+ const digest = new Uint8Array(
+ await crypto.subtle.digest(
+ "SHA-256",
+ new TextEncoder().encode(identity),
+ ),
+ );
+ return `wake_${Array.from(digest, (byte) => byte.toString(16).padStart(2, "0")).join("")}`;
+}
+
/**
* One `WAKE_QUEUE` message — deliberately minimal (plan
* minimal-wake-queue-unread-notice §1): just enough to rebuild the wake
* command from CURRENT D1 state at consume time. No `machineId`, `runtime`,
* `launchId`, message text, sender, or preview — all of that is re-derived
* by `buildUnreadWakeCommand` in `src/wake-worker` so a stale queue item
- * never wakes an old machine or carries stale content.
+ * never wakes an old machine or carries stale content. `launchId` is derived
+ * deterministically from the bot/message tuple so retry and reconnect replay
+ * retain one semantic command identity.
*/
export interface WakePayload {
messageId: string;
@@ -55,14 +82,14 @@ export interface WakePayload {
* surface (never a raw DO namespace — `src/web`/`src/wake-worker` cannot
* fetch a DO stub directly). This function POSTs an already-fully-built
* `HostCommand` to that worker's `/community-machine/by-id/<machineId>/forward-agent-wake`
- * route and normalizes the response to a boolean — it never inspects,
+ * route and normalizes the attempted socket-write count to a boolean — it never inspects,
* validates, or constructs any part of `command`, and it never exposes the
* DO-naming mechanics (no public `getMachineDoName` here or anywhere else).
*
* Error contract (load-bearing for the queue consumer's retry semantics):
- * - `{ sent: true }` — at least one live doName's DO forwarded the command
- * to an authenticated daemon WebSocket.
- * - `{ sent: false }` — the ws-do route responded 200 with `{ sent: 0 }`:
+ * - `{ attempted: true }` — at least one live doName's DO attempted the command
+ * on an authenticated daemon WebSocket. This is not a daemon receipt.
+ * - `{ attempted: false }` — the ws-do route responded 200 with `{ attempted: 0 }`:
* no active credential for this machine, or a live credential but no open
* WS (daemon offline). This is a known-permanent state for this attempt —
* the consumer must `ack()`, not `retry()`. Daemon reconnect warmup
@@ -70,14 +97,14 @@ export interface WakePayload {
* - throws — the ws-do route (or the service-binding fetch itself) returned
* non-2xx, or the fetch itself threw (network error/timeout). This is
* transient — the consumer must `retry()`. Never swallowed into
- * `{ sent: false }`, or a real outage would look identical to "daemon is
+ * `{ attempted: false }`, or a real outage would look identical to "daemon is
* just offline" and stop retrying.
*/
export async function sendWakeToMachine(
env: { WS_DO_WORKER: FetcherLike },
machineId: string,
command: HostCommand
-): Promise<{ sent: boolean }> {
+): Promise<{ attempted: boolean }> {
const path = `/community-machine/by-id/${encodeURIComponent(machineId)}/forward-agent-wake`;
const res = await env.WS_DO_WORKER.fetch(`http://internal${path}`, {
method: "POST",
@@ -89,8 +116,7 @@ export async function sendWakeToMachine(
throw new Error(`sendWakeToMachine: ws-do route returned ${res.status} for machine ${machineId}`);
}
- const data = (await res.json()) as { sent?: number };
- return { sent: (data.sent ?? 0) > 0 };
+ return { attempted: parseAttemptedCountReceipt(await res.json()) > 0 };
}
/** Why `buildUnreadWakeCommand` decided NOT to wake — every reason is a permanent, current-state miss the consumer must `ack()`, never `retry()`. */
@@ -194,7 +220,10 @@ export async function buildUnreadWakeCommand(
type: "agent:wake",
agentId: botCtx.botUserId,
config,
- launchId: nanoid(),
+ launchId: await unreadWakeLaunchId({
+ botUserId: botCtx.botUserId,
+ messageId: msg.id,
+ }),
unreadNotice,
};
@@ -306,8 +335,8 @@ async function writeWakeTriggerAudit(
/** Outcome of resolving ONE wake candidate — what every caller (the real queue consumer, and the dev-only inline stand-in) needs to decide what to log. */
export type DispatchOneWakeResult =
| { outcome: "skip"; reason: SkipReason }
- | { outcome: "sent" }
- | { outcome: "delivered_nowhere"; machineId: string };
+ | { outcome: "attempted" }
+ | { outcome: "attempted_nowhere"; machineId: string };
/**
* The ONE place that decides what happens for a single `{ messageId,
@@ -328,6 +357,6 @@ export async function dispatchOneUnreadWake(
): Promise<DispatchOneWakeResult> {
const result = await buildUnreadWakeCommand(db, input, env);
if (result.state === "skip") return { outcome: "skip", reason: result.reason };
- const { sent } = await sendWakeToMachine(env, result.machineId, result.command);
- return sent ? { outcome: "sent" } : { outcome: "delivered_nowhere", machineId: result.machineId };
+ const { attempted } = await sendWakeToMachine(env, result.machineId, result.command);
+ return attempted ? { outcome: "attempted" } : { outcome: "attempted_nowhere", machineId: result.machineId };
}
diff --git a/src/shared/src/db/queries/community/diagnostic-report.ts b/src/shared/src/db/queries/community/diagnostic-report.ts
index dd8c64fdb..2756e67a6 100644
--- a/src/shared/src/db/queries/community/diagnostic-report.ts
+++ b/src/shared/src/db/queries/community/diagnostic-report.ts
@@ -1,4 +1,4 @@
-import { and, eq, gt, isNull, lte, sql } from "drizzle-orm";
+import { and, asc, eq, gt, isNull, lte, sql } from "drizzle-orm";
import { alias } from "drizzle-orm/sqlite-core";
import { nanoid } from "nanoid";
import {
@@ -280,6 +280,42 @@ export async function getPendingDiagnosticReportForMachine(
);
}
+export async function listPendingDiagnosticReportsForMachine(
+ db: Database,
+ input: { machineId: string; nowMs: number }
+): Promise<DiagnosticReportRow[]> {
+ assertSafeEpoch(input.nowMs);
+ return db
+ .select()
+ .from(communityDiagnosticReport)
+ .where(
+ and(
+ eq(communityDiagnosticReport.machineId, input.machineId),
+ eq(communityDiagnosticReport.status, "pending"),
+ gt(communityDiagnosticReport.deadlineAt, input.nowMs)
+ )
+ )
+ .orderBy(asc(communityDiagnosticReport.deadlineAt));
+}
+
+export async function getNextPendingDiagnosticDeadlineForMachine(
+ db: Database,
+ input: { machineId: string }
+): Promise<number | null> {
+ const rows = await db
+ .select({ deadlineAt: communityDiagnosticReport.deadlineAt })
+ .from(communityDiagnosticReport)
+ .where(
+ and(
+ eq(communityDiagnosticReport.machineId, input.machineId),
+ eq(communityDiagnosticReport.status, "pending")
+ )
+ )
+ .orderBy(asc(communityDiagnosticReport.deadlineAt))
+ .limit(1);
+ return rows[0]?.deadlineAt ?? null;
+}
+
export async function getDiagnosticReportForMachine(
db: Database,
input: { reportId: string; machineId: string }
@@ -318,6 +354,24 @@ export async function timeoutPendingDiagnosticReport(
return rows[0] ?? null;
}
+export async function timeoutPendingDiagnosticReportsForMachine(
+ db: Database,
+ input: { machineId: string; nowMs: number }
+): Promise<DiagnosticReportRow[]> {
+ assertSafeEpoch(input.nowMs);
+ return db
+ .update(communityDiagnosticReport)
+ .set({ status: "failed", failureCode: "timeout", completedAt: input.nowMs })
+ .where(
+ and(
+ eq(communityDiagnosticReport.machineId, input.machineId),
+ eq(communityDiagnosticReport.status, "pending"),
+ lte(communityDiagnosticReport.deadlineAt, input.nowMs)
+ )
+ )
+ .returning();
+}
+
export async function failPendingDiagnosticReport(
db: Database,
input: {
diff --git a/src/shared/src/index.ts b/src/shared/src/index.ts
index c120e3292..70aa9816e 100644
--- a/src/shared/src/index.ts
+++ b/src/shared/src/index.ts
@@ -270,6 +270,8 @@ export {
AgentTypingStopMessageSchema,
AgentSessionMessageSchema,
AgentWakeAckMessageSchema,
+ MachineHeartbeatAckMessageSchema,
+ DiagnosticCommandAckMessageSchema,
COMMUNITY_RUNTIME_ID_MAX,
COMMUNITY_RUNTIME_VERSION_MAX,
COMMUNITY_RUNTIME_LIST_MAX,
@@ -369,6 +371,8 @@ export type {
AgentTypingStopMessage,
AgentSessionMessage,
AgentWakeAckMessage,
+ MachineHeartbeatAckMessage,
+ DiagnosticCommandAckMessage,
CommunityBotCreateRequest,
CommunityBotPatchRequest,
CommunityBotAddToServerRequest,
@@ -454,7 +458,15 @@ export type {
HostBotAuditEventFrame,
BotAuditEventPayload,
} from "./community-cli-contract";
-export { DM_SERVER, parseRef, formatRef, formatCanonicalRef, parseSeq, formatSeq } from "./community-cli-contract";
+export {
+ CONTROL_HEARTBEAT_CAPABILITY,
+ DM_SERVER,
+ parseRef,
+ formatRef,
+ formatCanonicalRef,
+ parseSeq,
+ formatSeq,
+} from "./community-cli-contract";
export type { CanonicalRefScope } from "./community-cli-contract";
export type {
@@ -482,7 +494,12 @@ export {
modelNameFromSelect,
} from "./community/bot-model";
-export { sendWakeToMachine, buildUnreadWakeCommand, dispatchOneUnreadWake } from "./community/wake-dispatch";
+export {
+ sendWakeToMachine,
+ buildUnreadWakeCommand,
+ dispatchOneUnreadWake,
+} from "./community/wake-dispatch";
+export { parseAttemptedCountReceipt } from "./transport-receipt";
export type { DispatchOneWakeResult } from "./community/wake-dispatch";
export type { WakePayload, BuildUnreadWakeResult } from "./community/wake-dispatch";
diff --git a/src/shared/src/schemas.ts b/src/shared/src/schemas.ts
index 4afd6baf1..eb6f73cf2 100644
--- a/src/shared/src/schemas.ts
+++ b/src/shared/src/schemas.ts
@@ -1,6 +1,7 @@
import { z } from "zod";
import { IssueStatus, TASK_TYPES } from "./constants";
import { sanitizeSlug } from "./utils/slug";
+import { DiagnosticReportIdSchema } from "./diagnostics-contract";
import {
ALLOWED_ICON_MIME_TYPES,
MAX_ATTACHMENTS_PER_MESSAGE,
@@ -899,6 +900,7 @@ export const CommunityMachineSummarySchema = z.object({
export const HostReadyMessageSchema = z.object({
type: z.literal("ready"),
runtimeReport: CommunityMachineRuntimeListSchema,
+ capabilities: z.array(z.string().min(1).max(64)).max(16).optional().default([]),
runningAgents: z.array(z.string()).default([]),
hostname: z.string().optional(),
platform: z.string().optional(),
@@ -1002,6 +1004,18 @@ export const AgentWakeAckMessageSchema = z.object({
});
export type AgentWakeAckMessage = z.infer<typeof AgentWakeAckMessageSchema>;
+export const MachineHeartbeatAckMessageSchema = z.strictObject({
+ type: z.literal("machine_heartbeat_ack"),
+ nonce: z.string().min(1).max(128),
+});
+export type MachineHeartbeatAckMessage = z.infer<typeof MachineHeartbeatAckMessageSchema>;
+
+export const DiagnosticCommandAckMessageSchema = z.strictObject({
+ type: z.literal("diagnostics_ack"),
+ reportId: DiagnosticReportIdSchema,
+});
+export type DiagnosticCommandAckMessage = z.infer<typeof DiagnosticCommandAckMessageSchema>;
+
export const CommunityPairTokenResponseSchema = z.object({
tokenId: z.string(),
diff --git a/src/shared/src/transport-receipt.ts b/src/shared/src/transport-receipt.ts
new file mode 100644
index 000000000..c453c6d58
--- /dev/null
+++ b/src/shared/src/transport-receipt.ts
@@ -0,0 +1,29 @@
+function isAttemptCount(value: unknown): value is number {
+ return Number.isSafeInteger(value) && (value as number) >= 0;
+}
+
+/**
+ * Parse a rolling transport-attempt receipt. `sent` is a temporary wire
+ * alias for independently deployed legacy consumers; it never means that
+ * the command was received or completed.
+ */
+export function parseAttemptedCountReceipt(
+ value: unknown,
+ options: { allowLegacySentOnly?: boolean } = {},
+): number {
+ if (!value || typeof value !== "object" || Array.isArray(value)) {
+ throw new Error("invalid attempted-count receipt");
+ }
+ const data = value as Record<string, unknown>;
+ const hasAttempted = Object.prototype.hasOwnProperty.call(data, "attempted");
+ const hasSent = Object.prototype.hasOwnProperty.call(data, "sent");
+ if (hasAttempted) {
+ if (!isAttemptCount(data.attempted)) throw new Error("invalid attempted count");
+ if (hasSent && (!isAttemptCount(data.sent) || data.sent !== data.attempted)) {
+ throw new Error("inconsistent attempted-count receipt");
+ }
+ return data.attempted;
+ }
+ if (hasSent && isAttemptCount(data.sent) && options.allowLegacySentOnly !== false) return data.sent;
+ throw new Error("missing attempted count");
+}
diff --git a/src/shared/test/community-cli-contract.test.ts b/src/shared/test/community-cli-contract.test.ts
index a47a6fc59..023320d4e 100644
--- a/src/shared/test/community-cli-contract.test.ts
+++ b/src/shared/test/community-cli-contract.test.ts
@@ -313,6 +313,18 @@ describe("HostCommand diagnostics contract", () => {
});
});
+describe("HostCommand control heartbeat contract", () => {
+ it("keeps a strict nonce-only arm", () => {
+ expect(HostCommandSchema.parse({ type: "machine:heartbeat", nonce: "nonce_1" }))
+ .toEqual({ type: "machine:heartbeat", nonce: "nonce_1" });
+ expect(HostCommandSchema.safeParse({
+ type: "machine:heartbeat",
+ nonce: "nonce_1",
+ machineId: "injected",
+ }).success).toBe(false);
+ });
+});
+
describe("HostCommand machine update contract", () => {
it("keeps the no-payload arm in the type and strict runtime schema", () => {
type UpdateCommand = { type: "machine:update" };
diff --git a/src/shared/test/community/wake-dispatch-build.test.ts b/src/shared/test/community/wake-dispatch-build.test.ts
index 3b6ab1883..5f8afb921 100644
--- a/src/shared/test/community/wake-dispatch-build.test.ts
+++ b/src/shared/test/community/wake-dispatch-build.test.ts
@@ -135,6 +135,59 @@ describe("buildUnreadWakeCommand", () => {
expect(mockResolveUnreadNoticeChannel).toHaveBeenCalledWith(fakeDb, { channelId: "ch_1" }, "bot_1");
});
+ it("reuses one launchId across queue retry, reconnect resync, and duplicate concurrent rebuilds", async () => {
+ seedHappyPath();
+
+ const initial = await buildUnreadWakeCommand(fakeDb, {
+ messageId: "msg_1",
+ botUserId: "bot_1",
+ });
+ const queueRetry = await buildUnreadWakeCommand(fakeDb, {
+ messageId: "msg_1",
+ botUserId: "bot_1",
+ });
+ const [reconnectResync, duplicate] = await Promise.all([
+ buildUnreadWakeCommand(fakeDb, {
+ messageId: "msg_1",
+ botUserId: "bot_1",
+ }),
+ buildUnreadWakeCommand(fakeDb, {
+ messageId: "msg_1",
+ botUserId: "bot_1",
+ }),
+ ]);
+
+ if (
+ initial.state !== "ready" ||
+ queueRetry.state !== "ready" ||
+ reconnectResync.state !== "ready" ||
+ duplicate.state !== "ready"
+ ) {
+ throw new Error("expected ready wakes");
+ }
+ expect(initial.command.launchId).toBe(
+ "wake_59f46f28b814c6f6b597cc0db2a5c9166d2fee5d3148a3e536a629cc227a8531",
+ );
+ expect(initial.command.launchId).not.toContain("bot_1");
+ expect(initial.command.launchId).not.toContain("msg_1");
+ expect(queueRetry.command.launchId).toBe(initial.command.launchId);
+ expect(reconnectResync.command.launchId).toBe(initial.command.launchId);
+ expect(duplicate.command.launchId).toBe(initial.command.launchId);
+ expect(
+ mockInsertBotAuditWakeTrigger.mock.calls
+ .slice(0, 4)
+ .map(([, audit]) => audit.launchId),
+ ).toEqual(Array(4).fill(initial.command.launchId));
+
+ seedHappyPath({ message: { id: "msg_2" } });
+ const nextMessage = await buildUnreadWakeCommand(fakeDb, {
+ messageId: "msg_2",
+ botUserId: "bot_1",
+ });
+ if (nextMessage.state !== "ready") throw new Error("expected ready wake");
+ expect(nextMessage.command.launchId).not.toBe(initial.command.launchId);
+ });
+
it("ready: resolves a DM scope — a DM is a type=dm channel, so it carries a channelId like any other", async () => {
seedHappyPath({
message: { channelId: "dm_ch_1" },
diff --git a/src/shared/test/community/wake-dispatch-one.test.ts b/src/shared/test/community/wake-dispatch-one.test.ts
index ad2a2a3d6..ac49ae44b 100644
--- a/src/shared/test/community/wake-dispatch-one.test.ts
+++ b/src/shared/test/community/wake-dispatch-one.test.ts
@@ -94,22 +94,22 @@ describe("dispatchOneUnreadWake", () => {
vi.clearAllMocks();
});
- it("resolves to { outcome: 'sent' } when the rebuild is ready and the daemon is online", async () => {
+ it("resolves to { outcome: 'attempted' } when a socket write is attempted", async () => {
seedReady();
- const env = makeEnv(async () => new Response(JSON.stringify({ sent: 1 }), { status: 200 }));
+ const env = makeEnv(async () => new Response(JSON.stringify({ attempted: 1 }), { status: 200 }));
const result = await dispatchOneUnreadWake(fakeDb, env, { messageId: "msg_1", botUserId: "bot_1" });
- expect(result).toEqual({ outcome: "sent" });
+ expect(result).toEqual({ outcome: "attempted" });
});
- it("resolves to { outcome: 'delivered_nowhere', machineId } when the daemon is offline", async () => {
+ it("resolves to { outcome: 'attempted_nowhere', machineId } when no socket write is attempted", async () => {
seedReady();
- const env = makeEnv(async () => new Response(JSON.stringify({ sent: 0 }), { status: 200 }));
+ const env = makeEnv(async () => new Response(JSON.stringify({ attempted: 0 }), { status: 200 }));
const result = await dispatchOneUnreadWake(fakeDb, env, { messageId: "msg_1", botUserId: "bot_1" });
- expect(result).toEqual({ outcome: "delivered_nowhere", machineId: "machine_1" });
+ expect(result).toEqual({ outcome: "attempted_nowhere", machineId: "machine_1" });
});
it("resolves to { outcome: 'skip', reason } without ever calling fetch when the rebuild is a skip", async () => {
diff --git a/src/shared/test/community/wake-dispatch.test.ts b/src/shared/test/community/wake-dispatch.test.ts
index 53da6667f..05c5e9952 100644
--- a/src/shared/test/community/wake-dispatch.test.ts
+++ b/src/shared/test/community/wake-dispatch.test.ts
@@ -20,30 +20,51 @@ describe("sendWakeToMachine", () => {
expect(url).toBe("http://internal/community-machine/by-id/machine-1/forward-agent-wake");
expect(init?.method).toBe("POST");
expect(JSON.parse(init!.body as string)).toEqual(dummyCommand);
- return new Response(JSON.stringify({ sent: 1 }), { status: 200 });
+ return new Response(JSON.stringify({ attempted: 1, sent: 1 }), { status: 200 });
});
const env = makeEnv(fetchMock);
const result = await sendWakeToMachine(env, "machine-1", dummyCommand);
- expect(result).toEqual({ sent: true });
+ expect(result).toEqual({ attempted: true });
expect(fetchMock).toHaveBeenCalledTimes(1);
});
- it("normalizes { sent: 0 } (daemon offline) to { sent: false }", async () => {
- const env = makeEnv(async () => new Response(JSON.stringify({ sent: 0 }), { status: 200 }));
+ it("normalizes { attempted: 0 } to { attempted: false }", async () => {
+ const env = makeEnv(async () => new Response(JSON.stringify({ attempted: 0 }), { status: 200 }));
const result = await sendWakeToMachine(env, "machine-1", dummyCommand);
- expect(result).toEqual({ sent: false });
+ expect(result).toEqual({ attempted: false });
});
- it("normalizes any positive sent count to { sent: true }", async () => {
- const env = makeEnv(async () => new Response(JSON.stringify({ sent: 3 }), { status: 200 }));
+ it("normalizes any positive attempted count to { attempted: true }", async () => {
+ const env = makeEnv(async () => new Response(JSON.stringify({ attempted: 3 }), { status: 200 }));
const result = await sendWakeToMachine(env, "machine-1", dummyCommand);
- expect(result).toEqual({ sent: true });
+ expect(result).toEqual({ attempted: true });
+ });
+
+ it("accepts a valid legacy sent-only receipt during rolling deployment", async () => {
+ const env = makeEnv(async () => new Response(JSON.stringify({ sent: 1 }), { status: 200 }));
+
+ await expect(sendWakeToMachine(env, "machine-1", dummyCommand)).resolves.toEqual({ attempted: true });
+ });
+
+ it.each([
+ ["missing fields", {}],
+ ["wrong attempted type", { attempted: "1" }],
+ ["negative attempted", { attempted: -1 }],
+ ["fractional attempted", { attempted: 0.5 }],
+ ["unsafe attempted", { attempted: Number.MAX_SAFE_INTEGER + 1 }],
+ ["invalid legacy sent", { sent: -1 }],
+ ["mismatched dual receipt", { attempted: 1, sent: 0 }],
+ ["invalid sent alias", { attempted: 1, sent: "1" }],
+ ])("throws on %s so the queue retries", async (_name, receipt) => {
+ const env = makeEnv(async () => new Response(JSON.stringify(receipt), { status: 200 }));
+
+ await expect(sendWakeToMachine(env, "machine-1", dummyCommand)).rejects.toThrow();
});
it("throws on non-2xx response (transient — consumer must retry)", async () => {
@@ -63,7 +84,7 @@ describe("sendWakeToMachine", () => {
it("encodes the machineId in the URL path", async () => {
const fetchMock = vi.fn(async (url: string) => {
expect(url).toBe("http://internal/community-machine/by-id/machine%2Fwith%2Fslash/forward-agent-wake");
- return new Response(JSON.stringify({ sent: 0 }), { status: 200 });
+ return new Response(JSON.stringify({ attempted: 0 }), { status: 200 });
});
const env = makeEnv(fetchMock);
diff --git a/src/shared/test/queries/community-diagnostic-report.test.ts b/src/shared/test/queries/community-diagnostic-report.test.ts
index 3a597859e..bd2733ce5 100644
--- a/src/shared/test/queries/community-diagnostic-report.test.ts
+++ b/src/shared/test/queries/community-diagnostic-report.test.ts
@@ -264,10 +264,45 @@ describe("diagnostic report query contract", () => {
expect(typeof q.getPendingDiagnosticReportForMachine).toBe("function");
expect(typeof q.getDiagnosticReportForMachine).toBe("function");
expect(typeof q.timeoutPendingDiagnosticReport).toBe("function");
+ expect(typeof q.listPendingDiagnosticReportsForMachine).toBe("function");
+ expect(typeof q.getNextPendingDiagnosticDeadlineForMachine).toBe("function");
+ expect(typeof q.timeoutPendingDiagnosticReportsForMachine).toBe("function");
expect(typeof q.failPendingDiagnosticReport).toBe("function");
expect(typeof q.finalizeDiagnosticReportUpload).toBe("function");
});
+ it("bulk timeout and reconnect reads stay scoped to one machine and preserve future rows", async () => {
+ const { db, sqlite } = realSnapshotDb();
+ const insert = sqlite.prepare(`
+ INSERT INTO community_diagnostic_report (
+ id, owner_user_id, agent_id, machine_id, client_nonce, rate_bucket,
+ status, failure_code, from_ms, created_at, deadline_at, completed_at,
+ r2_key, sha256, size_bytes, uploaded_at, object_expires_at
+ ) VALUES (?, ?, ?, ?, ?, ?, 'pending', NULL, ?, ?, ?, NULL, NULL, NULL, NULL, NULL, NULL)
+ `);
+ insert.run("dbr_expired", "owner_a", "bot_a", MACHINE_ID, "nonce_a_123456789", 1, FROM_MS, NOW_MS - 20_000, NOW_MS - 1);
+ insert.run("dbr_future", "owner_b", "bot_b", MACHINE_ID, "nonce_b_123456789", 2, FROM_MS, NOW_MS, NOW_MS + 20_000);
+ insert.run("dbr_other", "owner_c", "bot_c", "cm_other", "nonce_c_123456789", 3, FROM_MS, NOW_MS, NOW_MS + 10_000);
+
+ try {
+ await expect(q.timeoutPendingDiagnosticReportsForMachine(db, {
+ machineId: MACHINE_ID,
+ nowMs: NOW_MS,
+ })).resolves.toMatchObject([{ id: "dbr_expired", status: "failed", failureCode: "timeout" }]);
+ await expect(q.listPendingDiagnosticReportsForMachine(db, {
+ machineId: MACHINE_ID,
+ nowMs: NOW_MS,
+ })).resolves.toMatchObject([{ id: "dbr_future" }]);
+ await expect(q.getNextPendingDiagnosticDeadlineForMachine(db, {
+ machineId: MACHINE_ID,
+ })).resolves.toBe(NOW_MS + 20_000);
+ expect(sqlite.prepare(`SELECT status FROM community_diagnostic_report WHERE id='dbr_other'`).get())
+ .toEqual({ status: "pending" });
+ } finally {
+ sqlite.close();
+ }
+ });
+
it("first create is one atomic INSERT…SELECT from the live owner/bot/binding snapshot", async () => {
const created = { ...pendingRow };
const { db, chain } = queryDb([[created]]);
diff --git a/src/shared/test/schemas/community-runtime.test.ts b/src/shared/test/schemas/community-runtime.test.ts
index 7a543ac43..c5e932e75 100644
--- a/src/shared/test/schemas/community-runtime.test.ts
+++ b/src/shared/test/schemas/community-runtime.test.ts
@@ -3,6 +3,8 @@ import {
CommunityMachineRuntimeSchema,
CommunityMachineRuntimeListSchema,
HostReadyMessageSchema,
+ MachineHeartbeatAckMessageSchema,
+ DiagnosticCommandAckMessageSchema,
SessionErrorFrameSchema,
COMMUNITY_RUNTIME_ID_MAX,
COMMUNITY_RUNTIME_LIST_MAX,
@@ -125,6 +127,7 @@ describe("HostReadyMessageSchema", () => {
{ id: "claude", version: "1.0.0", status: "healthy" },
]);
expect(parsed.runningAgents).toEqual(["agent_a"]);
+ expect(parsed.capabilities).toEqual([]);
});
it("defaults runningAgents to an empty array", () => {
@@ -133,6 +136,20 @@ describe("HostReadyMessageSchema", () => {
runtimeReport: [],
});
expect(parsed.runningAgents).toEqual([]);
+ expect(parsed.capabilities).toEqual([]);
+ });
+
+ it("accepts bounded capability names and rejects an empty capability", () => {
+ expect(HostReadyMessageSchema.parse({
+ type: "ready",
+ runtimeReport: [],
+ capabilities: ["control-heartbeat-v1"],
+ }).capabilities).toEqual(["control-heartbeat-v1"]);
+ expect(HostReadyMessageSchema.safeParse({
+ type: "ready",
+ runtimeReport: [],
+ capabilities: [""],
+ }).success).toBe(false);
});
it("rejects the legacy string-only `runtimes` field (no runtimeReport)", () => {
@@ -164,6 +181,24 @@ describe("HostReadyMessageSchema", () => {
});
});
+describe("control-plane receipt schemas", () => {
+ it("strictly parses heartbeat and diagnostic ingress receipts", () => {
+ expect(MachineHeartbeatAckMessageSchema.parse({
+ type: "machine_heartbeat_ack",
+ nonce: "nonce_1",
+ })).toEqual({ type: "machine_heartbeat_ack", nonce: "nonce_1" });
+ expect(DiagnosticCommandAckMessageSchema.parse({
+ type: "diagnostics_ack",
+ reportId: "dbr_0123456789abcdef",
+ })).toEqual({ type: "diagnostics_ack", reportId: "dbr_0123456789abcdef" });
+ expect(MachineHeartbeatAckMessageSchema.safeParse({
+ type: "machine_heartbeat_ack",
+ nonce: "nonce_1",
+ extra: true,
+ }).success).toBe(false);
+ });
+});
+
describe("SessionErrorFrameSchema", () => {
it("accepts a runtime_not_available frame", () => {
const parsed = SessionErrorFrameSchema.parse({
diff --git a/src/shared/test/transport-receipt.test.ts b/src/shared/test/transport-receipt.test.ts
new file mode 100644
index 000000000..9a94db8ad
--- /dev/null
+++ b/src/shared/test/transport-receipt.test.ts
@@ -0,0 +1,23 @@
+import { describe, expect, it } from "vitest";
+import { parseAttemptedCountReceipt } from "../src/transport-receipt";
+
+describe("parseAttemptedCountReceipt", () => {
+ it("accepts attempted-only, equal dual-write, and optionally legacy sent-only receipts", () => {
+ expect(parseAttemptedCountReceipt({ attempted: 2 })).toBe(2);
+ expect(parseAttemptedCountReceipt({ attempted: 2, sent: 2 })).toBe(2);
+ expect(parseAttemptedCountReceipt({ sent: 2 })).toBe(2);
+ expect(() => parseAttemptedCountReceipt({ sent: 2 }, { allowLegacySentOnly: false })).toThrow();
+ });
+
+ it.each([
+ null,
+ {},
+ { attempted: -1 },
+ { attempted: 0.5 },
+ { attempted: Number.MAX_SAFE_INTEGER + 1 },
+ { attempted: 1, sent: 0 },
+ { attempted: 1, sent: "1" },
+ ])("rejects malformed or inconsistent receipt %#", (receipt) => {
+ expect(() => parseAttemptedCountReceipt(receipt)).toThrow();
+ });
+});
diff --git a/src/wake-worker/src/index.test.ts b/src/wake-worker/src/index.test.ts
index 96183d7be..0f141052a 100644
--- a/src/wake-worker/src/index.test.ts
+++ b/src/wake-worker/src/index.test.ts
@@ -2,8 +2,9 @@ import { describe, it, expect, vi, beforeEach } from "vitest"
const mockDispatchOneUnreadWake = vi.fn()
const mockCreateDb = vi.fn((..._a: unknown[]) => ({ __db: true }))
+const { mockLogInfo } = vi.hoisted(() => ({ mockLogInfo: vi.fn() }))
vi.mock("@alook/shared", () => {
- const noopLogger = { debug: () => { }, info: () => { }, warn: () => { }, error: () => { }, child() { return this } }
+ const noopLogger = { debug: () => { }, info: mockLogInfo, warn: () => { }, error: () => { }, child() { return this } }
return {
createLogger: () => noopLogger,
createDb: (...a: unknown[]) => mockCreateDb(...a),
@@ -24,8 +25,8 @@ describe("wake-worker queue consumer", () => {
vi.clearAllMocks()
})
- it("resolves via dispatchOneUnreadWake and acks on successful delivery", async () => {
- mockDispatchOneUnreadWake.mockResolvedValue({ outcome: "sent" })
+ it("resolves via dispatchOneUnreadWake and acks after an attempted write", async () => {
+ mockDispatchOneUnreadWake.mockResolvedValue({ outcome: "attempted" })
const msg = makeMsg({ messageId: "msg1", botUserId: "bot1" })
const batch = { messages: [msg] } as unknown as MessageBatch<unknown>
@@ -37,8 +38,8 @@ describe("wake-worker queue consumer", () => {
expect(msg.retry).not.toHaveBeenCalled()
})
- it("acks (does not retry) when daemon is offline (delivered_nowhere) — known-permanent state", async () => {
- mockDispatchOneUnreadWake.mockResolvedValue({ outcome: "delivered_nowhere", machineId: "m1" })
+ it("acks and logs when no socket write was attempted — reconnect resync owns recovery", async () => {
+ mockDispatchOneUnreadWake.mockResolvedValue({ outcome: "attempted_nowhere", machineId: "m1" })
const msg = makeMsg({ messageId: "msg1", botUserId: "bot1" })
const batch = { messages: [msg] } as unknown as MessageBatch<unknown>
@@ -46,6 +47,10 @@ describe("wake-worker queue consumer", () => {
expect(msg.ack).toHaveBeenCalledTimes(1)
expect(msg.retry).not.toHaveBeenCalled()
+ expect(mockLogInfo).toHaveBeenCalledWith("wake_attempted_nowhere", {
+ botUserId: "bot1",
+ machineId: "m1",
+ })
})
it("acks when the resolution is a skip", async () => {
@@ -93,7 +98,7 @@ describe("wake-worker queue consumer", () => {
it("processes every message in the batch independently — one failure doesn't block others", async () => {
mockDispatchOneUnreadWake
- .mockResolvedValueOnce({ outcome: "sent" })
+ .mockResolvedValueOnce({ outcome: "attempted" })
.mockRejectedValueOnce(new Error("boom"))
.mockResolvedValueOnce({ outcome: "skip", reason: "already_read" })
@@ -127,7 +132,7 @@ describe("wake-worker dev-only HTTP entrypoint (fetch)", () => {
}
it("resolves every candidate in the body via dispatchOneUnreadWake — the SAME function queue() calls", async () => {
- mockDispatchOneUnreadWake.mockResolvedValue({ outcome: "sent" })
+ mockDispatchOneUnreadWake.mockResolvedValue({ outcome: "attempted" })
const payloads = [
{ messageId: "msg1", botUserId: "bot1" },
{ messageId: "msg1", botUserId: "bot2" },
@@ -142,7 +147,7 @@ describe("wake-worker dev-only HTTP entrypoint (fetch)", () => {
expect(mockDispatchOneUnreadWake).toHaveBeenCalledWith({ __db: true }, env, payloads[1])
})
- it("returns 202 even when a skip/delivered_nowhere outcome resolves (no queue infra to ack/retry against)", async () => {
+ it("returns 202 even when a skip/attempted_nowhere outcome resolves (no queue infra to ack/retry against)", async () => {
mockDispatchOneUnreadWake.mockResolvedValue({ outcome: "skip", reason: "already_read" })
const res = await handler.fetch!(makeRequest("POST", [{ messageId: "msg1", botUserId: "bot1" }]), env)
@@ -151,7 +156,7 @@ describe("wake-worker dev-only HTTP entrypoint (fetch)", () => {
})
it("deduplicates candidates by stable messageId + botUserId within one request", async () => {
- mockDispatchOneUnreadWake.mockResolvedValue({ outcome: "sent" })
+ mockDispatchOneUnreadWake.mockResolvedValue({ outcome: "attempted" })
const first = { messageId: "msg:one", botUserId: "bot" }
const delimiterCollision = { messageId: "msg", botUserId: "one:bot" }
@@ -169,7 +174,7 @@ describe("wake-worker dev-only HTTP entrypoint (fetch)", () => {
const attempt = (attempts.get(item.botUserId) ?? 0) + 1
attempts.set(item.botUserId, attempt)
if (item.botUserId === "bot_flaky" && attempt === 1) throw new Error("D1 exploded")
- return { outcome: "sent" }
+ return { outcome: "attempted" }
})
const res = await handler.fetch!(
@@ -193,7 +198,7 @@ describe("wake-worker dev-only HTTP entrypoint (fetch)", () => {
const attempt = (attempts.get(item.botUserId) ?? 0) + 1
attempts.set(item.botUserId, attempt)
if (item.botUserId === "bot_flaky") throw new Error("D1 stayed down")
- return { outcome: "sent" }
+ return { outcome: "attempted" }
})
const res = await handler.fetch!(
@@ -252,8 +257,8 @@ describe("wake-worker dev-only HTTP entrypoint (fetch)", () => {
* details are covered by the `describe` blocks above.
*/
const RESOLUTION_SCENARIOS = [
- { name: "sent", outcome: { outcome: "sent" as const } },
- { name: "delivered_nowhere", outcome: { outcome: "delivered_nowhere" as const, machineId: "m1" } },
+ { name: "attempted", outcome: { outcome: "attempted" as const } },
+ { name: "attempted_nowhere", outcome: { outcome: "attempted_nowhere" as const, machineId: "m1" } },
{ name: "skip: already_read", outcome: { outcome: "skip" as const, reason: "already_read" as const } },
{ name: "skip: forbidden", outcome: { outcome: "skip" as const, reason: "forbidden" as const } },
{ name: "skip: muted", outcome: { outcome: "skip" as const, reason: "muted" as const } },
@@ -288,7 +293,7 @@ describe("wake-worker queue() vs fetch() — same resolution for the same candid
{ messageId: "msg2", botUserId: "bot2" },
]
- mockDispatchOneUnreadWake.mockRejectedValueOnce(new Error("boom")).mockResolvedValueOnce({ outcome: "sent" })
+ mockDispatchOneUnreadWake.mockRejectedValueOnce(new Error("boom")).mockResolvedValueOnce({ outcome: "attempted" })
const msgs = items.map((body) => ({ body, ack: vi.fn(), retry: vi.fn() }))
await handler.queue({ messages: msgs } as never, env)
expect(msgs[0]!.retry).toHaveBeenCalledTimes(1)
@@ -300,7 +305,7 @@ describe("wake-worker queue() vs fetch() — same resolution for the same candid
const attempt = (attempts.get(item.botUserId) ?? 0) + 1
attempts.set(item.botUserId, attempt)
if (item.botUserId === "bot1" && attempt === 1) throw new Error("boom")
- return { outcome: "sent" }
+ return { outcome: "attempted" }
})
const res = await handler.fetch!(new Request("http://internal/", { method: "POST", body: JSON.stringify(items) }), env)
expect(res.status).toBe(202)
diff --git a/src/wake-worker/src/index.ts b/src/wake-worker/src/index.ts
index 8606a3012..8e5c58fe0 100644
--- a/src/wake-worker/src/index.ts
+++ b/src/wake-worker/src/index.ts
@@ -19,11 +19,11 @@ async function resolveAndLog(db: Database, env: Env, item: WakePayload) {
if (result.outcome === "skip") {
// Every skip reason is a permanent current-state miss — caller must ack, never retry.
log.info("wake_skipped", { botUserId: item.botUserId, messageId: item.messageId, reason: result.reason })
- } else if (result.outcome === "delivered_nowhere") {
+ } else if (result.outcome === "attempted_nowhere") {
// ws-do resolved cleanly but the daemon is offline — a known-permanent
// state for this attempt (plan §3's error contract). Daemon reconnect
// warmup recovers; retrying would just spin, so this also acks.
- log.info("wake_delivered_nowhere", { botUserId: item.botUserId, machineId: result.machineId })
+ log.info("wake_attempted_nowhere", { botUserId: item.botUserId, machineId: result.machineId })
}
return result
}
diff --git a/src/web/AGENTS.md b/src/web/AGENTS.md
new file mode 100644
index 000000000..643577dfa
--- /dev/null
+++ b/src/web/AGENTS.md
@@ -0,0 +1,9 @@
+<!-- BEGIN:nextjs-agent-rules -->
+
+# This is NOT the Next.js you know
+
+This version has breaking changes — APIs, conventions, and file structure may all differ from your training data. Read the relevant guide in `node_modules/next/dist/docs/` (resolved from this file's directory; in monorepos the `next` package may not be visible from the repo root) before writing any code. Heed deprecation notices.
+
+This block is written and re-added by `next dev` — verify at `node_modules/next/dist/server/lib/generate-agent-files.js`. Removing it from a diff only re-creates the uncommitted change; committing it with your work keeps the tree clean.
+
+<!-- END:nextjs-agent-rules -->
diff --git a/src/web/CLAUDE.md b/src/web/CLAUDE.md
new file mode 100644
index 000000000..43c994c2d
--- /dev/null
+++ b/src/web/CLAUDE.md
@@ -0,0 +1 @@
diff --git a/src/web/migrations/0088_timeout_expired_diagnostic_reports.sql b/src/web/migrations/0088_timeout_expired_diagnostic_reports.sql
new file mode 100644
index 000000000..902e0f609
--- /dev/null
+++ b/src/web/migrations/0088_timeout_expired_diagnostic_reports.sql
@@ -0,0 +1,10 @@
+-- Close diagnostic rows that expired before receipt-driven alarms shipped.
+-- New reports are expired by the ws-do alarm and daemon reconnect resync path;
+-- this idempotent backfill is only the deploy-time bridge for legacy pending
+-- rows (including the two reports stranded by the 2026-08-17 incident).
+UPDATE community_diagnostic_report
+SET status = 'failed',
+ failure_code = 'timeout',
+ completed_at = CAST(strftime('%s', 'now') AS INTEGER) * 1000
+WHERE status = 'pending'
+ AND deadline_at <= CAST(strftime('%s', 'now') AS INTEGER) * 1000;
diff --git a/src/web/src/app/api/community/bots/[id]/diagnostics/route.test.ts b/src/web/src/app/api/community/bots/[id]/diagnostics/route.test.ts
index 8dacc9384..fad35e341 100644
--- a/src/web/src/app/api/community/bots/[id]/diagnostics/route.test.ts
+++ b/src/web/src/app/api/community/bots/[id]/diagnostics/route.test.ts
@@ -89,7 +89,7 @@ describe("POST /api/community/bots/:id/diagnostics", () => {
vi.clearAllMocks();
vi.spyOn(Date, "now").mockReturnValue(NOW);
createOrGet.mockResolvedValue({ kind: "created", report: row() });
- pushDiagnostic.mockResolvedValue({ kind: "delivered", sent: 1 });
+ pushDiagnostic.mockResolvedValue({ kind: "attempted", attempted: 1 });
});
it.each([
@@ -136,7 +136,7 @@ describe("POST /api/community/bots/:id/diagnostics", () => {
expect(getForOwner).not.toHaveBeenCalled();
});
- it("creates through the atomic owner gate and returns 202 accepted pending", async () => {
+ it("creates through the atomic owner gate and returns 202 unknown after a socket attempt", async () => {
const response = await POST(request(), context);
expect(createOrGet).toHaveBeenCalledWith({}, {
@@ -158,7 +158,7 @@ describe("POST /api/community/bots/:id/diagnostics", () => {
);
expect(response.status).toBe(202);
await expect(response.json()).resolves.toEqual({
- delivery: "accepted",
+ delivery: "unknown",
report: {
reportId: "dbr_0123456789abcdef",
status: "pending",
@@ -170,7 +170,7 @@ describe("POST /api/community/bots/:id/diagnostics", () => {
});
});
- it("turns definitive sent=0 into guarded offline and returns a 200 terminal envelope", async () => {
+ it("turns definitive attempted=0 into guarded offline and returns a 200 terminal envelope", async () => {
pushDiagnostic.mockResolvedValue({ kind: "offline" });
const failed = row({ status: "failed", failureCode: "offline", completedAt: NOW + 1 });
failPending.mockResolvedValue(failed);
@@ -197,7 +197,7 @@ describe("POST /api/community/bots/:id/diagnostics", () => {
});
});
- it("reads the authoritative owner row when the sent=0 offline CAS loses", async () => {
+ it("reads the authoritative owner row when the attempted=0 offline CAS loses", async () => {
pushDiagnostic.mockResolvedValue({ kind: "offline" });
failPending.mockResolvedValue(null);
getForOwner.mockResolvedValue(row({
@@ -219,7 +219,7 @@ describe("POST /api/community/bots/:id/diagnostics", () => {
});
});
- it("keeps delivery unknown when the sent=0 offline CAS loses to a still-pending row", async () => {
+ it("keeps delivery unknown when the attempted=0 offline CAS loses to a still-pending row", async () => {
pushDiagnostic.mockResolvedValue({ kind: "offline" });
failPending.mockResolvedValue(null);
getForOwner.mockResolvedValue(row());
diff --git a/src/web/src/app/api/community/bots/[id]/diagnostics/route.ts b/src/web/src/app/api/community/bots/[id]/diagnostics/route.ts
index e09c82b3d..d2c49fe3b 100644
--- a/src/web/src/app/api/community/bots/[id]/diagnostics/route.ts
+++ b/src/web/src/app/api/community/bots/[id]/diagnostics/route.ts
@@ -68,8 +68,8 @@ export const POST = withAuth(async (req: NextRequest, ctx) => {
fromMs: report.fromMs,
deadlineAt: report.deadlineAt,
});
- if (delivery.kind === "delivered") {
- return reportEnvelope("accepted", report, nowMs, 202);
+ if (delivery.kind === "attempted") {
+ return reportEnvelope("unknown", report, nowMs, 202);
}
if (delivery.kind === "ambiguous") {
return reportEnvelope("unknown", report, nowMs, 202);
diff --git a/src/web/src/app/api/community/daemon/resync-diagnostics/route.test.ts b/src/web/src/app/api/community/daemon/resync-diagnostics/route.test.ts
new file mode 100644
index 000000000..c78c5f23b
--- /dev/null
+++ b/src/web/src/app/api/community/daemon/resync-diagnostics/route.test.ts
@@ -0,0 +1,94 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { NextRequest } from "next/server";
+
+const findCredential = vi.fn();
+const timeoutPending = vi.fn();
+const listPending = vi.fn();
+const pushDiagnostic = vi.fn();
+
+vi.mock("@opennextjs/cloudflare", () => ({
+ getCloudflareContext: vi.fn(async () => ({ env: { DB: {}, WS_DO_WORKER: {} } })),
+}));
+vi.mock("@/lib/db", () => ({ getDb: vi.fn(() => ({})) }));
+vi.mock("@/lib/community/diagnostic-report-push", () => ({
+ pushDiagnosticReportToMachine: (...args: unknown[]) => pushDiagnostic(...args),
+}));
+vi.mock("@alook/shared", async () => {
+ const actual = await vi.importActual<typeof import("@alook/shared")>("@alook/shared");
+ return {
+ ...actual,
+ queries: {
+ ...actual.queries,
+ communityMachine: {
+ ...actual.queries.communityMachine,
+ findActiveCredentialByBearer: (...args: unknown[]) => findCredential(...args),
+ },
+ communityDiagnosticReport: {
+ ...actual.queries.communityDiagnosticReport,
+ timeoutPendingDiagnosticReportsForMachine: (...args: unknown[]) => timeoutPending(...args),
+ listPendingDiagnosticReportsForMachine: (...args: unknown[]) => listPending(...args),
+ },
+ },
+ };
+});
+
+import { POST } from "./route";
+
+const NOW = 1_800_000_000_000;
+const pending = (id: string, deadlineAt: number) => ({
+ id,
+ agentId: `bot_${id}`,
+ fromMs: NOW - 86_400_000,
+ deadlineAt,
+});
+
+function request(): NextRequest {
+ return new NextRequest("http://localhost/api/community/daemon/resync-diagnostics", {
+ method: "POST",
+ headers: { Authorization: "Bearer cmk_ok" },
+ });
+}
+
+describe("POST /api/community/daemon/resync-diagnostics", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ vi.spyOn(Date, "now").mockReturnValue(NOW);
+ findCredential.mockResolvedValue({
+ credentialId: "cmk_ok",
+ userId: "owner_1",
+ machineId: "cm_1",
+ });
+ timeoutPending.mockResolvedValue([]);
+ listPending.mockResolvedValue([]);
+ });
+
+ it("expires overdue rows, then re-derives and re-attempts every pending report from D1", async () => {
+ listPending.mockResolvedValue([
+ pending("dbr_one", NOW + 10_000),
+ pending("dbr_two", NOW + 20_000),
+ ]);
+ pushDiagnostic
+ .mockResolvedValueOnce({ kind: "attempted", attempted: 1 })
+ .mockResolvedValueOnce({ kind: "ambiguous" });
+
+ const response = await POST(request());
+
+ expect(timeoutPending).toHaveBeenCalledWith({}, { machineId: "cm_1", nowMs: NOW });
+ expect(listPending).toHaveBeenCalledWith({}, { machineId: "cm_1", nowMs: NOW });
+ expect(pushDiagnostic).toHaveBeenNthCalledWith(1, expect.anything(), "cm_1", {
+ reportId: "dbr_one",
+ agentId: "bot_dbr_one",
+ fromMs: NOW - 86_400_000,
+ deadlineAt: NOW + 10_000,
+ });
+ expect(response.status).toBe(200);
+ await expect(response.json()).resolves.toEqual({ pending: 2, attempted: 1, ambiguous: 1 });
+ });
+
+ it("returns an empty recovery set without attempting delivery", async () => {
+ const response = await POST(request());
+
+ await expect(response.json()).resolves.toEqual({ pending: 0, attempted: 0, ambiguous: 0 });
+ expect(pushDiagnostic).not.toHaveBeenCalled();
+ });
+});
diff --git a/src/web/src/app/api/community/daemon/resync-diagnostics/route.ts b/src/web/src/app/api/community/daemon/resync-diagnostics/route.ts
new file mode 100644
index 000000000..c81de75f9
--- /dev/null
+++ b/src/web/src/app/api/community/daemon/resync-diagnostics/route.ts
@@ -0,0 +1,33 @@
+import { NextResponse } from "next/server";
+import { queries } from "@alook/shared";
+import { getDb } from "@/lib/db";
+import { pushDiagnosticReportToMachine } from "@/lib/community/diagnostic-report-push";
+import { withCommunityDaemonAuth } from "@/lib/middleware/community-daemon-auth";
+
+export const POST = withCommunityDaemonAuth(async (_request, ctx) => {
+ const db = getDb(ctx.env.DB);
+ const nowMs = Date.now();
+ await queries.communityDiagnosticReport.timeoutPendingDiagnosticReportsForMachine(db, {
+ machineId: ctx.machineId,
+ nowMs,
+ });
+ const reports = await queries.communityDiagnosticReport.listPendingDiagnosticReportsForMachine(db, {
+ machineId: ctx.machineId,
+ nowMs,
+ });
+
+ let attempted = 0;
+ let ambiguous = 0;
+ for (const report of reports) {
+ const outcome = await pushDiagnosticReportToMachine(ctx.env, ctx.machineId, {
+ reportId: report.id,
+ agentId: report.agentId,
+ fromMs: report.fromMs,
+ deadlineAt: report.deadlineAt,
+ });
+ if (outcome.kind === "attempted") attempted++;
+ else if (outcome.kind === "ambiguous") ambiguous++;
+ }
+
+ return NextResponse.json({ pending: reports.length, attempted, ambiguous });
+});
diff --git a/src/web/src/app/api/community/daemon/resync-wakes/route.test.ts b/src/web/src/app/api/community/daemon/resync-wakes/route.test.ts
index 064acd484..76fffdb95 100644
--- a/src/web/src/app/api/community/daemon/resync-wakes/route.test.ts
+++ b/src/web/src/app/api/community/daemon/resync-wakes/route.test.ts
@@ -51,11 +51,11 @@ describe("POST /api/community/daemon/resync-wakes", () => {
});
});
- it("returns { woken: 0 } and does nothing when the machine has no bots", async () => {
+ it("returns { attempted: 0 } and does nothing when the machine has no bots", async () => {
mockListBotsForMachine.mockResolvedValue([]);
const res = await POST(req({ Authorization: "Bearer cmk_ok" }));
expect(res.status).toBe(200);
- expect(await res.json()).toEqual({ woken: 0 });
+ expect(await res.json()).toEqual({ attempted: 0 });
expect(mockGetLatestUnreadMessageForAgent).not.toHaveBeenCalled();
expect(mockDispatchOneUnreadWake).not.toHaveBeenCalled();
});
@@ -67,7 +67,7 @@ describe("POST /api/community/daemon/resync-wakes", () => {
mockGetLatestUnreadMessageForAgent.mockResolvedValue({
messageId: "msg_1",
});
- mockDispatchOneUnreadWake.mockResolvedValue({ outcome: "sent" });
+ mockDispatchOneUnreadWake.mockResolvedValue({ outcome: "attempted" });
const res = await POST(req({ Authorization: "Bearer cmk_ok" }));
@@ -76,7 +76,7 @@ describe("POST /api/community/daemon/resync-wakes", () => {
expect.anything(),
{ messageId: "msg_1", botUserId: "bot_1" },
);
- expect(await res.json()).toEqual({ woken: 1 });
+ expect(await res.json()).toEqual({ attempted: 1 });
});
it("skips bots with no pending unread — not counted, dispatch never called for them", async () => {
@@ -88,10 +88,10 @@ describe("POST /api/community/daemon/resync-wakes", () => {
const res = await POST(req({ Authorization: "Bearer cmk_ok" }));
expect(mockDispatchOneUnreadWake).not.toHaveBeenCalled();
- expect(await res.json()).toEqual({ woken: 0 });
+ expect(await res.json()).toEqual({ attempted: 0 });
});
- it("does not count a skip/delivered_nowhere outcome as woken", async () => {
+ it("does not count a skip/attempted_nowhere outcome as attempted", async () => {
mockListBotsForMachine.mockResolvedValue([
{ id: "bot_1", name: "a", description: "" },
{ id: "bot_2", name: "b", description: "" },
@@ -102,13 +102,13 @@ describe("POST /api/community/daemon/resync-wakes", () => {
mockDispatchOneUnreadWake
.mockResolvedValueOnce({ outcome: "skip", reason: "already_read" })
.mockResolvedValueOnce({
- outcome: "delivered_nowhere",
+ outcome: "attempted_nowhere",
machineId: "cm_1",
});
const res = await POST(req({ Authorization: "Bearer cmk_ok" }));
- expect(await res.json()).toEqual({ woken: 0 });
+ expect(await res.json()).toEqual({ attempted: 0 });
});
it("401 without Authorization", async () => {
@@ -123,11 +123,11 @@ describe("POST /api/community/daemon/resync-wakes", () => {
expect(res.status).toBe(401);
});
- it("Mellicent regression: bot on a server whose only unread lives in a non-participated forum_post — feeder returns null → woken: 0", async () => {
+ it("Mellicent regression: bot on a server whose only unread lives in a non-participated forum_post — feeder returns null → attempted: 0", async () => {
// Post-fix behaviour: `getLatestUnreadMessageForAgent` applies the
// thread-participation post-filter (`plans/agent-unread-visibility-unify.md`),
// so a message in a forum_post the bot isn't a participant of no longer
- // surfaces. The feeder returns null → resync-wakes reports woken: 0 and
+ // surfaces. The feeder returns null → resync-wakes reports attempted: 0 and
// never touches the wake dispatcher.
mockListBotsForMachine.mockResolvedValue([
{ id: "bot_Y", name: "Mellicent", description: "" },
@@ -136,7 +136,7 @@ describe("POST /api/community/daemon/resync-wakes", () => {
const res = await POST(req({ Authorization: "Bearer cmk_ok" }));
- expect(await res.json()).toEqual({ woken: 0 });
+ expect(await res.json()).toEqual({ attempted: 0 });
expect(mockDispatchOneUnreadWake).not.toHaveBeenCalled();
});
});
diff --git a/src/web/src/app/api/community/daemon/resync-wakes/route.ts b/src/web/src/app/api/community/daemon/resync-wakes/route.ts
index 3b6c59b81..299745357 100644
--- a/src/web/src/app/api/community/daemon/resync-wakes/route.ts
+++ b/src/web/src/app/api/community/daemon/resync-wakes/route.ts
@@ -9,7 +9,7 @@ import { withCommunityDaemonAuth } from "@/lib/middleware/community-daemon-auth"
* Daemon-initiated recovery for the "message sent while the daemon was
* offline" gap: `WAKE_QUEUE`'s consumer acks (never retries) a wake whose
* daemon was unreachable at delivery time (`dispatchOneUnreadWake`'s
- * `delivered_nowhere` outcome) — that queue item is gone for good. Rather
+ * `attempted_nowhere` outcome) — that queue item is gone for good. Rather
* than the server pushing a catch-up wake on its own when the daemon's WS
* reconnects, the DAEMON proactively calls this route right after it opens
* its control-plane connection (`channel.onOpen()`), asking "do any of my
@@ -29,7 +29,7 @@ export const POST = withCommunityDaemonAuth(async (_req, ctx) => {
{ route: "community/daemon/resync-wakes:list-bots" },
)
- let woken = 0
+ let attempted = 0
for (const bot of bots) {
const latest = await withD1Retry(
() => queries.communityAgentInbox.getLatestUnreadMessageForAgent(db, bot.id),
@@ -37,8 +37,8 @@ export const POST = withCommunityDaemonAuth(async (_req, ctx) => {
)
if (!latest) continue
const result = await dispatchOneUnreadWake(db, ctx.env, { messageId: latest.messageId, botUserId: bot.id })
- if (result.outcome === "sent") woken++
+ if (result.outcome === "attempted") attempted++
}
- return NextResponse.json({ woken })
+ return NextResponse.json({ attempted })
})
diff --git a/src/web/src/lib/community/diagnostic-report-push.test.ts b/src/web/src/lib/community/diagnostic-report-push.test.ts
index dfee51260..90d224501 100644
--- a/src/web/src/lib/community/diagnostic-report-push.test.ts
+++ b/src/web/src/lib/community/diagnostic-report-push.test.ts
@@ -19,11 +19,11 @@ describe("pushDiagnosticReportToMachine", () => {
beforeEach(() => wsDoFetch.mockReset());
it("uses only the purpose-built diagnostics route and narrow body", async () => {
- wsDoFetch.mockResolvedValue(Response.json({ sent: 1 }));
+ wsDoFetch.mockResolvedValue(Response.json({ attempted: 1, sent: 1 }));
const outcome = await pushDiagnosticReportToMachine(env, "cm one", command);
- expect(outcome).toEqual({ kind: "delivered", sent: 1 });
+ expect(outcome).toEqual({ kind: "attempted", attempted: 1 });
const [, path, init, audit] = wsDoFetch.mock.calls[0]!;
expect(path).toBe("/community-machine/by-id/cm%20one/forward-diagnostics-collect");
expect(init.method).toBe("POST");
@@ -32,14 +32,14 @@ describe("pushDiagnosticReportToMachine", () => {
expect(audit).toEqual({ label: "cm one", type: "diagnostics:collect" });
});
- it("distinguishes delivered, definite offline, and ambiguous transport", async () => {
- wsDoFetch.mockResolvedValueOnce(Response.json({ sent: 2 }));
+ it("distinguishes attempted, definite offline, and ambiguous transport", async () => {
+ wsDoFetch.mockResolvedValueOnce(Response.json({ attempted: 2 }));
await expect(pushDiagnosticReportToMachine(env, "cm_1", command)).resolves.toEqual({
- kind: "delivered",
- sent: 2,
+ kind: "attempted",
+ attempted: 2,
});
- wsDoFetch.mockResolvedValueOnce(Response.json({ sent: 0 }));
+ wsDoFetch.mockResolvedValueOnce(Response.json({ attempted: 0 }));
await expect(pushDiagnosticReportToMachine(env, "cm_1", command)).resolves.toEqual({
kind: "offline",
});
@@ -56,10 +56,12 @@ describe("pushDiagnosticReportToMachine", () => {
});
it.each([
- ["missing sent", {}],
- ["negative sent", { sent: -1 }],
- ["fractional sent", { sent: 0.5 }],
- ["string sent", { sent: "1" }],
+ ["missing attempted", {}],
+ ["legacy sent-only without deadline proof", { sent: 1 }],
+ ["negative attempted", { attempted: -1 }],
+ ["fractional attempted", { attempted: 0.5 }],
+ ["string attempted", { attempted: "1" }],
+ ["mismatched sent alias", { attempted: 1, sent: 0 }],
])("fails closed on a 2xx response with %s", async (_label, response) => {
wsDoFetch.mockResolvedValue(Response.json(response));
diff --git a/src/web/src/lib/community/diagnostic-report-push.ts b/src/web/src/lib/community/diagnostic-report-push.ts
index 63dd6779a..235d96f71 100644
--- a/src/web/src/lib/community/diagnostic-report-push.ts
+++ b/src/web/src/lib/community/diagnostic-report-push.ts
@@ -1,8 +1,9 @@
+import { parseAttemptedCountReceipt } from "@alook/shared";
import type { DiagnosticCollectPayload } from "@alook/shared";
import { wsDoFetch } from "@/lib/broadcast";
export type DiagnosticDeliveryOutcome =
- | { kind: "delivered"; sent: number }
+ | { kind: "attempted"; attempted: number }
| { kind: "offline" }
| { kind: "ambiguous" };
@@ -24,13 +25,14 @@ export async function pushDiagnosticReportToMachine(
{ label: machineId, type: "diagnostics:collect" },
);
if (!response.ok) return { kind: "ambiguous" };
- const result = await response.json() as { sent?: unknown };
- if (!Number.isSafeInteger(result.sent) || (result.sent as number) < 0) {
- return { kind: "ambiguous" };
- }
- return (result.sent as number) === 0
+ const attempted = parseAttemptedCountReceipt(await response.json(), {
+ // A legacy sent-only ws-do predates deterministic deadline ownership,
+ // so it cannot establish a safe offline/attempted result for new web.
+ allowLegacySentOnly: false,
+ });
+ return attempted === 0
? { kind: "offline" }
- : { kind: "delivered", sent: result.sent as number };
+ : { kind: "attempted", attempted };
} catch {
return { kind: "ambiguous" };
}
diff --git a/src/ws-do/src/routes/community-machine-delivery.test.ts b/src/ws-do/src/routes/community-machine-delivery.test.ts
index 1e0c173d4..bd598a275 100644
--- a/src/ws-do/src/routes/community-machine-delivery.test.ts
+++ b/src/ws-do/src/routes/community-machine-delivery.test.ts
@@ -111,7 +111,7 @@ describe("ws-do router", () => {
mockWithD1Retry.mockImplementation(async (fn) => fn())
})
- it("zero active doNames → { sent: 0 } without touching any DO", async () => {
+ it("zero active doNames → dual-write zero without touching any DO", async () => {
const req = new Request("http://localhost/community-machine/by-id/machine-1/forward-agent-wake", {
method: "POST",
body: JSON.stringify({ type: "agent:wake" }),
@@ -120,13 +120,13 @@ describe("ws-do router", () => {
expect(mockGetActiveDoNamesForMachine).toHaveBeenCalledWith({}, "machine-1")
expect(res.status).toBe(200)
- expect(await res.json()).toEqual({ sent: 0 })
+ expect(await res.json()).toEqual({ attempted: 0, sent: 0 })
expect(doMock.stubFetch).not.toHaveBeenCalled()
})
- it("single active doName, daemon connected → forwards and aggregates sent count", async () => {
+ it("single active doName, daemon connected → forwards and aggregates attempted count", async () => {
mockGetActiveDoNamesForMachine.mockResolvedValue(["do-abc"])
- doMock.stubFetch.mockResolvedValue(new Response(JSON.stringify({ sent: 1 }), { status: 200 }))
+ doMock.stubFetch.mockResolvedValue(new Response(JSON.stringify({ attempted: 1, sent: 1 }), { status: 200 }))
const req = new Request("http://localhost/community-machine/by-id/machine-1/forward-agent-wake", {
method: "POST",
@@ -139,12 +139,12 @@ describe("ws-do router", () => {
const stubReq = doMock.stubFetch.mock.calls[0][0] as Request
expect(stubReq.url).toBe("http://internal/forward-agent-wake")
expect(res.status).toBe(200)
- expect(await res.json()).toEqual({ sent: 1 })
+ expect(await res.json()).toEqual({ attempted: 1, sent: 1 })
})
- it("daemon offline (DO reports { sent: 0 }) → does not count as delivered", async () => {
+ it("daemon offline (DO reports { attempted: 0 }) → does not count a write", async () => {
mockGetActiveDoNamesForMachine.mockResolvedValue(["do-abc"])
- doMock.stubFetch.mockResolvedValue(new Response(JSON.stringify({ sent: 0 }), { status: 200 }))
+ doMock.stubFetch.mockResolvedValue(new Response(JSON.stringify({ attempted: 0 }), { status: 200 }))
const req = new Request("http://localhost/community-machine/by-id/machine-1/forward-agent-wake", {
method: "POST",
@@ -153,16 +153,16 @@ describe("ws-do router", () => {
const res = await handler.fetch(req, env as any)
expect(res.status).toBe(200)
- expect(await res.json()).toEqual({ sent: 0 })
+ expect(await res.json()).toEqual({ attempted: 0, sent: 0 })
})
- it("multi-doName fan-out: both DOs hit, aggregate sums delivered counts", async () => {
+ it("multi-doName fan-out: both DOs hit, aggregate sums attempted counts", async () => {
mockGetActiveDoNamesForMachine.mockResolvedValue(["do-a", "do-b"])
let call = 0
doMock.stubFetch.mockImplementation(() => {
call++
- const sent = call === 1 ? 0 : 1
- return Promise.resolve(new Response(JSON.stringify({ sent }), { status: 200 }))
+ const attempted = call === 1 ? 0 : 1
+ return Promise.resolve(new Response(JSON.stringify({ attempted }), { status: 200 }))
})
const req = new Request("http://localhost/community-machine/by-id/machine-1/forward-agent-wake", {
@@ -174,16 +174,16 @@ describe("ws-do router", () => {
expect(doMock.idFromName).toHaveBeenCalledWith("community-machine:do-a")
expect(doMock.idFromName).toHaveBeenCalledWith("community-machine:do-b")
expect(res.status).toBe(200)
- expect(await res.json()).toEqual({ sent: 1 })
+ expect(await res.json()).toEqual({ attempted: 1, sent: 1 })
})
- it("DO fetch throws — tolerated, other doNames still evaluated", async () => {
+ it("DO fetch throws — other doNames are evaluated but the aggregate remains retryable", async () => {
mockGetActiveDoNamesForMachine.mockResolvedValue(["do-a", "do-b"])
let call = 0
doMock.stubFetch.mockImplementation(() => {
call++
if (call === 1) return Promise.reject(new Error("network error"))
- return Promise.resolve(new Response(JSON.stringify({ sent: 1 }), { status: 200 }))
+ return Promise.resolve(new Response(JSON.stringify({ attempted: 1 }), { status: 200 }))
})
const req = new Request("http://localhost/community-machine/by-id/machine-1/forward-agent-wake", {
@@ -192,8 +192,9 @@ describe("ws-do router", () => {
})
const res = await handler.fetch(req, env as any)
- expect(res.status).toBe(200)
- expect(await res.json()).toEqual({ sent: 1 })
+ expect(doMock.stubFetch).toHaveBeenCalledTimes(2)
+ expect(res.status).toBe(503)
+ expect(await res.json()).toEqual({ error: "failed to forward agent wake" })
})
it("DO fetch throws with no delivery → retryable 503", async () => {
@@ -210,13 +211,13 @@ describe("ws-do router", () => {
expect(await res.json()).toEqual({ error: "failed to forward agent wake" })
})
- it("non-2xx or malformed DO responses with no delivery → retryable 503", async () => {
+ it("non-2xx or malformed DO responses → retryable 503", async () => {
mockGetActiveDoNamesForMachine.mockResolvedValue(["do-a", "do-b"])
let call = 0
doMock.stubFetch.mockImplementation(() => {
call++
if (call === 1) return Promise.resolve(new Response("oops", { status: 502 }))
- return Promise.resolve(new Response(JSON.stringify({ sent: "bad" }), { status: 200 }))
+ return Promise.resolve(new Response(JSON.stringify({ attempted: "bad" }), { status: 200 }))
})
const req = new Request("http://localhost/community-machine/by-id/machine-1/forward-agent-wake", {
@@ -229,7 +230,38 @@ describe("ws-do router", () => {
expect(await res.json()).toEqual({ error: "failed to forward agent wake" })
})
- it("DB lookup failure → retryable 503, not offline { sent: 0 }", async () => {
+ it("accepts a legacy sent-only inner DO and dual-writes the outer receipt", async () => {
+ mockGetActiveDoNamesForMachine.mockResolvedValue(["do-legacy"])
+ doMock.stubFetch.mockResolvedValue(new Response(JSON.stringify({ sent: 1 }), { status: 200 }))
+
+ const res = await handler.fetch(new Request(
+ "http://localhost/community-machine/by-id/machine-1/forward-agent-wake",
+ { method: "POST", body: JSON.stringify({ type: "agent:wake" }) },
+ ), env as any)
+
+ expect(res.status).toBe(200)
+ expect(await res.json()).toEqual({ attempted: 1, sent: 1 })
+ })
+
+ it.each([
+ ["missing", {}],
+ ["negative", { attempted: -1 }],
+ ["fractional", { attempted: 0.5 }],
+ ["mismatched", { attempted: 1, sent: 0 }],
+ ])("maps a %s inner receipt to retryable 503", async (_name, receipt) => {
+ mockGetActiveDoNamesForMachine.mockResolvedValue(["do-bad"])
+ doMock.stubFetch.mockResolvedValue(new Response(JSON.stringify(receipt), { status: 200 }))
+
+ const res = await handler.fetch(new Request(
+ "http://localhost/community-machine/by-id/machine-1/forward-agent-wake",
+ { method: "POST", body: JSON.stringify({ type: "agent:wake" }) },
+ ), env as any)
+
+ expect(res.status).toBe(503)
+ expect(await res.json()).toEqual({ error: "failed to forward agent wake" })
+ })
+
+ it("DB lookup failure → retryable 503, not { attempted: 0 }", async () => {
mockGetActiveDoNamesForMachine.mockRejectedValue(new Error("d1 unreachable"))
const req = new Request("http://localhost/community-machine/by-id/machine-1/forward-agent-wake", {
@@ -254,7 +286,7 @@ describe("ws-do router", () => {
return fn()
}
})
- doMock.stubFetch.mockResolvedValue(new Response(JSON.stringify({ sent: 1 }), { status: 200 }))
+ doMock.stubFetch.mockResolvedValue(new Response(JSON.stringify({ attempted: 1 }), { status: 200 }))
const res = await handler.fetch(new Request(
"http://localhost/community-machine/by-id/machine-1/forward-agent-wake",
@@ -267,7 +299,7 @@ describe("ws-do router", () => {
})
expect(doMock.stubFetch).toHaveBeenCalledTimes(1)
expect(res.status).toBe(200)
- expect(await res.json()).toEqual({ sent: 1 })
+ expect(await res.json()).toEqual({ attempted: 1, sent: 1 })
})
it("maps exhausted SQLITE_BUSY retries to the existing 503 response", async () => {
diff --git a/src/ws-do/src/routes/community-machine-delivery.ts b/src/ws-do/src/routes/community-machine-delivery.ts
index b75cd046a..b4684eecf 100644
--- a/src/ws-do/src/routes/community-machine-delivery.ts
+++ b/src/ws-do/src/routes/community-machine-delivery.ts
@@ -1,4 +1,4 @@
-import { queries, withD1Retry } from "@alook/shared"
+import { parseAttemptedCountReceipt, queries, withD1Retry } from "@alook/shared"
import type { RouterContext } from "../router-context"
export async function handleMachinePush({ request, env, url, traceId, log }: RouterContext): Promise<Response | null> {
@@ -56,10 +56,9 @@ export async function handleMachineWake({ request, env, url, traceId, log }: Rou
// of the `/push` route above, for the minimal-wake-queue-unread-notice
// wake path. Forwards an already-built `HostCommand` (`agent:wake`)
// verbatim to every live DO for this machine's active credential(s),
- // then aggregates by parsing each DO's own `{ sent: N }` response —
- // unlike `/push`, we must NOT just count `res.ok`: a 200 with
- // `{ sent: 0 }` means that DO has no authenticated daemon socket at all,
- // which must not be reported as delivered.
+ // then aggregates each DO's rolling `{ attempted: N, sent: N }` receipt.
+ // A successful socket write is only an attempt; daemon `agent_wake_ack`
+ // is the receipt and unread D1 state remains the reconnect truth.
const forwardAgentWake = url.pathname.match(/^\/community-machine\/by-id\/([^/]+)\/forward-agent-wake$/)
if (!forwardAgentWake || request.method !== "POST") return null
@@ -80,10 +79,10 @@ export async function handleMachineWake({ request, env, url, traceId, log }: Rou
return Response.json({ error: "failed to resolve machine" }, { status: 503 })
}
if (doNames.length === 0) {
- return Response.json({ sent: 0 })
+ return Response.json({ attempted: 0, sent: 0 })
}
const bodyText = await request.text()
- let delivered = 0
+ let attempted = 0
let transientFailure = false
for (const dn of doNames) {
const doId = env.WS_DO.idFromName("community-machine:" + dn)
@@ -100,18 +99,13 @@ export async function handleMachineWake({ request, env, url, traceId, log }: Rou
transientFailure = true
continue
}
- const data = (await res.json()) as { sent?: unknown }
- if (typeof data.sent !== "number" || !Number.isFinite(data.sent) || data.sent < 0) {
- transientFailure = true
- continue
- }
- delivered += data.sent
+ attempted += parseAttemptedCountReceipt(await res.json())
} catch {
transientFailure = true
}
}
- if (delivered === 0 && transientFailure) {
+ if (transientFailure) {
return Response.json({ error: "failed to forward agent wake" }, { status: 503 })
}
- return Response.json({ sent: delivered })
+ return Response.json({ attempted, sent: attempted })
}
diff --git a/src/ws-do/src/routes/community-machine-diagnostics.test.ts b/src/ws-do/src/routes/community-machine-diagnostics.test.ts
index 97f0fff8c..7896362bf 100644
--- a/src/ws-do/src/routes/community-machine-diagnostics.test.ts
+++ b/src/ws-do/src/routes/community-machine-diagnostics.test.ts
@@ -50,19 +50,23 @@ describe("POST /community-machine/by-id/:machineId/forward-diagnostics-collect",
expect(context.doMock.stubFetch).not.toHaveBeenCalled();
});
- it("constructs the exact HostCommand and attempts every active credential", async () => {
+ it("constructs the exact HostCommand, accepts a legacy inner receipt, and dual-writes for old web", async () => {
getActiveDoNamesForMachine.mockResolvedValue(["do-a", "do-b", "do-c"]);
context.doMock.stubFetch
+ .mockResolvedValueOnce(Response.json({ registered: true }))
+ .mockResolvedValueOnce(Response.json({ attempted: 0, sent: 0 }))
.mockResolvedValueOnce(Response.json({ sent: 0 }))
- .mockRejectedValueOnce(new Error("credential transport failed"))
- .mockResolvedValueOnce(Response.json({ sent: 1 }));
+ .mockResolvedValueOnce(Response.json({ attempted: 1, sent: 1 }));
const response = await handler.fetch(request(payload), context.env as never);
expect(response.status).toBe(200);
- await expect(response.json()).resolves.toEqual({ sent: 1 });
- expect(context.doMock.stubFetch).toHaveBeenCalledTimes(3);
- const forwarded = context.doMock.stubFetch.mock.calls.map(([value]) => value as Request);
+ await expect(response.json()).resolves.toEqual({ attempted: 1, sent: 1 });
+ expect(context.doMock.stubFetch).toHaveBeenCalledTimes(4);
+ const deadline = context.doMock.stubFetch.mock.calls[0]![0] as Request;
+ expect(deadline.url).toBe("http://internal/register-diagnostic-deadline?machineId=cm_1");
+ await expect(deadline.clone().json()).resolves.toEqual({ deadlineAt: payload.deadlineAt });
+ const forwarded = context.doMock.stubFetch.mock.calls.slice(1).map(([value]) => value as Request);
expect(forwarded.every((value) => value.url === "http://internal/forward-diagnostics-collect")).toBe(true);
expect(await forwarded[0].clone().json()).toEqual({ type: "diagnostics:collect", ...payload });
expect(forwarded.some((value) => value.url.endsWith("/push"))).toBe(false);
@@ -71,26 +75,118 @@ describe("POST /community-machine/by-id/:machineId/forward-diagnostics-collect",
it("returns definite offline only when every credential reports zero", async () => {
getActiveDoNamesForMachine.mockResolvedValue(["do-a", "do-b"]);
context.doMock.stubFetch
- .mockResolvedValueOnce(Response.json({ sent: 0 }))
- .mockResolvedValueOnce(Response.json({ sent: 0 }));
+ .mockResolvedValueOnce(Response.json({ registered: true }))
+ .mockResolvedValueOnce(Response.json({ attempted: 0, sent: 0 }))
+ .mockResolvedValueOnce(Response.json({ attempted: 0, sent: 0 }));
const response = await handler.fetch(request(payload), context.env as never);
expect(response.status).toBe(200);
- await expect(response.json()).resolves.toEqual({ sent: 0 });
- expect(context.doMock.stubFetch).toHaveBeenCalledTimes(2);
+ await expect(response.json()).resolves.toEqual({ attempted: 0, sent: 0 });
+ expect(context.doMock.stubFetch).toHaveBeenCalledTimes(3);
});
it("returns ambiguous when no delivery succeeds and any credential is non-definitive", async () => {
getActiveDoNamesForMachine.mockResolvedValue(["do-a", "do-b", "do-c"]);
context.doMock.stubFetch
- .mockResolvedValueOnce(Response.json({ sent: 0 }))
+ .mockResolvedValueOnce(Response.json({ registered: true }))
+ .mockResolvedValueOnce(Response.json({ attempted: 0 }))
.mockResolvedValueOnce(new Response("unavailable", { status: 503 }))
- .mockResolvedValueOnce(Response.json({ sent: "one" }));
+ .mockResolvedValueOnce(Response.json({ attempted: "one" }));
+
+ const response = await handler.fetch(request(payload), context.env as never);
+
+ expect(response.status).toBe(503);
+ expect(context.doMock.stubFetch).toHaveBeenCalledTimes(4);
+ });
+
+ it("returns ambiguous when one credential succeeds but another forward fails", async () => {
+ getActiveDoNamesForMachine.mockResolvedValue(["do-a", "do-b"]);
+ context.doMock.stubFetch
+ .mockResolvedValueOnce(Response.json({ registered: true }))
+ .mockResolvedValueOnce(Response.json({ attempted: 1, sent: 1 }))
+ .mockRejectedValueOnce(new Error("credential transport failed"));
+
+ const response = await handler.fetch(request(payload), context.env as never);
+
+ expect(response.status).toBe(503);
+ await expect(response.json()).resolves.toEqual({ error: "diagnostic delivery ambiguous" });
+ });
+
+ it.each([
+ ["missing", {}],
+ ["negative", { attempted: -1 }],
+ ["fractional", { attempted: 0.5 }],
+ ["mismatched", { attempted: 1, sent: 0 }],
+ ])("returns ambiguous for a %s inner receipt", async (_label, receipt) => {
+ getActiveDoNamesForMachine.mockResolvedValue(["do-a"]);
+ context.doMock.stubFetch
+ .mockResolvedValueOnce(Response.json({ registered: true }))
+ .mockResolvedValueOnce(Response.json(receipt));
+
+ const response = await handler.fetch(request(payload), context.env as never);
+
+ expect(response.status).toBe(503);
+ await expect(response.json()).resolves.toEqual({ error: "diagnostic delivery ambiguous" });
+ });
+
+ it("dual-writes zero after deadline registration when there are no active credentials", async () => {
+ context.doMock.stubFetch.mockResolvedValueOnce(Response.json({ registered: true }));
+
+ const response = await handler.fetch(request(payload), context.env as never);
+
+ expect(response.status).toBe(200);
+ await expect(response.json()).resolves.toEqual({ attempted: 0, sent: 0 });
+ });
+
+ it("registers the deadline before an active-credential lookup failure", async () => {
+ context.doMock.stubFetch.mockResolvedValueOnce(Response.json({ registered: true }));
+ getActiveDoNamesForMachine.mockRejectedValueOnce(new Error("D1 unavailable"));
+
+ const response = await handler.fetch(request(payload), context.env as never);
+
+ expect(response.status).toBe(503);
+ expect(context.doMock.stubFetch).toHaveBeenCalledOnce();
+ const deadline = context.doMock.stubFetch.mock.calls[0]![0] as Request;
+ expect(deadline.url).toBe("http://internal/register-diagnostic-deadline?machineId=cm_1");
+ });
+
+ it("keeps the registered deadline when every active credential forward throws", async () => {
+ getActiveDoNamesForMachine.mockResolvedValue(["do-a", "do-b"]);
+ context.doMock.stubFetch
+ .mockResolvedValueOnce(Response.json({ registered: true }))
+ .mockRejectedValueOnce(new Error("do-a unavailable"))
+ .mockRejectedValueOnce(new Error("do-b unavailable"));
const response = await handler.fetch(request(payload), context.env as never);
expect(response.status).toBe(503);
expect(context.doMock.stubFetch).toHaveBeenCalledTimes(3);
+ expect((context.doMock.stubFetch.mock.calls[0]![0] as Request).url)
+ .toContain("/register-diagnostic-deadline");
+ });
+
+ it("fails explicitly before lookup or fanout when deadline registration fails", async () => {
+ context.doMock.stubFetch.mockResolvedValueOnce(new Response("unavailable", { status: 503 }));
+
+ const response = await handler.fetch(request(payload), context.env as never);
+
+ expect(response.status).toBe(503);
+ await expect(response.json()).resolves.toEqual({ error: "diagnostic deadline registration failed" });
+ expect(getActiveDoNamesForMachine).not.toHaveBeenCalled();
+ });
+
+ it.each([
+ ["registered false", { registered: false }],
+ ["missing receipt", {}],
+ ["extra field", { registered: true, attempted: 1 }],
+ ])("rejects a 200 deadline response with %s", async (_label, receipt) => {
+ context.doMock.stubFetch.mockResolvedValueOnce(Response.json(receipt));
+
+ const response = await handler.fetch(request(payload), context.env as never);
+
+ expect(response.status).toBe(503);
+ await expect(response.json()).resolves.toEqual({ error: "diagnostic deadline registration failed" });
+ expect(getActiveDoNamesForMachine).not.toHaveBeenCalled();
});
});
diff --git a/src/ws-do/src/routes/community-machine-diagnostics.ts b/src/ws-do/src/routes/community-machine-diagnostics.ts
index 8320af876..9bda837a5 100644
--- a/src/ws-do/src/routes/community-machine-diagnostics.ts
+++ b/src/ws-do/src/routes/community-machine-diagnostics.ts
@@ -2,6 +2,7 @@ import {
createDb,
DiagnosticCollectCommandSchema,
DiagnosticCollectPayloadSchema,
+ parseAttemptedCountReceipt,
queries,
} from "@alook/shared";
import type { RouterContext } from "../router-context";
@@ -35,6 +36,38 @@ export async function handleMachineDiagnostics(
return Response.json({ error: "invalid payload" }, { status: 400 });
}
+ // Deadline ownership is deterministic by machineId, independent of the
+ // currently active credential doName. Register before D1 lookup/fanout so
+ // lookup failures, credential rotation, and ambiguous socket writes cannot
+ // strand the durable report row forever.
+ try {
+ const deadlineId = env.WS_DO.idFromName(`community-machine-deadline:${machineId}`);
+ const deadlineStub = env.WS_DO.get(deadlineId);
+ const registration = await deadlineStub.fetch(new Request(
+ `http://internal/register-diagnostic-deadline?machineId=${encodeURIComponent(machineId)}`,
+ {
+ method: "POST",
+ headers: { "content-type": "application/json" },
+ body: JSON.stringify({ deadlineAt: command.data.deadlineAt }),
+ },
+ ));
+ if (!registration.ok) {
+ return Response.json({ error: "diagnostic deadline registration failed" }, { status: 503 });
+ }
+ const receipt = await registration.json() as unknown;
+ if (
+ !receipt ||
+ typeof receipt !== "object" ||
+ Object.keys(receipt).length !== 1 ||
+ (receipt as Record<string, unknown>).registered !== true
+ ) {
+ return Response.json({ error: "diagnostic deadline registration failed" }, { status: 503 });
+ }
+ } catch (err) {
+ reqLog.error("failed to register diagnostic deadline", { err });
+ return Response.json({ error: "diagnostic deadline registration failed" }, { status: 503 });
+ }
+
let doNames: string[];
try {
const db = createDb((env as unknown as { DB: D1Database }).DB);
@@ -43,10 +76,10 @@ export async function handleMachineDiagnostics(
reqLog.error("failed to resolve machine doNames for diagnostics", { err });
return Response.json({ error: "failed to resolve machine" }, { status: 503 });
}
- if (doNames.length === 0) return Response.json({ sent: 0 });
+ if (doNames.length === 0) return Response.json({ attempted: 0, sent: 0 });
const body = JSON.stringify(command.data);
- let delivered = 0;
+ let attempted = 0;
let ambiguous = false;
for (const doName of doNames) {
const id = env.WS_DO.idFromName(`community-machine:${doName}`);
@@ -64,19 +97,14 @@ export async function handleMachineDiagnostics(
ambiguous = true;
continue;
}
- const result = await response.json() as { sent?: unknown };
- if (!Number.isSafeInteger(result.sent) || (result.sent as number) < 0) {
- ambiguous = true;
- continue;
- }
- delivered += result.sent as number;
+ attempted += parseAttemptedCountReceipt(await response.json());
} catch {
ambiguous = true;
}
}
- if (delivered === 0 && ambiguous) {
+ if (ambiguous) {
return Response.json({ error: "diagnostic delivery ambiguous" }, { status: 503 });
}
- return Response.json({ sent: delivered });
+ return Response.json({ attempted, sent: attempted });
}
diff --git a/src/ws-do/src/ws-durable.ts b/src/ws-do/src/ws-durable.ts
index dd22b7fe5..99e66bd04 100644
--- a/src/ws-do/src/ws-durable.ts
+++ b/src/ws-do/src/ws-durable.ts
@@ -103,6 +103,7 @@ export class WebSocketDurableObject extends DurableObject<Env> {
message,
(parsed) => handleCommunityMachineMessage(
this.domainContext(),
+ ws,
parsed,
{
fanOutMachineUpdated: (userId, machineId) =>
diff --git a/src/ws-do/src/ws-durable/internal.ts b/src/ws-do/src/ws-durable/internal.ts
index 8b4d87fdd..bd861ba1e 100644
--- a/src/ws-do/src/ws-durable/internal.ts
+++ b/src/ws-do/src/ws-durable/internal.ts
@@ -15,6 +15,9 @@ export type ConnectionState =
machineId: string
userId: string
authenticated: boolean
+ controlHeartbeat?: boolean
+ lastHeartbeatAckAt?: number
+ pendingHeartbeatNonce?: string
}
export type UserConnectionState = Extract<ConnectionState, { type: "user" }>
@@ -34,6 +37,7 @@ export interface CommunityMachineHandle {
export const IDENTITY_KEY = "community-machine-identity"
export const HANDLE_KEY = "community-machine-handle"
export const RUNTIME_ERROR_KEY = "community-machine-runtime-error"
+export const DIAGNOSTIC_ALARM_HINT_KEY = "community-machine-diagnostic-alarm-hint"
const RESTART_PENDING_PREFIX = "reset-pending:"
export const restartPendingKey = (launchId: string) => RESTART_PENDING_PREFIX + launchId
diff --git a/src/ws-do/src/ws-durable/machine-control.test.ts b/src/ws-do/src/ws-durable/machine-control.test.ts
index afd555800..44e32e8e8 100644
--- a/src/ws-do/src/ws-durable/machine-control.test.ts
+++ b/src/ws-do/src/ws-durable/machine-control.test.ts
@@ -291,7 +291,7 @@ describe("WebSocketDurableObject", () => {
body,
}))
- await expect(response.json()).resolves.toEqual({ sent: 1 })
+ await expect(response.json()).resolves.toEqual({ attempted: 1, sent: 1 })
expect(ws.send).toHaveBeenCalledWith(body)
})
@@ -320,11 +320,34 @@ describe("WebSocketDurableObject", () => {
}))
expect(response.status).toBe(200)
- await expect(response.json()).resolves.toEqual({ sent: 1 })
+ await expect(response.json()).resolves.toEqual({ attempted: 1, sent: 1 })
expect(ws.send).toHaveBeenCalledOnce()
expect(JSON.parse(ws.send.mock.calls[0]![0] as string)).toEqual(payload)
})
+ it("registers a strict machine-keyed diagnostic deadline without forwarding a command", async () => {
+ vi.spyOn(Date, "now").mockReturnValue(1_700_000_000_000)
+ const { durable, getWebSockets, store, storage } = createDO()
+ getWebSockets.mockReturnValue([])
+
+ const response = await durable.fetch(new Request(
+ "http://internal/register-diagnostic-deadline?machineId=cm_1",
+ {
+ method: "POST",
+ headers: { "content-type": "application/json" },
+ body: JSON.stringify({ deadlineAt: 1_700_087_000_000 }),
+ },
+ ))
+
+ expect(response.status).toBe(200)
+ await expect(response.json()).resolves.toEqual({ registered: true })
+ expect(store.get("community-machine-diagnostic-alarm-hint")).toEqual({
+ machineId: "cm_1",
+ deadlineAt: 1_700_087_000_000,
+ })
+ expect(storage.setAlarm).toHaveBeenCalledWith(1_700_087_000_000)
+ })
+
it.each([
["malformed JSON", "{"],
["missing field", JSON.stringify({ type: "diagnostics:collect", agentId: "bot_1" })],
diff --git a/src/ws-do/src/ws-durable/machine-control.ts b/src/ws-do/src/ws-durable/machine-control.ts
index 72535cb04..3054f30bd 100644
--- a/src/ws-do/src/ws-durable/machine-control.ts
+++ b/src/ws-do/src/ws-durable/machine-control.ts
@@ -12,7 +12,7 @@ import type {
RestartAttribution,
WsDurableContext,
} from "./internal"
-import { scheduleHeartbeatAlarm } from "./machine-lifecycle"
+import { registerDiagnosticDeadline, scheduleHeartbeatAlarm } from "./machine-lifecycle"
export type MachineControlHooks = Readonly<{
recordPendingRestarts: (body: string) => Promise<void>
@@ -92,9 +92,33 @@ export async function handleMachineControlFetch(
if (url.pathname === "/forward-agent-wake" && request.method === "POST") {
const body = await request.text()
- const sent = forwardToCommunityMachine(context, body)
+ const attempted = forwardToCommunityMachine(context, body)
await hooks.clearRuntimeErrorOverlay().catch(() => { })
- return new Response(JSON.stringify({ sent }), {
+ return new Response(JSON.stringify({ attempted, sent: attempted }), {
+ headers: { "Content-Type": "application/json" },
+ })
+ }
+
+ if (url.pathname === "/register-diagnostic-deadline" && request.method === "POST") {
+ let raw: unknown
+ try {
+ raw = await request.json()
+ } catch {
+ return new Response(JSON.stringify({ registered: false }), { status: 400 })
+ }
+ const machineId = url.searchParams.get("machineId")
+ const record = raw && typeof raw === "object" ? raw as Record<string, unknown> : null
+ if (
+ !machineId ||
+ !record ||
+ Object.keys(record).length !== 1 ||
+ !Number.isSafeInteger(record.deadlineAt) ||
+ (record.deadlineAt as number) < 0
+ ) {
+ return new Response(JSON.stringify({ registered: false }), { status: 400 })
+ }
+ await registerDiagnosticDeadline(context, machineId, record.deadlineAt as number)
+ return new Response(JSON.stringify({ registered: true }), {
headers: { "Content-Type": "application/json" },
})
}
@@ -104,14 +128,14 @@ export async function handleMachineControlFetch(
try {
raw = await request.json()
} catch {
- return new Response(JSON.stringify({ sent: 0 }), { status: 400 })
+ return new Response(JSON.stringify({ attempted: 0 }), { status: 400 })
}
const command = DiagnosticCollectCommandSchema.safeParse(raw)
if (!command.success) {
- return new Response(JSON.stringify({ sent: 0 }), { status: 400 })
+ return new Response(JSON.stringify({ attempted: 0 }), { status: 400 })
}
- const sent = forwardToCommunityMachine(context, JSON.stringify(command.data))
- return new Response(JSON.stringify({ sent }), {
+ const attempted = forwardToCommunityMachine(context, JSON.stringify(command.data))
+ return new Response(JSON.stringify({ attempted, sent: attempted }), {
headers: { "Content-Type": "application/json" },
})
}
diff --git a/src/ws-do/src/ws-durable/machine-frames.test.ts b/src/ws-do/src/ws-durable/machine-frames.test.ts
index e76554864..91715e40a 100644
--- a/src/ws-do/src/ws-durable/machine-frames.test.ts
+++ b/src/ws-do/src/ws-durable/machine-frames.test.ts
@@ -183,6 +183,37 @@ describe("WebSocketDurableObject", () => {
})
})
+ it.each([
+ ["legacy ready", undefined, false],
+ ["heartbeat-capable ready", ["control-heartbeat-v1"], true],
+ ] as const)("capability-gates the app heartbeat lease for %s", async (_name, capabilities, expected) => {
+ vi.spyOn(Date, "now").mockReturnValue(9_000_000)
+ const { durable, store } = createDO()
+ store.set("community-machine-identity", {
+ userId: "u_1",
+ machineId: "cm_1",
+ credentialHash: "hash",
+ })
+ const ws = createMockWebSocket()
+ ws.serializeAttachment({
+ type: "community-machine",
+ machineId: "cm_1",
+ userId: "u_1",
+ authenticated: true,
+ })
+
+ await durable.webSocketMessage(ws as any, JSON.stringify({
+ type: "ready",
+ runtimeReport: [],
+ runningAgents: [],
+ ...(capabilities ? { capabilities } : {}),
+ }))
+
+ expect(ws._attachment).toMatchObject({ controlHeartbeat: expected })
+ if (expected) expect((ws._attachment as any).lastHeartbeatAckAt).toBe(9_000_000)
+ else expect((ws._attachment as any).lastHeartbeatAckAt).toBeUndefined()
+ })
+
it("silently drops a wrapped `{ready:{...}}` frame (legacy shape — regression guard)", async () => {
const { durable, store } = createDO()
store.set("community-machine-identity", {
@@ -214,6 +245,35 @@ describe("WebSocketDurableObject", () => {
})
})
+ describe("community-machine — control-plane receipt scope", () => {
+ it("consumes but does not accept a diagnostics receipt from a mismatched machine socket", async () => {
+ const { durable, store } = createDO()
+ store.set("community-machine-identity", {
+ userId: "u_1",
+ machineId: "cm_1",
+ credentialHash: "hash",
+ })
+ const ws = createMockWebSocket()
+ ws.serializeAttachment({
+ type: "community-machine",
+ machineId: "cm_other",
+ userId: "u_1",
+ authenticated: true,
+ })
+ mockLogDebug.mockClear()
+
+ await durable.webSocketMessage(ws as any, JSON.stringify({
+ type: "diagnostics_ack",
+ reportId: "dbr_0123456789abcdef",
+ }))
+
+ expect(mockLogDebug).not.toHaveBeenCalledWith(
+ "diagnostics command receipted",
+ expect.anything(),
+ )
+ })
+ })
+
describe("community-machine — ready frame reconciles bot activity", () => {
beforeEach(() => {
mockUpsertMachineByMachineId.mockReset()
diff --git a/src/ws-do/src/ws-durable/machine-frames.ts b/src/ws-do/src/ws-durable/machine-frames.ts
index 05538b8dc..0461fe053 100644
--- a/src/ws-do/src/ws-durable/machine-frames.ts
+++ b/src/ws-do/src/ws-durable/machine-frames.ts
@@ -1,6 +1,10 @@
import { IDENTITY_KEY } from "./internal"
import type { CommunityMachineIdentity, WsDurableContext } from "./internal"
import { handleReadyFrame } from "./machine-ready"
+import {
+ handleDiagnosticCommandAck,
+ handleMachineHeartbeatAck,
+} from "./machine-lifecycle"
import {
handleAgentCommandAck,
handleAgentSessionFrame,
@@ -16,6 +20,7 @@ import {
export async function handleCommunityMachineMessage(
context: WsDurableContext,
+ ws: WebSocket,
parsed: unknown,
hooks: MachineRestartHooks,
): Promise<void> {
@@ -25,6 +30,9 @@ export async function handleCommunityMachineMessage(
return
}
+ if (await handleMachineHeartbeatAck(context, ws, parsed, identity)) return
+ if (handleDiagnosticCommandAck(context, ws, parsed, identity)) return
+
// The router is intentionally first-match. Keep identity before every
// frame and preserve ack → session error → activity → typing → typing stop
// → agent session → audit → strict ready. A recognized frame is consumed
@@ -70,5 +78,6 @@ export async function handleCommunityMachineMessage(
context,
parsed,
identity,
+ ws,
)
}
diff --git a/src/ws-do/src/ws-durable/machine-lifecycle.test.ts b/src/ws-do/src/ws-durable/machine-lifecycle.test.ts
index fac7a3866..896b78268 100644
--- a/src/ws-do/src/ws-durable/machine-lifecycle.test.ts
+++ b/src/ws-do/src/ws-durable/machine-lifecycle.test.ts
@@ -45,6 +45,8 @@ import {
mockToSummary,
mockTouchBotRefreshContext,
mockTouchMachineHeartbeat,
+ mockTimeoutPendingDiagnosticReportsForMachine,
+ mockGetNextPendingDiagnosticDeadlineForMachine,
mockUpdateProfile,
mockUpsertMachineByMachineId,
resetHarness
@@ -98,8 +100,9 @@ describe("WebSocketDurableObject", () => {
expect(store.has("community-machine-handle")).toBe(false)
})
- it("null return (credential revoked or already offline) does NOT broadcast and leaves the alarm armed", async () => {
+ it("revoked credential on normal close is terminal local cleanup without a broadcast", async () => {
const { durable, store, ctx } = createDO()
+ store.set("community-machine-handle", { userId: "u_1", machineId: "cm_1" })
store.set("community-machine-identity", {
userId: "u_1",
machineId: "cm_1",
@@ -116,16 +119,17 @@ describe("WebSocketDurableObject", () => {
})
mockStubFetch.mockClear()
+ ;(ctx.storage.setAlarm as any).mockClear?.()
+ ;(ctx.storage.deleteAlarm as any).mockClear?.()
await durable.webSocketClose(ws as any)
expect(mockMarkMachineOffline).toHaveBeenCalledTimes(1)
// No broadcast fired — the guarded UPDATE returned zero rows.
expect(mockStubFetch).not.toHaveBeenCalled()
- // Alarm armed as the safety-net fallback (setAlarm was called; not deleted).
- expect(ctx.storage.setAlarm).toHaveBeenCalled()
+ expect(ctx.storage.setAlarm).not.toHaveBeenCalled()
expect(ctx.storage.deleteAlarm).not.toHaveBeenCalled()
- // Storage keys retained — a different DO instance may own the row now.
- expect(store.has("community-machine-identity")).toBe(true)
+ expect(store.has("community-machine-handle")).toBe(false)
+ expect(store.has("community-machine-identity")).toBe(false)
})
it("missing IDENTITY_KEY (never fully accepted) is a clean no-op — no markMachineOffline, no alarm", async () => {
@@ -289,6 +293,143 @@ describe("WebSocketDurableObject", () => {
})
describe("community-machine — alarm presence + backfill", () => {
+ it("capability-gated heartbeat requires the matching nonce and ignores duplicate/stale acks", async () => {
+ vi.spyOn(Date, "now").mockReturnValue(10_000_000)
+ const { durable, store, getWebSockets } = createDO()
+ store.set("community-machine-handle", { userId: "u_1", machineId: "cm_1" })
+ store.set("community-machine-identity", {
+ userId: "u_1",
+ machineId: "cm_1",
+ credentialHash: "hash",
+ })
+ const ws = createMockWebSocket()
+ ws.serializeAttachment({
+ type: "community-machine",
+ machineId: "cm_1",
+ userId: "u_1",
+ authenticated: true,
+ controlHeartbeat: true,
+ lastHeartbeatAckAt: 9_940_000,
+ })
+ getWebSockets.mockReturnValue([ws])
+
+ await durable.alarm()
+
+ expect(ws.send).toHaveBeenCalledOnce()
+ const heartbeat = JSON.parse(ws.send.mock.calls[0]![0] as string) as {
+ type: string
+ nonce: string
+ }
+ expect(heartbeat.type).toBe("machine:heartbeat")
+ expect((ws._attachment as any).pendingHeartbeatNonce).toBe(heartbeat.nonce)
+
+ await durable.webSocketMessage(ws as any, JSON.stringify({
+ type: "machine_heartbeat_ack",
+ nonce: "stale-nonce",
+ }))
+ expect((ws._attachment as any).pendingHeartbeatNonce).toBe(heartbeat.nonce)
+
+ await durable.webSocketMessage(ws as any, JSON.stringify({
+ type: "machine_heartbeat_ack",
+ nonce: heartbeat.nonce,
+ }))
+ expect(ws._attachment).toMatchObject({
+ lastHeartbeatAckAt: 10_000_000,
+ pendingHeartbeatNonce: undefined,
+ })
+
+ await durable.webSocketMessage(ws as any, JSON.stringify({
+ type: "machine_heartbeat_ack",
+ nonce: heartbeat.nonce,
+ }))
+ expect((ws._attachment as any).lastHeartbeatAckAt).toBe(10_000_000)
+ })
+
+ it("expires a retained capability socket after the app-level lease and flips D1 once", async () => {
+ vi.spyOn(Date, "now").mockReturnValue(20_000_000)
+ const { durable, store, getWebSockets } = createDO()
+ store.set("community-machine-handle", { userId: "u_1", machineId: "cm_1" })
+ store.set("community-machine-identity", {
+ userId: "u_1",
+ machineId: "cm_1",
+ credentialHash: "hash",
+ })
+ const ws = createMockWebSocket()
+ ws.serializeAttachment({
+ type: "community-machine",
+ machineId: "cm_1",
+ userId: "u_1",
+ authenticated: true,
+ controlHeartbeat: true,
+ lastHeartbeatAckAt: 19_800_000,
+ })
+ getWebSockets.mockReturnValue([ws])
+ mockMarkMachineOffline.mockResolvedValueOnce({
+ id: "cm_1",
+ userId: "u_1",
+ status: "offline",
+ lastSeenAt: "now",
+ })
+
+ await durable.alarm()
+
+ expect(ws.close).toHaveBeenCalledWith(1011, "Heartbeat lease expired")
+ expect(mockTouchMachineHeartbeat).not.toHaveBeenCalled()
+ expect(mockMarkMachineOffline).toHaveBeenCalledOnce()
+ expect(store.has("community-machine-handle")).toBe(false)
+ expect(store.has("community-machine-identity")).toBe(false)
+ })
+
+ it("forgets an expired retained socket whose credential no longer owns the D1 row", async () => {
+ vi.spyOn(Date, "now").mockReturnValue(20_000_000)
+ const { durable, store, getWebSockets } = createDO()
+ store.set("community-machine-handle", { userId: "u_1", machineId: "cm_1" })
+ store.set("community-machine-identity", {
+ userId: "u_1",
+ machineId: "cm_1",
+ credentialHash: "revoked-credential",
+ })
+ const ws = createMockWebSocket()
+ ws.serializeAttachment({
+ type: "community-machine",
+ machineId: "cm_1",
+ userId: "u_1",
+ authenticated: true,
+ controlHeartbeat: true,
+ lastHeartbeatAckAt: 19_800_000,
+ })
+ getWebSockets.mockReturnValue([ws])
+ mockMarkMachineOffline.mockResolvedValueOnce(null)
+ mockStubFetch.mockClear()
+
+ await durable.alarm()
+
+ expect(ws.close).toHaveBeenCalledWith(1011, "Heartbeat lease expired")
+ expect(mockMarkMachineOffline).toHaveBeenCalledOnce()
+ expect(mockStubFetch).not.toHaveBeenCalled()
+ expect(store.has("community-machine-handle")).toBe(false)
+ expect(store.has("community-machine-identity")).toBe(false)
+ })
+
+ it("machine-keyed deadline owner expires D1 without a socket, poll, or daemon reconnect", async () => {
+ vi.spyOn(Date, "now").mockReturnValue(30_000_000)
+ const { durable, store, getWebSockets } = createDO()
+ getWebSockets.mockReturnValue([])
+ store.set("community-machine-diagnostic-alarm-hint", {
+ machineId: "cm_1",
+ deadlineAt: 29_999_999,
+ })
+ mockGetNextPendingDiagnosticDeadlineForMachine.mockResolvedValueOnce(null)
+
+ await durable.alarm()
+
+ expect(mockTimeoutPendingDiagnosticReportsForMachine).toHaveBeenCalledWith({}, {
+ machineId: "cm_1",
+ nowMs: 30_000_000,
+ })
+ expect(store.has("community-machine-diagnostic-alarm-hint")).toBe(false)
+ })
+
it("live-WS + status=offline row: markMachineOnlineIfOffline flips it back online and broadcasts (post-deploy backfill)", async () => {
const { durable, store, getWebSockets, ctx } = createDO()
store.set("community-machine-identity", {
@@ -395,11 +536,9 @@ describe("WebSocketDurableObject", () => {
expect(mockStubFetch).toHaveBeenCalled()
expect(store.has("community-machine-handle")).toBe(false)
expect(store.has("community-machine-identity")).toBe(false)
- // No further alarm reschedule after the terminal offline flip.
- // (setAlarm may have been called on the earlier setup path; we assert
- // deleteAlarm was NOT called since alarm() doesn't need to explicitly
- // delete — it just doesn't reschedule.)
- expect(ctx.storage.deleteAlarm).not.toHaveBeenCalled()
+ // Combined machine/diagnostics scheduling explicitly clears the alarm
+ // after the terminal offline flip leaves no remaining work.
+ expect(ctx.storage.deleteAlarm).toHaveBeenCalled()
})
it("no-live-WS + stale row + no identity (mid-lifecycle wipe): still broadcasts offline using stored handle so UI sees the transition", async () => {
@@ -430,7 +569,7 @@ describe("WebSocketDurableObject", () => {
expect(mockStubFetch).toHaveBeenCalled()
// Storage keys dropped — this DO's presence lifecycle is done.
expect(store.has("community-machine-handle")).toBe(false)
- expect(ctx.storage.deleteAlarm).not.toHaveBeenCalled()
+ expect(ctx.storage.deleteAlarm).toHaveBeenCalled()
})
it("no-live-WS + fresh row: reschedules alarm to exact stale moment, no broadcast, no DB flip", async () => {
@@ -557,7 +696,7 @@ describe("WebSocketDurableObject", () => {
expect(storage.setAlarm).toHaveBeenCalledWith(5_060_000)
})
- it("cleans lifecycle storage when a stale offline flip rejects", async () => {
+ it("retains lifecycle storage and retries when a stale offline flip rejects", async () => {
vi.spyOn(Date, "now").mockReturnValue(6_000_000)
const { durable, getWebSockets, store, storage } = createDO()
getWebSockets.mockReturnValue([])
@@ -577,11 +716,10 @@ describe("WebSocketDurableObject", () => {
mockMarkMachineOffline.mockRejectedValueOnce(new Error("d1 unavailable"))
await expect(durable.alarm()).resolves.toBeUndefined()
- expect(storage.delete.mock.calls).toEqual([
- ["community-machine-handle"],
- ["community-machine-identity"],
- ])
- expect(storage.setAlarm).not.toHaveBeenCalled()
+ expect(storage.delete).not.toHaveBeenCalled()
+ expect(store.has("community-machine-handle")).toBe(true)
+ expect(store.has("community-machine-identity")).toBe(true)
+ expect(storage.setAlarm).toHaveBeenCalledWith(6_060_000)
})
it("does not broadcast a stale null flip and still cleans lifecycle storage", async () => {
diff --git a/src/ws-do/src/ws-durable/machine-lifecycle.ts b/src/ws-do/src/ws-durable/machine-lifecycle.ts
index 0adde6d26..cdfef1167 100644
--- a/src/ws-do/src/ws-durable/machine-lifecycle.ts
+++ b/src/ws-do/src/ws-durable/machine-lifecycle.ts
@@ -1,11 +1,17 @@
import {
COMMUNITY_MACHINE_HEARTBEAT_MS,
COMMUNITY_MACHINE_OFFLINE_THRESHOLD_MS,
+ DiagnosticCommandAckMessageSchema,
+ MachineHeartbeatAckMessageSchema,
WS_EVENTS,
createDb,
queries,
} from "@alook/shared"
-import { HANDLE_KEY, IDENTITY_KEY } from "./internal"
+import {
+ DIAGNOSTIC_ALARM_HINT_KEY,
+ HANDLE_KEY,
+ IDENTITY_KEY,
+} from "./internal"
import type {
CommunityMachineConnectionState,
CommunityMachineHandle,
@@ -15,11 +21,143 @@ import type {
} from "./internal"
import { notifyUserDO } from "./presence-typing"
+type DiagnosticAlarmHint = Readonly<{ machineId: string; deadlineAt: number }>
+
+function authenticatedMachineSockets(context: WsDurableContext): WebSocket[] {
+ return context.ctx.getWebSockets().filter((socket) => {
+ const state = socket.deserializeAttachment() as ConnectionState
+ return state?.type === "community-machine" && state.authenticated
+ })
+}
+
export async function scheduleHeartbeatAlarm(context: WsDurableContext): Promise<void> {
+ const now = Date.now()
+ const hint = await context.ctx.storage.get<DiagnosticAlarmHint>(DIAGNOSTIC_ALARM_HINT_KEY)
+ const handle = await context.ctx.storage.get<CommunityMachineHandle>(HANDLE_KEY)
+ const identity = await context.ctx.storage.get<CommunityMachineIdentity>(IDENTITY_KEY)
+ const hasMachineSocket = authenticatedMachineSockets(context).length > 0
+ const candidates: number[] = []
+ if (hasMachineSocket || handle || identity) candidates.push(now + COMMUNITY_MACHINE_HEARTBEAT_MS)
+ if (hint) candidates.push(Math.max(now, hint.deadlineAt))
+ if (candidates.length === 0) {
+ await context.ctx.storage.deleteAlarm()
+ return
+ }
+ const want = Math.min(...candidates)
const current = await context.ctx.storage.getAlarm()
- const want = Date.now() + COMMUNITY_MACHINE_HEARTBEAT_MS
- if (current == null || current > want) {
- await context.ctx.storage.setAlarm(want)
+ if (current == null || current > want) await context.ctx.storage.setAlarm(want)
+}
+
+export async function registerDiagnosticDeadline(
+ context: WsDurableContext,
+ machineId: string,
+ deadlineAt: number,
+): Promise<void> {
+ const current = await context.ctx.storage.get<DiagnosticAlarmHint>(DIAGNOSTIC_ALARM_HINT_KEY)
+ if (!current || current.machineId !== machineId || deadlineAt < current.deadlineAt) {
+ await context.ctx.storage.put<DiagnosticAlarmHint>(DIAGNOSTIC_ALARM_HINT_KEY, {
+ machineId,
+ deadlineAt,
+ })
+ }
+ await scheduleHeartbeatAlarm(context)
+}
+
+export async function handleMachineHeartbeatAck(
+ _context: WsDurableContext,
+ ws: WebSocket,
+ parsed: unknown,
+ identity: CommunityMachineIdentity,
+): Promise<boolean> {
+ const result = MachineHeartbeatAckMessageSchema.safeParse(parsed)
+ if (!result.success) return false
+ const state = ws.deserializeAttachment() as ConnectionState
+ if (
+ state?.type !== "community-machine" ||
+ !state.authenticated ||
+ state.userId !== identity.userId ||
+ state.machineId !== identity.machineId ||
+ state.controlHeartbeat !== true ||
+ state.pendingHeartbeatNonce !== result.data.nonce
+ ) return true
+ ws.serializeAttachment({
+ ...state,
+ lastHeartbeatAckAt: Date.now(),
+ pendingHeartbeatNonce: undefined,
+ } satisfies CommunityMachineConnectionState)
+ return true
+}
+
+export function handleDiagnosticCommandAck(
+ context: WsDurableContext,
+ ws: WebSocket,
+ parsed: unknown,
+ identity: CommunityMachineIdentity,
+): boolean {
+ const result = DiagnosticCommandAckMessageSchema.safeParse(parsed)
+ if (!result.success) return false
+ const state = ws.deserializeAttachment() as ConnectionState
+ if (
+ state?.type !== "community-machine" ||
+ !state.authenticated ||
+ state.userId !== identity.userId ||
+ state.machineId !== identity.machineId
+ ) return true
+ context.log.debug("diagnostics command receipted", {
+ machineId: identity.machineId,
+ reportId: result.data.reportId,
+ })
+ return true
+}
+
+async function markIdentityOffline(
+ context: WsDurableContext,
+ identity: CommunityMachineIdentity,
+): Promise<boolean> {
+ const db = createDb(context.env.DB)
+ const flipped = await queries.communityMachine.markMachineOffline(db, {
+ userId: identity.userId,
+ machineId: identity.machineId,
+ credentialHash: identity.credentialHash,
+ })
+ if (!flipped) return false
+ await notifyUserDO(context, identity.userId, {
+ type: WS_EVENTS.MACHINE_STATUS,
+ machineId: identity.machineId,
+ status: "offline",
+ lastSeenAt: flipped.lastSeenAt ?? new Date().toISOString(),
+ }).catch(() => { })
+ await context.ctx.storage.delete(HANDLE_KEY)
+ await context.ctx.storage.delete(IDENTITY_KEY)
+ return true
+}
+
+async function sweepDiagnosticDeadlines(context: WsDurableContext, nowMs: number): Promise<void> {
+ const hint = await context.ctx.storage.get<DiagnosticAlarmHint>(DIAGNOSTIC_ALARM_HINT_KEY)
+ if (!hint || hint.deadlineAt > nowMs) return
+ try {
+ const db = createDb(context.env.DB)
+ await queries.communityDiagnosticReport.timeoutPendingDiagnosticReportsForMachine(db, {
+ machineId: hint.machineId,
+ nowMs,
+ })
+ const next = await queries.communityDiagnosticReport.getNextPendingDiagnosticDeadlineForMachine(db, {
+ machineId: hint.machineId,
+ })
+ if (next == null) await context.ctx.storage.delete(DIAGNOSTIC_ALARM_HINT_KEY)
+ else await context.ctx.storage.put<DiagnosticAlarmHint>(DIAGNOSTIC_ALARM_HINT_KEY, {
+ machineId: hint.machineId,
+ deadlineAt: next,
+ })
+ } catch (err) {
+ context.log.warn("diagnostics deadline sweep failed", {
+ machineId: hint.machineId,
+ err: String(err),
+ })
+ await context.ctx.storage.put<DiagnosticAlarmHint>(DIAGNOSTIC_ALARM_HINT_KEY, {
+ machineId: hint.machineId,
+ deadlineAt: nowMs + COMMUNITY_MACHINE_HEARTBEAT_MS,
+ })
}
}
@@ -30,60 +168,62 @@ export async function handleCommunityMachineClose(
): Promise<void> {
context.log.info("community machine websocket closed", { machineId: state.machineId, userId: state.userId })
const identity = await context.ctx.storage.get<CommunityMachineIdentity>(IDENTITY_KEY)
- if (!identity) return
- const replacementIsLive = context.ctx.getWebSockets().some((socket) => {
+ if (!identity) {
+ await scheduleHeartbeatAlarm(context)
+ return
+ }
+ const replacementIsLive = authenticatedMachineSockets(context).some((socket) => {
if (socket === closingSocket) return false
- const candidate = socket.deserializeAttachment() as ConnectionState
- return candidate?.type === "community-machine"
- && candidate.authenticated
- && candidate.userId === state.userId
- && candidate.machineId === state.machineId
+ const candidate = socket.deserializeAttachment() as CommunityMachineConnectionState
+ return candidate.userId === state.userId && candidate.machineId === state.machineId
})
if (replacementIsLive) return
try {
- const db = createDb(context.env.DB)
- const flipped = await queries.communityMachine.markMachineOffline(db, {
- userId: identity.userId,
- machineId: identity.machineId,
- credentialHash: identity.credentialHash,
- })
- if (flipped) {
- await notifyUserDO(context, identity.userId, {
- type: WS_EVENTS.MACHINE_STATUS,
- machineId: identity.machineId,
- status: "offline",
- lastSeenAt: flipped.lastSeenAt ?? new Date().toISOString(),
- }).catch(() => { })
- await context.ctx.storage.deleteAlarm()
+ const flipped = await markIdentityOffline(context, identity)
+ if (!flipped) {
+ // The guarded row belongs to a newer credential (or is already
+ // offline), so this close is terminal for the old DO identity.
await context.ctx.storage.delete(HANDLE_KEY)
await context.ctx.storage.delete(IDENTITY_KEY)
- } else {
- await context.ctx.storage.setAlarm(Date.now() + COMMUNITY_MACHINE_OFFLINE_THRESHOLD_MS)
+ return
}
} catch (err) {
context.log.warn("markMachineOffline failed on webSocketClose", { err: String(err) })
await context.ctx.storage.setAlarm(Date.now() + COMMUNITY_MACHINE_OFFLINE_THRESHOLD_MS)
}
+ await scheduleHeartbeatAlarm(context)
}
export async function handleMachineAlarm(context: WsDurableContext): Promise<void> {
- const sockets = context.ctx.getWebSockets()
+ const now = Date.now()
+ await sweepDiagnosticDeadlines(context, now)
+
const liveMachines: Array<{ userId: string; machineId: string }> = []
- for (const ws of sockets) {
- const s = ws.deserializeAttachment() as ConnectionState
- if (s?.type === "community-machine" && s.authenticated) {
- liveMachines.push({ userId: s.userId, machineId: s.machineId })
+ let expiredCapableSocket = false
+ for (const ws of authenticatedMachineSockets(context)) {
+ const state = ws.deserializeAttachment() as CommunityMachineConnectionState
+ if (state.controlHeartbeat === true) {
+ const lastAckAt = state.lastHeartbeatAckAt ?? 0
+ if (now - lastAckAt >= COMMUNITY_MACHINE_OFFLINE_THRESHOLD_MS) {
+ expiredCapableSocket = true
+ try { ws.close(1011, "Heartbeat lease expired") } catch { }
+ continue
+ }
+ const nonce = state.pendingHeartbeatNonce ?? crypto.randomUUID()
+ ws.serializeAttachment({ ...state, pendingHeartbeatNonce: nonce })
+ try { ws.send(JSON.stringify({ type: "machine:heartbeat", nonce })) } catch { }
}
+ liveMachines.push({ userId: state.userId, machineId: state.machineId })
}
if (liveMachines.length > 0) {
const db = createDb(context.env.DB)
const identity = await context.ctx.storage.get<CommunityMachineIdentity>(IDENTITY_KEY)
- for (const m of liveMachines) {
+ for (const machine of liveMachines) {
try {
- await queries.communityMachine.touchMachineHeartbeat(db, m.userId, m.machineId)
- } catch { /* ok */ }
- if (identity && identity.userId === m.userId && identity.machineId === m.machineId) {
+ await queries.communityMachine.touchMachineHeartbeat(db, machine.userId, machine.machineId)
+ } catch { }
+ if (identity && identity.userId === machine.userId && identity.machineId === machine.machineId) {
try {
const backfilled = await queries.communityMachine.markMachineOnlineIfOffline(db, {
userId: identity.userId,
@@ -99,48 +239,67 @@ export async function handleMachineAlarm(context: WsDurableContext): Promise<voi
}).catch(() => { })
}
} catch (err) {
- context.log.warn("markMachineOnlineIfOffline (alarm live-WS) failed", { err: String(err) })
+ context.log.warn("markMachineOnlineIfOffline (alarm fresh-WS) failed", { err: String(err) })
}
}
}
- await context.ctx.storage.setAlarm(Date.now() + COMMUNITY_MACHINE_HEARTBEAT_MS)
+ await scheduleHeartbeatAlarm(context)
return
}
const stored = await context.ctx.storage.get<CommunityMachineHandle>(HANDLE_KEY)
- if (!stored) return
const identity = await context.ctx.storage.get<CommunityMachineIdentity>(IDENTITY_KEY)
+ if (expiredCapableSocket && identity) {
+ try {
+ const flipped = await markIdentityOffline(context, identity)
+ if (!flipped) {
+ // A guarded no-op means this credential no longer owns the D1 row
+ // (or another actor already won the offline transition). Forget only
+ // this DO's stale lifecycle state; do not broadcast over the current
+ // credential's presence.
+ await context.ctx.storage.delete(HANDLE_KEY)
+ await context.ctx.storage.delete(IDENTITY_KEY)
+ }
+ } catch (err) {
+ context.log.warn("markMachineOffline (heartbeat lease) failed", { err: String(err) })
+ await context.ctx.storage.setAlarm(now + COMMUNITY_MACHINE_HEARTBEAT_MS)
+ return
+ }
+ await scheduleHeartbeatAlarm(context)
+ return
+ }
+ if (!stored) {
+ await scheduleHeartbeatAlarm(context)
+ return
+ }
+
const db = createDb(context.env.DB)
- const machine = await queries.communityMachine.getMachineByIdForUser(
- db,
- stored.userId,
- stored.machineId
- )
+ const machine = await queries.communityMachine.getMachineByIdForUser(db, stored.userId, stored.machineId)
if (!machine) {
await context.ctx.storage.delete(HANDLE_KEY)
await context.ctx.storage.delete(IDENTITY_KEY)
+ await scheduleHeartbeatAlarm(context)
return
}
const lastSeen = machine.lastSeenAt ? Date.parse(machine.lastSeenAt) : 0
- const elapsed = Date.now() - lastSeen
+ const elapsed = now - lastSeen
if (elapsed >= COMMUNITY_MACHINE_OFFLINE_THRESHOLD_MS) {
if (identity) {
+ let flipped: boolean
try {
- const flipped = await queries.communityMachine.markMachineOffline(db, {
- userId: identity.userId,
- machineId: identity.machineId,
- credentialHash: identity.credentialHash,
- })
- if (flipped) {
- await notifyUserDO(context, stored.userId, {
- type: WS_EVENTS.MACHINE_STATUS,
- machineId: stored.machineId,
- status: "offline",
- lastSeenAt: flipped.lastSeenAt ?? new Date().toISOString(),
- }).catch(() => { })
- }
+ flipped = await markIdentityOffline(context, identity)
} catch (err) {
context.log.warn("markMachineOffline (alarm stale-flip) failed", { err: String(err) })
+ await context.ctx.storage.setAlarm(now + COMMUNITY_MACHINE_HEARTBEAT_MS)
+ return
+ }
+ // This is the legacy no-socket stale-row cleanup path. Once its D1
+ // observation is stale, drop the local lifecycle record even when the
+ // guarded flip was already won elsewhere. A thrown update is transient
+ // and retains the identity for the retry path above.
+ if (!flipped) {
+ await context.ctx.storage.delete(HANDLE_KEY)
+ await context.ctx.storage.delete(IDENTITY_KEY)
}
} else {
await notifyUserDO(context, stored.userId, {
@@ -149,10 +308,10 @@ export async function handleMachineAlarm(context: WsDurableContext): Promise<voi
status: "offline",
lastSeenAt: machine.lastSeenAt ?? new Date().toISOString(),
}).catch(() => { })
+ await context.ctx.storage.delete(HANDLE_KEY)
}
- await context.ctx.storage.delete(HANDLE_KEY)
- await context.ctx.storage.delete(IDENTITY_KEY)
+ await scheduleHeartbeatAlarm(context)
return
}
- await context.ctx.storage.setAlarm(Date.now() + (COMMUNITY_MACHINE_OFFLINE_THRESHOLD_MS - elapsed))
+ await context.ctx.storage.setAlarm(now + (COMMUNITY_MACHINE_OFFLINE_THRESHOLD_MS - elapsed))
}
diff --git a/src/ws-do/src/ws-durable/machine-ready.ts b/src/ws-do/src/ws-durable/machine-ready.ts
index b17d90a04..8cb3067d0 100644
--- a/src/ws-do/src/ws-durable/machine-ready.ts
+++ b/src/ws-do/src/ws-durable/machine-ready.ts
@@ -1,4 +1,5 @@
import {
+ CONTROL_HEARTBEAT_CAPABILITY,
createDb,
HostReadyMessageSchema,
queries,
@@ -59,10 +60,26 @@ export async function handleReadyFrame(
context: WsDurableContext,
parsed: unknown,
identity: CommunityMachineIdentity,
+ ws: WebSocket,
): Promise<boolean> {
const readyParse = HostReadyMessageSchema.safeParse(parsed)
if (!readyParse.success) return false
const ready = readyParse.data
+ const state = ws.deserializeAttachment() as import("./internal").ConnectionState
+ if (
+ state?.type === "community-machine" &&
+ state.authenticated &&
+ state.userId === identity.userId &&
+ state.machineId === identity.machineId
+ ) {
+ const controlHeartbeat = (ready.capabilities ?? []).includes(CONTROL_HEARTBEAT_CAPABILITY)
+ ws.serializeAttachment({
+ ...state,
+ controlHeartbeat,
+ ...(controlHeartbeat ? { lastHeartbeatAckAt: Date.now() } : {}),
+ pendingHeartbeatNonce: undefined,
+ })
+ }
try {
const hostname = ready.hostname ?? ""
const platform = ready.platform ?? ""
diff --git a/src/ws-do/src/ws-durable/test-harness.ts b/src/ws-do/src/ws-durable/test-harness.ts
index 7536a448c..f448fb183 100644
--- a/src/ws-do/src/ws-durable/test-harness.ts
+++ b/src/ws-do/src/ws-durable/test-harness.ts
@@ -87,6 +87,8 @@ export const mockUpsertMachineByMachineId = vi.fn()
export const mockTouchMachineHeartbeat = vi.fn()
export const mockMarkMachineOffline = vi.fn()
export const mockMarkMachineOnlineIfOffline = vi.fn()
+export const mockTimeoutPendingDiagnosticReportsForMachine = vi.fn().mockResolvedValue([])
+export const mockGetNextPendingDiagnosticDeadlineForMachine = vi.fn().mockResolvedValue(null)
export const mockGetCoMemberUserIds = vi.fn<(db: unknown, userId: string) => Promise<string[]>>().mockResolvedValue([])
export const mockGetFriendUserIds = vi.fn<(db: unknown, userId: string) => Promise<string[]>>().mockResolvedValue([])
export const mockGetChannelForMember = vi.fn()
@@ -184,12 +186,13 @@ vi.mock("@alook/shared", async () => {
// flat top-level fields.
const HostReadyMessageSchema = {
safeParse(v: unknown) {
- const m = v as { type?: unknown; runtimeReport?: unknown; runningAgents?: unknown }
+ const m = v as { type?: unknown; runtimeReport?: unknown; capabilities?: unknown; runningAgents?: unknown }
if (m?.type !== "ready") return { success: false } as const
if (!Array.isArray(m?.runtimeReport)) return { success: false } as const
const data: Record<string, unknown> = {
type: "ready",
runtimeReport: m.runtimeReport,
+ capabilities: Array.isArray(m.capabilities) ? m.capabilities : [],
runningAgents: Array.isArray(m.runningAgents) ? m.runningAgents : [],
}
for (const k of ["hostname", "platform", "arch", "osRelease", "daemonVersion"]) {
@@ -199,6 +202,22 @@ vi.mock("@alook/shared", async () => {
return { success: true as const, data }
},
}
+ const MachineHeartbeatAckMessageSchema = {
+ safeParse(v: unknown) {
+ const m = v as { type?: unknown; nonce?: unknown }
+ return m?.type === "machine_heartbeat_ack" && typeof m.nonce === "string" && m.nonce.length > 0
+ ? { success: true as const, data: { type: "machine_heartbeat_ack" as const, nonce: m.nonce } }
+ : { success: false as const }
+ },
+ }
+ const DiagnosticCommandAckMessageSchema = {
+ safeParse(v: unknown) {
+ const m = v as { type?: unknown; reportId?: unknown }
+ return m?.type === "diagnostics_ack" && typeof m.reportId === "string" && /^dbr_[A-Za-z0-9_-]+$/.test(m.reportId)
+ ? { success: true as const, data: { type: "diagnostics_ack" as const, reportId: m.reportId } }
+ : { success: false as const }
+ },
+ }
const AgentActivityMessageSchema = {
safeParse(v: unknown) {
const m = v as { type?: unknown; agentId?: unknown; state?: unknown }
@@ -374,6 +393,10 @@ vi.mock("@alook/shared", async () => {
reconcileBotActivityFromRunningAgents: (...a: any[]) =>
mockReconcileBotActivityFromRunningAgents(...(a as [unknown, string, string[]])),
},
+ communityDiagnosticReport: {
+ timeoutPendingDiagnosticReportsForMachine: (...a: any[]) => mockTimeoutPendingDiagnosticReportsForMachine(...a),
+ getNextPendingDiagnosticDeadlineForMachine: (...a: any[]) => mockGetNextPendingDiagnosticDeadlineForMachine(...a),
+ },
communityUserProfile: {
updateProfile: (...a: any[]) =>
mockUpdateProfile(...(a as [unknown, string, { statusEmoji?: string | null; statusText?: string | null }])),
@@ -421,6 +444,9 @@ vi.mock("@alook/shared", async () => {
getUserInternal: (...a: [unknown, string]) => mockGetUserInternal(...a),
},
},
+ CONTROL_HEARTBEAT_CAPABILITY: "control-heartbeat-v1",
+ MachineHeartbeatAckMessageSchema,
+ DiagnosticCommandAckMessageSchema,
}
})
@@ -473,6 +499,8 @@ export function resetHarness() {
mockUpdateProfile.mockResolvedValue({})
mockGetProfile.mockResolvedValue(null)
mockReconcileBotActivityFromRunningAgents.mockResolvedValue([])
+ mockTimeoutPendingDiagnosticReportsForMachine.mockResolvedValue([])
+ mockGetNextPendingDiagnosticDeadlineForMachine.mockResolvedValue(null)
// Daemon-auth binds the token to a machine row for daemonId+workspace;
// default to a present row so auth-flow success tests pass. Tests that
// exercise a missing machine override this.