Skip to content

armgit / alook

Public Imported from GitHub
0

Multi-Agent Collaboration for Team and Friends

fix(daemon): deduplicate wake admissions (#485)

Gustavo Yedfbe43d0f114
diff --git a/src/daemon/src/credentials/credentialProxy.test.ts b/src/daemon/src/credentials/credentialProxy.test.ts
index bd757b493..d1fd434e3 100644
--- a/src/daemon/src/credentials/credentialProxy.test.ts
+++ b/src/daemon/src/credentials/credentialProxy.test.ts
@@ -658,6 +658,54 @@ describe("startCredentialProxy (zero-trust end to end)", () => {
// an inboxPull payload, but the response itself must still be forwarded.
expect(seen).toEqual([]);
});
+
+ it("captures each inbox observation token at pull start even when responses finish out of order", async () => {
+ const pending: http.ServerResponse[] = [];
+ const upstreamServer = http.createServer((_req, res) => pending.push(res));
+ await new Promise<void>((resolve) => upstreamServer.listen(0, "127.0.0.1", resolve));
+ const address = upstreamServer.address();
+ const upstreamUrl = `http://127.0.0.1:${typeof address === "object" && address ? address.port : 0}`;
+ upstreamClose = () => new Promise<void>((resolve) => upstreamServer.close(() => resolve()));
+ const seen: number[] = [];
+ let generation = 0;
+ const broker = new CredentialBroker({ upstreamBaseUrl: upstreamUrl });
+ proxy = await startCredentialProxy(broker, {
+ onInboxPullStart: () => ++generation,
+ onInboxPullResponse: (_agentId, _messages, token) => seen.push(token as number),
+ });
+ const reg = broker.mint("agent-1", "l", ["read"], REAL_KEY);
+
+ const first = post(proxy.url, reg.voucher, "/api/community/users/me/inbox/pull");
+ const second = post(proxy.url, reg.voucher, "/api/community/users/me/inbox/pull");
+ await vi.waitFor(() => expect(pending).toHaveLength(2));
+ pending[1]!.writeHead(200, { "content-type": "application/json" });
+ pending[1]!.end(JSON.stringify({ messages: [{ seq: "#2", channel: "/s#0001/c" }] }));
+ await vi.waitFor(() => expect(seen).toEqual([2]));
+ pending[0]!.writeHead(200, { "content-type": "application/json" });
+ pending[0]!.end(JSON.stringify({ messages: [{ seq: "#1", channel: "/s#0001/c" }] }));
+ await Promise.all([first, second]);
+
+ expect(seen).toEqual([2, 1]);
+ });
+
+ it("keeps a real inbox pull fail-open when onInboxPullStart throws synchronously", async () => {
+ const upstream = await startUpstream();
+ upstreamClose = upstream.close;
+ const broker = new CredentialBroker({ upstreamBaseUrl: upstream.url });
+ proxy = await startCredentialProxy(broker, {
+ onInboxPullStart: () => {
+ throw new Error("observation unavailable");
+ },
+ onInboxPullResponse: vi.fn(),
+ });
+ const reg = broker.mint("agent-1", "l", ["read"], REAL_KEY);
+
+ const response = await post(proxy.url, reg.voucher, "/api/community/users/me/inbox/pull");
+
+ expect(response.status).toBe(200);
+ expect(upstream.seen).toHaveLength(1);
+ expect(upstream.seen[0]?.path).toBe("/api/community/users/me/inbox/pull");
+ });
});
/** A throwaway upstream that never responds — for timeout/leak tests below. */
diff --git a/src/daemon/src/credentials/credentialProxy.ts b/src/daemon/src/credentials/credentialProxy.ts
index bee06411f..c1bab5a7e 100644
--- a/src/daemon/src/credentials/credentialProxy.ts
+++ b/src/daemon/src/credentials/credentialProxy.ts
@@ -315,7 +315,8 @@ export interface CredentialProxyOptions {
* surface the pulled messages — used by the daemon to write timeline entries
* regardless of whether the agent is an in-process stub or a real subprocess.
*/
- onInboxPullResponse?: (agentId: string, messages: Message[]) => void;
+ onInboxPullStart?: (agentId: string) => unknown;
+ onInboxPullResponse?: (agentId: string, messages: Message[], observationToken?: unknown) => void;
/**
* Called once per successfully-authorized upstream proxy request, BEFORE the
* upstream is dispatched. Fires only on `verdict.ok === true` — never on
@@ -530,6 +531,15 @@ export async function startCredentialProxy(
// `/api/inboxPull` verb is deleted (flat-delete step). inboxSnapshot is
// deliberately EXCLUDED (peek, no `{ messages }` to record).
const isInboxPull = onPull && pathname === "/api/community/users/me/inbox/pull";
+ let inboxPullObservationToken: unknown;
+ if (isInboxPull) {
+ try {
+ inboxPullObservationToken = options.onInboxPullStart?.(reg.agentId);
+ } catch {
+ // Observational callback — a generation snapshot failure must never
+ // interrupt the agent's real inbox pull.
+ }
+ }
if (onProxyRequest) {
try {
@@ -595,7 +605,7 @@ export async function startCredentialProxy(
res.end(body);
try {
const parsed = JSON.parse(body.toString()) as { messages?: Message[] };
- if (parsed.messages) onPull(reg.agentId, parsed.messages);
+ if (parsed.messages) onPull(reg.agentId, parsed.messages, inboxPullObservationToken);
} catch { /* best-effort */ }
});
} else {
diff --git a/src/daemon/src/daemon/createDaemon.test.ts b/src/daemon/src/daemon/createDaemon.test.ts
index 209b3e5d7..79b78e4dc 100644
--- a/src/daemon/src/daemon/createDaemon.test.ts
+++ b/src/daemon/src/daemon/createDaemon.test.ts
@@ -227,24 +227,39 @@ describe("createDaemon", () => {
launchId: "launch_2",
unreadNotice: { kind: "unread_notice", channel: "/demo#1234/general", latestSeq: 9 },
}));
- await vi.waitFor(() => expect(deliver).toHaveBeenCalledTimes(3));
+ await new Promise((resolve) => setTimeout(resolve, 0));
runTimer(1);
- expect(deliver).toHaveBeenCalledTimes(3);
+ expect(deliver).toHaveBeenCalledTimes(2);
await arm(10);
- sockets[0]!.emit("message", JSON.stringify({ type: "agent:stop", agentId: "bot_1" }));
+ sockets[0]!.emit("message", JSON.stringify({
+ type: "agent:wake",
+ agentId: "bot_1",
+ config: { version: 1, runtime: "mock", model: { kind: "default" }, mode: { kind: "default" } },
+ launchId: "launch_duplicate",
+ unreadNotice: { kind: "unread_notice", channel: "/demo#1234/general", latestSeq: 9 },
+ }));
+ await new Promise((resolve) => setTimeout(resolve, 0));
runTimer(2);
expect(deliver).toHaveBeenCalledTimes(3);
+ expect(deliver).toHaveBeenLastCalledWith("bot_1", expect.objectContaining({
+ text: expect.stringContaining("/demo#1234/general#10"),
+ }));
await arm(11);
- sockets[0]!.emit("message", JSON.stringify({ type: "bot:removed", botId: "bot_1" }));
+ sockets[0]!.emit("message", JSON.stringify({ type: "agent:stop", agentId: "bot_1" }));
runTimer(3);
expect(deliver).toHaveBeenCalledTimes(3);
await arm(12);
- await daemon.stop();
+ sockets[0]!.emit("message", JSON.stringify({ type: "bot:removed", botId: "bot_1" }));
runTimer(4);
expect(deliver).toHaveBeenCalledTimes(3);
+
+ await arm(13);
+ await daemon.stop();
+ runTimer(5);
+ expect(deliver).toHaveBeenCalledTimes(3);
} finally {
// stop is idempotent enough for the failure path and keeps the loopback
// server from leaking if an assertion above throws early.
diff --git a/src/daemon/src/daemon/createDaemon.ts b/src/daemon/src/daemon/createDaemon.ts
index 0e2db9e2d..f7d732567 100644
--- a/src/daemon/src/daemon/createDaemon.ts
+++ b/src/daemon/src/daemon/createDaemon.ts
@@ -408,7 +408,13 @@ export async function createDaemon(opts: CreateDaemonOptions): Promise<RunningDa
const broker = new CredentialBroker({ upstreamBaseUrl: opts.serverUrl });
const proxy = await startCredentialProxy(broker, {
- onInboxPullResponse: (agentId, messages) => timeline.appendEntryForAgent(agentId, messages),
+ onInboxPullStart: (agentId) => channelRef?.modelSeenGeneration(agentId),
+ onInboxPullResponse: (agentId, messages, observationToken) => {
+ timeline.appendEntryForAgent(agentId, messages);
+ if (typeof observationToken === "number") {
+ channelRef?.recordModelSeen(agentId, messages, observationToken);
+ }
+ },
onMessageReminderArm: (input) =>
reminderSchedulerRef?.arm(input) ?? { armed: false, reason: "reminder_scheduler_unavailable" },
// Bot audit log — Producer B (authoritative for `alook <sub>`). Fires
@@ -908,14 +914,14 @@ export async function createDaemon(opts: CreateDaemonOptions): Promise<RunningDa
handleDiagnosticCommand: opts.handleDiagnosticCommand,
reportDiagnosticFailure: opts.reportDiagnosticFailure,
}));
- // Local reminder bookkeeping observes the original FIFO control stream
- // before AgentRouter registers its listener in router.start(). It never
- // consumes commands; AgentRouter remains the sole server-wake dispatcher.
+ // Local reminder bookkeeping observes real desired-watermark advances even
+ // when an active agent coalesces the wake before AgentRouter. Replayed/old
+ // wakes do not create a second observation or enter agent routing.
+ channel.onWakeDesiredAdvance((cmd) => {
+ reminderSchedulerRef?.observe(cmd.agentId, cmd.unreadNotice.channel, cmd.unreadNotice.latestSeq);
+ });
channel.onCommand((cmd) => {
switch (cmd.type) {
- case "agent:wake":
- reminderSchedulerRef?.observe(cmd.agentId, cmd.unreadNotice.channel, cmd.unreadNotice.latestSeq);
- break;
case "agent:stop":
reminderSchedulerRef?.clearAgent(cmd.agentId);
break;
diff --git a/src/daemon/src/manager/wakeCoordinator.test.ts b/src/daemon/src/manager/wakeCoordinator.test.ts
new file mode 100644
index 000000000..c622b7687
--- /dev/null
+++ b/src/daemon/src/manager/wakeCoordinator.test.ts
@@ -0,0 +1,344 @@
+import { describe, expect, it, vi } from "vitest";
+import type { HostCommand } from "@alook/shared";
+import { WakeCoordinator } from "./wakeCoordinator.js";
+
+type Wake = Extract<HostCommand, { type: "agent:wake" }>;
+
+function wake(agentId: string, channel: string, seq: number, launchId = `${agentId}-${channel}-${seq}`): Wake {
+ return {
+ type: "agent:wake",
+ agentId,
+ launchId,
+ config: {} as Wake["config"],
+ unreadNotice: { kind: "unread_notice", channel, latestSeq: seq },
+ };
+}
+
+const tick = () => new Promise<void>((resolve) => setTimeout(resolve, 0));
+
+describe("WakeCoordinator", () => {
+ it("admits only one active turn per agent across channels", async () => {
+ const coordinator = new WakeCoordinator();
+ let release!: () => void;
+ const blocked = new Promise<void>((resolve) => { release = resolve; });
+ const dispatch = vi.fn(async () => blocked);
+
+ const first = coordinator.run(wake("a1", "/s/c1", 1), dispatch);
+ await tick();
+ await expect(coordinator.run(wake("a1", "/s/c2", 1), dispatch)).resolves.toMatchObject({ state: "suppressed" });
+ expect(dispatch).toHaveBeenCalledTimes(1);
+ release();
+ await expect(first).resolves.toEqual({ state: "accepted" });
+ });
+
+ it("coalesces seq 5 to 6 during active work and pull coverage prevents an idle rewake", async () => {
+ const coordinator = new WakeCoordinator();
+ const dispatch = vi.fn(async () => {});
+ await coordinator.run(wake("a1", "/s/c", 5), dispatch);
+ await coordinator.run(wake("a1", "/s/c", 6), dispatch);
+ expect(dispatch).toHaveBeenCalledTimes(1);
+
+ const generation = coordinator.modelSeenGeneration("a1");
+ expect(coordinator.recordModelSeen("a1", [{ channel: "/s/c", seq: "#6" }], generation)).toBe(true);
+ coordinator.recordAgentActivity("a1", "idle");
+ await tick();
+ expect(dispatch).toHaveBeenCalledTimes(1);
+ });
+
+ it("suppresses an old wake when inbox pull arrived first", async () => {
+ const coordinator = new WakeCoordinator();
+ const dispatch = vi.fn(async () => {});
+ coordinator.recordModelSeen("a1", [{ channel: "/s/c", seq: "#9" }], 0);
+ await expect(coordinator.run(wake("a1", "/s/c", 9), dispatch)).resolves.toEqual({
+ state: "suppressed",
+ coveredSeq: 9,
+ });
+ expect(dispatch).not.toHaveBeenCalled();
+ });
+
+ it("admits a higher seq after pull coverage and idle", async () => {
+ const coordinator = new WakeCoordinator();
+ const dispatch = vi.fn(async () => {});
+ coordinator.recordModelSeen("a1", [{ channel: "/s/c", seq: "#5" }], 0);
+ coordinator.recordAgentActivity("a1", "idle");
+ await expect(coordinator.run(wake("a1", "/s/c", 6), dispatch)).resolves.toEqual({ state: "accepted" });
+ expect(dispatch).toHaveBeenCalledTimes(1);
+ });
+
+ it("re-arms after an error wake ack", async () => {
+ const coordinator = new WakeCoordinator();
+ const dispatch = vi.fn(async () => {});
+ await coordinator.run(wake("a1", "/s/c", 5, "failed"), dispatch);
+ coordinator.recordDeliveryAck("a1", "failed", "error");
+ await coordinator.run(wake("a1", "/s/c", 5, "retry"), dispatch);
+ expect(dispatch).toHaveBeenCalledTimes(2);
+ });
+
+ it("fences a pull that began before stop while preserving committed seen", () => {
+ const coordinator = new WakeCoordinator();
+ coordinator.recordModelSeen("a1", [{ channel: "/s/c", seq: "#3" }], 0);
+ const staleGeneration = coordinator.modelSeenGeneration("a1");
+ coordinator.invalidate("a1", false);
+ expect(coordinator.recordModelSeen("a1", [{ channel: "/s/c", seq: "#8" }], staleGeneration)).toBe(false);
+ expect(coordinator.modelSeenGeneration("a1")).toBe(1);
+ });
+
+ it("keeps different agents independent", async () => {
+ const coordinator = new WakeCoordinator();
+ let release!: () => void;
+ const blocked = new Promise<void>((resolve) => { release = resolve; });
+ const dispatchA = vi.fn(async () => blocked);
+ const dispatchB = vi.fn(async () => {});
+ const first = coordinator.run(wake("a1", "/s/c", 1), dispatchA);
+ await tick();
+ await expect(coordinator.run(wake("a2", "/s/c", 1), dispatchB)).resolves.toEqual({ state: "accepted" });
+ expect(dispatchB).toHaveBeenCalledOnce();
+ release();
+ await first;
+ });
+
+ it("admits the same desired vector only once across repeated idle transitions", async () => {
+ const coordinator = new WakeCoordinator();
+ const dispatch = vi.fn(async () => {});
+ await coordinator.run(wake("a1", "/s/c", 5), dispatch);
+ await coordinator.run(wake("a1", "/s/c", 6), dispatch);
+ coordinator.recordAgentActivity("a1", "idle");
+ await tick();
+ expect(dispatch).toHaveBeenCalledTimes(2);
+ coordinator.recordAgentActivity("a1", "running");
+ coordinator.recordAgentActivity("a1", "idle");
+ await tick();
+ expect(dispatch).toHaveBeenCalledTimes(2);
+ });
+
+ it("suppresses a same-seq duplicate after compensation admission", async () => {
+ const coordinator = new WakeCoordinator();
+ const dispatch = vi.fn(async () => {});
+ await coordinator.run(wake("a1", "/s/c", 5), dispatch);
+ await coordinator.run(wake("a1", "/s/c", 6), dispatch);
+ coordinator.recordAgentActivity("a1", "idle");
+ await tick();
+
+ await expect(coordinator.run(wake("a1", "/s/c", 6, "duplicate"), dispatch)).resolves.toEqual({
+ state: "suppressed",
+ coveredSeq: 6,
+ });
+ expect(dispatch).toHaveBeenCalledTimes(2);
+ });
+
+ it("does not re-admit a pending vector after an unrelated pull", async () => {
+ const coordinator = new WakeCoordinator();
+ const dispatch = vi.fn(async () => {});
+ await coordinator.run(wake("a1", "/s/c1", 1), dispatch);
+ await coordinator.run(wake("a1", "/s/c1", 2), dispatch);
+ coordinator.recordAgentActivity("a1", "idle");
+ await tick();
+ expect(dispatch).toHaveBeenCalledTimes(2);
+
+ const generation = coordinator.modelSeenGeneration("a1");
+ coordinator.recordModelSeen("a1", [{ channel: "/s/unrelated", seq: "#9" }], generation);
+ coordinator.recordAgentActivity("a1", "running");
+ coordinator.recordAgentActivity("a1", "idle");
+ await tick();
+ expect(dispatch).toHaveBeenCalledTimes(2);
+ });
+
+ it("covers every pending channel in one agent admission", async () => {
+ const coordinator = new WakeCoordinator();
+ const dispatch = vi.fn(async () => {});
+ await coordinator.run(wake("a1", "/s/c0", 1), dispatch);
+ await coordinator.run(wake("a1", "/s/c1", 5), dispatch);
+ await coordinator.run(wake("a1", "/s/c2", 7), dispatch);
+
+ coordinator.recordAgentActivity("a1", "idle");
+ await tick();
+ expect(dispatch).toHaveBeenCalledTimes(2);
+ coordinator.recordAgentActivity("a1", "running");
+ coordinator.recordAgentActivity("a1", "idle");
+ await tick();
+ expect(dispatch).toHaveBeenCalledTimes(2);
+ });
+
+ it("rolls back an errored admission vector without reviving an old covered launch", async () => {
+ const coordinator = new WakeCoordinator();
+ const dispatch = vi.fn(async () => {});
+ await coordinator.run(wake("a1", "/s/c0", 1), dispatch);
+ await coordinator.run(wake("a1", "/s/c1", 5), dispatch);
+ await coordinator.run(wake("a1", "/s/c2", 7, "compensation"), dispatch);
+ coordinator.recordAgentActivity("a1", "idle");
+ await tick();
+ expect(dispatch).toHaveBeenCalledTimes(2);
+
+ coordinator.recordDeliveryAck("a1", "compensation", "error");
+ await tick();
+ expect(dispatch).toHaveBeenCalledTimes(2);
+ await expect(coordinator.run(wake("a1", "/s/c1", 5, "retry"), dispatch)).resolves.toEqual({
+ state: "accepted",
+ });
+ expect(dispatch).toHaveBeenCalledTimes(3);
+ await expect(coordinator.run(wake("a1", "/s/c2", 7, "duplicate"), dispatch)).resolves.toEqual({
+ state: "suppressed",
+ coveredSeq: 7,
+ });
+ });
+
+ it("auto-admits a different-launch wake coalesced behind a failed provisional admission", async () => {
+ const coordinator = new WakeCoordinator();
+ let release!: () => void;
+ const blocked = new Promise<void>((resolve) => { release = resolve; });
+ const dispatch = vi.fn(async (command: Wake) => {
+ if (command.launchId === "launch_a") await blocked;
+ });
+
+ const first = coordinator.run(wake("a1", "/s/c", 5, "launch_a"), dispatch);
+ await tick();
+ await expect(coordinator.run(wake("a1", "/s/c", 5, "launch_b"), dispatch)).resolves.toMatchObject({
+ state: "suppressed",
+ });
+ coordinator.recordDeliveryAck("a1", "launch_a", "error");
+ expect(dispatch).toHaveBeenCalledTimes(1);
+ release();
+ await first;
+
+ expect(dispatch.mock.calls.map(([command]) => command.launchId)).toEqual(["launch_a", "launch_b"]);
+ });
+
+ it("does not let a stale lower-seq launch replace the current retry candidate", async () => {
+ const coordinator = new WakeCoordinator();
+ let release!: () => void;
+ const blocked = new Promise<void>((resolve) => { release = resolve; });
+ const dispatch = vi.fn(async (command: Wake) => {
+ if (command.launchId === "launch_a") await blocked;
+ });
+
+ const first = coordinator.run(wake("a1", "/s/c", 5, "launch_a"), dispatch);
+ await tick();
+ await coordinator.run(wake("a1", "/s/c", 5, "launch_b"), dispatch);
+ await coordinator.run(wake("a1", "/s/c", 4, "launch_c"), dispatch);
+ coordinator.recordDeliveryAck("a1", "launch_a", "error");
+ release();
+ await first;
+
+ expect(dispatch.mock.calls.map(([command]) => command.launchId)).toEqual(["launch_a", "launch_b"]);
+ });
+
+ it("fences a rejected old admission from a new lifecycle admission", async () => {
+ const coordinator = new WakeCoordinator();
+ let rejectA!: (error: Error) => void;
+ let releaseB!: () => void;
+ const blockedA = new Promise<void>((_resolve, reject) => { rejectA = reject; });
+ const blockedB = new Promise<void>((resolve) => { releaseB = resolve; });
+ const dispatch = vi.fn(async (command: Wake) => {
+ if (command.launchId === "launch_a") await blockedA;
+ if (command.launchId === "launch_b") await blockedB;
+ });
+
+ const first = coordinator.run(wake("a1", "/s/c", 5, "launch_a"), dispatch);
+ await tick();
+ coordinator.invalidate("a1", true);
+ const second = coordinator.run(wake("a1", "/s/c", 6, "launch_b"), dispatch);
+ await tick();
+ await coordinator.run(wake("a1", "/s/c", 6, "launch_c"), dispatch);
+
+ rejectA(new Error("old lifecycle failed"));
+ await expect(first).rejects.toThrow("old lifecycle failed");
+ expect(dispatch.mock.calls.map(([command]) => command.launchId)).toEqual(["launch_a", "launch_b"]);
+
+ coordinator.recordDeliveryAck("a1", "launch_b", "error");
+ releaseB();
+ await second;
+ expect(dispatch.mock.calls.map(([command]) => command.launchId)).toEqual([
+ "launch_a",
+ "launch_b",
+ "launch_c",
+ ]);
+ });
+
+ it("reconciles pending desired work when idle arrives before dispatch returns", async () => {
+ const coordinator = new WakeCoordinator();
+ let release!: () => void;
+ const blocked = new Promise<void>((resolve) => { release = resolve; });
+ const dispatch = vi.fn(async (command: Wake) => {
+ if (command.launchId === "launch_a") await blocked;
+ });
+
+ const first = coordinator.run(wake("a1", "/s/c", 5, "launch_a"), dispatch);
+ await tick();
+ await coordinator.run(wake("a1", "/s/c", 6, "launch_b"), dispatch);
+ coordinator.recordAgentActivity("a1", "idle");
+ release();
+ await first;
+
+ expect(dispatch.mock.calls.map(([command]) => command.launchId)).toEqual(["launch_a", "launch_b"]);
+ });
+
+ it("does not spin-retry a permanently failed admission without a different launch", async () => {
+ const coordinator = new WakeCoordinator();
+ const dispatch = vi.fn(async (command: Wake) => {
+ coordinator.recordDeliveryAck(command.agentId, command.launchId, "error");
+ });
+
+ await coordinator.run(wake("a1", "/s/c", 5, "launch_a"), dispatch);
+ coordinator.recordAgentActivity("a1", "idle");
+ await tick();
+ expect(dispatch).toHaveBeenCalledTimes(1);
+ });
+
+ it("does not fall back to the old failed launch when its replacement also fails", async () => {
+ const coordinator = new WakeCoordinator();
+ let release!: () => void;
+ const blocked = new Promise<void>((resolve) => { release = resolve; });
+ const dispatch = vi.fn(async (command: Wake) => {
+ if (command.launchId === "launch_a") {
+ await blocked;
+ } else {
+ coordinator.recordDeliveryAck(command.agentId, command.launchId, "error");
+ }
+ });
+
+ const first = coordinator.run(wake("a1", "/s/c", 5, "launch_a"), dispatch);
+ await tick();
+ await coordinator.run(wake("a1", "/s/c", 5, "launch_b"), dispatch);
+ coordinator.recordDeliveryAck("a1", "launch_a", "error");
+ release();
+ await first;
+ coordinator.recordAgentActivity("a1", "idle");
+ await tick();
+
+ expect(dispatch.mock.calls.map(([command]) => command.launchId)).toEqual(["launch_a", "launch_b"]);
+ });
+
+ it("bounds automatic retries when covered launches on different channels both fail", async () => {
+ const coordinator = new WakeCoordinator();
+ let release!: () => void;
+ const blocked = new Promise<void>((resolve) => { release = resolve; });
+ const dispatch = vi.fn(async (command: Wake) => {
+ if (command.launchId === "launch_a") await blocked;
+ coordinator.recordDeliveryAck(command.agentId, command.launchId, "error");
+ });
+
+ const first = coordinator.run(wake("a1", "/s/c1", 5, "launch_a"), dispatch);
+ await tick();
+ await coordinator.run(wake("a1", "/s/c2", 7, "launch_b"), dispatch);
+ release();
+ await first;
+ await tick();
+
+ expect(dispatch.mock.calls.map(([command]) => command.launchId)).toEqual(["launch_a", "launch_b"]);
+ });
+
+ it("keeps failed-launch tracking bounded across unique external re-arms", async () => {
+ const coordinator = new WakeCoordinator();
+ const dispatch = vi.fn(async (command: Wake) => {
+ coordinator.recordDeliveryAck(command.agentId, command.launchId, "error");
+ });
+
+ for (let attempt = 1; attempt <= 100; attempt++) {
+ await coordinator.run(wake("a1", "/s/c", 5, `launch_${attempt}`), dispatch);
+ }
+ coordinator.recordAgentActivity("a1", "idle");
+ await tick();
+
+ expect(dispatch).toHaveBeenCalledTimes(100);
+ });
+});
diff --git a/src/daemon/src/manager/wakeCoordinator.ts b/src/daemon/src/manager/wakeCoordinator.ts
new file mode 100644
index 000000000..f81e1a452
--- /dev/null
+++ b/src/daemon/src/manager/wakeCoordinator.ts
@@ -0,0 +1,282 @@
+import type { HostCommand } from "@alook/shared";
+
+type WakeCommand = Extract<HostCommand, { type: "agent:wake" }>;
+
+export type WakeCoordinationResult =
+ | { state: "accepted" }
+ | { state: "suppressed"; coveredSeq: number };
+
+interface ScopeState {
+ desiredSeq: number;
+ admittedSeq: number;
+ modelSeenSeq: number;
+ command: WakeCommand | null;
+}
+
+interface AdmissionCoverage {
+ previousSeq: number;
+ admittedSeq: number;
+}
+
+interface ActiveAdmission {
+ launchId: string;
+ coverage: Map<string, AdmissionCoverage>;
+}
+
+interface AgentState {
+ active: boolean;
+ admitting: boolean;
+ activeAdmission: ActiveAdmission | null;
+ failedAdmissionLaunchId: string | null;
+ coalescedReplacement: WakeCommand | null;
+ retryAfterFailure: WakeCommand | null;
+ observationGeneration: number;
+ scopes: Map<string, ScopeState>;
+ dispatch: ((command: WakeCommand) => void | Promise<void>) | null;
+}
+
+/**
+ * One active wake/turn admission per agent. Channels carry only desired and
+ * model-seen watermarks; they never become independent steer lanes.
+ */
+export class WakeCoordinator {
+ private readonly agents = new Map<string, AgentState>();
+
+ async run(
+ command: WakeCommand,
+ dispatch: (command: WakeCommand) => void | Promise<void>,
+ onDesiredAdvance?: (command: WakeCommand) => void,
+ ): Promise<WakeCoordinationResult> {
+ const state = this.agent(command.agentId);
+ const channel = command.unreadNotice.channel;
+ const seq = command.unreadNotice.latestSeq;
+ const scope = state.scopes.get(channel);
+ const coveredSeq = Math.max(scope?.modelSeenSeq ?? 0, scope?.admittedSeq ?? 0);
+ if (seq <= coveredSeq) {
+ this.rememberReplacement(state, command);
+ return { state: "suppressed", coveredSeq };
+ }
+
+ const desiredAdvanced = !scope || seq > scope.desiredSeq;
+ if (!scope || seq >= scope.desiredSeq) {
+ state.scopes.set(channel, {
+ desiredSeq: Math.max(scope?.desiredSeq ?? 0, seq),
+ admittedSeq: scope?.admittedSeq ?? 0,
+ modelSeenSeq: scope?.modelSeenSeq ?? 0,
+ command,
+ });
+ }
+ if (desiredAdvanced) onDesiredAdvance?.(command);
+ state.dispatch = dispatch;
+
+ // The active agent will pull the global inbox. Higher watermarks only
+ // advance desired state; idle reconciliation decides whether one rewake is
+ // needed after observing what the turn actually pulled.
+ if (state.active || state.admitting) {
+ this.rememberReplacement(state, command);
+ return { state: "suppressed", coveredSeq: seq };
+ }
+
+ await this.admit(command.agentId, command);
+ return { state: "accepted" };
+ }
+
+ modelSeenGeneration(agentId: string): number {
+ return this.agent(agentId).observationGeneration;
+ }
+
+ recordModelSeen(
+ agentId: string,
+ messages: ReadonlyArray<{ channel: string; seq: string }>,
+ generation: number,
+ ): boolean {
+ const state = this.agent(agentId);
+ if (generation !== state.observationGeneration) return false;
+ for (const message of messages) {
+ const seq = Number(message.seq.startsWith("#") ? message.seq.slice(1) : message.seq);
+ if (!Number.isSafeInteger(seq) || seq <= 0) continue;
+ const scope = state.scopes.get(message.channel);
+ if (scope) {
+ scope.modelSeenSeq = Math.max(scope.modelSeenSeq, seq);
+ } else {
+ // Pull may beat a delayed/replayed WS frame. Preserve that observation
+ // even when no wake for the scope has reached ingress yet.
+ state.scopes.set(message.channel, {
+ desiredSeq: 0,
+ admittedSeq: 0,
+ modelSeenSeq: seq,
+ command: null,
+ });
+ }
+ }
+ return true;
+ }
+
+ /** Runtime activity is the admission lifecycle, not command dispatch time. */
+ recordAgentActivity(agentId: string, activity: string): void {
+ const state = this.agent(agentId);
+ if (activity === "starting" || activity === "running") {
+ state.active = true;
+ return;
+ }
+ if (activity !== "idle") return;
+ state.active = false;
+ state.activeAdmission = null;
+ const next = this.nextPending(state);
+ if (next && state.dispatch) void this.admit(agentId, next);
+ }
+
+ recordDeliveryAck(agentId: string, launchId: string, status: "ok" | "error"): void {
+ const state = this.agents.get(agentId);
+ const admission = state?.activeAdmission;
+ if (!state || !admission || admission.launchId !== launchId) return;
+ if (status === "ok") return;
+ // A failed delivery never occupies admission coverage. Desired watermarks
+ // remain, so a later transport retry or idle reconciliation can re-arm.
+ for (const [channel, coverage] of admission.coverage) {
+ const scope = state.scopes.get(channel);
+ if (scope?.admittedSeq === coverage.admittedSeq) {
+ scope.admittedSeq = coverage.previousSeq;
+ }
+ }
+ state.active = false;
+ state.failedAdmissionLaunchId = launchId;
+ state.retryAfterFailure = state.coalescedReplacement;
+ state.coalescedReplacement = null;
+ state.activeAdmission = null;
+ if (!state.admitting && state.retryAfterFailure) {
+ void this.retryFailedAdmission(agentId, state).catch(() => {});
+ }
+ }
+
+ invalidate(agentId: string, clearModelSeen: boolean): void {
+ const state = this.agent(agentId);
+ state.observationGeneration += 1;
+ state.active = false;
+ state.admitting = false;
+ state.activeAdmission = null;
+ state.failedAdmissionLaunchId = null;
+ state.coalescedReplacement = null;
+ state.retryAfterFailure = null;
+ if (clearModelSeen) {
+ state.scopes.clear();
+ return;
+ }
+ // Stop/model switch preserve messages actually shown to the model, while
+ // dropping any pre-stop admission/desired coverage.
+ for (const scope of state.scopes.values()) {
+ scope.desiredSeq = scope.modelSeenSeq;
+ scope.admittedSeq = scope.modelSeenSeq;
+ }
+ }
+
+ private agent(agentId: string): AgentState {
+ let state = this.agents.get(agentId);
+ if (!state) {
+ state = {
+ active: false,
+ admitting: false,
+ activeAdmission: null,
+ failedAdmissionLaunchId: null,
+ coalescedReplacement: null,
+ retryAfterFailure: null,
+ observationGeneration: 0,
+ scopes: new Map(),
+ dispatch: null,
+ };
+ this.agents.set(agentId, state);
+ }
+ return state;
+ }
+
+ private nextPending(state: AgentState): WakeCommand | null {
+ let next: ScopeState | null = null;
+ for (const scope of state.scopes.values()) {
+ if (scope.desiredSeq <= Math.max(scope.modelSeenSeq, scope.admittedSeq)) continue;
+ if (!scope.command) continue;
+ if (scope.command.launchId === state.failedAdmissionLaunchId) continue;
+ if (!next || scope.desiredSeq > next.desiredSeq) next = scope;
+ }
+ return next?.command ?? null;
+ }
+
+ private async admit(agentId: string, preferred?: WakeCommand): Promise<void> {
+ const state = this.agent(agentId);
+ if (state.active || state.admitting || !state.dispatch) return;
+ const admissionGeneration = state.observationGeneration;
+ const command = preferred ?? this.nextPending(state);
+ if (!command) return;
+ const commandScope = state.scopes.get(command.unreadNotice.channel);
+ if (commandScope && command.unreadNotice.latestSeq >= commandScope.desiredSeq) {
+ commandScope.command = command;
+ }
+ const coverage = new Map<string, AdmissionCoverage>();
+ for (const [channel, scope] of state.scopes) {
+ if (scope.desiredSeq <= Math.max(scope.modelSeenSeq, scope.admittedSeq)) continue;
+ coverage.set(channel, {
+ previousSeq: scope.admittedSeq,
+ admittedSeq: scope.desiredSeq,
+ });
+ scope.admittedSeq = scope.desiredSeq;
+ scope.command = command;
+ }
+ if (coverage.size === 0) return;
+ state.admitting = true;
+ state.active = true;
+ state.failedAdmissionLaunchId = null;
+ state.coalescedReplacement = null;
+ state.activeAdmission = { launchId: command.launchId, coverage };
+ let dispatchError: unknown;
+ try {
+ await state.dispatch(command);
+ } catch (error) {
+ if (state.observationGeneration === admissionGeneration) {
+ for (const [channel, admitted] of coverage) {
+ const scope = state.scopes.get(channel);
+ if (scope?.admittedSeq === admitted.admittedSeq) {
+ scope.admittedSeq = admitted.previousSeq;
+ }
+ }
+ state.active = false;
+ state.failedAdmissionLaunchId = command.launchId;
+ state.retryAfterFailure = state.coalescedReplacement;
+ state.coalescedReplacement = null;
+ state.activeAdmission = null;
+ }
+ dispatchError = error;
+ } finally {
+ if (state.observationGeneration === admissionGeneration) {
+ state.admitting = false;
+ }
+ }
+ if (state.observationGeneration === admissionGeneration) {
+ await this.retryFailedAdmission(agentId, state);
+ if (!state.active && !state.admitting) {
+ const next = this.nextPending(state);
+ if (next) await this.admit(agentId, next);
+ }
+ }
+ if (dispatchError) throw dispatchError;
+ }
+
+ private rememberReplacement(state: AgentState, command: WakeCommand): void {
+ const failedOrActiveLaunchId = state.activeAdmission?.launchId ?? state.failedAdmissionLaunchId;
+ if (!failedOrActiveLaunchId || command.launchId === failedOrActiveLaunchId) return;
+ const scope = state.scopes.get(command.unreadNotice.channel);
+ // Only a command at the scope's current desired watermark may replace a
+ // provisional admission. A delayed lower-seq frame must not displace a
+ // valid same-watermark retry candidate that arrived before it.
+ if (!scope || command.unreadNotice.latestSeq < scope.desiredSeq) return;
+ state.coalescedReplacement = command;
+ if (state.failedAdmissionLaunchId) state.retryAfterFailure = command;
+ }
+
+ private async retryFailedAdmission(agentId: string, state: AgentState): Promise<void> {
+ if (state.active || state.admitting) return;
+ const replacement = state.retryAfterFailure;
+ state.retryAfterFailure = null;
+ if (!replacement) return;
+ state.failedAdmissionLaunchId = null;
+ await this.admit(agentId, replacement);
+ }
+}
diff --git a/src/daemon/src/server/wsControlChannel.test.ts b/src/daemon/src/server/wsControlChannel.test.ts
index 043fa9467..7815df8cb 100644
--- a/src/daemon/src/server/wsControlChannel.test.ts
+++ b/src/daemon/src/server/wsControlChannel.test.ts
@@ -465,6 +465,21 @@ describe("WsControlChannel — downlink HostCommand validation (convergence #6)"
return { sockets, received };
}
+ it("fences an old inbox pull before awaiting lifecycle listeners", async () => {
+ const { ch } = makeChannel();
+ let release!: () => void;
+ const blocked = new Promise<void>((resolve) => { release = resolve; });
+ ch.onCommand((command) => command.type === "agent:stop" ? blocked : undefined);
+ const staleGeneration = ch.modelSeenGeneration("bot_1");
+
+ const stopping = ch.ingestCommand({ type: "agent:stop", agentId: "bot_1" });
+ expect(ch.modelSeenGeneration("bot_1")).toBe(staleGeneration + 1);
+ expect(ch.recordModelSeen("bot_1", [{ channel: "/demo#1234/general", seq: "#9" }], staleGeneration))
+ .toBe(false);
+ release();
+ await stopping;
+ });
+
it("dispatches each valid arm unchanged (happy path — the transparent gate)", () => {
for (const frame of [realWake, realReset, realNap, realModelSwitch, realStop]) {
const { sockets, received } = driven();
diff --git a/src/daemon/src/server/wsControlChannel.ts b/src/daemon/src/server/wsControlChannel.ts
index 700ca35e0..9bbbc545e 100644
--- a/src/daemon/src/server/wsControlChannel.ts
+++ b/src/daemon/src/server/wsControlChannel.ts
@@ -36,6 +36,7 @@ import type {
} from "./contract.js";
import { HostCommandSchema } from "./contract.js";
import { createLogger, type Logger } from "../logger.js";
+import { WakeCoordinator } from "../manager/wakeCoordinator.js";
// Re-export so existing importers of these from this module keep working.
export type { WebSocketLike, WebSocketFactory } from "./contract.js";
@@ -50,6 +51,7 @@ export const WS_CONTROL_COMMAND_CONSUMED = Symbol("ws-control-command-consumed")
type CommandListener = (
cmd: HostCommand,
) => void | Promise<void> | typeof WS_CONTROL_COMMAND_CONSUMED;
+type WakeDesiredListener = (cmd: Extract<HostCommand, { type: "agent:wake" }>) => void;
/** Heartbeat: how often we ping the server WebSocket. */
const DEFAULT_PING_INTERVAL_MS = 15_000;
@@ -151,6 +153,7 @@ export class WsControlChannel implements HostControlChannel {
// Multiple listeners so consumers can layer behavior (e.g. bot-cache pre-hook
// + AgentRouter's real handler) without monkey-patching this class.
private commandCbs: CommandListener[] = [];
+ private wakeDesiredCbs: WakeDesiredListener[] = [];
private resyncHooks: Array<() => void> = [];
private ws: WebSocketLike | null = null;
private attempt = 0;
@@ -160,6 +163,7 @@ export class WsControlChannel implements HostControlChannel {
private pongDeadline = 0;
private resyncProvider: ResyncProvider | null = null;
private readonly log: Logger;
+ private readonly wakeCoordinator = new WakeCoordinator();
constructor(private readonly opts: WsControlChannelOpts) {
this.log = opts.logger ?? createLogger({ header: "@alook/daemon:ws" });
@@ -195,6 +199,46 @@ export class WsControlChannel implements HostControlChannel {
this.commandCbs.push(cb);
}
+ /** Observe only wakes that advance an agent/channel desired watermark. */
+ onWakeDesiredAdvance(cb: WakeDesiredListener): void {
+ this.wakeDesiredCbs.push(cb);
+ }
+
+ /**
+ * Inject a command through the same schema and semantic ingress as a WS
+ * frame, so alternate control transports cannot bypass observers, routing,
+ * lifecycle handling, or duplicate suppression.
+ */
+ async ingestCommand(frame: unknown): Promise<boolean> {
+ const parsed = HostCommandSchema.safeParse(frame);
+ if (!parsed.success) {
+ this.log.warn("dropped malformed HostCommand frame", {
+ type: typeof frame === "object" && frame !== null && "type" in frame
+ ? String((frame as { type?: unknown }).type)
+ : "unknown",
+ issues: parsed.error.issues.map((issue) => ({
+ path: issue.path.join("."),
+ code: issue.code,
+ })),
+ });
+ return false;
+ }
+ await this.dispatchIngressCommand(parsed.data as HostCommand);
+ return true;
+ }
+
+ modelSeenGeneration(agentId: string): number {
+ return this.wakeCoordinator.modelSeenGeneration(agentId);
+ }
+
+ recordModelSeen(
+ agentId: string,
+ messages: ReadonlyArray<{ channel: string; seq: string }>,
+ generation = this.wakeCoordinator.modelSeenGeneration(agentId),
+ ): boolean {
+ return this.wakeCoordinator.recordModelSeen(agentId, messages, generation);
+ }
+
/**
* The resync provider builds the current-state snapshot the server needs on
* every (re)connect. Only one provider makes sense; the last registration
@@ -237,6 +281,7 @@ export class WsControlChannel implements HostControlChannel {
}
async reportAgentActivity(info: { agentId: AgentId; state: AgentActivityState }): Promise<void> {
+ this.wakeCoordinator.recordAgentActivity(info.agentId, info.state);
this.sendFrame({ type: "agent_activity", ...info });
}
@@ -283,6 +328,7 @@ export class WsControlChannel implements HostControlChannel {
status: AgentCommandAckStatus;
error?: AgentCommandAckError;
}): Promise<void> {
+ this.wakeCoordinator.recordDeliveryAck(info.agentId, info.launchId, info.status);
this.sendFrame({ type: "agent_wake_ack", ...info });
}
@@ -390,19 +436,13 @@ export class WsControlChannel implements HostControlChannel {
// schema gate and short-circuits regardless of command validity.
this.attempt = 0;
- // Validate the downlink shape before dispatch — the mirror of ws-do's
- // per-frame `safeParse` on the uplink. A malformed/half-written frame (or a
- // producer bug) must be DROPPED + logged here, never handed to the router's
- // arms as a lie. Valid frames dispatch exactly as before (transparent gate).
- const parsed = HostCommandSchema.safeParse(frame);
- if (!parsed.success) {
- this.log.warn("dropped malformed HostCommand frame", {
- type: frame.type,
- issues: parsed.error.issues.map((i) => ({ path: i.path.join("."), code: i.code })),
- });
- return;
- }
- const cmd = parsed.data as HostCommand;
+ void this.ingestCommand(frame).catch((err: unknown) => {
+ this.log.warn("command ingress failed", { type: frame.type, err: describeErr(err) });
+ });
+ }
+
+ private async dispatchListeners(cmd: HostCommand): Promise<void> {
+ const pending: Promise<void>[] = [];
for (const cb of this.commandCbs) {
// Each listener is fire-and-forget; failures in one must not skip the
// next. Catch rejections explicitly — a bare `void cb(cmd)` on an async
@@ -411,13 +451,57 @@ export class WsControlChannel implements HostControlChannel {
try {
const result = cb(cmd);
if (result === WS_CONTROL_COMMAND_CONSUMED) break;
- Promise.resolve(result).catch((err: unknown) => {
+ pending.push(Promise.resolve(result).catch((err: unknown) => {
this.log.warn("command listener threw", { type: cmd.type, err: describeErr(err) });
- });
+ }));
} catch (err) {
this.log.warn("command listener threw synchronously", { type: cmd.type, err: describeErr(err) });
}
}
+ await Promise.all(pending);
+ }
+
+ private async dispatchIngressCommand(cmd: HostCommand): Promise<void> {
+ if (cmd.type === "agent:wake") {
+ const result = await this.wakeCoordinator.run(cmd, (accepted) =>
+ this.dispatchListeners(accepted), (advanced) => {
+ for (const cb of this.wakeDesiredCbs) {
+ try {
+ cb(advanced);
+ } catch (err) {
+ this.log.warn("wake desired observer threw", { err: describeErr(err) });
+ }
+ }
+ });
+ if (result.state === "suppressed") {
+ await this.reportWakeAck({
+ agentId: cmd.agentId,
+ launchId: cmd.launchId,
+ status: "ok",
+ });
+ this.log.debug("duplicate wake suppressed", {
+ agentId: cmd.agentId,
+ channel: cmd.unreadNotice.channel,
+ latestSeq: cmd.unreadNotice.latestSeq,
+ coveredSeq: result.coveredSeq,
+ });
+ }
+ return;
+ }
+
+ if (cmd.type === "machine:reset_all") {
+ for (const reset of cmd.resets) this.wakeCoordinator.invalidate(reset.agentId, true);
+ await this.dispatchListeners(cmd);
+ return;
+ }
+ if (cmd.type === "agent:stop" || cmd.type === "agent:model_switch") {
+ this.wakeCoordinator.invalidate(cmd.agentId, false);
+ } else if (cmd.type === "agent:reset" || cmd.type === "agent:nap") {
+ this.wakeCoordinator.invalidate(cmd.agentId, true);
+ } else if (cmd.type === "bot:removed") {
+ this.wakeCoordinator.invalidate(cmd.botId, true);
+ }
+ await this.dispatchListeners(cmd);
}
private onSocketClosed(code?: number, reason?: unknown): void {