Multi-Agent Collaboration for Team and Friends
TypeScript agent-automationagent-orchestrationagent-workflowai-agentsai-collaborationai-memoryai-workforceautonomous-agentsclaude-codeclaude-code-skillscodexcoding-agentmulti-agentmultiple-agentone-person-businessone-person-companyone-person-teamopencodesolopreneur
fix(daemon): detect a deaf-but-running agent via unanswered-delivery timeout
Claudette3bc142006e83
diff --git a/src/daemon/src/manager/agentRouter.test.ts b/src/daemon/src/manager/agentRouter.test.tsindex edc562c66..b969db0e6 100644--- a/src/daemon/src/manager/agentRouter.test.ts+++ b/src/daemon/src/manager/agentRouter.test.ts@@ -164,7 +164,9 @@ describe("AgentRouter — agent:wake", () => {expect(wakeAcks.length).toBe(2);});- it("logs an honest-ack diagnostic when deliver produced NO executable effect, keeping the wire ack ok", async () => {+ const NO_EFFECT_DIAG = "agent:wake produced no effect (coalesced onto running agent)";++ it("logs an honest-ack diagnostic when a no-effect wake lands on a running agent, keeping the wire ack ok", async () => {const { mgr } = fakeManager({ a1: "running" }, false);const { ch, wakeAcks, fire } = fakeChannel();const logger = stubLogger();@@ -181,11 +183,11 @@ describe("AgentRouter — agent:wake", () => {// Diagnostic surfaced (batch B), but the wire ack is still ok — reachability// honesty is batch D, this router only reports the local "no effect" fact.- expect(logger.calls.info.some(([m]) => m === "agent:wake produced no effect (coalesced only)")).toBe(true);+ expect(logger.calls.info.some(([m]) => m === NO_EFFECT_DIAG)).toBe(true);expect(wakeAcks).toEqual([{ agentId: "a1", launchId: "l1", status: "ok" }]);});- it("does NOT log the honest-ack diagnostic when deliver produced an effect", async () => {+ it("does NOT log the diagnostic when deliver produced an effect", async () => {const { mgr } = fakeManager({ a1: "idle" }, true);const { ch, fire } = fakeChannel();const logger = stubLogger();@@ -200,7 +202,28 @@ describe("AgentRouter — agent:wake", () => {unreadNotice: { kind: "unread_notice", channel: "/demo/general", latestSeq: 7 },});- expect(logger.calls.info.some(([m]) => m === "agent:wake produced no effect (coalesced only)")).toBe(false);+ expect(logger.calls.info.some(([m]) => m === NO_EFFECT_DIAG)).toBe(false);+ });++ it("does NOT log the diagnostic for a benign no-effect coalesce on a non-running agent (starting)", async () => {+ // A no-effect wake while starting/stopping/reset-window is routine+ // queue-and-drain, not the deaf-orphan fault — must stay quiet (batch A+ // narrowing per Claudette's gate observation).+ const { mgr } = fakeManager({ a1: "starting" }, false);+ const { ch, fire } = fakeChannel();+ const logger = stubLogger();+ const router = new AgentRouter({ manager: mgr, channel: ch, runtimeReport: [{ id: "mock" }], logger });+ await router.start();++ await fire({+ type: "agent:wake",+ agentId: "a1",+ config: { version: 1, runtime: "mock", model: { kind: "default" }, mode: { kind: "default" } },+ launchId: "l1",+ unreadNotice: { kind: "unread_notice", channel: "/demo/general", latestSeq: 7 },+ });++ expect(logger.calls.info.some(([m]) => m === NO_EFFECT_DIAG)).toBe(false);});});diff --git a/src/daemon/src/manager/agentRouter.ts b/src/daemon/src/manager/agentRouter.tsindex 2db4d6142..c718d62a9 100644--- a/src/daemon/src/manager/agentRouter.ts+++ b/src/daemon/src/manager/agentRouter.ts@@ -411,22 +411,24 @@ export class AgentRouter {seq: cmd.unreadNotice.latestSeq,text,});- // Honest-ack diagnostic (plans/daemon-fsm-desync.md batch B): a wake- // that produced NO executable effect was only coalesced into the- // inbox. Benign for the queue-and-drain states (starting/stopping/- // reset-window/per-turn), but on a `running` agent it can mean the- // message is stacking behind a process that will never drain it (the- // deaf-orphan class). Surface it as a daemon-local log line so the- // stall stops masquerading as a healthy `status:"ok"` ack. This does- // NOT change the wire ack status — reachability-based honesty is+ // Honest-ack diagnostic (plans/daemon-fsm-desync.md batch B, narrowed+ // in batch A): a wake that produced NO executable effect was only+ // coalesced into the inbox. Coalescing is BENIGN and routine for the+ // queue-and-drain states (starting/stopping/reset-window/per-turn) —+ // logging those is just noise on a busy-spawning agent. The+ // pathological shape is a wake landing on an ALREADY-`running` agent+ // that produces nothing: the message may be stacking behind a process+ // that will never drain it (the deaf-orphan class). So gate the+ // diagnostic on `beforeStatus === "running"` to keep the signal+ // pointed at the real fault. Surfaced as a daemon-local log line so+ // the stall stops masquerading as a healthy `status:"ok"` ack; this+ // does NOT change the wire ack status — reachability-based honesty is// batch D's job (it needs a liveness probe this router deliberately// avoids). `producedEffect` is `false` for a legacy mock manager- // whose `deliver` returns void, so the diagnostic simply won't fire- // in those tests.- if (producedEffect === false) {- this.log.info("agent:wake produced no effect (coalesced only)", {+ // whose `deliver` returns void, so the diagnostic won't fire there.+ if (producedEffect === false && beforeStatus === "running") {+ this.log.info("agent:wake produced no effect (coalesced onto running agent)", {agentId: cmd.agentId,- beforeStatus,latestSeq: cmd.unreadNotice.latestSeq,});}diff --git a/src/daemon/src/manager/managerPolicy.test.ts b/src/daemon/src/manager/managerPolicy.test.tsindex 627cd9b95..d84e8d36c 100644--- a/src/daemon/src/manager/managerPolicy.test.ts+++ b/src/daemon/src/manager/managerPolicy.test.ts@@ -376,6 +376,72 @@ describe("reduceManager — tick: stall + idle hibernation", () => {});});+// Batch A (plans/daemon-fsm-desync.md): the suspected-deaf detector catches the+// gated-no-further-turn_end permanent orphan that slips BOTH existing tick+// predicates. Reproduces Olivia's wedge: a gated agent turn-ends idle, a later+// wake drain-sends (stamping lastDeliverAt, re-arming turnActive), then the+// process goes deaf — no progress, no turn_end, no exit ever follows.+describe("reduceManager — tick: suspected-deaf detection (batch A)", () => {+ // Build the exact orphan tuple: running, turnActive (from the drain-send),+ // inbox empty (drained), lastDeliverAt > lastProgressAt, past the threshold.+ function toDeafOrphan(staleMs = 100) {+ let s = createInitialManagerState(staleMs);+ s = register(s, "a", PERSISTENT_GATED);+ s = reduceManager(s, { type: "wake", agentId: "a", message: { text: "m1" }, nowMs: 0 }).state;+ s = reduceManager(s, { type: "spawned", agentId: "a", nowMs: 0 }).state;+ s = reduceManager(s, { type: "session", agentId: "a", sessionId: "sess-1" }).state;+ // Turn ends → turnActive false, idleSince armed, lastProgressAt=10.+ s = reduceManager(s, { type: "turn_end", agentId: "a", nowMs: 10 }).state;+ // A later wake on the now-idle turn drain-sends into the (about-to-go-deaf)+ // process: stamps lastDeliverAt=20, re-arms turnActive, drains the inbox.+ s = reduceManager(s, { type: "wake", agentId: "a", message: { text: "m2" }, nowMs: 20 }).state;+ return s;+ }++ it("terminates a gated agent whose delivery got no process response past the threshold", () => {+ const s = toDeafOrphan();+ expect(s.agents.a.turnActive).toBe(true);+ expect(s.agents.a.inbox.length).toBe(0); // drained — slips stalled's gated sub-clause+ expect(s.agents.a.lastDeliverAt).toBe(20);+ expect(s.agents.a.lastProgressAt).toBe(10); // deliver newer than last progress++ const r = reduceManager(s, { type: "tick", nowMs: 200 });+ expect(r.effects).toEqual([{ type: "terminate_stalled", agentId: "a" }]);+ expect(r.state.agents.a.status).toBe("stopping");+ });++ it("does NOT terminate before the stale window elapses after delivery", () => {+ const s = toDeafOrphan();+ expect(reduceManager(s, { type: "tick", nowMs: 100 }).effects).toEqual([]); // 100-20 < 100+ });++ it("does NOT flag a healthy agent whose process reported progress after the delivery", () => {+ let s = toDeafOrphan();+ // Process answers: a progress event lands after the deliver → lastProgressAt+ // (30) overtakes lastDeliverAt (20), so the suspicion clears on its own.+ s = reduceManager(s, { type: "progress", agentId: "a", nowMs: 30 }).state;+ expect(reduceManager(s, { type: "tick", nowMs: 200 }).effects).toEqual([]);+ });++ it("does NOT flag a fresh agent that never had a delivery (lastDeliverAt null)", () => {+ let s = createInitialManagerState(100);+ s = register(s, "a", PERSISTENT_GATED);+ s = reduceManager(s, { type: "wake", agentId: "a", message: { text: "m1" }, nowMs: 0 }).state;+ s = reduceManager(s, { type: "spawned", agentId: "a", nowMs: 0 }).state;+ expect(s.agents.a.lastDeliverAt).toBeNull();+ expect(reduceManager(s, { type: "tick", nowMs: 200 }).effects).toEqual([]);+ });++ it("clears lastDeliverAt across a respawn (no cross-lifecycle stale suspicion)", () => {+ let s = toDeafOrphan();+ // The orphan gets terminated → exit → onExit; then respawn. lastDeliverAt+ // must be null again so the next lifecycle starts clean.+ s = reduceManager(s, { type: "tick", nowMs: 200 }).state; // → stopping + terminate+ s = reduceManager(s, { type: "exit", agentId: "a" }).state;+ expect(s.agents.a.lastDeliverAt).toBeNull();+ });+});+describe("reduceManager — reset_session", () => {it("nulls sessionId on a known agent without changing status/turnActive", () => {let s = createInitialManagerState();diff --git a/src/daemon/src/manager/managerPolicy.ts b/src/daemon/src/manager/managerPolicy.tsindex 90798f24b..f291983d6 100644--- a/src/daemon/src/manager/managerPolicy.ts+++ b/src/daemon/src/manager/managerPolicy.ts@@ -60,6 +60,17 @@ export interface AgentState {turnActive: boolean;/** ms timestamp of the last observed progress (event), for stall detection. */lastProgressAt: number;+ /**+ * ms timestamp of the last `send` effect emitted toward the live process+ * (drain-send / onTurnEnd drain / gated flush); null if none this lifecycle.+ * Paired with `lastProgressAt` to detect a deaf process: a `send` was+ * dispatched (`lastDeliverAt > lastProgressAt`) yet no process-reported+ * event followed within the stale window — the message went into a stdin+ * nothing is reading. This is the ONLY signal that catches the+ * gated-no-turn_end permanent orphan, which slips both existing onTick+ * predicates. See plans/daemon-fsm-desync.md batch A.+ */+ lastDeliverAt: number | null;/** ms timestamp since which the agent has been idle (running, no turn, empty inbox); null if not idle. */idleSince: number | null;/**@@ -184,13 +195,19 @@ export function reduceManager(state: ManagerState, event: ManagerEvent): ReduceRreturn withAgent(state, event.agentId, (a) => a ?? freshAgent(event.agentId, event.caps), []);case "wake":- return onWake(state, event.agentId, event.message);+ return onWake(state, event.agentId, event.message, event.nowMs);case "spawned":return mutate(state, event.agentId, (a) => {a.status = "running";a.turnActive = true;a.lastProgressAt = event.nowMs;+ // Fresh process ⇒ no outstanding unanswered delivery yet. Clearing keeps+ // the suspected-deaf detector (onTick) from carrying a prior lifecycle's+ // `lastDeliverAt` across a respawn. Redundant with `lastProgressAt` being+ // bumped here (a stale deliver would already be < progress), but explicit+ // so the fresh-lifecycle invariant doesn't rely on that coincidence.+ a.lastDeliverAt = null;a.idleSince = null;// Belt for the idle-branch reset path where `deliver` triggered a// spawn directly (no `exit` fires there). The kill-and-respawn path@@ -241,7 +258,7 @@ export function reduceManager(state: ManagerState, event: ManagerEvent): ReduceRreturn onTick(state, event.nowMs);case "runtime_signal":- return onRuntimeSignal(state, event.agentId, event.kind);+ return onRuntimeSignal(state, event.agentId, event.kind, event.nowMs);}}@@ -249,7 +266,7 @@ export function reduceManager(state: ManagerState, event: ManagerEvent): ReduceR/* Event handlers *//* ------------------------------------------------------------------ */-function onWake(state: ManagerState, agentId: string, message: AgentMsg): ReduceResult {+function onWake(state: ManagerState, agentId: string, message: AgentMsg, nowMs: number): ReduceResult {const existing = state.agents[agentId];const agent = existing ? clone(existing) : null;if (!agent) {@@ -306,6 +323,7 @@ function onWake(state: ManagerState, agentId: string, message: AgentMsg): Reduceconst text = drainInboxToPrompt(agent);const mode = agent.turnActive ? "busy" : "idle";if (mode === "idle") agent.turnActive = true;+ agent.lastDeliverAt = nowMs;return commit(state, agent, [{ type: "send", agentId, text, mode }]);}// Per-turn or no stdin ⇒ keep queued; delivered after exit / next spawn.@@ -336,6 +354,7 @@ function onTurnEnd(state: ManagerState, agentId: string, nowMs: number): ReduceRif (agent.inbox.length > 0 && agent.caps.supportsStdinNotification) {const text = drainInboxToPrompt(agent);agent.turnActive = true;+ agent.lastDeliverAt = nowMs;return commit(state, agent, [{ type: "send", agentId, text, mode: "idle" }]);}@@ -360,7 +379,7 @@ function onTurnEnd(state: ManagerState, agentId: string, nowMs: number): ReduceR* 3. If the branch calls for it, attempt a flush via* `reduceApmGatedFlushReadiness`, reading the now-fully-updated `apm`.*/-function onRuntimeSignal(state: ManagerState, agentId: string, kind: string): ReduceResult {+function onRuntimeSignal(state: ManagerState, agentId: string, kind: string, nowMs: number): ReduceResult {const existing = state.agents[agentId];if (!existing) return { state, effects: [] };const agent = clone(existing);@@ -428,6 +447,7 @@ function onRuntimeSignal(state: ManagerState, agentId: string, kind: string): Re});if (readiness.shouldNotify) {const text = drainInboxToPrompt(agent);+ agent.lastDeliverAt = nowMs;return commit(state, agent, [{ type: "send", agentId, text, mode: "busy" }]);}return commit(state, agent, [@@ -457,6 +477,9 @@ function onExit(state: ManagerState, agentId: string): ReduceResult {// prevents a dead process's `compacting` / outstanding-tool-use flags from// silently carrying into the next spawn and re-blocking `onWake`.agent.apm = createInitialApmGatedSteeringState();+ // The dead process can't answer any prior delivery — drop the outstanding+ // deliver marker so the suspected-deaf detector starts clean for the respawn.+ agent.lastDeliverAt = null;// Per-turn: if more messages queued, immediately respawn for the next batch.if (agent.inbox.length > 0) {@@ -491,7 +514,28 @@ function onTick(state: ManagerState, nowMs: number): ReduceResult {(a.caps.supportsStdinNotification &&a.caps.busyDeliveryMode === "gated" &&a.inbox.length > 0));- if (stalled) {+ // Suspected-deaf (plans/daemon-fsm-desync.md batch A): a `send` was+ // dispatched (`lastDeliverAt` set, and NEWER than the last process-reported+ // progress) yet the stale window elapsed with no event following it — the+ // message went into a stdin nothing is reading. This is the ONE trigger+ // that catches the `gated`-no-`turn_end` permanent orphan, which slips BOTH+ // existing predicates: `stalled`'s gated sub-clause needs `inbox.length > 0`+ // but the inbox was drained empty by the very send that stamped+ // `lastDeliverAt`, and `idleEligible` needs `!turnActive` but the drain-send+ // set `turnActive = true`. Reuses the existing `terminate_stalled` recovery+ // (SIGKILL → exit → onExit drain-respawn) — batch A adds a trigger, not a+ // new recovery path. Batch D upgrades this same suspicion to a confirmed+ // diagnosis via a liveness probe before any destructive action on the+ // runtime kinds that lack the SIGKILL backstop (SDK); here it only fires for+ // a stdin-capable persistent agent whose recovery is physically reliable.+ const suspectedDeaf =+ a.status === "running" &&+ a.caps.lifecycleKind === "persistent" &&+ a.caps.supportsStdinNotification &&+ a.lastDeliverAt !== null &&+ a.lastDeliverAt > a.lastProgressAt &&+ nowMs - a.lastDeliverAt >= state.staleThresholdMs;+ if (stalled || suspectedDeaf) {agents[id] = { ...a, status: "stopping", idleSince: null };effects.push({ type: "terminate_stalled", agentId: id });continue;@@ -528,6 +572,7 @@ function freshAgent(agentId: string, caps: AgentRuntimeCaps): AgentState {sessionId: null,turnActive: false,lastProgressAt: 0,+ lastDeliverAt: null,idleSince: null,resetting: false,apm: createInitialApmGatedSteeringState(),