Skip to content

armgit / alook

Public Imported from GitHub
0

Multi-Agent Collaboration for Team and Friends

fix(daemon): detect a deaf-but-running agent via unanswered-delivery timeout

Claudette3bc142006e83
diff --git a/src/daemon/src/manager/agentRouter.test.ts b/src/daemon/src/manager/agentRouter.test.ts
index edc562c66..b969db0e6 100644
--- a/src/daemon/src/manager/agentRouter.test.ts
+++ b/src/daemon/src/manager/agentRouter.test.ts
@@ -164,7 +164,9 @@ describe("AgentRouter — agent:wake", () => {
expect(wakeAcks.length).toBe(2);
});
- it("logs an honest-ack diagnostic when deliver produced NO executable effect, keeping the wire ack ok", async () => {
+ const NO_EFFECT_DIAG = "agent:wake produced no effect (coalesced onto running agent)";
+
+ it("logs an honest-ack diagnostic when a no-effect wake lands on a running agent, keeping the wire ack ok", async () => {
const { mgr } = fakeManager({ a1: "running" }, false);
const { ch, wakeAcks, fire } = fakeChannel();
const logger = stubLogger();
@@ -181,11 +183,11 @@ describe("AgentRouter — agent:wake", () => {
// Diagnostic surfaced (batch B), but the wire ack is still ok — reachability
// honesty is batch D, this router only reports the local "no effect" fact.
- expect(logger.calls.info.some(([m]) => m === "agent:wake produced no effect (coalesced only)")).toBe(true);
+ expect(logger.calls.info.some(([m]) => m === NO_EFFECT_DIAG)).toBe(true);
expect(wakeAcks).toEqual([{ agentId: "a1", launchId: "l1", status: "ok" }]);
});
- it("does NOT log the honest-ack diagnostic when deliver produced an effect", async () => {
+ it("does NOT log the diagnostic when deliver produced an effect", async () => {
const { mgr } = fakeManager({ a1: "idle" }, true);
const { ch, fire } = fakeChannel();
const logger = stubLogger();
@@ -200,7 +202,28 @@ describe("AgentRouter — agent:wake", () => {
unreadNotice: { kind: "unread_notice", channel: "/demo/general", latestSeq: 7 },
});
- expect(logger.calls.info.some(([m]) => m === "agent:wake produced no effect (coalesced only)")).toBe(false);
+ expect(logger.calls.info.some(([m]) => m === NO_EFFECT_DIAG)).toBe(false);
+ });
+
+ it("does NOT log the diagnostic for a benign no-effect coalesce on a non-running agent (starting)", async () => {
+ // A no-effect wake while starting/stopping/reset-window is routine
+ // queue-and-drain, not the deaf-orphan fault — must stay quiet (batch A
+ // narrowing per Claudette's gate observation).
+ const { mgr } = fakeManager({ a1: "starting" }, false);
+ const { ch, fire } = fakeChannel();
+ const logger = stubLogger();
+ const router = new AgentRouter({ manager: mgr, channel: ch, runtimeReport: [{ id: "mock" }], logger });
+ await router.start();
+
+ await fire({
+ type: "agent:wake",
+ agentId: "a1",
+ config: { version: 1, runtime: "mock", model: { kind: "default" }, mode: { kind: "default" } },
+ launchId: "l1",
+ unreadNotice: { kind: "unread_notice", channel: "/demo/general", latestSeq: 7 },
+ });
+
+ expect(logger.calls.info.some(([m]) => m === NO_EFFECT_DIAG)).toBe(false);
});
});
diff --git a/src/daemon/src/manager/agentRouter.ts b/src/daemon/src/manager/agentRouter.ts
index 2db4d6142..c718d62a9 100644
--- a/src/daemon/src/manager/agentRouter.ts
+++ b/src/daemon/src/manager/agentRouter.ts
@@ -411,22 +411,24 @@ export class AgentRouter {
seq: cmd.unreadNotice.latestSeq,
text,
});
- // Honest-ack diagnostic (plans/daemon-fsm-desync.md batch B): a wake
- // that produced NO executable effect was only coalesced into the
- // inbox. Benign for the queue-and-drain states (starting/stopping/
- // reset-window/per-turn), but on a `running` agent it can mean the
- // message is stacking behind a process that will never drain it (the
- // deaf-orphan class). Surface it as a daemon-local log line so the
- // stall stops masquerading as a healthy `status:"ok"` ack. This does
- // NOT change the wire ack status — reachability-based honesty is
+ // Honest-ack diagnostic (plans/daemon-fsm-desync.md batch B, narrowed
+ // in batch A): a wake that produced NO executable effect was only
+ // coalesced into the inbox. Coalescing is BENIGN and routine for the
+ // queue-and-drain states (starting/stopping/reset-window/per-turn) —
+ // logging those is just noise on a busy-spawning agent. The
+ // pathological shape is a wake landing on an ALREADY-`running` agent
+ // that produces nothing: the message may be stacking behind a process
+ // that will never drain it (the deaf-orphan class). So gate the
+ // diagnostic on `beforeStatus === "running"` to keep the signal
+ // pointed at the real fault. Surfaced as a daemon-local log line so
+ // the stall stops masquerading as a healthy `status:"ok"` ack; this
+ // does NOT change the wire ack status — reachability-based honesty is
// batch D's job (it needs a liveness probe this router deliberately
// avoids). `producedEffect` is `false` for a legacy mock manager
- // whose `deliver` returns void, so the diagnostic simply won't fire
- // in those tests.
- if (producedEffect === false) {
- this.log.info("agent:wake produced no effect (coalesced only)", {
+ // whose `deliver` returns void, so the diagnostic won't fire there.
+ if (producedEffect === false && beforeStatus === "running") {
+ this.log.info("agent:wake produced no effect (coalesced onto running agent)", {
agentId: cmd.agentId,
- beforeStatus,
latestSeq: cmd.unreadNotice.latestSeq,
});
}
diff --git a/src/daemon/src/manager/managerPolicy.test.ts b/src/daemon/src/manager/managerPolicy.test.ts
index 627cd9b95..d84e8d36c 100644
--- a/src/daemon/src/manager/managerPolicy.test.ts
+++ b/src/daemon/src/manager/managerPolicy.test.ts
@@ -376,6 +376,72 @@ describe("reduceManager — tick: stall + idle hibernation", () => {
});
});
+// Batch A (plans/daemon-fsm-desync.md): the suspected-deaf detector catches the
+// gated-no-further-turn_end permanent orphan that slips BOTH existing tick
+// predicates. Reproduces Olivia's wedge: a gated agent turn-ends idle, a later
+// wake drain-sends (stamping lastDeliverAt, re-arming turnActive), then the
+// process goes deaf — no progress, no turn_end, no exit ever follows.
+describe("reduceManager — tick: suspected-deaf detection (batch A)", () => {
+ // Build the exact orphan tuple: running, turnActive (from the drain-send),
+ // inbox empty (drained), lastDeliverAt > lastProgressAt, past the threshold.
+ function toDeafOrphan(staleMs = 100) {
+ let s = createInitialManagerState(staleMs);
+ s = register(s, "a", PERSISTENT_GATED);
+ s = reduceManager(s, { type: "wake", agentId: "a", message: { text: "m1" }, nowMs: 0 }).state;
+ s = reduceManager(s, { type: "spawned", agentId: "a", nowMs: 0 }).state;
+ s = reduceManager(s, { type: "session", agentId: "a", sessionId: "sess-1" }).state;
+ // Turn ends → turnActive false, idleSince armed, lastProgressAt=10.
+ s = reduceManager(s, { type: "turn_end", agentId: "a", nowMs: 10 }).state;
+ // A later wake on the now-idle turn drain-sends into the (about-to-go-deaf)
+ // process: stamps lastDeliverAt=20, re-arms turnActive, drains the inbox.
+ s = reduceManager(s, { type: "wake", agentId: "a", message: { text: "m2" }, nowMs: 20 }).state;
+ return s;
+ }
+
+ it("terminates a gated agent whose delivery got no process response past the threshold", () => {
+ const s = toDeafOrphan();
+ expect(s.agents.a.turnActive).toBe(true);
+ expect(s.agents.a.inbox.length).toBe(0); // drained — slips stalled's gated sub-clause
+ expect(s.agents.a.lastDeliverAt).toBe(20);
+ expect(s.agents.a.lastProgressAt).toBe(10); // deliver newer than last progress
+
+ const r = reduceManager(s, { type: "tick", nowMs: 200 });
+ expect(r.effects).toEqual([{ type: "terminate_stalled", agentId: "a" }]);
+ expect(r.state.agents.a.status).toBe("stopping");
+ });
+
+ it("does NOT terminate before the stale window elapses after delivery", () => {
+ const s = toDeafOrphan();
+ expect(reduceManager(s, { type: "tick", nowMs: 100 }).effects).toEqual([]); // 100-20 < 100
+ });
+
+ it("does NOT flag a healthy agent whose process reported progress after the delivery", () => {
+ let s = toDeafOrphan();
+ // Process answers: a progress event lands after the deliver → lastProgressAt
+ // (30) overtakes lastDeliverAt (20), so the suspicion clears on its own.
+ s = reduceManager(s, { type: "progress", agentId: "a", nowMs: 30 }).state;
+ expect(reduceManager(s, { type: "tick", nowMs: 200 }).effects).toEqual([]);
+ });
+
+ it("does NOT flag a fresh agent that never had a delivery (lastDeliverAt null)", () => {
+ let s = createInitialManagerState(100);
+ s = register(s, "a", PERSISTENT_GATED);
+ s = reduceManager(s, { type: "wake", agentId: "a", message: { text: "m1" }, nowMs: 0 }).state;
+ s = reduceManager(s, { type: "spawned", agentId: "a", nowMs: 0 }).state;
+ expect(s.agents.a.lastDeliverAt).toBeNull();
+ expect(reduceManager(s, { type: "tick", nowMs: 200 }).effects).toEqual([]);
+ });
+
+ it("clears lastDeliverAt across a respawn (no cross-lifecycle stale suspicion)", () => {
+ let s = toDeafOrphan();
+ // The orphan gets terminated → exit → onExit; then respawn. lastDeliverAt
+ // must be null again so the next lifecycle starts clean.
+ s = reduceManager(s, { type: "tick", nowMs: 200 }).state; // → stopping + terminate
+ s = reduceManager(s, { type: "exit", agentId: "a" }).state;
+ expect(s.agents.a.lastDeliverAt).toBeNull();
+ });
+});
+
describe("reduceManager — reset_session", () => {
it("nulls sessionId on a known agent without changing status/turnActive", () => {
let s = createInitialManagerState();
diff --git a/src/daemon/src/manager/managerPolicy.ts b/src/daemon/src/manager/managerPolicy.ts
index 90798f24b..f291983d6 100644
--- a/src/daemon/src/manager/managerPolicy.ts
+++ b/src/daemon/src/manager/managerPolicy.ts
@@ -60,6 +60,17 @@ export interface AgentState {
turnActive: boolean;
/** ms timestamp of the last observed progress (event), for stall detection. */
lastProgressAt: number;
+ /**
+ * ms timestamp of the last `send` effect emitted toward the live process
+ * (drain-send / onTurnEnd drain / gated flush); null if none this lifecycle.
+ * Paired with `lastProgressAt` to detect a deaf process: a `send` was
+ * dispatched (`lastDeliverAt > lastProgressAt`) yet no process-reported
+ * event followed within the stale window — the message went into a stdin
+ * nothing is reading. This is the ONLY signal that catches the
+ * gated-no-turn_end permanent orphan, which slips both existing onTick
+ * predicates. See plans/daemon-fsm-desync.md batch A.
+ */
+ lastDeliverAt: number | null;
/** ms timestamp since which the agent has been idle (running, no turn, empty inbox); null if not idle. */
idleSince: number | null;
/**
@@ -184,13 +195,19 @@ export function reduceManager(state: ManagerState, event: ManagerEvent): ReduceR
return withAgent(state, event.agentId, (a) => a ?? freshAgent(event.agentId, event.caps), []);
case "wake":
- return onWake(state, event.agentId, event.message);
+ return onWake(state, event.agentId, event.message, event.nowMs);
case "spawned":
return mutate(state, event.agentId, (a) => {
a.status = "running";
a.turnActive = true;
a.lastProgressAt = event.nowMs;
+ // Fresh process ⇒ no outstanding unanswered delivery yet. Clearing keeps
+ // the suspected-deaf detector (onTick) from carrying a prior lifecycle's
+ // `lastDeliverAt` across a respawn. Redundant with `lastProgressAt` being
+ // bumped here (a stale deliver would already be < progress), but explicit
+ // so the fresh-lifecycle invariant doesn't rely on that coincidence.
+ a.lastDeliverAt = null;
a.idleSince = null;
// Belt for the idle-branch reset path where `deliver` triggered a
// spawn directly (no `exit` fires there). The kill-and-respawn path
@@ -241,7 +258,7 @@ export function reduceManager(state: ManagerState, event: ManagerEvent): ReduceR
return onTick(state, event.nowMs);
case "runtime_signal":
- return onRuntimeSignal(state, event.agentId, event.kind);
+ return onRuntimeSignal(state, event.agentId, event.kind, event.nowMs);
}
}
@@ -249,7 +266,7 @@ export function reduceManager(state: ManagerState, event: ManagerEvent): ReduceR
/* Event handlers */
/* ------------------------------------------------------------------ */
-function onWake(state: ManagerState, agentId: string, message: AgentMsg): ReduceResult {
+function onWake(state: ManagerState, agentId: string, message: AgentMsg, nowMs: number): ReduceResult {
const existing = state.agents[agentId];
const agent = existing ? clone(existing) : null;
if (!agent) {
@@ -306,6 +323,7 @@ function onWake(state: ManagerState, agentId: string, message: AgentMsg): Reduce
const text = drainInboxToPrompt(agent);
const mode = agent.turnActive ? "busy" : "idle";
if (mode === "idle") agent.turnActive = true;
+ agent.lastDeliverAt = nowMs;
return commit(state, agent, [{ type: "send", agentId, text, mode }]);
}
// Per-turn or no stdin ⇒ keep queued; delivered after exit / next spawn.
@@ -336,6 +354,7 @@ function onTurnEnd(state: ManagerState, agentId: string, nowMs: number): ReduceR
if (agent.inbox.length > 0 && agent.caps.supportsStdinNotification) {
const text = drainInboxToPrompt(agent);
agent.turnActive = true;
+ agent.lastDeliverAt = nowMs;
return commit(state, agent, [{ type: "send", agentId, text, mode: "idle" }]);
}
@@ -360,7 +379,7 @@ function onTurnEnd(state: ManagerState, agentId: string, nowMs: number): ReduceR
* 3. If the branch calls for it, attempt a flush via
* `reduceApmGatedFlushReadiness`, reading the now-fully-updated `apm`.
*/
-function onRuntimeSignal(state: ManagerState, agentId: string, kind: string): ReduceResult {
+function onRuntimeSignal(state: ManagerState, agentId: string, kind: string, nowMs: number): ReduceResult {
const existing = state.agents[agentId];
if (!existing) return { state, effects: [] };
const agent = clone(existing);
@@ -428,6 +447,7 @@ function onRuntimeSignal(state: ManagerState, agentId: string, kind: string): Re
});
if (readiness.shouldNotify) {
const text = drainInboxToPrompt(agent);
+ agent.lastDeliverAt = nowMs;
return commit(state, agent, [{ type: "send", agentId, text, mode: "busy" }]);
}
return commit(state, agent, [
@@ -457,6 +477,9 @@ function onExit(state: ManagerState, agentId: string): ReduceResult {
// prevents a dead process's `compacting` / outstanding-tool-use flags from
// silently carrying into the next spawn and re-blocking `onWake`.
agent.apm = createInitialApmGatedSteeringState();
+ // The dead process can't answer any prior delivery — drop the outstanding
+ // deliver marker so the suspected-deaf detector starts clean for the respawn.
+ agent.lastDeliverAt = null;
// Per-turn: if more messages queued, immediately respawn for the next batch.
if (agent.inbox.length > 0) {
@@ -491,7 +514,28 @@ function onTick(state: ManagerState, nowMs: number): ReduceResult {
(a.caps.supportsStdinNotification &&
a.caps.busyDeliveryMode === "gated" &&
a.inbox.length > 0));
- if (stalled) {
+ // Suspected-deaf (plans/daemon-fsm-desync.md batch A): a `send` was
+ // dispatched (`lastDeliverAt` set, and NEWER than the last process-reported
+ // progress) yet the stale window elapsed with no event following it — the
+ // message went into a stdin nothing is reading. This is the ONE trigger
+ // that catches the `gated`-no-`turn_end` permanent orphan, which slips BOTH
+ // existing predicates: `stalled`'s gated sub-clause needs `inbox.length > 0`
+ // but the inbox was drained empty by the very send that stamped
+ // `lastDeliverAt`, and `idleEligible` needs `!turnActive` but the drain-send
+ // set `turnActive = true`. Reuses the existing `terminate_stalled` recovery
+ // (SIGKILL → exit → onExit drain-respawn) — batch A adds a trigger, not a
+ // new recovery path. Batch D upgrades this same suspicion to a confirmed
+ // diagnosis via a liveness probe before any destructive action on the
+ // runtime kinds that lack the SIGKILL backstop (SDK); here it only fires for
+ // a stdin-capable persistent agent whose recovery is physically reliable.
+ const suspectedDeaf =
+ a.status === "running" &&
+ a.caps.lifecycleKind === "persistent" &&
+ a.caps.supportsStdinNotification &&
+ a.lastDeliverAt !== null &&
+ a.lastDeliverAt > a.lastProgressAt &&
+ nowMs - a.lastDeliverAt >= state.staleThresholdMs;
+ if (stalled || suspectedDeaf) {
agents[id] = { ...a, status: "stopping", idleSince: null };
effects.push({ type: "terminate_stalled", agentId: id });
continue;
@@ -528,6 +572,7 @@ function freshAgent(agentId: string, caps: AgentRuntimeCaps): AgentState {
sessionId: null,
turnActive: false,
lastProgressAt: 0,
+ lastDeliverAt: null,
idleSince: null,
resetting: false,
apm: createInitialApmGatedSteeringState(),