Multi-Agent Collaboration for Team and Friends
TypeScript agent-automationagent-orchestrationagent-workflowai-agentsai-collaborationai-memoryai-workforceautonomous-agentsclaude-codeclaude-code-skillscodexcoding-agentmulti-agentmultiple-agentone-person-businessone-person-companyone-person-teamopencodesolopreneur
fix(daemon): re-deliver a stranded held inbox when the process is idle (batch L2)
Gus6d385def1792
diff --git a/src/daemon/src/manager/managerPolicy.test.ts b/src/daemon/src/manager/managerPolicy.test.tsindex 58e8f7af0..5c8c701ab 100644--- a/src/daemon/src/manager/managerPolicy.test.ts+++ b/src/daemon/src/manager/managerPolicy.test.ts@@ -442,6 +442,121 @@ describe("reduceManager — tick: suspected-deaf detection (batch A)", () => {});});+// Batch L2 (plans/daemon-fsm-desync.md): the stranded-held-inbox re-delivery.+// Reproduces Olivia 2026-07-31: a mid-turn wake is gated_held into the inbox,+// then the turn's `turn_end` runtime event is lost (a wake raced the turn close+// by ~1s), so `turnActive` stays stuck true while the process is demonstrably+// idle (`apm.phase === "idle"`) — the held message strands forever. The GENTLE+// backstop re-delivers it (a `send`, no kill), checked BEFORE `stalled` so a+// recoverable strand is fed rather than terminated.+describe("reduceManager — tick: stranded-held-inbox re-delivery (batch L2)", () => {+ // Build the exact wedge tuple: running, turnActive=true, apm.phase idle,+ // inbox non-empty (a held mid-turn wake), heldSince stamped — the state Olivia+ // was stuck in. staleThresholdMs kept huge so `stalled`/`suspectedDeaf` can't+ // fire and confound the assertion; strandedInboxThresholdMs is the 4th arg.+ function toStrandedWedge(strandedMs = 100) {+ let s = createInitialManagerState(1_000_000, 300_000, 120_000, strandedMs);+ s = register(s, "a", PERSISTENT_GATED);+ s = reduceManager(s, { type: "wake", agentId: "a", message: { text: "m1" }, nowMs: 0 }).state;+ s = reduceManager(s, { type: "spawned", agentId: "a", nowMs: 0 }).state; // running, turnActive, apm.phase idle+ s = reduceManager(s, { type: "session", agentId: "a", sessionId: "sess-1" }).state;+ // A mid-turn wake at t=10 is gated-held (turnActive true) — lands in inbox,+ // stamps heldSince=10. This is the message that will strand.+ s = reduceManager(s, { type: "wake", agentId: "a", message: { text: "held" }, nowMs: 10 }).state;+ return s;+ }++ it("re-delivers a held message once the process has sat idle holding it past the threshold", () => {+ const s = toStrandedWedge();+ expect(s.agents.a.turnActive).toBe(true);+ expect(s.agents.a.apm.phase).toBe("idle");+ expect(s.agents.a.inbox.length).toBe(1);+ expect(s.agents.a.heldSince).toBe(10);++ const r = reduceManager(s, { type: "tick", nowMs: 200 }); // 200-10 >= 100+ // GENTLE: a plain idle send draining the inbox — NOT a terminate.+ expect(r.effects).toEqual([{ type: "send", agentId: "a", text: "held", mode: "idle" }]);+ expect(r.state.agents.a.status).toBe("running"); // no kill, still running+ expect(r.state.agents.a.inbox.length).toBe(0); // drained+ expect(r.state.agents.a.heldSince).toBeNull(); // strand cleared+ // Upgrade chain armed: lastDeliverAt stamped so suspectedDeaf takes over if+ // this re-deliver also gets no response.+ expect(r.state.agents.a.lastDeliverAt).toBe(200);+ expect(r.state.agents.a.turnActive).toBe(true);+ });++ it("does NOT re-deliver before the stranded threshold elapses", () => {+ const s = toStrandedWedge();+ expect(reduceManager(s, { type: "tick", nowMs: 50 }).effects).toEqual([]); // 50-10 < 100+ });++ it("measures from the OLDEST held wake — a later coalescing wake does not reset the clock", () => {+ let s = toStrandedWedge();+ // A second held wake at t=90 coalesces in; heldSince must stay 10, not jump.+ s = reduceManager(s, { type: "wake", agentId: "a", message: { text: "held2" }, nowMs: 90 }).state;+ expect(s.agents.a.heldSince).toBe(10);+ expect(s.agents.a.inbox.length).toBe(2);+ // At t=115 the oldest strand is 105ms (>=100) → fire, even though the newest+ // wake is only 25ms old. Both held messages drain together.+ const r = reduceManager(s, { type: "tick", nowMs: 115 });+ expect(r.effects).toEqual([{ type: "send", agentId: "a", text: "held\nheld2", mode: "idle" }]);+ expect(r.state.agents.a.heldSince).toBeNull();+ });++ it("re-delivers (does not kill) BEFORE stalled would escalate — gentle-first ordering", () => {+ // Both stranded (L2) and stalled key on running+turnActive+gated+inbox>0;+ // with a small stale threshold too, L2 must win by running first + continue.+ let s = createInitialManagerState(100, 300_000, 120_000, 100); // staleMs==strandedMs==100+ s = register(s, "a", PERSISTENT_GATED);+ s = reduceManager(s, { type: "wake", agentId: "a", message: { text: "m1" }, nowMs: 0 }).state;+ s = reduceManager(s, { type: "spawned", agentId: "a", nowMs: 0 }).state;+ s = reduceManager(s, { type: "wake", agentId: "a", message: { text: "held" }, nowMs: 10 }).state;+ const r = reduceManager(s, { type: "tick", nowMs: 200 });+ expect(r.effects).toEqual([{ type: "send", agentId: "a", text: "held", mode: "idle" }]);+ expect(r.effects.some((e) => e.type === "terminate_stalled")).toBe(false);+ });++ it("does NOT re-deliver when the process reports a turn genuinely in flight (apm.phase not idle)", () => {+ let s = toStrandedWedge();+ // A tool_call moves apm.phase off idle — the process IS mid-work, so holding+ // is correct and a raw send could collide. L2 must not fire.+ s = reduceManager(s, { type: "runtime_signal", agentId: "a", kind: "tool_call", nowMs: 20 }).state;+ expect(s.agents.a.apm.phase).not.toBe("idle");+ expect(reduceManager(s, { type: "tick", nowMs: 200 }).effects).toEqual([]);+ });++ it("clears heldSince on a normal onTurnEnd redeliver so no stale strand lingers", () => {+ let s = toStrandedWedge();+ expect(s.agents.a.heldSince).toBe(10);+ // The lost turn_end finally arrives (or a real one does): onTurnEnd drains+ // the held inbox itself → heldSince must clear, so L2 won't double-fire.+ const r = reduceManager(s, { type: "turn_end", agentId: "a", nowMs: 30 });+ expect(r.effects).toEqual([{ type: "send", agentId: "a", text: "held", mode: "idle" }]);+ expect(r.state.agents.a.heldSince).toBeNull();+ });++ it("UPGRADE CHAIN: a re-deliver that STILL gets no response escalates to suspectedDeaf (why L2 needs no kill branch)", () => {+ // strandedMs=100, staleMs=100: L2 re-delivers first (gentle), and if the+ // process is genuinely deaf the re-deliver's stamped lastDeliverAt arms+ // suspectedDeaf, which terminates+respawns after the stale window.+ let s = createInitialManagerState(100, 300_000, 120_000, 100);+ s = register(s, "a", PERSISTENT_GATED);+ s = reduceManager(s, { type: "wake", agentId: "a", message: { text: "m1" }, nowMs: 0 }).state;+ s = reduceManager(s, { type: "spawned", agentId: "a", nowMs: 0 }).state;+ s = reduceManager(s, { type: "wake", agentId: "a", message: { text: "held" }, nowMs: 10 }).state;+ // t=200: L2 re-delivers (gentle send, no kill), stamps lastDeliverAt=200.+ const r1 = reduceManager(s, { type: "tick", nowMs: 200 });+ expect(r1.effects).toEqual([{ type: "send", agentId: "a", text: "held", mode: "idle" }]);+ s = r1.state;+ expect(s.agents.a.lastDeliverAt).toBe(200);+ // The process never answers the re-deliver → no progress after 200. At+ // t=400 (>=100 past the deliver) suspectedDeaf fires: NOW it's a kill.+ const r2 = reduceManager(s, { type: "tick", nowMs: 400 });+ expect(r2.effects).toEqual([{ type: "terminate_stalled", agentId: "a" }]);+ expect(r2.state.agents.a.status).toBe("stopping");+ });+});+// Batch D (plans/daemon-fsm-desync.md): the reset-stuck reconcile. The reset// window (`resetting`) closes only via `enterStable` at a stable running/idle// state; if the converging event never arrives the agent wedges in `starting`diff --git a/src/daemon/src/manager/managerPolicy.ts b/src/daemon/src/manager/managerPolicy.tsindex 702cbba1a..b48ccb22a 100644--- a/src/daemon/src/manager/managerPolicy.ts+++ b/src/daemon/src/manager/managerPolicy.ts@@ -73,6 +73,19 @@ export interface AgentState {lastDeliverAt: number | null;/** ms timestamp since which the agent has been idle (running, no turn, empty inbox); null if not idle. */idleSince: number | null;+ /**+ * ms timestamp at which the OLDEST currently-held mid-turn wake was gated+ * into the inbox (`onWake`'s `gated_hold` branch); null when nothing is being+ * held. This is the ground-truth "how long has a message been stranded"+ * clock the L2 stranded-inbox backstop keys on — deliberately NOT reused from+ * `lastDeliverAt` (the held path never sends, so that stays null) or+ * `lastProgressAt` (means "last progress event", not "how long held", and can+ * drift). SET on the first hold while none is outstanding; CLEARED (→ null)+ * the moment the inbox is drained (onTurnEnd redeliver, onWake steer, the L2+ * re-deliver) or the lifecycle ends (exit/reset). See+ * plans/daemon-fsm-desync.md batch L2.+ */+ heldSince: number | null;/*** True during an owner-triggered reset window. Gates every inbound wake to* inbox-only (no `spawn`, no `send`, no `gated_hold`) EXCEPT when the@@ -132,6 +145,13 @@ export interface ManagerState {* `DEFAULT_RESET_STUCK_THRESHOLD_MS`.*/resetStuckThresholdMs: number;+ /**+ * Stranded-held-inbox threshold: a `running`+`turnActive` gated agent whose+ * process self-reports idle (`apm.phase === "idle"`) but still holds a+ * non-empty inbox past this long ⇒ the tick re-delivers (no kill). See+ * `DEFAULT_STRANDED_INBOX_THRESHOLD_MS`.+ */+ strandedInboxThresholdMs: number;}/* ------------------------------------------------------------------ */@@ -201,6 +221,21 @@ export const DEFAULT_IDLE_TIMEOUT_MS = 300_000;* See plans/daemon-fsm-desync.md batch D.*/export const DEFAULT_RESET_STUCK_THRESHOLD_MS = 120_000;+/**+ * Stranded-held-inbox threshold: how long a `running` + `turnActive` gated+ * agent whose process self-reports `apm.phase === "idle"` (the turn actually+ * ended) may sit with a NON-EMPTY inbox before the tick re-delivers it. This is+ * the "held message never flushed" backstop: onWake holds a mid-turn wake into+ * the inbox expecting `onTurnEnd`/`onRuntimeSignal` to flush it at the next safe+ * boundary, but if the `turn_end` runtime event is lost (e.g. a wake races the+ * turn's close by ~1s, plans/daemon-fsm-desync.md), `turnActive` stays stuck+ * true and the held message strands forever with the live process idle. A+ * SEPARATE constant from `staleThresholdMs`: that measures "a running turn made+ * no progress" (→ terminate); this measures "the process is demonstrably idle+ * yet we're still holding its inbox" (→ gently re-deliver, no kill). Different+ * semantics, independently tunable. See plans/daemon-fsm-desync.md batch L2.+ */+export const DEFAULT_STRANDED_INBOX_THRESHOLD_MS = 30_000;export interface ReduceResult {state: ManagerState;@@ -211,8 +246,9 @@ export function createInitialManagerState(staleThresholdMs = DEFAULT_STALE_THRESHOLD_MS,idleTimeoutMs = DEFAULT_IDLE_TIMEOUT_MS,resetStuckThresholdMs = DEFAULT_RESET_STUCK_THRESHOLD_MS,+ strandedInboxThresholdMs = DEFAULT_STRANDED_INBOX_THRESHOLD_MS,): ManagerState {- return { agents: {}, staleThresholdMs, idleTimeoutMs, resetStuckThresholdMs };+ return { agents: {}, staleThresholdMs, idleTimeoutMs, resetStuckThresholdMs, strandedInboxThresholdMs };}/* ------------------------------------------------------------------ */@@ -344,6 +380,12 @@ function onWake(state: ManagerState, agentId: string, message: AgentMsg, nowMs:// `onRuntimeSignal` flushes it at the next safe boundary.const isGatedMidTurn = agent.caps.busyDeliveryMode === "gated" && agent.turnActive;if (isGatedMidTurn) {+ // Start the stranding clock on the FIRST held message while none is+ // outstanding; leave it if already set so it measures from the OLDEST+ // held wake (a later wake coalescing in must not reset the timer, or a+ // steady wake stream would keep the strand invisible to L2 forever). The+ // matching clear is in `drainInboxToPrompt` (inbox emptied ⇒ null).+ if (agent.heldSince === null) agent.heldSince = nowMs;return commit(state, agent, [{type: "gated_hold",@@ -538,6 +580,64 @@ function onTick(state: ManagerState, nowMs: number): ReduceResult {for (const id of Object.keys(agents)) {const a = agents[id];+ // Stranded-held-inbox re-delivery (plans/daemon-fsm-desync.md batch L2):+ // the GENTLE backstop, checked BEFORE `stalled` so a recoverable strand is+ // re-delivered rather than escalated to a kill. onWake holds a mid-turn+ // wake into the inbox (`gated_hold`) expecting a later `turn_end` /+ // safe-boundary signal to flush it. If that `turn_end` runtime event is+ // lost — e.g. a wake races the turn's close by ~1s (Olivia 2026-07-31:+ // wake 09:41:08, end_turn 09:41:09) — `turnActive` stays stuck true and the+ // held message strands forever while the live process sits IDLE. The tell+ // that it's safe to re-deliver is `apm.phase === "idle"`: the process itself+ // reports no turn in flight (same field `gated_hold`'s `blockedReason`+ // reads), so a `send` now can't collide with an in-flight tool call /+ // thinking block / compaction. Emit a plain `send` (idle mode) draining the+ // inbox — the SAME effect `onTurnEnd` would have emitted had the signal+ // arrived; NO kill. Distinct from `stalled` (turn-genuinely-stuck →+ // terminate) and `suspectedDeaf` (a send went unanswered): here nothing was+ // ever sent and the process is provably idle, so the correct action is to+ // finally deliver, not to destroy.+ //+ // STRANDING ANCHOR = `heldSince` (a dedicated field), NOT `lastDeliverAt` or+ // `lastProgressAt`. The held path never issues a `send`, so `lastDeliverAt`+ // stays the `spawned`-time null — keying on it (as `suspectedDeaf` does)+ // would make this predicate NEVER fire, the exact trap that dooms `stalled`+ // to key on `lastProgressAt`. And `lastProgressAt` means "last progress+ // EVENT", not "how long held" — a stray progress event would drift it. So+ // `onWake` stamps `heldSince` when it first holds a wake, and+ // `drainInboxToPrompt` clears it; `nowMs - heldSince` is precisely "how long+ // the oldest held message has been stranded". Threshold sits well above a+ // normal turn's length so a wake that merely raced the turn's close by ~1s+ // isn't misread as stranded.+ //+ // UPGRADE CHAIN: on re-deliver set `turnActive = true` + `lastDeliverAt =+ // nowMs` (mirrors `onTurnEnd`'s drain). If the process was merely un-poked,+ // it now works and emits progress → self-healed. If it's actually deaf, this+ // send goes unanswered and `lastDeliverAt > lastProgressAt` arms+ // `suspectedDeaf`, which terminates+respawns after the stale window. So the+ // gentle re-deliver and the existing kill-path compose into "try feeding+ // first, escalate to kill only if feeding gets no response" — which is why+ // L2 needs no kill branch of its own (that would be fire-and-assume).+ const strandedHeld =+ a.status === "running" &&+ a.turnActive &&+ a.caps.busyDeliveryMode === "gated" &&+ a.caps.supportsStdinNotification &&+ a.apm.phase === "idle" &&+ a.inbox.length > 0 &&+ a.heldSince !== null &&+ nowMs - a.heldSince >= state.strandedInboxThresholdMs;+ if (strandedHeld) {+ const drainedAgent = clone(a);+ const text = drainInboxToPrompt(drainedAgent);+ drainedAgent.turnActive = true;+ drainedAgent.lastDeliverAt = nowMs;+ drainedAgent.idleSince = null;+ agents[id] = drainedAgent;+ effects.push({ type: "send", agentId: id, text, mode: "idle" });+ continue;+ }+// Stalled recovery: running, turn in flight, no progress past threshold.// Gated runtimes are normally excluded from restart (their tool-batch/// compaction/review boundaries do the flushing), BUT a silent gated turn@@ -651,6 +751,7 @@ function freshAgent(agentId: string, caps: AgentRuntimeCaps): AgentState {lastProgressAt: 0,lastDeliverAt: null,idleSince: null,+ heldSince: null,resetting: false,resettingSince: null,apm: createInitialApmGatedSteeringState(),@@ -694,6 +795,12 @@ function drainInboxToPrompt(agent: AgentState): string {}}agent.inbox = [];+ // The inbox is now empty, so nothing is stranded — clear the hold clock. This+ // is the single chokepoint that empties the inbox (every drain path calls it),+ // so clearing `heldSince` here covers onWake-steer / onTurnEnd-redeliver /+ // onExit-respawn / the L2 re-deliver at once, keeping the invariant+ // "heldSince != null ⇔ inbox non-empty and being held" without per-site edits.+ agent.heldSince = null;return unique.join("\n");}diff --git a/src/daemon/src/manager/managerRuntime.ts b/src/daemon/src/manager/managerRuntime.tsindex eec9979ec..5edb376fe 100644--- a/src/daemon/src/manager/managerRuntime.ts+++ b/src/daemon/src/manager/managerRuntime.ts@@ -13,6 +13,7 @@import {reduceManager,createInitialManagerState,+ DEFAULT_STRANDED_INBOX_THRESHOLD_MS,type ManagerState,type ManagerEvent,type ManagerEffect,@@ -91,6 +92,8 @@ export interface ManagerRuntimeOpts {idleTimeoutMs?: number;/** Reset-stuck reconcile threshold (ms): `resetting` stuck this long ⇒ escalate. */resetStuckThresholdMs?: number;+ /** Stranded-held-inbox threshold (ms): a running+turnActive gated agent whose process is idle yet holds a non-empty inbox this long ⇒ re-deliver (no kill). */+ strandedInboxThresholdMs?: number;/*** Handshake watchdog (ms): a spawned session that never emits its first* `runtime_event` (the handshake) within this window is treated as a@@ -582,13 +585,19 @@ export class AgentProcessManager {staleThresholdMs: 120_000,idleTimeoutMs: 300_000,resetStuckThresholdMs: 120_000,+ strandedInboxThresholdMs: DEFAULT_STRANDED_INBOX_THRESHOLD_MS,handshakeTimeoutMs: 60_000,stampWakePromptTime: false,...opts,};this.now = opts.now ?? (() => Date.now());this.log = opts.logger ?? createLogger({ header: "@alook/daemon:manager" });- this.state = createInitialManagerState(this.opts.staleThresholdMs, this.opts.idleTimeoutMs, this.opts.resetStuckThresholdMs);+ this.state = createInitialManagerState(+ this.opts.staleThresholdMs,+ this.opts.idleTimeoutMs,+ this.opts.resetStuckThresholdMs,+ this.opts.strandedInboxThresholdMs,+ );}/**