Skip to content

armgit / alook

Public Imported from GitHub
0

Multi-Agent Collaboration for Team and Friends

fix(daemon): re-deliver a stranded held inbox when the process is idle (batch L2)

Gus6d385def1792
diff --git a/src/daemon/src/manager/managerPolicy.test.ts b/src/daemon/src/manager/managerPolicy.test.ts
index 58e8f7af0..5c8c701ab 100644
--- a/src/daemon/src/manager/managerPolicy.test.ts
+++ b/src/daemon/src/manager/managerPolicy.test.ts
@@ -442,6 +442,121 @@ describe("reduceManager — tick: suspected-deaf detection (batch A)", () => {
});
});
+// Batch L2 (plans/daemon-fsm-desync.md): the stranded-held-inbox re-delivery.
+// Reproduces Olivia 2026-07-31: a mid-turn wake is gated_held into the inbox,
+// then the turn's `turn_end` runtime event is lost (a wake raced the turn close
+// by ~1s), so `turnActive` stays stuck true while the process is demonstrably
+// idle (`apm.phase === "idle"`) — the held message strands forever. The GENTLE
+// backstop re-delivers it (a `send`, no kill), checked BEFORE `stalled` so a
+// recoverable strand is fed rather than terminated.
+describe("reduceManager — tick: stranded-held-inbox re-delivery (batch L2)", () => {
+ // Build the exact wedge tuple: running, turnActive=true, apm.phase idle,
+ // inbox non-empty (a held mid-turn wake), heldSince stamped — the state Olivia
+ // was stuck in. staleThresholdMs kept huge so `stalled`/`suspectedDeaf` can't
+ // fire and confound the assertion; strandedInboxThresholdMs is the 4th arg.
+ function toStrandedWedge(strandedMs = 100) {
+ let s = createInitialManagerState(1_000_000, 300_000, 120_000, strandedMs);
+ s = register(s, "a", PERSISTENT_GATED);
+ s = reduceManager(s, { type: "wake", agentId: "a", message: { text: "m1" }, nowMs: 0 }).state;
+ s = reduceManager(s, { type: "spawned", agentId: "a", nowMs: 0 }).state; // running, turnActive, apm.phase idle
+ s = reduceManager(s, { type: "session", agentId: "a", sessionId: "sess-1" }).state;
+ // A mid-turn wake at t=10 is gated-held (turnActive true) — lands in inbox,
+ // stamps heldSince=10. This is the message that will strand.
+ s = reduceManager(s, { type: "wake", agentId: "a", message: { text: "held" }, nowMs: 10 }).state;
+ return s;
+ }
+
+ it("re-delivers a held message once the process has sat idle holding it past the threshold", () => {
+ const s = toStrandedWedge();
+ expect(s.agents.a.turnActive).toBe(true);
+ expect(s.agents.a.apm.phase).toBe("idle");
+ expect(s.agents.a.inbox.length).toBe(1);
+ expect(s.agents.a.heldSince).toBe(10);
+
+ const r = reduceManager(s, { type: "tick", nowMs: 200 }); // 200-10 >= 100
+ // GENTLE: a plain idle send draining the inbox — NOT a terminate.
+ expect(r.effects).toEqual([{ type: "send", agentId: "a", text: "held", mode: "idle" }]);
+ expect(r.state.agents.a.status).toBe("running"); // no kill, still running
+ expect(r.state.agents.a.inbox.length).toBe(0); // drained
+ expect(r.state.agents.a.heldSince).toBeNull(); // strand cleared
+ // Upgrade chain armed: lastDeliverAt stamped so suspectedDeaf takes over if
+ // this re-deliver also gets no response.
+ expect(r.state.agents.a.lastDeliverAt).toBe(200);
+ expect(r.state.agents.a.turnActive).toBe(true);
+ });
+
+ it("does NOT re-deliver before the stranded threshold elapses", () => {
+ const s = toStrandedWedge();
+ expect(reduceManager(s, { type: "tick", nowMs: 50 }).effects).toEqual([]); // 50-10 < 100
+ });
+
+ it("measures from the OLDEST held wake — a later coalescing wake does not reset the clock", () => {
+ let s = toStrandedWedge();
+ // A second held wake at t=90 coalesces in; heldSince must stay 10, not jump.
+ s = reduceManager(s, { type: "wake", agentId: "a", message: { text: "held2" }, nowMs: 90 }).state;
+ expect(s.agents.a.heldSince).toBe(10);
+ expect(s.agents.a.inbox.length).toBe(2);
+ // At t=115 the oldest strand is 105ms (>=100) → fire, even though the newest
+ // wake is only 25ms old. Both held messages drain together.
+ const r = reduceManager(s, { type: "tick", nowMs: 115 });
+ expect(r.effects).toEqual([{ type: "send", agentId: "a", text: "held\nheld2", mode: "idle" }]);
+ expect(r.state.agents.a.heldSince).toBeNull();
+ });
+
+ it("re-delivers (does not kill) BEFORE stalled would escalate — gentle-first ordering", () => {
+ // Both stranded (L2) and stalled key on running+turnActive+gated+inbox>0;
+ // with a small stale threshold too, L2 must win by running first + continue.
+ let s = createInitialManagerState(100, 300_000, 120_000, 100); // staleMs==strandedMs==100
+ s = register(s, "a", PERSISTENT_GATED);
+ s = reduceManager(s, { type: "wake", agentId: "a", message: { text: "m1" }, nowMs: 0 }).state;
+ s = reduceManager(s, { type: "spawned", agentId: "a", nowMs: 0 }).state;
+ s = reduceManager(s, { type: "wake", agentId: "a", message: { text: "held" }, nowMs: 10 }).state;
+ const r = reduceManager(s, { type: "tick", nowMs: 200 });
+ expect(r.effects).toEqual([{ type: "send", agentId: "a", text: "held", mode: "idle" }]);
+ expect(r.effects.some((e) => e.type === "terminate_stalled")).toBe(false);
+ });
+
+ it("does NOT re-deliver when the process reports a turn genuinely in flight (apm.phase not idle)", () => {
+ let s = toStrandedWedge();
+ // A tool_call moves apm.phase off idle — the process IS mid-work, so holding
+ // is correct and a raw send could collide. L2 must not fire.
+ s = reduceManager(s, { type: "runtime_signal", agentId: "a", kind: "tool_call", nowMs: 20 }).state;
+ expect(s.agents.a.apm.phase).not.toBe("idle");
+ expect(reduceManager(s, { type: "tick", nowMs: 200 }).effects).toEqual([]);
+ });
+
+ it("clears heldSince on a normal onTurnEnd redeliver so no stale strand lingers", () => {
+ let s = toStrandedWedge();
+ expect(s.agents.a.heldSince).toBe(10);
+ // The lost turn_end finally arrives (or a real one does): onTurnEnd drains
+ // the held inbox itself → heldSince must clear, so L2 won't double-fire.
+ const r = reduceManager(s, { type: "turn_end", agentId: "a", nowMs: 30 });
+ expect(r.effects).toEqual([{ type: "send", agentId: "a", text: "held", mode: "idle" }]);
+ expect(r.state.agents.a.heldSince).toBeNull();
+ });
+
+ it("UPGRADE CHAIN: a re-deliver that STILL gets no response escalates to suspectedDeaf (why L2 needs no kill branch)", () => {
+ // strandedMs=100, staleMs=100: L2 re-delivers first (gentle), and if the
+ // process is genuinely deaf the re-deliver's stamped lastDeliverAt arms
+ // suspectedDeaf, which terminates+respawns after the stale window.
+ let s = createInitialManagerState(100, 300_000, 120_000, 100);
+ s = register(s, "a", PERSISTENT_GATED);
+ s = reduceManager(s, { type: "wake", agentId: "a", message: { text: "m1" }, nowMs: 0 }).state;
+ s = reduceManager(s, { type: "spawned", agentId: "a", nowMs: 0 }).state;
+ s = reduceManager(s, { type: "wake", agentId: "a", message: { text: "held" }, nowMs: 10 }).state;
+ // t=200: L2 re-delivers (gentle send, no kill), stamps lastDeliverAt=200.
+ const r1 = reduceManager(s, { type: "tick", nowMs: 200 });
+ expect(r1.effects).toEqual([{ type: "send", agentId: "a", text: "held", mode: "idle" }]);
+ s = r1.state;
+ expect(s.agents.a.lastDeliverAt).toBe(200);
+ // The process never answers the re-deliver → no progress after 200. At
+ // t=400 (>=100 past the deliver) suspectedDeaf fires: NOW it's a kill.
+ const r2 = reduceManager(s, { type: "tick", nowMs: 400 });
+ expect(r2.effects).toEqual([{ type: "terminate_stalled", agentId: "a" }]);
+ expect(r2.state.agents.a.status).toBe("stopping");
+ });
+});
+
// Batch D (plans/daemon-fsm-desync.md): the reset-stuck reconcile. The reset
// window (`resetting`) closes only via `enterStable` at a stable running/idle
// state; if the converging event never arrives the agent wedges in `starting`
diff --git a/src/daemon/src/manager/managerPolicy.ts b/src/daemon/src/manager/managerPolicy.ts
index 702cbba1a..b48ccb22a 100644
--- a/src/daemon/src/manager/managerPolicy.ts
+++ b/src/daemon/src/manager/managerPolicy.ts
@@ -73,6 +73,19 @@ export interface AgentState {
lastDeliverAt: number | null;
/** ms timestamp since which the agent has been idle (running, no turn, empty inbox); null if not idle. */
idleSince: number | null;
+ /**
+ * ms timestamp at which the OLDEST currently-held mid-turn wake was gated
+ * into the inbox (`onWake`'s `gated_hold` branch); null when nothing is being
+ * held. This is the ground-truth "how long has a message been stranded"
+ * clock the L2 stranded-inbox backstop keys on — deliberately NOT reused from
+ * `lastDeliverAt` (the held path never sends, so that stays null) or
+ * `lastProgressAt` (means "last progress event", not "how long held", and can
+ * drift). SET on the first hold while none is outstanding; CLEARED (→ null)
+ * the moment the inbox is drained (onTurnEnd redeliver, onWake steer, the L2
+ * re-deliver) or the lifecycle ends (exit/reset). See
+ * plans/daemon-fsm-desync.md batch L2.
+ */
+ heldSince: number | null;
/**
* True during an owner-triggered reset window. Gates every inbound wake to
* inbox-only (no `spawn`, no `send`, no `gated_hold`) EXCEPT when the
@@ -132,6 +145,13 @@ export interface ManagerState {
* `DEFAULT_RESET_STUCK_THRESHOLD_MS`.
*/
resetStuckThresholdMs: number;
+ /**
+ * Stranded-held-inbox threshold: a `running`+`turnActive` gated agent whose
+ * process self-reports idle (`apm.phase === "idle"`) but still holds a
+ * non-empty inbox past this long ⇒ the tick re-delivers (no kill). See
+ * `DEFAULT_STRANDED_INBOX_THRESHOLD_MS`.
+ */
+ strandedInboxThresholdMs: number;
}
/* ------------------------------------------------------------------ */
@@ -201,6 +221,21 @@ export const DEFAULT_IDLE_TIMEOUT_MS = 300_000;
* See plans/daemon-fsm-desync.md batch D.
*/
export const DEFAULT_RESET_STUCK_THRESHOLD_MS = 120_000;
+/**
+ * Stranded-held-inbox threshold: how long a `running` + `turnActive` gated
+ * agent whose process self-reports `apm.phase === "idle"` (the turn actually
+ * ended) may sit with a NON-EMPTY inbox before the tick re-delivers it. This is
+ * the "held message never flushed" backstop: onWake holds a mid-turn wake into
+ * the inbox expecting `onTurnEnd`/`onRuntimeSignal` to flush it at the next safe
+ * boundary, but if the `turn_end` runtime event is lost (e.g. a wake races the
+ * turn's close by ~1s, plans/daemon-fsm-desync.md), `turnActive` stays stuck
+ * true and the held message strands forever with the live process idle. A
+ * SEPARATE constant from `staleThresholdMs`: that measures "a running turn made
+ * no progress" (→ terminate); this measures "the process is demonstrably idle
+ * yet we're still holding its inbox" (→ gently re-deliver, no kill). Different
+ * semantics, independently tunable. See plans/daemon-fsm-desync.md batch L2.
+ */
+export const DEFAULT_STRANDED_INBOX_THRESHOLD_MS = 30_000;
export interface ReduceResult {
state: ManagerState;
@@ -211,8 +246,9 @@ export function createInitialManagerState(
staleThresholdMs = DEFAULT_STALE_THRESHOLD_MS,
idleTimeoutMs = DEFAULT_IDLE_TIMEOUT_MS,
resetStuckThresholdMs = DEFAULT_RESET_STUCK_THRESHOLD_MS,
+ strandedInboxThresholdMs = DEFAULT_STRANDED_INBOX_THRESHOLD_MS,
): ManagerState {
- return { agents: {}, staleThresholdMs, idleTimeoutMs, resetStuckThresholdMs };
+ return { agents: {}, staleThresholdMs, idleTimeoutMs, resetStuckThresholdMs, strandedInboxThresholdMs };
}
/* ------------------------------------------------------------------ */
@@ -344,6 +380,12 @@ function onWake(state: ManagerState, agentId: string, message: AgentMsg, nowMs:
// `onRuntimeSignal` flushes it at the next safe boundary.
const isGatedMidTurn = agent.caps.busyDeliveryMode === "gated" && agent.turnActive;
if (isGatedMidTurn) {
+ // Start the stranding clock on the FIRST held message while none is
+ // outstanding; leave it if already set so it measures from the OLDEST
+ // held wake (a later wake coalescing in must not reset the timer, or a
+ // steady wake stream would keep the strand invisible to L2 forever). The
+ // matching clear is in `drainInboxToPrompt` (inbox emptied ⇒ null).
+ if (agent.heldSince === null) agent.heldSince = nowMs;
return commit(state, agent, [
{
type: "gated_hold",
@@ -538,6 +580,64 @@ function onTick(state: ManagerState, nowMs: number): ReduceResult {
for (const id of Object.keys(agents)) {
const a = agents[id];
+ // Stranded-held-inbox re-delivery (plans/daemon-fsm-desync.md batch L2):
+ // the GENTLE backstop, checked BEFORE `stalled` so a recoverable strand is
+ // re-delivered rather than escalated to a kill. onWake holds a mid-turn
+ // wake into the inbox (`gated_hold`) expecting a later `turn_end` /
+ // safe-boundary signal to flush it. If that `turn_end` runtime event is
+ // lost — e.g. a wake races the turn's close by ~1s (Olivia 2026-07-31:
+ // wake 09:41:08, end_turn 09:41:09) — `turnActive` stays stuck true and the
+ // held message strands forever while the live process sits IDLE. The tell
+ // that it's safe to re-deliver is `apm.phase === "idle"`: the process itself
+ // reports no turn in flight (same field `gated_hold`'s `blockedReason`
+ // reads), so a `send` now can't collide with an in-flight tool call /
+ // thinking block / compaction. Emit a plain `send` (idle mode) draining the
+ // inbox — the SAME effect `onTurnEnd` would have emitted had the signal
+ // arrived; NO kill. Distinct from `stalled` (turn-genuinely-stuck →
+ // terminate) and `suspectedDeaf` (a send went unanswered): here nothing was
+ // ever sent and the process is provably idle, so the correct action is to
+ // finally deliver, not to destroy.
+ //
+ // STRANDING ANCHOR = `heldSince` (a dedicated field), NOT `lastDeliverAt` or
+ // `lastProgressAt`. The held path never issues a `send`, so `lastDeliverAt`
+ // stays the `spawned`-time null — keying on it (as `suspectedDeaf` does)
+ // would make this predicate NEVER fire, the exact trap that dooms `stalled`
+ // to key on `lastProgressAt`. And `lastProgressAt` means "last progress
+ // EVENT", not "how long held" — a stray progress event would drift it. So
+ // `onWake` stamps `heldSince` when it first holds a wake, and
+ // `drainInboxToPrompt` clears it; `nowMs - heldSince` is precisely "how long
+ // the oldest held message has been stranded". Threshold sits well above a
+ // normal turn's length so a wake that merely raced the turn's close by ~1s
+ // isn't misread as stranded.
+ //
+ // UPGRADE CHAIN: on re-deliver set `turnActive = true` + `lastDeliverAt =
+ // nowMs` (mirrors `onTurnEnd`'s drain). If the process was merely un-poked,
+ // it now works and emits progress → self-healed. If it's actually deaf, this
+ // send goes unanswered and `lastDeliverAt > lastProgressAt` arms
+ // `suspectedDeaf`, which terminates+respawns after the stale window. So the
+ // gentle re-deliver and the existing kill-path compose into "try feeding
+ // first, escalate to kill only if feeding gets no response" — which is why
+ // L2 needs no kill branch of its own (that would be fire-and-assume).
+ const strandedHeld =
+ a.status === "running" &&
+ a.turnActive &&
+ a.caps.busyDeliveryMode === "gated" &&
+ a.caps.supportsStdinNotification &&
+ a.apm.phase === "idle" &&
+ a.inbox.length > 0 &&
+ a.heldSince !== null &&
+ nowMs - a.heldSince >= state.strandedInboxThresholdMs;
+ if (strandedHeld) {
+ const drainedAgent = clone(a);
+ const text = drainInboxToPrompt(drainedAgent);
+ drainedAgent.turnActive = true;
+ drainedAgent.lastDeliverAt = nowMs;
+ drainedAgent.idleSince = null;
+ agents[id] = drainedAgent;
+ effects.push({ type: "send", agentId: id, text, mode: "idle" });
+ continue;
+ }
+
// Stalled recovery: running, turn in flight, no progress past threshold.
// Gated runtimes are normally excluded from restart (their tool-batch/
// compaction/review boundaries do the flushing), BUT a silent gated turn
@@ -651,6 +751,7 @@ function freshAgent(agentId: string, caps: AgentRuntimeCaps): AgentState {
lastProgressAt: 0,
lastDeliverAt: null,
idleSince: null,
+ heldSince: null,
resetting: false,
resettingSince: null,
apm: createInitialApmGatedSteeringState(),
@@ -694,6 +795,12 @@ function drainInboxToPrompt(agent: AgentState): string {
}
}
agent.inbox = [];
+ // The inbox is now empty, so nothing is stranded — clear the hold clock. This
+ // is the single chokepoint that empties the inbox (every drain path calls it),
+ // so clearing `heldSince` here covers onWake-steer / onTurnEnd-redeliver /
+ // onExit-respawn / the L2 re-deliver at once, keeping the invariant
+ // "heldSince != null ⇔ inbox non-empty and being held" without per-site edits.
+ agent.heldSince = null;
return unique.join("\n");
}
diff --git a/src/daemon/src/manager/managerRuntime.ts b/src/daemon/src/manager/managerRuntime.ts
index eec9979ec..5edb376fe 100644
--- a/src/daemon/src/manager/managerRuntime.ts
+++ b/src/daemon/src/manager/managerRuntime.ts
@@ -13,6 +13,7 @@
import {
reduceManager,
createInitialManagerState,
+ DEFAULT_STRANDED_INBOX_THRESHOLD_MS,
type ManagerState,
type ManagerEvent,
type ManagerEffect,
@@ -91,6 +92,8 @@ export interface ManagerRuntimeOpts {
idleTimeoutMs?: number;
/** Reset-stuck reconcile threshold (ms): `resetting` stuck this long ⇒ escalate. */
resetStuckThresholdMs?: number;
+ /** Stranded-held-inbox threshold (ms): a running+turnActive gated agent whose process is idle yet holds a non-empty inbox this long ⇒ re-deliver (no kill). */
+ strandedInboxThresholdMs?: number;
/**
* Handshake watchdog (ms): a spawned session that never emits its first
* `runtime_event` (the handshake) within this window is treated as a
@@ -582,13 +585,19 @@ export class AgentProcessManager {
staleThresholdMs: 120_000,
idleTimeoutMs: 300_000,
resetStuckThresholdMs: 120_000,
+ strandedInboxThresholdMs: DEFAULT_STRANDED_INBOX_THRESHOLD_MS,
handshakeTimeoutMs: 60_000,
stampWakePromptTime: false,
...opts,
};
this.now = opts.now ?? (() => Date.now());
this.log = opts.logger ?? createLogger({ header: "@alook/daemon:manager" });
- this.state = createInitialManagerState(this.opts.staleThresholdMs, this.opts.idleTimeoutMs, this.opts.resetStuckThresholdMs);
+ this.state = createInitialManagerState(
+ this.opts.staleThresholdMs,
+ this.opts.idleTimeoutMs,
+ this.opts.resetStuckThresholdMs,
+ this.opts.strandedInboxThresholdMs,
+ );
}
/**