Multi-Agent Collaboration for Team and Friends
TypeScript agent-automationagent-orchestrationagent-workflowai-agentsai-collaborationai-memoryai-workforceautonomous-agentsclaude-codeclaude-code-skillscodexcoding-agentmulti-agentmultiple-agentone-person-businessone-person-companyone-person-teamopencodesolopreneur
fix(daemon): reconcile a reset that never converges (batch D)
Gusb256bac68d3e
diff --git a/src/daemon/src/manager/managerPolicy.test.ts b/src/daemon/src/manager/managerPolicy.test.tsindex 07be8126d..58e8f7af0 100644--- a/src/daemon/src/manager/managerPolicy.test.ts+++ b/src/daemon/src/manager/managerPolicy.test.ts@@ -442,6 +442,91 @@ describe("reduceManager — tick: suspected-deaf detection (batch A)", () => {});});+// Batch D (plans/daemon-fsm-desync.md): the reset-stuck reconcile. The reset+// window (`resetting`) closes only via `enterStable` at a stable running/idle+// state; if the converging event never arrives the agent wedges in `starting`+// with `resetting` stuck true — invisible to the three running-keyed onTick+// predicates. This watchdog is the ONLY thing that catches it.+describe("reduceManager — tick: reset-stuck reconcile (batch D)", () => {+ // Build a reset that respawned but never reached `running`: begin_reset →+ // queued rewake → exit (onExit respawns into `starting`, resetting STAYS true+ // per batch C) → no `spawned` ever arrives. resettingSince is the begin_reset+ // stamp; the respawn is still inside the same never-closed window.+ function toResetStuckOrphan(resetStuckMs = 100) {+ let s = createInitialManagerState(100_000, 100_000, resetStuckMs);+ s = register(s, "a", PERSISTENT_GATED);+ s = reduceManager(s, { type: "wake", agentId: "a", message: { text: "m1" }, nowMs: 0 }).state;+ s = reduceManager(s, { type: "spawned", agentId: "a", nowMs: 0 }).state; // running+ s = reduceManager(s, { type: "begin_reset", agentId: "a", nowMs: 10 }).state; // resetting, resettingSince=10+ s = reduceManager(s, { type: "rewake_after_reset", agentId: "a", message: { text: "rewake" } }).state;+ s = reduceManager(s, { type: "exit", agentId: "a" }).state; // onExit → respawn, status=starting, resetting stays+ return s;+ }++ it("escalates a reset that never converged (resetting stuck in starting past the threshold)", () => {+ const s = toResetStuckOrphan();+ expect(s.agents.a.status).toBe("starting");+ expect(s.agents.a.resetting).toBe(true);+ expect(s.agents.a.resettingSince).toBe(10);++ // now - resettingSince = 200 - 10 = 190 >= 100 → escalate.+ const r = reduceManager(s, { type: "tick", nowMs: 200 });+ expect(r.effects).toEqual([{ type: "terminate_stalled", agentId: "a" }]);+ expect(r.state.agents.a.status).toBe("stopping");+ });++ it("does NOT escalate before the reset-stuck threshold elapses", () => {+ const s = toResetStuckOrphan();+ // now - resettingSince = 100 - 10 = 90 < 100 → not yet.+ expect(reduceManager(s, { type: "tick", nowMs: 100 }).effects).toEqual([]);+ });++ it("does NOT re-escalate every tick while the forced exit is in flight (stopping guard)", () => {+ let s = toResetStuckOrphan();+ s = reduceManager(s, { type: "tick", nowMs: 200 }).state; // → stopping + terminate+ expect(s.agents.a.status).toBe("stopping");+ // Next tick before the exit lands: resetting still true, but status is+ // stopping → guard blocks a second terminate_stalled (no storm).+ expect(reduceManager(s, { type: "tick", nowMs: 210 }).effects).toEqual([]);+ });++ it("stops firing once the reset converges (enterStable clears resetting)", () => {+ let s = toResetStuckOrphan();+ // The respawn finally reaches running → spawned → enterStable clears the+ // reset window. The reconcile must go quiet.+ s = reduceManager(s, { type: "spawned", agentId: "a", nowMs: 150 }).state;+ expect(s.agents.a.resetting).toBe(false);+ expect(s.agents.a.resettingSince).toBeNull();+ expect(reduceManager(s, { type: "tick", nowMs: 10_000 }).effects).toEqual([]);+ });++ it("re-escalates on a later tick if the forced exit's respawn wedges again", () => {+ let s = toResetStuckOrphan();+ s = reduceManager(s, { type: "tick", nowMs: 200 }).state; // → stopping + terminate+ // A new wake arrives during the still-open reset window — gated to inbox+ // (onWake's reset gate), NOT delivered. So the forced exit's onExit has+ // queued work → respawns into `starting` again, resetting still true (the+ // reset STILL hasn't converged). Without the new wake the first onExit would+ // have drained the sole rewake and settled idle = converged (correct); the+ // re-wedge only exists when there's fresh queued work each cycle.+ s = reduceManager(s, { type: "wake", agentId: "a", message: { text: "m2" }, nowMs: 210 }).state;+ s = reduceManager(s, { type: "exit", agentId: "a" }).state;+ expect(s.agents.a.status).toBe("starting");+ expect(s.agents.a.resetting).toBe(true);+ // A later tick past the (unchanged, begin_reset-stamped) window fires anew.+ const r = reduceManager(s, { type: "tick", nowMs: 400 });+ expect(r.effects).toEqual([{ type: "terminate_stalled", agentId: "a" }]);+ });++ it("does NOT escalate a normal (converged) agent with resetting false", () => {+ let s = createInitialManagerState(100_000, 100_000, 100);+ s = register(s, "a", PERSISTENT_GATED);+ s = reduceManager(s, { type: "wake", agentId: "a", message: { text: "m1" }, nowMs: 0 }).state;+ s = reduceManager(s, { type: "spawned", agentId: "a", nowMs: 0 }).state; // running, resetting=false+ expect(reduceManager(s, { type: "tick", nowMs: 10_000 }).effects).toEqual([]);+ });+});+describe("reduceManager — reset_session", () => {it("nulls sessionId on a known agent without changing status/turnActive", () => {let s = createInitialManagerState();diff --git a/src/daemon/src/manager/managerPolicy.ts b/src/daemon/src/manager/managerPolicy.tsindex bf542c24d..702cbba1a 100644--- a/src/daemon/src/manager/managerPolicy.ts+++ b/src/daemon/src/manager/managerPolicy.ts@@ -126,6 +126,12 @@ export interface ManagerState {* Its sessionId is preserved so the next wake resumes. 0/∞ disables.*/idleTimeoutMs: number;+ /**+ * Reset-window convergence threshold: `resetting` stuck true past this long+ * (measured from `resettingSince`) ⇒ the reconcile watchdog escalates. See+ * `DEFAULT_RESET_STUCK_THRESHOLD_MS`.+ */+ resetStuckThresholdMs: number;}/* ------------------------------------------------------------------ */@@ -184,6 +190,17 @@ export type ManagerEffect =/** Default thresholds (ms). */export const DEFAULT_STALE_THRESHOLD_MS = 120_000;export const DEFAULT_IDLE_TIMEOUT_MS = 300_000;+/**+ * Reset-window convergence threshold: how long after a reset was initiated+ * (`begin_reset` stamps `resettingSince`) the agent may stay non-stable+ * (`resetting` still true, never reached `running`/`idle` via `enterStable`)+ * before the reconcile watchdog forces it out. Deliberately a SEPARATE constant+ * from `staleThresholdMs` — that one measures "a running turn made no progress",+ * this one measures "a restart never converged". Different physical processes,+ * independently tunable; sharing a constant would couple two unrelated timeouts.+ * See plans/daemon-fsm-desync.md batch D.+ */+export const DEFAULT_RESET_STUCK_THRESHOLD_MS = 120_000;export interface ReduceResult {state: ManagerState;@@ -193,8 +210,9 @@ export interface ReduceResult {export function createInitialManagerState(staleThresholdMs = DEFAULT_STALE_THRESHOLD_MS,idleTimeoutMs = DEFAULT_IDLE_TIMEOUT_MS,+ resetStuckThresholdMs = DEFAULT_RESET_STUCK_THRESHOLD_MS,): ManagerState {- return { agents: {}, staleThresholdMs, idleTimeoutMs };+ return { agents: {}, staleThresholdMs, idleTimeoutMs, resetStuckThresholdMs };}/* ------------------------------------------------------------------ */@@ -562,6 +580,44 @@ function onTick(state: ManagerState, nowMs: number): ReduceResult {continue;}+ // Reset-stuck reconcile (plans/daemon-fsm-desync.md batch D): the reset+ // window (`resetting`) is closed exclusively by `enterStable` when the agent+ // reaches a stable `running`/`idle` state. If the converging event never+ // arrives — an idle-branch spawn whose process never emits `spawned`, or an+ // onExit-respawn that wedges in `starting` — `resetting` stays true forever+ // with an aging `resettingSince`, and every wake gets gated to the inbox+ // (onWake's reset gate) and never delivered = a permanent orphan. The three+ // predicates above can't catch it: they all require `status === "running"`,+ // but a reset-stuck agent is stuck in `starting`/`stopping` (never reached+ // running, so `enterStable` never ran, so `resetting` is still true). This+ // is the ONE belief-vs-reality gap the running-keyed detectors leave open.+ // Escalate by forcing an `exit`: `onExit` drains any queued rewake and+ // respawns (or settles idle), and its `enterStable` atomically closes the+ // window — recovery REUSES C's single-owner clear, no second clear path.+ // The `status !== "stopping"` guard is what makes this NOT fire-and-assume+ // AND storm-free: the escalate flips status to `stopping`, so it can't+ // re-issue a `terminate_stalled` every tick while the (guaranteed) exit is+ // in flight — the running-keyed predicates get this for free by leaving+ // `running`, but `resetStuck` keys on `resetting` (which `enterStable`, not+ // this branch, clears) so it needs the explicit guard. Re-escalation still+ // happens correctly when it's warranted: if the forced exit's respawn wedges+ // AGAIN, onExit puts the agent back in `starting` (not `stopping`) with+ // `resetting` still true and `resettingSince` still aging, so a later tick+ // fires this anew — the recovery keeps trying until the reset converges to a+ // stable state (`enterStable` clears `resetting` → this stops firing).+ // Mutually exclusive with stalled/suspectedDeaf by the disjoint `status`+ // precondition (they need `running`; a reset-stuck orphan is in `starting`).+ const resetStuck =+ a.resetting &&+ a.status !== "stopping" &&+ a.resettingSince !== null &&+ nowMs - a.resettingSince >= state.resetStuckThresholdMs;+ if (resetStuck) {+ agents[id] = { ...a, status: "stopping", idleSince: null };+ effects.push({ type: "terminate_stalled", agentId: id });+ continue;+ }+// Idle hibernation: a persistent keep-alive process that has sat idle (turn// ended, inbox empty) past the timeout is stopped to free resources. Its// sessionId is preserved, so the next wake resumes it.diff --git a/src/daemon/src/manager/managerRuntime.test.ts b/src/daemon/src/manager/managerRuntime.test.tsindex 188f84ca1..416a511a3 100644--- a/src/daemon/src/manager/managerRuntime.test.ts+++ b/src/daemon/src/manager/managerRuntime.test.ts@@ -82,7 +82,7 @@ function fakeSession(): FakeSession {return s;}-function makeManager(opts: { logger?: Logger; tickIntervalMs?: number; idleTimeoutMs?: number; staleThresholdMs?: number; handshakeTimeoutMs?: number; now?: () => number; onBotAuditEvent?: (agentId: string, event: unknown, context: { sessionId: string | null; launchId: string | null }) => void } = {}) {+function makeManager(opts: { logger?: Logger; tickIntervalMs?: number; idleTimeoutMs?: number; staleThresholdMs?: number; resetStuckThresholdMs?: number; handshakeTimeoutMs?: number; now?: () => number; onBotAuditEvent?: (agentId: string, event: unknown, context: { sessionId: string | null; launchId: string | null }) => void } = {}) {const session = fakeSession();const factory: SessionFactory = () => session;const onRuntimeSpawnFailed = vi.fn();@@ -1166,6 +1166,57 @@ describe("AgentProcessManager — error audit emission", () => {expect(errCalls).toHaveLength(0);});+ it("adds a stuck-reset correlation trace for a reborn error while the reset window is wedged — WITHOUT suppressing the error (batch D)", () => {+ // Batch D: the stuck-reset trace is purely additive. A reborn (non-+ // superseded) session's genuine error must STILL surface as an audit row+ // (batch C's 命门, unchanged); on top of that, if the agent's reset window+ // has been stuck past the reconcile threshold, a diagnostic `warn` line+ // correlates the error with the wedged reset. The gate judge is untouched.+ let clock = 0;+ const onBotAuditEvent = vi.fn();+ const logger = stubLogger();+ const { mgr } = makeManager({ onBotAuditEvent, logger, now: () => clock, resetStuckThresholdMs: 100 });++ // Put the agent into a reset window that has aged past the threshold.+ mgr.deliver("a1", { seq: 1, text: "hello" });+ clock = 10;+ mgr.markResetting("a1"); // resetting=true, resettingSince=10+ clock = 200; // now - resettingSince = 190 >= 100 → stuck++ // Reborn (superseded=false) error fires while the window is wedged.+ (mgr as unknown as {+ onRuntimeEvent: (a: string, e: unknown, r: string, superseded: boolean) => void;+ }).onRuntimeEvent("a1", { kind: "error", message: "reborn hit a rate limit" }, "codex", false);++ // 命门: the error STILL surfaces (additive trace never suppresses).+ const errCalls = onBotAuditEvent.mock.calls.filter(+ ([, ev]) => (ev as { kind?: string })?.kind === "error",+ );+ expect(errCalls).toHaveLength(1);+ // AND the correlation trace fired.+ expect(logger.calls.warn.some(([m]) => m === "runtime error during a stuck reset window")).toBe(true);+ });++ it("does NOT add the stuck-reset trace for a reborn error when the reset window is NOT stuck (batch D)", () => {+ let clock = 0;+ const onBotAuditEvent = vi.fn();+ const logger = stubLogger();+ const { mgr } = makeManager({ onBotAuditEvent, logger, now: () => clock, resetStuckThresholdMs: 100 });++ mgr.deliver("a1", { seq: 1, text: "hello" });+ clock = 10;+ mgr.markResetting("a1"); // resettingSince=10+ clock = 50; // now - resettingSince = 40 < 100 → NOT stuck++ (mgr as unknown as {+ onRuntimeEvent: (a: string, e: unknown, r: string, superseded: boolean) => void;+ }).onRuntimeEvent("a1", { kind: "error", message: "reborn hit a rate limit" }, "codex", false);++ // Error still surfaces (it's a real reborn error), but no stuck-reset trace.+ expect(onBotAuditEvent.mock.calls.filter(([, ev]) => (ev as { kind?: string })?.kind === "error")).toHaveLength(1);+ expect(logger.calls.warn.some(([m]) => m === "runtime error during a stuck reset window")).toBe(false);+ });+it("scrubs secrets out of an error message before it becomes an audit row", () => {const onBotAuditEvent = vi.fn();const { mgr, session } = makeManager({ onBotAuditEvent });diff --git a/src/daemon/src/manager/managerRuntime.ts b/src/daemon/src/manager/managerRuntime.tsindex 5265602d7..eec9979ec 100644--- a/src/daemon/src/manager/managerRuntime.ts+++ b/src/daemon/src/manager/managerRuntime.ts@@ -89,6 +89,8 @@ export interface ManagerRuntimeOpts {staleThresholdMs?: number;/** Idle hibernation timeout (ms): stop a persistent process idle this long. */idleTimeoutMs?: number;+ /** Reset-stuck reconcile threshold (ms): `resetting` stuck this long ⇒ escalate. */+ resetStuckThresholdMs?: number;/*** Handshake watchdog (ms): a spawned session that never emits its first* `runtime_event` (the handshake) within this window is treated as a@@ -579,13 +581,14 @@ export class AgentProcessManager {tickIntervalMs: 5_000,staleThresholdMs: 120_000,idleTimeoutMs: 300_000,+ resetStuckThresholdMs: 120_000,handshakeTimeoutMs: 60_000,stampWakePromptTime: false,...opts,};this.now = opts.now ?? (() => Date.now());this.log = opts.logger ?? createLogger({ header: "@alook/daemon:manager" });- this.state = createInitialManagerState(this.opts.staleThresholdMs, this.opts.idleTimeoutMs);+ this.state = createInitialManagerState(this.opts.staleThresholdMs, this.opts.idleTimeoutMs, this.opts.resetStuckThresholdMs);}/**@@ -1355,6 +1358,27 @@ export class AgentProcessManager {// surfaces. See plans/daemon-fsm-desync.md batch C (reader-C fix).if (ev.kind === "error" && !sessionSuperseded) {this.emitErrorAudit(agentId, "runtime", "runtime_error", ev.message ?? "Runtime error");+ // Stuck-reset correlation trace (plans/daemon-fsm-desync.md batch D):+ // PURELY ADDITIVE — this runs AFTER the error has already been decided to+ // surface (gate above unchanged), and only annotates. If this error fired+ // while the agent's reset window is wedged (`resetting` still true past+ // the reconcile threshold), leave a diagnostic line correlating the two:+ // an error emerging from a bot whose restart never converged is a+ // higher-signal event than a lone error, and the reconcile watchdog is+ // about to (or already did) escalate it. Never suppresses — a reborn's+ // genuine error keeps surfacing exactly as batch C made it.+ const agent = this.state.agents[agentId];+ if (+ agent?.resetting &&+ agent.resettingSince !== null &&+ this.now() - agent.resettingSince >= this.state.resetStuckThresholdMs+ ) {+ this.log.warn("runtime error during a stuck reset window", {+ agentId,+ resettingForMs: this.now() - agent.resettingSince,+ message: ev.message ?? "Runtime error",+ });+ }}// Bot audit hook — thinking + non-Bash tool_call, no correlation.// Context carries the sessionId/launchId learned so far this launch so