Skip to content

armgit / alook

Public Imported from GitHub
0

Multi-Agent Collaboration for Team and Friends

fix(daemon): reconcile a reset that never converges (batch D)

Gusb256bac68d3e
diff --git a/src/daemon/src/manager/managerPolicy.test.ts b/src/daemon/src/manager/managerPolicy.test.ts
index 07be8126d..58e8f7af0 100644
--- a/src/daemon/src/manager/managerPolicy.test.ts
+++ b/src/daemon/src/manager/managerPolicy.test.ts
@@ -442,6 +442,91 @@ describe("reduceManager — tick: suspected-deaf detection (batch A)", () => {
});
});
+// Batch D (plans/daemon-fsm-desync.md): the reset-stuck reconcile. The reset
+// window (`resetting`) closes only via `enterStable` at a stable running/idle
+// state; if the converging event never arrives the agent wedges in `starting`
+// with `resetting` stuck true — invisible to the three running-keyed onTick
+// predicates. This watchdog is the ONLY thing that catches it.
+describe("reduceManager — tick: reset-stuck reconcile (batch D)", () => {
+ // Build a reset that respawned but never reached `running`: begin_reset →
+ // queued rewake → exit (onExit respawns into `starting`, resetting STAYS true
+ // per batch C) → no `spawned` ever arrives. resettingSince is the begin_reset
+ // stamp; the respawn is still inside the same never-closed window.
+ function toResetStuckOrphan(resetStuckMs = 100) {
+ let s = createInitialManagerState(100_000, 100_000, resetStuckMs);
+ s = register(s, "a", PERSISTENT_GATED);
+ s = reduceManager(s, { type: "wake", agentId: "a", message: { text: "m1" }, nowMs: 0 }).state;
+ s = reduceManager(s, { type: "spawned", agentId: "a", nowMs: 0 }).state; // running
+ s = reduceManager(s, { type: "begin_reset", agentId: "a", nowMs: 10 }).state; // resetting, resettingSince=10
+ s = reduceManager(s, { type: "rewake_after_reset", agentId: "a", message: { text: "rewake" } }).state;
+ s = reduceManager(s, { type: "exit", agentId: "a" }).state; // onExit → respawn, status=starting, resetting stays
+ return s;
+ }
+
+ it("escalates a reset that never converged (resetting stuck in starting past the threshold)", () => {
+ const s = toResetStuckOrphan();
+ expect(s.agents.a.status).toBe("starting");
+ expect(s.agents.a.resetting).toBe(true);
+ expect(s.agents.a.resettingSince).toBe(10);
+
+ // now - resettingSince = 200 - 10 = 190 >= 100 → escalate.
+ const r = reduceManager(s, { type: "tick", nowMs: 200 });
+ expect(r.effects).toEqual([{ type: "terminate_stalled", agentId: "a" }]);
+ expect(r.state.agents.a.status).toBe("stopping");
+ });
+
+ it("does NOT escalate before the reset-stuck threshold elapses", () => {
+ const s = toResetStuckOrphan();
+ // now - resettingSince = 100 - 10 = 90 < 100 → not yet.
+ expect(reduceManager(s, { type: "tick", nowMs: 100 }).effects).toEqual([]);
+ });
+
+ it("does NOT re-escalate every tick while the forced exit is in flight (stopping guard)", () => {
+ let s = toResetStuckOrphan();
+ s = reduceManager(s, { type: "tick", nowMs: 200 }).state; // → stopping + terminate
+ expect(s.agents.a.status).toBe("stopping");
+ // Next tick before the exit lands: resetting still true, but status is
+ // stopping → guard blocks a second terminate_stalled (no storm).
+ expect(reduceManager(s, { type: "tick", nowMs: 210 }).effects).toEqual([]);
+ });
+
+ it("stops firing once the reset converges (enterStable clears resetting)", () => {
+ let s = toResetStuckOrphan();
+ // The respawn finally reaches running → spawned → enterStable clears the
+ // reset window. The reconcile must go quiet.
+ s = reduceManager(s, { type: "spawned", agentId: "a", nowMs: 150 }).state;
+ expect(s.agents.a.resetting).toBe(false);
+ expect(s.agents.a.resettingSince).toBeNull();
+ expect(reduceManager(s, { type: "tick", nowMs: 10_000 }).effects).toEqual([]);
+ });
+
+ it("re-escalates on a later tick if the forced exit's respawn wedges again", () => {
+ let s = toResetStuckOrphan();
+ s = reduceManager(s, { type: "tick", nowMs: 200 }).state; // → stopping + terminate
+ // A new wake arrives during the still-open reset window — gated to inbox
+ // (onWake's reset gate), NOT delivered. So the forced exit's onExit has
+ // queued work → respawns into `starting` again, resetting still true (the
+ // reset STILL hasn't converged). Without the new wake the first onExit would
+ // have drained the sole rewake and settled idle = converged (correct); the
+ // re-wedge only exists when there's fresh queued work each cycle.
+ s = reduceManager(s, { type: "wake", agentId: "a", message: { text: "m2" }, nowMs: 210 }).state;
+ s = reduceManager(s, { type: "exit", agentId: "a" }).state;
+ expect(s.agents.a.status).toBe("starting");
+ expect(s.agents.a.resetting).toBe(true);
+ // A later tick past the (unchanged, begin_reset-stamped) window fires anew.
+ const r = reduceManager(s, { type: "tick", nowMs: 400 });
+ expect(r.effects).toEqual([{ type: "terminate_stalled", agentId: "a" }]);
+ });
+
+ it("does NOT escalate a normal (converged) agent with resetting false", () => {
+ let s = createInitialManagerState(100_000, 100_000, 100);
+ s = register(s, "a", PERSISTENT_GATED);
+ s = reduceManager(s, { type: "wake", agentId: "a", message: { text: "m1" }, nowMs: 0 }).state;
+ s = reduceManager(s, { type: "spawned", agentId: "a", nowMs: 0 }).state; // running, resetting=false
+ expect(reduceManager(s, { type: "tick", nowMs: 10_000 }).effects).toEqual([]);
+ });
+});
+
describe("reduceManager — reset_session", () => {
it("nulls sessionId on a known agent without changing status/turnActive", () => {
let s = createInitialManagerState();
diff --git a/src/daemon/src/manager/managerPolicy.ts b/src/daemon/src/manager/managerPolicy.ts
index bf542c24d..702cbba1a 100644
--- a/src/daemon/src/manager/managerPolicy.ts
+++ b/src/daemon/src/manager/managerPolicy.ts
@@ -126,6 +126,12 @@ export interface ManagerState {
* Its sessionId is preserved so the next wake resumes. 0/∞ disables.
*/
idleTimeoutMs: number;
+ /**
+ * Reset-window convergence threshold: `resetting` stuck true past this long
+ * (measured from `resettingSince`) ⇒ the reconcile watchdog escalates. See
+ * `DEFAULT_RESET_STUCK_THRESHOLD_MS`.
+ */
+ resetStuckThresholdMs: number;
}
/* ------------------------------------------------------------------ */
@@ -184,6 +190,17 @@ export type ManagerEffect =
/** Default thresholds (ms). */
export const DEFAULT_STALE_THRESHOLD_MS = 120_000;
export const DEFAULT_IDLE_TIMEOUT_MS = 300_000;
+/**
+ * Reset-window convergence threshold: how long after a reset was initiated
+ * (`begin_reset` stamps `resettingSince`) the agent may stay non-stable
+ * (`resetting` still true, never reached `running`/`idle` via `enterStable`)
+ * before the reconcile watchdog forces it out. Deliberately a SEPARATE constant
+ * from `staleThresholdMs` — that one measures "a running turn made no progress",
+ * this one measures "a restart never converged". Different physical processes,
+ * independently tunable; sharing a constant would couple two unrelated timeouts.
+ * See plans/daemon-fsm-desync.md batch D.
+ */
+export const DEFAULT_RESET_STUCK_THRESHOLD_MS = 120_000;
export interface ReduceResult {
state: ManagerState;
@@ -193,8 +210,9 @@ export interface ReduceResult {
export function createInitialManagerState(
staleThresholdMs = DEFAULT_STALE_THRESHOLD_MS,
idleTimeoutMs = DEFAULT_IDLE_TIMEOUT_MS,
+ resetStuckThresholdMs = DEFAULT_RESET_STUCK_THRESHOLD_MS,
): ManagerState {
- return { agents: {}, staleThresholdMs, idleTimeoutMs };
+ return { agents: {}, staleThresholdMs, idleTimeoutMs, resetStuckThresholdMs };
}
/* ------------------------------------------------------------------ */
@@ -562,6 +580,44 @@ function onTick(state: ManagerState, nowMs: number): ReduceResult {
continue;
}
+ // Reset-stuck reconcile (plans/daemon-fsm-desync.md batch D): the reset
+ // window (`resetting`) is closed exclusively by `enterStable` when the agent
+ // reaches a stable `running`/`idle` state. If the converging event never
+ // arrives — an idle-branch spawn whose process never emits `spawned`, or an
+ // onExit-respawn that wedges in `starting` — `resetting` stays true forever
+ // with an aging `resettingSince`, and every wake gets gated to the inbox
+ // (onWake's reset gate) and never delivered = a permanent orphan. The three
+ // predicates above can't catch it: they all require `status === "running"`,
+ // but a reset-stuck agent is stuck in `starting`/`stopping` (never reached
+ // running, so `enterStable` never ran, so `resetting` is still true). This
+ // is the ONE belief-vs-reality gap the running-keyed detectors leave open.
+ // Escalate by forcing an `exit`: `onExit` drains any queued rewake and
+ // respawns (or settles idle), and its `enterStable` atomically closes the
+ // window — recovery REUSES C's single-owner clear, no second clear path.
+ // The `status !== "stopping"` guard is what makes this NOT fire-and-assume
+ // AND storm-free: the escalate flips status to `stopping`, so it can't
+ // re-issue a `terminate_stalled` every tick while the (guaranteed) exit is
+ // in flight — the running-keyed predicates get this for free by leaving
+ // `running`, but `resetStuck` keys on `resetting` (which `enterStable`, not
+ // this branch, clears) so it needs the explicit guard. Re-escalation still
+ // happens correctly when it's warranted: if the forced exit's respawn wedges
+ // AGAIN, onExit puts the agent back in `starting` (not `stopping`) with
+ // `resetting` still true and `resettingSince` still aging, so a later tick
+ // fires this anew — the recovery keeps trying until the reset converges to a
+ // stable state (`enterStable` clears `resetting` → this stops firing).
+ // Mutually exclusive with stalled/suspectedDeaf by the disjoint `status`
+ // precondition (they need `running`; a reset-stuck orphan is in `starting`).
+ const resetStuck =
+ a.resetting &&
+ a.status !== "stopping" &&
+ a.resettingSince !== null &&
+ nowMs - a.resettingSince >= state.resetStuckThresholdMs;
+ if (resetStuck) {
+ agents[id] = { ...a, status: "stopping", idleSince: null };
+ effects.push({ type: "terminate_stalled", agentId: id });
+ continue;
+ }
+
// Idle hibernation: a persistent keep-alive process that has sat idle (turn
// ended, inbox empty) past the timeout is stopped to free resources. Its
// sessionId is preserved, so the next wake resumes it.
diff --git a/src/daemon/src/manager/managerRuntime.test.ts b/src/daemon/src/manager/managerRuntime.test.ts
index 188f84ca1..416a511a3 100644
--- a/src/daemon/src/manager/managerRuntime.test.ts
+++ b/src/daemon/src/manager/managerRuntime.test.ts
@@ -82,7 +82,7 @@ function fakeSession(): FakeSession {
return s;
}
-function makeManager(opts: { logger?: Logger; tickIntervalMs?: number; idleTimeoutMs?: number; staleThresholdMs?: number; handshakeTimeoutMs?: number; now?: () => number; onBotAuditEvent?: (agentId: string, event: unknown, context: { sessionId: string | null; launchId: string | null }) => void } = {}) {
+function makeManager(opts: { logger?: Logger; tickIntervalMs?: number; idleTimeoutMs?: number; staleThresholdMs?: number; resetStuckThresholdMs?: number; handshakeTimeoutMs?: number; now?: () => number; onBotAuditEvent?: (agentId: string, event: unknown, context: { sessionId: string | null; launchId: string | null }) => void } = {}) {
const session = fakeSession();
const factory: SessionFactory = () => session;
const onRuntimeSpawnFailed = vi.fn();
@@ -1166,6 +1166,57 @@ describe("AgentProcessManager — error audit emission", () => {
expect(errCalls).toHaveLength(0);
});
+ it("adds a stuck-reset correlation trace for a reborn error while the reset window is wedged — WITHOUT suppressing the error (batch D)", () => {
+ // Batch D: the stuck-reset trace is purely additive. A reborn (non-
+ // superseded) session's genuine error must STILL surface as an audit row
+ // (batch C's 命门, unchanged); on top of that, if the agent's reset window
+ // has been stuck past the reconcile threshold, a diagnostic `warn` line
+ // correlates the error with the wedged reset. The gate judge is untouched.
+ let clock = 0;
+ const onBotAuditEvent = vi.fn();
+ const logger = stubLogger();
+ const { mgr } = makeManager({ onBotAuditEvent, logger, now: () => clock, resetStuckThresholdMs: 100 });
+
+ // Put the agent into a reset window that has aged past the threshold.
+ mgr.deliver("a1", { seq: 1, text: "hello" });
+ clock = 10;
+ mgr.markResetting("a1"); // resetting=true, resettingSince=10
+ clock = 200; // now - resettingSince = 190 >= 100 → stuck
+
+ // Reborn (superseded=false) error fires while the window is wedged.
+ (mgr as unknown as {
+ onRuntimeEvent: (a: string, e: unknown, r: string, superseded: boolean) => void;
+ }).onRuntimeEvent("a1", { kind: "error", message: "reborn hit a rate limit" }, "codex", false);
+
+ // 命门: the error STILL surfaces (additive trace never suppresses).
+ const errCalls = onBotAuditEvent.mock.calls.filter(
+ ([, ev]) => (ev as { kind?: string })?.kind === "error",
+ );
+ expect(errCalls).toHaveLength(1);
+ // AND the correlation trace fired.
+ expect(logger.calls.warn.some(([m]) => m === "runtime error during a stuck reset window")).toBe(true);
+ });
+
+ it("does NOT add the stuck-reset trace for a reborn error when the reset window is NOT stuck (batch D)", () => {
+ let clock = 0;
+ const onBotAuditEvent = vi.fn();
+ const logger = stubLogger();
+ const { mgr } = makeManager({ onBotAuditEvent, logger, now: () => clock, resetStuckThresholdMs: 100 });
+
+ mgr.deliver("a1", { seq: 1, text: "hello" });
+ clock = 10;
+ mgr.markResetting("a1"); // resettingSince=10
+ clock = 50; // now - resettingSince = 40 < 100 → NOT stuck
+
+ (mgr as unknown as {
+ onRuntimeEvent: (a: string, e: unknown, r: string, superseded: boolean) => void;
+ }).onRuntimeEvent("a1", { kind: "error", message: "reborn hit a rate limit" }, "codex", false);
+
+ // Error still surfaces (it's a real reborn error), but no stuck-reset trace.
+ expect(onBotAuditEvent.mock.calls.filter(([, ev]) => (ev as { kind?: string })?.kind === "error")).toHaveLength(1);
+ expect(logger.calls.warn.some(([m]) => m === "runtime error during a stuck reset window")).toBe(false);
+ });
+
it("scrubs secrets out of an error message before it becomes an audit row", () => {
const onBotAuditEvent = vi.fn();
const { mgr, session } = makeManager({ onBotAuditEvent });
diff --git a/src/daemon/src/manager/managerRuntime.ts b/src/daemon/src/manager/managerRuntime.ts
index 5265602d7..eec9979ec 100644
--- a/src/daemon/src/manager/managerRuntime.ts
+++ b/src/daemon/src/manager/managerRuntime.ts
@@ -89,6 +89,8 @@ export interface ManagerRuntimeOpts {
staleThresholdMs?: number;
/** Idle hibernation timeout (ms): stop a persistent process idle this long. */
idleTimeoutMs?: number;
+ /** Reset-stuck reconcile threshold (ms): `resetting` stuck this long ⇒ escalate. */
+ resetStuckThresholdMs?: number;
/**
* Handshake watchdog (ms): a spawned session that never emits its first
* `runtime_event` (the handshake) within this window is treated as a
@@ -579,13 +581,14 @@ export class AgentProcessManager {
tickIntervalMs: 5_000,
staleThresholdMs: 120_000,
idleTimeoutMs: 300_000,
+ resetStuckThresholdMs: 120_000,
handshakeTimeoutMs: 60_000,
stampWakePromptTime: false,
...opts,
};
this.now = opts.now ?? (() => Date.now());
this.log = opts.logger ?? createLogger({ header: "@alook/daemon:manager" });
- this.state = createInitialManagerState(this.opts.staleThresholdMs, this.opts.idleTimeoutMs);
+ this.state = createInitialManagerState(this.opts.staleThresholdMs, this.opts.idleTimeoutMs, this.opts.resetStuckThresholdMs);
}
/**
@@ -1355,6 +1358,27 @@ export class AgentProcessManager {
// surfaces. See plans/daemon-fsm-desync.md batch C (reader-C fix).
if (ev.kind === "error" && !sessionSuperseded) {
this.emitErrorAudit(agentId, "runtime", "runtime_error", ev.message ?? "Runtime error");
+ // Stuck-reset correlation trace (plans/daemon-fsm-desync.md batch D):
+ // PURELY ADDITIVE — this runs AFTER the error has already been decided to
+ // surface (gate above unchanged), and only annotates. If this error fired
+ // while the agent's reset window is wedged (`resetting` still true past
+ // the reconcile threshold), leave a diagnostic line correlating the two:
+ // an error emerging from a bot whose restart never converged is a
+ // higher-signal event than a lone error, and the reconcile watchdog is
+ // about to (or already did) escalate it. Never suppresses — a reborn's
+ // genuine error keeps surfacing exactly as batch C made it.
+ const agent = this.state.agents[agentId];
+ if (
+ agent?.resetting &&
+ agent.resettingSince !== null &&
+ this.now() - agent.resettingSince >= this.state.resetStuckThresholdMs
+ ) {
+ this.log.warn("runtime error during a stuck reset window", {
+ agentId,
+ resettingForMs: this.now() - agent.resettingSince,
+ message: ev.message ?? "Runtime error",
+ });
+ }
}
// Bot audit hook — thinking + non-Bash tool_call, no correlation.
// Context carries the sessionId/launchId learned so far this launch so