Skip to content

armgit / alook

Public Imported from GitHub
0

Multi-Agent Collaboration for Team and Friends

daemon: B1 crashed-turn tag — endReason/terminationCause on turn_end (runtime-error auto-rewake followup)

Gus19f31f2cc3e9
diff --git a/src/daemon/src/drivers/claudeEventNormalizer.test.ts b/src/daemon/src/drivers/claudeEventNormalizer.test.ts
index 2949dd97c..b4e34d378 100644
--- a/src/daemon/src/drivers/claudeEventNormalizer.test.ts
+++ b/src/daemon/src/drivers/claudeEventNormalizer.test.ts
@@ -73,4 +73,17 @@ describe("ClaudeEventNormalizer.normalizeLine", () => {
expect(out.some((e) => e.kind === "error" && (e as any).message === "boom")).toBe(true);
expect(out.some((e) => e.kind === "turn_end")).toBe(true);
});
+
+ it("errored result emits `error` BEFORE the trailing `turn_end` (the ordering B1's errored-turn marker relies on)", () => {
+ // managerRuntime buffers the `error`'s message on the way past, then reads
+ // it out when the trailing `turn_end` arrives to stamp `endReason:"errored"`.
+ // That handoff only works if error precedes turn_end in the SAME batch.
+ // See plans/daemon-runtime-error-rewake.md B1.
+ const out = new ClaudeEventNormalizer().normalizeLine(
+ J({ type: "result", is_error: true, result: "boom", session_id: "s1" }),
+ );
+ const kinds = out.map((e) => e.kind);
+ expect(kinds.indexOf("error")).toBeGreaterThanOrEqual(0);
+ expect(kinds.indexOf("error")).toBeLessThan(kinds.indexOf("turn_end"));
+ });
});
diff --git a/src/daemon/src/manager/managerPolicy.ts b/src/daemon/src/manager/managerPolicy.ts
index cc2ceb0a1..536c7c427 100644
--- a/src/daemon/src/manager/managerPolicy.ts
+++ b/src/daemon/src/manager/managerPolicy.ts
@@ -165,7 +165,26 @@ export type ManagerEvent =
| { type: "spawned"; agentId: string; nowMs: number }
| { type: "session"; agentId: string; sessionId: string }
| { type: "progress"; agentId: string; nowMs: number }
- | { type: "turn_end"; agentId: string; nowMs: number }
+ /**
+ * `endReason:"errored"` is set when a turn ended NON-cleanly — either a
+ * mid-turn runtime `error` OR a `terminate_stalled` kill (managerRuntime
+ * buffers the cause, stamps it here on the trailing turn_end). Absent = clean
+ * turn-end. STRICTLY BINARY (single literal, not a union) so the rewake gate
+ * keyed on `=== "errored"` can't be silently widened — red line 1. The CAUSE
+ * rides on the separate `terminationCause` field (for B2 policy branching:
+ * `killed_stalled` gets a tighter rewake bound than `runtime_error` since an
+ * input-caused hang recurs deterministically); `errorDetail` is free-text.
+ * B1 only RECORDS these (trace); the rewake policy that consumes them is B2.
+ * See plans/daemon-runtime-error-rewake.md.
+ */
+ | {
+ type: "turn_end";
+ agentId: string;
+ nowMs: number;
+ endReason?: "errored";
+ terminationCause?: "runtime_error" | "killed_stalled";
+ errorDetail?: string;
+ }
| { type: "exit"; agentId: string }
| { type: "tick"; nowMs: number }
/**
diff --git a/src/daemon/src/manager/managerRuntime.test.ts b/src/daemon/src/manager/managerRuntime.test.ts
index 3f7c65670..b1b3e4e72 100644
--- a/src/daemon/src/manager/managerRuntime.test.ts
+++ b/src/daemon/src/manager/managerRuntime.test.ts
@@ -1950,4 +1950,170 @@ describe("AgentProcessManager — onFsmTransition trace (observability, zero beh
vi.useRealTimers();
}
});
+
+ // ---- B1: crashed-turn tag (plans/daemon-runtime-error-rewake.md) ----------
+ // The trailing turn_end after a mid-turn `error` carries `endReason:"errored"`
+ // + `errorDetail` into the trace, so a crashed turn is externally
+ // distinguishable from a clean nap/idle (red line 7). B1 only RECORDS it —
+ // onTurnEnd behavior is unchanged (verified in the "clean" case below).
+
+ it("a mid-turn error then turn_end stamps endReason=errored + terminationCause=runtime_error + errorDetail (red line 6/7)", () => {
+ const recs: Record<string, unknown>[] = [];
+ const { mgr, session } = makeWithTrace((r) => recs.push(r));
+ mgr.deliver("a1", { seq: 1, text: "hi" });
+ session.fire("runtime_event", { kind: "session_init", sessionId: "s1" });
+ // Genuinely interrupt the turn: an `error` event (as the normalizer emits
+ // from an is_error result) FOLLOWED by the trailing turn_end.
+ session.fire("runtime_event", { kind: "error", message: "boom: model overloaded" });
+ session.fire("runtime_event", { kind: "turn_end" });
+
+ const turnEnd = recs.find((r) => r.event === "turn_end" && r.agentId === "a1");
+ expect(turnEnd).toBeTruthy();
+ expect(turnEnd!.endReason).toBe("errored");
+ expect(turnEnd!.terminationCause).toBe("runtime_error");
+ expect(turnEnd!.errorDetail).toBe("boom: model overloaded");
+ });
+
+ it("a CLEAN turn_end (no preceding error/kill) carries no endReason/terminationCause/errorDetail (byte-for-byte the old row)", () => {
+ const recs: Record<string, unknown>[] = [];
+ const { mgr, session } = makeWithTrace((r) => recs.push(r));
+ mgr.deliver("a1", { seq: 1, text: "hi" });
+ session.fire("runtime_event", { kind: "session_init", sessionId: "s1" });
+ session.fire("runtime_event", { kind: "turn_end" });
+
+ const turnEnd = recs.find((r) => r.event === "turn_end" && r.agentId === "a1");
+ expect(turnEnd).toBeTruthy();
+ expect(turnEnd!.endReason).toBeUndefined();
+ expect(turnEnd!.terminationCause).toBeUndefined();
+ expect(turnEnd!.errorDetail).toBeUndefined();
+ });
+
+ it("an intentional-kill (superseded) death-rattle error does NOT tag its turn_end errored (red line 5 — reset/nap is not a crash)", () => {
+ const recs: Record<string, unknown>[] = [];
+ const { mgr, session } = makeWithTrace((r) => recs.push(r));
+ mgr.deliver("a1", { seq: 1, text: "hi" });
+ session.fire("runtime_event", { kind: "session_init", sessionId: "s1" });
+ // Enter the reset/nap kill window, THEN the dying process rattles. The
+ // marker is gated by the same `!sessionSuperseded` as the audit, so it must
+ // NOT set — a nap must stay indistinguishable-from-clean, not read as crash.
+ mgr.markResetting("a1");
+ session.fire("runtime_event", { kind: "error", message: "turn interrupted" });
+ session.fire("runtime_event", { kind: "turn_end" });
+
+ const turnEnd = recs.find((r) => r.event === "turn_end" && r.agentId === "a1");
+ expect(turnEnd).toBeTruthy();
+ expect(turnEnd!.endReason).toBeUndefined();
+ expect(turnEnd!.terminationCause).toBeUndefined();
+ });
+
+ it("a mid-turn error followed by a hard exit (no turn_end) clears the marker so the NEXT turn is not mis-tagged (3a marker-leak guard)", () => {
+ const recs: Record<string, unknown>[] = [];
+ const { mgr, session } = makeWithTrace((r) => recs.push(r));
+ mgr.deliver("a1", { seq: 1, text: "hi" });
+ session.fire("runtime_event", { kind: "session_init", sessionId: "s1" });
+ // Error with NO trailing turn_end, then a hard process exit (bypasses the
+ // normalizer) — the marker would otherwise leak in the map.
+ session.fire("runtime_event", { kind: "error", message: "crashed hard" });
+ session.fire("exit");
+
+ // A fresh wake spawns a new session; its clean turn_end must NOT inherit the
+ // stale marker.
+ recs.length = 0;
+ mgr.deliver("a1", { seq: 2, text: "again" });
+ session.fire("runtime_event", { kind: "session_init", sessionId: "s2" });
+ session.fire("runtime_event", { kind: "turn_end" });
+
+ const turnEnd = recs.find((r) => r.event === "turn_end" && r.agentId === "a1");
+ expect(turnEnd).toBeTruthy();
+ expect(turnEnd!.endReason).toBeUndefined();
+ expect(turnEnd!.terminationCause).toBeUndefined();
+ });
+
+ it("red line 6 — a REAL stall (no progress past threshold → terminate_stalled) tags the turn_end killed_stalled, NOT injected error (Blair's actual case)", async () => {
+ vi.useFakeTimers();
+ try {
+ let now = 0;
+ const recs: Record<string, unknown>[] = [];
+ // Default fakeDriver (per_turn) satisfies the `stalled` predicate's first
+ // sub-clause — the same setup the proven "terminate_stalled from the stall
+ // watchdog" test uses. We only add the trace sink.
+ const session = fakeSession();
+ const mgr = new AgentProcessManager({
+ driverFor: () => fakeDriver("codex"),
+ baseContextFor: () => ({ workingDirectory: "/tmp", agentId: "a1", standingPrompt: "", config: {} as LaunchContext["config"], credentialProxy: {} as LaunchContext["credentialProxy"] }),
+ sessionFactory: () => session,
+ now: () => now,
+ tickIntervalMs: 5,
+ staleThresholdMs: 100, // wedge → terminate_stalled after 100ms of no progress
+ onFsmTransition: ((r: Record<string, unknown>) => recs.push(r)) as never,
+ });
+ mgr.start();
+ mgr.register("a1");
+ mgr.deliver("a1", { seq: 1, text: "hi" });
+ session.startResolver?.();
+ await Promise.resolve();
+ session.fire("runtime_event", { kind: "session_init", sessionId: "s1" });
+ // Turn is in flight (turnActive) and makes NO progress. No error injected —
+ // this is a genuine hang, exactly Blair's case.
+ now = 200; // past staleThreshold=100 → stalled watchdog fires terminate_stalled
+ await vi.advanceTimersByTimeAsync(10);
+ const killTick = recs.find((r) => Array.isArray(r.effects) && (r.effects as string[]).includes("terminate_stalled"));
+ expect(killTick).toBeTruthy(); // the stall kill actually fired
+
+ // The SIGKILL makes the process emit its trailing turn_end (no error
+ // rattle needed — that's the whole point of keying on cause, not rattle).
+ recs.length = 0;
+ session.fire("runtime_event", { kind: "turn_end" });
+ const turnEnd = recs.find((r) => r.event === "turn_end" && r.agentId === "a1");
+ expect(turnEnd).toBeTruthy();
+ expect(turnEnd!.endReason).toBe("errored");
+ expect(turnEnd!.terminationCause).toBe("killed_stalled");
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("a VOLUNTARY idle-timeout stop (which also flips status→stopping) does NOT produce a tagged turn_end (red line 2 — cause, not bare status)", async () => {
+ // Cecilia's "don't treat 'not seen' as 'won't happen'": idle-hibernation's
+ // `stop(idle_timeout)` flips status→stopping too, but it is a CLEAN end. We
+ // key on the terminate_stalled EFFECT, not on status, so no marker is set →
+ // any turn_end around a voluntary stop stays untagged.
+ vi.useFakeTimers();
+ try {
+ let now = 0;
+ const recs: Record<string, unknown>[] = [];
+ const persistentDriver = {
+ ...fakeDriver("codex"),
+ lifecycle: { kind: "persistent", start: "immediate", exit: "natural", inFlightWake: "queue" } as never,
+ } as Driver;
+ const session = fakeSession();
+ const mgr = new AgentProcessManager({
+ driverFor: () => persistentDriver,
+ baseContextFor: () => ({ workingDirectory: "/tmp", agentId: "a1", standingPrompt: "", config: {} as LaunchContext["config"], credentialProxy: {} as LaunchContext["credentialProxy"] }),
+ sessionFactory: () => session,
+ now: () => now,
+ tickIntervalMs: 5,
+ idleTimeoutMs: 50, // idle → voluntary stop(idle_timeout), flips stopping
+ stoppingStuckThresholdMs: 1_000_000, // don't force_exit during the test
+ onFsmTransition: ((r: Record<string, unknown>) => recs.push(r)) as never,
+ });
+ mgr.start();
+ mgr.register("a1");
+ mgr.deliver("a1", { seq: 1, text: "hi" });
+ session.startResolver?.();
+ await Promise.resolve();
+ session.fire("runtime_event", { kind: "session_init", sessionId: "s1" });
+ session.fire("runtime_event", { kind: "turn_end" }); // clean end → idle
+ recs.length = 0;
+ now = 100; // past idleTimeout=50 → idle-hibernation issues a voluntary stop
+ await vi.advanceTimersByTimeAsync(10);
+ const stoppingRec = recs.find((r) => r.status === "stopping" && r.agentId === "a1");
+ expect(stoppingRec).toBeTruthy(); // it DID flip stopping (voluntary)
+ // No terminate_stalled marker was set, so no tagged turn_end can appear.
+ const tagged = recs.find((r) => r.event === "turn_end" && r.endReason === "errored");
+ expect(tagged).toBeUndefined();
+ } finally {
+ vi.useRealTimers();
+ }
+ });
});
diff --git a/src/daemon/src/manager/managerRuntime.ts b/src/daemon/src/manager/managerRuntime.ts
index 624e711a9..bf3b31a61 100644
--- a/src/daemon/src/manager/managerRuntime.ts
+++ b/src/daemon/src/manager/managerRuntime.ts
@@ -185,6 +185,18 @@ export interface ManagerRuntimeOpts {
sinceDeliverMs: number | null;
/** `nowMs - stoppingSince` (null unless in `stopping`) — stopping-stuck backstop clock. */
sinceStoppingMs: number | null;
+ /**
+ * Present only on a `turn_end` that ended NON-cleanly. `endReason:"errored"`
+ * is the binary crashed-vs-clean judgement; `terminationCause` is the cause
+ * (`runtime_error` = mid-turn runtime error; `killed_stalled` = stall
+ * watchdog kill) for B2 policy branching; `errorDetail` is free-text. Absent
+ * on a clean turn_end and every non-turn_end event — makes a non-clean turn
+ * externally distinguishable from a clean nap/idle in the trace (B1 red line
+ * 7). See plans/daemon-runtime-error-rewake.md.
+ */
+ endReason?: "errored";
+ terminationCause?: "runtime_error" | "killed_stalled";
+ errorDetail?: string;
}) => void;
/**
* Optional context-timeline recorder. When provided, the manager logs each
@@ -584,6 +596,31 @@ export class AgentProcessManager {
pid: number | null;
}
>();
+ /**
+ * agentId → why the current turn ended NON-cleanly, buffered until the
+ * trailing `turn_end` reads+clears it. Two symmetric marker points feed it
+ * (Cecilia 架构#352 / Claudette #353 — key on CAUSE, never on bare `status`,
+ * so a voluntary idle-timeout `stop` that also flips `stopping` is never
+ * misread as a crash):
+ * - `runtime_error`: a runtime `error` event mid-turn, set ONLY in the
+ * `!sessionSuperseded` branch (an intentional reset/nap death-rattle never
+ * marks). Carries the free-text `detail` for lossless fsm-trace.
+ * - `killed_stalled`: the `terminate_stalled` effect (the stall watchdog
+ * SIGKILLing a wedged turn — Blair's actual case). By construction, no
+ * dependence on whether the runtime emits a trailing error rattle.
+ * At the trailing `turn_end` this becomes `endReason:"errored"` (the binary
+ * can-rewake judgement) + `terminationCause` (the cause, for B2 policy
+ * branching). A kill's marker is authoritative: if a `killed_stalled` marker
+ * is present, a following `runtime_error` rattle must NOT downgrade it (the
+ * kill is the real cause; the rattle is its side effect). Cleared on the
+ * `exit` teardown too, so a hard exit with no trailing turn_end can't leak a
+ * stale marker onto the next turn (B1 3a). See
+ * plans/daemon-runtime-error-rewake.md.
+ */
+ private readonly nonCleanEndMarker = new Map<
+ string,
+ { cause: "runtime_error" | "killed_stalled"; detail?: string }
+ >();
private readonly opts: Required<
Omit<
ManagerRuntimeOpts,
@@ -1032,6 +1069,17 @@ export class AgentProcessManager {
sinceProgressMs: nowMs - a.lastProgressAt,
sinceDeliverMs: a.lastDeliverAt === null ? null : nowMs - a.lastDeliverAt,
sinceStoppingMs: a.stoppingSince === null ? null : nowMs - a.stoppingSince,
+ // Non-clean-turn tag, carried on the `turn_end` event itself (not agent
+ // state). Spread so a clean turn_end / any other event omits the keys
+ // entirely rather than emitting explicit undefineds. B1.
+ ...(event.type === "turn_end" && (event as { endReason?: "errored" }).endReason === "errored"
+ ? {
+ endReason: "errored" as const,
+ terminationCause: (event as { terminationCause?: "runtime_error" | "killed_stalled" })
+ .terminationCause,
+ errorDetail: (event as { errorDetail?: string }).errorDetail,
+ }
+ : {}),
});
};
const agentId = (event as { agentId?: string }).agentId;
@@ -1124,6 +1172,14 @@ export class AgentProcessManager {
// termination doesn't produce two contradictory "session ended" lines.
const spawnState = this.activeSpawnState.get(effect.agentId);
if (spawnState) spawnState.suppressExitLog = true;
+ // `terminate_stalled` is the stall watchdog SIGKILLing a wedged turn —
+ // an INVOLUNTARY kill (Blair's case). Mark it by cause so the trailing
+ // turn_end tags `killed_stalled`, independent of whether the runtime
+ // emits a death-rattle error. `stop` (voluntary idle-timeout) does NOT
+ // mark — that's a clean end (Cecilia 架构#352 red line 2). See B1.
+ if (effect.type === "terminate_stalled") {
+ this.nonCleanEndMarker.set(effect.agentId, { cause: "killed_stalled" });
+ }
this.logSessionEnded(effect.agentId, effect.type === "stop" ? "stopped" : "terminate_stalled");
this.opts.onAgentLocallyStopped?.({ agentId: effect.agentId, reason: effect.type });
break;
@@ -1394,7 +1450,14 @@ export class AgentProcessManager {
this.flushThinkingAudit(agentId);
this.sessions.delete(agentId);
this.liveSessions.delete(agentId);
- if (this.activeSpawnState.get(agentId) === state) this.activeSpawnState.delete(agentId);
+ if (this.activeSpawnState.get(agentId) === state) {
+ this.activeSpawnState.delete(agentId);
+ // Clear a non-clean-end marker (either cause) that never saw its trailing
+ // `turn_end` (a hard exit bypasses the normalizer, B1 3a) so it can't
+ // leak onto the next turn. Same `=== state` guard as above: a late exit
+ // from a stale/superseded spawn must NOT drop a fresh session's marker.
+ this.nonCleanEndMarker.delete(agentId);
+ }
this.dispatch({ type: "exit", agentId });
});
@@ -1555,6 +1618,16 @@ export class AgentProcessManager {
// surfaces. See plans/daemon-fsm-desync.md batch C (reader-C fix).
if (ev.kind === "error" && !sessionSuperseded) {
this.emitErrorAudit(agentId, "runtime", "runtime_error", ev.message ?? "Runtime error");
+ // Mark this turn non-clean (cause=runtime_error) so the trailing
+ // `turn_end` (a separate normalizer event, B1) carries `endReason:"errored"`
+ // + `terminationCause` + detail into the FSM/trace. Gated by the SAME
+ // `!sessionSuperseded` as the audit above, so an intentional reset/nap
+ // death-rattle never marks. A stall-kill's rattle also reaches here — but
+ // if `killed_stalled` was already marked (the kill IS the cause), DON'T
+ // downgrade it: the rattle is the kill's side effect, not the reason.
+ if (this.nonCleanEndMarker.get(agentId)?.cause !== "killed_stalled") {
+ this.nonCleanEndMarker.set(agentId, { cause: "runtime_error", detail: ev.message ?? "Runtime error" });
+ }
// Stuck-reset correlation trace (plans/daemon-fsm-desync.md batch D):
// PURELY ADDITIVE — this runs AFTER the error has already been decided to
// surface (gate above unchanged), and only annotates. If this error fired
@@ -1647,7 +1720,25 @@ export class AgentProcessManager {
this.dispatch({ type: "runtime_signal", agentId, kind: ev.kind, nowMs: this.now() });
if (ev.kind === "turn_end") {
this.logSessionEnded(agentId, "turn_end");
- this.dispatch({ type: "turn_end", agentId, nowMs: this.now() });
+ // Read+clear the non-clean-end marker set this turn by a preceding runtime
+ // `error` OR a `terminate_stalled` kill (B1). Present ⇒ the turn is ending
+ // NON-cleanly; carry the binary judgement (`endReason:"errored"`) + the
+ // cause (`terminationCause`, for B2 policy branching) + free-text detail
+ // into the FSM so onTurnEnd + fsm-trace can tell it from a clean end.
+ const marker = this.nonCleanEndMarker.get(agentId);
+ this.nonCleanEndMarker.delete(agentId);
+ this.dispatch(
+ marker !== undefined
+ ? {
+ type: "turn_end",
+ agentId,
+ nowMs: this.now(),
+ endReason: "errored",
+ terminationCause: marker.cause,
+ errorDetail: marker.detail,
+ }
+ : { type: "turn_end", agentId, nowMs: this.now() },
+ );
}
}
}