Multi-Agent Collaboration for Team and Friends
TypeScript agent-automationagent-orchestrationagent-workflowai-agentsai-collaborationai-memoryai-workforceautonomous-agentsclaude-codeclaude-code-skillscodexcoding-agentmulti-agentmultiple-agentone-person-businessone-person-companyone-person-teamopencodesolopreneur
daemon: B1 crashed-turn tag — endReason/terminationCause on turn_end (runtime-error auto-rewake followup)
Gus19f31f2cc3e9
diff --git a/src/daemon/src/drivers/claudeEventNormalizer.test.ts b/src/daemon/src/drivers/claudeEventNormalizer.test.tsindex 2949dd97c..b4e34d378 100644--- a/src/daemon/src/drivers/claudeEventNormalizer.test.ts+++ b/src/daemon/src/drivers/claudeEventNormalizer.test.ts@@ -73,4 +73,17 @@ describe("ClaudeEventNormalizer.normalizeLine", () => {expect(out.some((e) => e.kind === "error" && (e as any).message === "boom")).toBe(true);expect(out.some((e) => e.kind === "turn_end")).toBe(true);});++ it("errored result emits `error` BEFORE the trailing `turn_end` (the ordering B1's errored-turn marker relies on)", () => {+ // managerRuntime buffers the `error`'s message on the way past, then reads+ // it out when the trailing `turn_end` arrives to stamp `endReason:"errored"`.+ // That handoff only works if error precedes turn_end in the SAME batch.+ // See plans/daemon-runtime-error-rewake.md B1.+ const out = new ClaudeEventNormalizer().normalizeLine(+ J({ type: "result", is_error: true, result: "boom", session_id: "s1" }),+ );+ const kinds = out.map((e) => e.kind);+ expect(kinds.indexOf("error")).toBeGreaterThanOrEqual(0);+ expect(kinds.indexOf("error")).toBeLessThan(kinds.indexOf("turn_end"));+ });});diff --git a/src/daemon/src/manager/managerPolicy.ts b/src/daemon/src/manager/managerPolicy.tsindex cc2ceb0a1..536c7c427 100644--- a/src/daemon/src/manager/managerPolicy.ts+++ b/src/daemon/src/manager/managerPolicy.ts@@ -165,7 +165,26 @@ export type ManagerEvent =| { type: "spawned"; agentId: string; nowMs: number }| { type: "session"; agentId: string; sessionId: string }| { type: "progress"; agentId: string; nowMs: number }- | { type: "turn_end"; agentId: string; nowMs: number }+ /**+ * `endReason:"errored"` is set when a turn ended NON-cleanly — either a+ * mid-turn runtime `error` OR a `terminate_stalled` kill (managerRuntime+ * buffers the cause, stamps it here on the trailing turn_end). Absent = clean+ * turn-end. STRICTLY BINARY (single literal, not a union) so the rewake gate+ * keyed on `=== "errored"` can't be silently widened — red line 1. The CAUSE+ * rides on the separate `terminationCause` field (for B2 policy branching:+ * `killed_stalled` gets a tighter rewake bound than `runtime_error` since an+ * input-caused hang recurs deterministically); `errorDetail` is free-text.+ * B1 only RECORDS these (trace); the rewake policy that consumes them is B2.+ * See plans/daemon-runtime-error-rewake.md.+ */+ | {+ type: "turn_end";+ agentId: string;+ nowMs: number;+ endReason?: "errored";+ terminationCause?: "runtime_error" | "killed_stalled";+ errorDetail?: string;+ }| { type: "exit"; agentId: string }| { type: "tick"; nowMs: number }/**diff --git a/src/daemon/src/manager/managerRuntime.test.ts b/src/daemon/src/manager/managerRuntime.test.tsindex 3f7c65670..b1b3e4e72 100644--- a/src/daemon/src/manager/managerRuntime.test.ts+++ b/src/daemon/src/manager/managerRuntime.test.ts@@ -1950,4 +1950,170 @@ describe("AgentProcessManager — onFsmTransition trace (observability, zero behvi.useRealTimers();}});++ // ---- B1: crashed-turn tag (plans/daemon-runtime-error-rewake.md) ----------+ // The trailing turn_end after a mid-turn `error` carries `endReason:"errored"`+ // + `errorDetail` into the trace, so a crashed turn is externally+ // distinguishable from a clean nap/idle (red line 7). B1 only RECORDS it —+ // onTurnEnd behavior is unchanged (verified in the "clean" case below).++ it("a mid-turn error then turn_end stamps endReason=errored + terminationCause=runtime_error + errorDetail (red line 6/7)", () => {+ const recs: Record<string, unknown>[] = [];+ const { mgr, session } = makeWithTrace((r) => recs.push(r));+ mgr.deliver("a1", { seq: 1, text: "hi" });+ session.fire("runtime_event", { kind: "session_init", sessionId: "s1" });+ // Genuinely interrupt the turn: an `error` event (as the normalizer emits+ // from an is_error result) FOLLOWED by the trailing turn_end.+ session.fire("runtime_event", { kind: "error", message: "boom: model overloaded" });+ session.fire("runtime_event", { kind: "turn_end" });++ const turnEnd = recs.find((r) => r.event === "turn_end" && r.agentId === "a1");+ expect(turnEnd).toBeTruthy();+ expect(turnEnd!.endReason).toBe("errored");+ expect(turnEnd!.terminationCause).toBe("runtime_error");+ expect(turnEnd!.errorDetail).toBe("boom: model overloaded");+ });++ it("a CLEAN turn_end (no preceding error/kill) carries no endReason/terminationCause/errorDetail (byte-for-byte the old row)", () => {+ const recs: Record<string, unknown>[] = [];+ const { mgr, session } = makeWithTrace((r) => recs.push(r));+ mgr.deliver("a1", { seq: 1, text: "hi" });+ session.fire("runtime_event", { kind: "session_init", sessionId: "s1" });+ session.fire("runtime_event", { kind: "turn_end" });++ const turnEnd = recs.find((r) => r.event === "turn_end" && r.agentId === "a1");+ expect(turnEnd).toBeTruthy();+ expect(turnEnd!.endReason).toBeUndefined();+ expect(turnEnd!.terminationCause).toBeUndefined();+ expect(turnEnd!.errorDetail).toBeUndefined();+ });++ it("an intentional-kill (superseded) death-rattle error does NOT tag its turn_end errored (red line 5 — reset/nap is not a crash)", () => {+ const recs: Record<string, unknown>[] = [];+ const { mgr, session } = makeWithTrace((r) => recs.push(r));+ mgr.deliver("a1", { seq: 1, text: "hi" });+ session.fire("runtime_event", { kind: "session_init", sessionId: "s1" });+ // Enter the reset/nap kill window, THEN the dying process rattles. The+ // marker is gated by the same `!sessionSuperseded` as the audit, so it must+ // NOT set — a nap must stay indistinguishable-from-clean, not read as crash.+ mgr.markResetting("a1");+ session.fire("runtime_event", { kind: "error", message: "turn interrupted" });+ session.fire("runtime_event", { kind: "turn_end" });++ const turnEnd = recs.find((r) => r.event === "turn_end" && r.agentId === "a1");+ expect(turnEnd).toBeTruthy();+ expect(turnEnd!.endReason).toBeUndefined();+ expect(turnEnd!.terminationCause).toBeUndefined();+ });++ it("a mid-turn error followed by a hard exit (no turn_end) clears the marker so the NEXT turn is not mis-tagged (3a marker-leak guard)", () => {+ const recs: Record<string, unknown>[] = [];+ const { mgr, session } = makeWithTrace((r) => recs.push(r));+ mgr.deliver("a1", { seq: 1, text: "hi" });+ session.fire("runtime_event", { kind: "session_init", sessionId: "s1" });+ // Error with NO trailing turn_end, then a hard process exit (bypasses the+ // normalizer) — the marker would otherwise leak in the map.+ session.fire("runtime_event", { kind: "error", message: "crashed hard" });+ session.fire("exit");++ // A fresh wake spawns a new session; its clean turn_end must NOT inherit the+ // stale marker.+ recs.length = 0;+ mgr.deliver("a1", { seq: 2, text: "again" });+ session.fire("runtime_event", { kind: "session_init", sessionId: "s2" });+ session.fire("runtime_event", { kind: "turn_end" });++ const turnEnd = recs.find((r) => r.event === "turn_end" && r.agentId === "a1");+ expect(turnEnd).toBeTruthy();+ expect(turnEnd!.endReason).toBeUndefined();+ expect(turnEnd!.terminationCause).toBeUndefined();+ });++ it("red line 6 — a REAL stall (no progress past threshold → terminate_stalled) tags the turn_end killed_stalled, NOT injected error (Blair's actual case)", async () => {+ vi.useFakeTimers();+ try {+ let now = 0;+ const recs: Record<string, unknown>[] = [];+ // Default fakeDriver (per_turn) satisfies the `stalled` predicate's first+ // sub-clause — the same setup the proven "terminate_stalled from the stall+ // watchdog" test uses. We only add the trace sink.+ const session = fakeSession();+ const mgr = new AgentProcessManager({+ driverFor: () => fakeDriver("codex"),+ baseContextFor: () => ({ workingDirectory: "/tmp", agentId: "a1", standingPrompt: "", config: {} as LaunchContext["config"], credentialProxy: {} as LaunchContext["credentialProxy"] }),+ sessionFactory: () => session,+ now: () => now,+ tickIntervalMs: 5,+ staleThresholdMs: 100, // wedge → terminate_stalled after 100ms of no progress+ onFsmTransition: ((r: Record<string, unknown>) => recs.push(r)) as never,+ });+ mgr.start();+ mgr.register("a1");+ mgr.deliver("a1", { seq: 1, text: "hi" });+ session.startResolver?.();+ await Promise.resolve();+ session.fire("runtime_event", { kind: "session_init", sessionId: "s1" });+ // Turn is in flight (turnActive) and makes NO progress. No error injected —+ // this is a genuine hang, exactly Blair's case.+ now = 200; // past staleThreshold=100 → stalled watchdog fires terminate_stalled+ await vi.advanceTimersByTimeAsync(10);+ const killTick = recs.find((r) => Array.isArray(r.effects) && (r.effects as string[]).includes("terminate_stalled"));+ expect(killTick).toBeTruthy(); // the stall kill actually fired++ // The SIGKILL makes the process emit its trailing turn_end (no error+ // rattle needed — that's the whole point of keying on cause, not rattle).+ recs.length = 0;+ session.fire("runtime_event", { kind: "turn_end" });+ const turnEnd = recs.find((r) => r.event === "turn_end" && r.agentId === "a1");+ expect(turnEnd).toBeTruthy();+ expect(turnEnd!.endReason).toBe("errored");+ expect(turnEnd!.terminationCause).toBe("killed_stalled");+ } finally {+ vi.useRealTimers();+ }+ });++ it("a VOLUNTARY idle-timeout stop (which also flips status→stopping) does NOT produce a tagged turn_end (red line 2 — cause, not bare status)", async () => {+ // Cecilia's "don't treat 'not seen' as 'won't happen'": idle-hibernation's+ // `stop(idle_timeout)` flips status→stopping too, but it is a CLEAN end. We+ // key on the terminate_stalled EFFECT, not on status, so no marker is set →+ // any turn_end around a voluntary stop stays untagged.+ vi.useFakeTimers();+ try {+ let now = 0;+ const recs: Record<string, unknown>[] = [];+ const persistentDriver = {+ ...fakeDriver("codex"),+ lifecycle: { kind: "persistent", start: "immediate", exit: "natural", inFlightWake: "queue" } as never,+ } as Driver;+ const session = fakeSession();+ const mgr = new AgentProcessManager({+ driverFor: () => persistentDriver,+ baseContextFor: () => ({ workingDirectory: "/tmp", agentId: "a1", standingPrompt: "", config: {} as LaunchContext["config"], credentialProxy: {} as LaunchContext["credentialProxy"] }),+ sessionFactory: () => session,+ now: () => now,+ tickIntervalMs: 5,+ idleTimeoutMs: 50, // idle → voluntary stop(idle_timeout), flips stopping+ stoppingStuckThresholdMs: 1_000_000, // don't force_exit during the test+ onFsmTransition: ((r: Record<string, unknown>) => recs.push(r)) as never,+ });+ mgr.start();+ mgr.register("a1");+ mgr.deliver("a1", { seq: 1, text: "hi" });+ session.startResolver?.();+ await Promise.resolve();+ session.fire("runtime_event", { kind: "session_init", sessionId: "s1" });+ session.fire("runtime_event", { kind: "turn_end" }); // clean end → idle+ recs.length = 0;+ now = 100; // past idleTimeout=50 → idle-hibernation issues a voluntary stop+ await vi.advanceTimersByTimeAsync(10);+ const stoppingRec = recs.find((r) => r.status === "stopping" && r.agentId === "a1");+ expect(stoppingRec).toBeTruthy(); // it DID flip stopping (voluntary)+ // No terminate_stalled marker was set, so no tagged turn_end can appear.+ const tagged = recs.find((r) => r.event === "turn_end" && r.endReason === "errored");+ expect(tagged).toBeUndefined();+ } finally {+ vi.useRealTimers();+ }+ });});diff --git a/src/daemon/src/manager/managerRuntime.ts b/src/daemon/src/manager/managerRuntime.tsindex 624e711a9..bf3b31a61 100644--- a/src/daemon/src/manager/managerRuntime.ts+++ b/src/daemon/src/manager/managerRuntime.ts@@ -185,6 +185,18 @@ export interface ManagerRuntimeOpts {sinceDeliverMs: number | null;/** `nowMs - stoppingSince` (null unless in `stopping`) — stopping-stuck backstop clock. */sinceStoppingMs: number | null;+ /**+ * Present only on a `turn_end` that ended NON-cleanly. `endReason:"errored"`+ * is the binary crashed-vs-clean judgement; `terminationCause` is the cause+ * (`runtime_error` = mid-turn runtime error; `killed_stalled` = stall+ * watchdog kill) for B2 policy branching; `errorDetail` is free-text. Absent+ * on a clean turn_end and every non-turn_end event — makes a non-clean turn+ * externally distinguishable from a clean nap/idle in the trace (B1 red line+ * 7). See plans/daemon-runtime-error-rewake.md.+ */+ endReason?: "errored";+ terminationCause?: "runtime_error" | "killed_stalled";+ errorDetail?: string;}) => void;/*** Optional context-timeline recorder. When provided, the manager logs each@@ -584,6 +596,31 @@ export class AgentProcessManager {pid: number | null;}>();+ /**+ * agentId → why the current turn ended NON-cleanly, buffered until the+ * trailing `turn_end` reads+clears it. Two symmetric marker points feed it+ * (Cecilia 架构#352 / Claudette #353 — key on CAUSE, never on bare `status`,+ * so a voluntary idle-timeout `stop` that also flips `stopping` is never+ * misread as a crash):+ * - `runtime_error`: a runtime `error` event mid-turn, set ONLY in the+ * `!sessionSuperseded` branch (an intentional reset/nap death-rattle never+ * marks). Carries the free-text `detail` for lossless fsm-trace.+ * - `killed_stalled`: the `terminate_stalled` effect (the stall watchdog+ * SIGKILLing a wedged turn — Blair's actual case). By construction, no+ * dependence on whether the runtime emits a trailing error rattle.+ * At the trailing `turn_end` this becomes `endReason:"errored"` (the binary+ * can-rewake judgement) + `terminationCause` (the cause, for B2 policy+ * branching). A kill's marker is authoritative: if a `killed_stalled` marker+ * is present, a following `runtime_error` rattle must NOT downgrade it (the+ * kill is the real cause; the rattle is its side effect). Cleared on the+ * `exit` teardown too, so a hard exit with no trailing turn_end can't leak a+ * stale marker onto the next turn (B1 3a). See+ * plans/daemon-runtime-error-rewake.md.+ */+ private readonly nonCleanEndMarker = new Map<+ string,+ { cause: "runtime_error" | "killed_stalled"; detail?: string }+ >();private readonly opts: Required<Omit<ManagerRuntimeOpts,@@ -1032,6 +1069,17 @@ export class AgentProcessManager {sinceProgressMs: nowMs - a.lastProgressAt,sinceDeliverMs: a.lastDeliverAt === null ? null : nowMs - a.lastDeliverAt,sinceStoppingMs: a.stoppingSince === null ? null : nowMs - a.stoppingSince,+ // Non-clean-turn tag, carried on the `turn_end` event itself (not agent+ // state). Spread so a clean turn_end / any other event omits the keys+ // entirely rather than emitting explicit undefineds. B1.+ ...(event.type === "turn_end" && (event as { endReason?: "errored" }).endReason === "errored"+ ? {+ endReason: "errored" as const,+ terminationCause: (event as { terminationCause?: "runtime_error" | "killed_stalled" })+ .terminationCause,+ errorDetail: (event as { errorDetail?: string }).errorDetail,+ }+ : {}),});};const agentId = (event as { agentId?: string }).agentId;@@ -1124,6 +1172,14 @@ export class AgentProcessManager {// termination doesn't produce two contradictory "session ended" lines.const spawnState = this.activeSpawnState.get(effect.agentId);if (spawnState) spawnState.suppressExitLog = true;+ // `terminate_stalled` is the stall watchdog SIGKILLing a wedged turn —+ // an INVOLUNTARY kill (Blair's case). Mark it by cause so the trailing+ // turn_end tags `killed_stalled`, independent of whether the runtime+ // emits a death-rattle error. `stop` (voluntary idle-timeout) does NOT+ // mark — that's a clean end (Cecilia 架构#352 red line 2). See B1.+ if (effect.type === "terminate_stalled") {+ this.nonCleanEndMarker.set(effect.agentId, { cause: "killed_stalled" });+ }this.logSessionEnded(effect.agentId, effect.type === "stop" ? "stopped" : "terminate_stalled");this.opts.onAgentLocallyStopped?.({ agentId: effect.agentId, reason: effect.type });break;@@ -1394,7 +1450,14 @@ export class AgentProcessManager {this.flushThinkingAudit(agentId);this.sessions.delete(agentId);this.liveSessions.delete(agentId);- if (this.activeSpawnState.get(agentId) === state) this.activeSpawnState.delete(agentId);+ if (this.activeSpawnState.get(agentId) === state) {+ this.activeSpawnState.delete(agentId);+ // Clear a non-clean-end marker (either cause) that never saw its trailing+ // `turn_end` (a hard exit bypasses the normalizer, B1 3a) so it can't+ // leak onto the next turn. Same `=== state` guard as above: a late exit+ // from a stale/superseded spawn must NOT drop a fresh session's marker.+ this.nonCleanEndMarker.delete(agentId);+ }this.dispatch({ type: "exit", agentId });});@@ -1555,6 +1618,16 @@ export class AgentProcessManager {// surfaces. See plans/daemon-fsm-desync.md batch C (reader-C fix).if (ev.kind === "error" && !sessionSuperseded) {this.emitErrorAudit(agentId, "runtime", "runtime_error", ev.message ?? "Runtime error");+ // Mark this turn non-clean (cause=runtime_error) so the trailing+ // `turn_end` (a separate normalizer event, B1) carries `endReason:"errored"`+ // + `terminationCause` + detail into the FSM/trace. Gated by the SAME+ // `!sessionSuperseded` as the audit above, so an intentional reset/nap+ // death-rattle never marks. A stall-kill's rattle also reaches here — but+ // if `killed_stalled` was already marked (the kill IS the cause), DON'T+ // downgrade it: the rattle is the kill's side effect, not the reason.+ if (this.nonCleanEndMarker.get(agentId)?.cause !== "killed_stalled") {+ this.nonCleanEndMarker.set(agentId, { cause: "runtime_error", detail: ev.message ?? "Runtime error" });+ }// Stuck-reset correlation trace (plans/daemon-fsm-desync.md batch D):// PURELY ADDITIVE — this runs AFTER the error has already been decided to// surface (gate above unchanged), and only annotates. If this error fired@@ -1647,7 +1720,25 @@ export class AgentProcessManager {this.dispatch({ type: "runtime_signal", agentId, kind: ev.kind, nowMs: this.now() });if (ev.kind === "turn_end") {this.logSessionEnded(agentId, "turn_end");- this.dispatch({ type: "turn_end", agentId, nowMs: this.now() });+ // Read+clear the non-clean-end marker set this turn by a preceding runtime+ // `error` OR a `terminate_stalled` kill (B1). Present ⇒ the turn is ending+ // NON-cleanly; carry the binary judgement (`endReason:"errored"`) + the+ // cause (`terminationCause`, for B2 policy branching) + free-text detail+ // into the FSM so onTurnEnd + fsm-trace can tell it from a clean end.+ const marker = this.nonCleanEndMarker.get(agentId);+ this.nonCleanEndMarker.delete(agentId);+ this.dispatch(+ marker !== undefined+ ? {+ type: "turn_end",+ agentId,+ nowMs: this.now(),+ endReason: "errored",+ terminationCause: marker.cause,+ errorDetail: marker.detail,+ }+ : { type: "turn_end", agentId, nowMs: this.now() },+ );}}}