Multi-Agent Collaboration for Team and Friends
TypeScript agent-automationagent-orchestrationagent-workflowai-agentsai-collaborationai-memoryai-workforceautonomous-agentsclaude-codeclaude-code-skillscodexcoding-agentmulti-agentmultiple-agentone-person-businessone-person-companyone-person-teamopencodesolopreneur
feat(daemon): FSM transition trace for wedge diagnosis (observability, zero behavior change)
Gus4829ca58a96c
diff --git a/src/daemon/src/daemon/createDaemon.ts b/src/daemon/src/daemon/createDaemon.tsindex f11805e0e..e8522e641 100644--- a/src/daemon/src/daemon/createDaemon.ts+++ b/src/daemon/src/daemon/createDaemon.ts@@ -26,6 +26,7 @@* — the `driverFor` is INJECTED by the caller.*/import { homedir } from "os";+import { appendFileSync } from "node:fs";import { WsControlChannel } from "../server/wsControlChannel.js";import { CredentialBroker, startCredentialProxy } from "../credentials/index.js";import { AgentProcessManager, AgentRouter, createTypingScopeTracker } from "../manager/index.js";@@ -586,6 +587,21 @@ export async function createDaemon(opts: CreateDaemonOptions): Promise<RunningDa// `ready` frame's `runningAgents` reflects what's actually live and the// server's reconciler safety net can flip stale pills to idle.onAgentLocallyStopped: (info) => router?.markLocallyStopped(info.agentId),+ // FSM transition trace → file, ONLY when ALOOK_FSM_TRACE is set (opt-in,+ // zero cost otherwise). One JSON line per reduce so a wedge that logs+ // nothing else is reconstructable from its FSM history. See+ // plans/daemon-fsm-desync.md — the "no log when it breaks" fix.+ ...(process.env.ALOOK_FSM_TRACE+ ? {+ onFsmTransition: (rec: Record<string, unknown>) => {+ try {+ appendFileSync(process.env.ALOOK_FSM_TRACE as string, JSON.stringify(rec) + "\n");+ } catch {+ /* never let tracing break the daemon */+ }+ },+ }+ : {}),// Only the "pi" runtime declares `Driver.createSession` today (in-process// SDK, no child process) — this is only ever consulted for that case.sdkDriverDepsFor: (ctx) => createPiSdkDriverDeps(ctx),diff --git a/src/daemon/src/manager/managerRuntime.test.ts b/src/daemon/src/manager/managerRuntime.test.tsindex 416a511a3..04bd376c1 100644--- a/src/daemon/src/manager/managerRuntime.test.ts+++ b/src/daemon/src/manager/managerRuntime.test.ts@@ -1687,3 +1687,85 @@ describe("onBotAuditEvent — integration through onRuntimeEvent (T9/T10)", () =expect(toolCalls).toHaveLength(0);});});++// FSM transition trace (plans/daemon-fsm-desync.md): pure-observability hook+// used to make a wedge that logs nothing else reconstructable. Two guarantees:+// it fires per dispatch with the fields the wedge-triage needs, and it does NOT+// change behavior (effects identical whether or not the hook is wired).+describe("AgentProcessManager — onFsmTransition trace (observability, zero behavior change)", () => {+ function makeWithTrace(trace?: (rec: Record<string, unknown>) => void) {+ const session = fakeSession();+ const mgr = new AgentProcessManager({+ driverFor: () => fakeDriver("codex"),+ baseContextFor: () => ({ workingDirectory: "/tmp", agentId: "a1", standingPrompt: "", config: {} as LaunchContext["config"], credentialProxy: {} as LaunchContext["credentialProxy"] }),+ sessionFactory: () => session,+ onFsmTransition: trace as never,+ });+ mgr.register("a1");+ return { mgr, session };+ }++ it("fires once per agent-scoped dispatch with the wedge-triage fields", () => {+ const recs: Record<string, unknown>[] = [];+ const { mgr } = makeWithTrace((r) => recs.push(r));+ mgr.deliver("a1", { seq: 1, text: "hello" });+ expect(recs.length).toBeGreaterThan(0);+ const wake = recs.find((r) => r.event === "wake");+ expect(wake).toBeTruthy();+ // Every field the triage needs to split the three "why no watchdog" exits.+ for (const k of ["agentId", "event", "status", "turnActive", "inbox", "lastDeliverAt", "lastProgressAt", "resetting", "resettingSince", "apmPhase", "effects", "nowMs"]) {+ expect(wake).toHaveProperty(k);+ }+ expect(wake!.agentId).toBe("a1");+ expect(Array.isArray(wake!.effects)).toBe(true);+ });++ it("does NOT change behavior — the observed effect sequence is identical with and without the hook", () => {+ // deliver() returns a boolean (produced-effect), so compare the effect+ // SEQUENCE the trace observed instead: it reflects exactly what the reducer+ // emitted. A wired hook must not perturb that sequence.+ const seq: string[][] = [];+ const withHook = makeWithTrace((r) => seq.push(r.effects as string[]));+ const produced1 = withHook.mgr.deliver("a1", { seq: 1, text: "hi" });+ const without = makeWithTrace(undefined);+ const produced2 = without.mgr.deliver("a1", { seq: 1, text: "hi" });+ // Same producedEffect result …+ expect(produced1).toBe(produced2);+ // … and the wake dispatch emitted a spawn (single-flight from idle),+ // observed identically through the trace.+ expect(seq.some((effs) => effs.includes("spawn"))).toBe(true);+ });++ it("fans out a per-agent record on every TICK with derived watchdog inputs (so 'why no watchdog fired' is answerable)", async () => {+ vi.useFakeTimers();+ try {+ let now = 0;+ const recs: Record<string, unknown>[] = [];+ const session = fakeSession();+ const mgr = new AgentProcessManager({+ driverFor: () => fakeDriver("codex"),+ baseContextFor: () => ({ workingDirectory: "/tmp", agentId: "a1", standingPrompt: "", config: {} as LaunchContext["config"], credentialProxy: {} as LaunchContext["credentialProxy"] }),+ sessionFactory: () => session,+ now: () => now,+ tickIntervalMs: 5,+ staleThresholdMs: 100,+ onFsmTransition: ((r: Record<string, unknown>) => recs.push(r)) as never,+ });+ mgr.start(); // arm the tick timer+ mgr.register("a1");+ mgr.deliver("a1", { seq: 1, text: "hi" });+ recs.length = 0;+ now = 50;+ await vi.advanceTimersByTimeAsync(10); // fires ticks+ const tickRecs = recs.filter((r) => r.event === "tick" && r.agentId === "a1");+ expect(tickRecs.length).toBeGreaterThan(0);+ // The derived inputs that let triage judge WHY a watchdog didn't fire.+ const t = tickRecs[0];+ expect(t).toHaveProperty("sinceProgressMs");+ expect(t).toHaveProperty("sinceDeliverMs");+ expect(typeof t.sinceProgressMs).toBe("number");+ } finally {+ vi.useRealTimers();+ }+ });+});diff --git a/src/daemon/src/manager/managerRuntime.ts b/src/daemon/src/manager/managerRuntime.tsindex eec9979ec..b8e38bcf1 100644--- a/src/daemon/src/manager/managerRuntime.ts+++ b/src/daemon/src/manager/managerRuntime.ts@@ -153,6 +153,32 @@ export interface ManagerRuntimeOpts {* so `ready.runningAgents` stays aligned with what's actually live.*/onAgentLocallyStopped?: (info: { agentId: string; reason: "stop" | "terminate_stalled" }) => void;+ /**+ * Pure-observability FSM transition trace. Called once per `dispatch` reduce+ * (for events carrying an agentId) with the post-reduce key fields + the+ * effect kinds produced. Wired in `createDaemon` to append to a file when+ * `ALOOK_FSM_TRACE` is set — lets a wedge that produces no other log be+ * reconstructed from its FSM history. No behavior change; omit ⇒ no-op.+ */+ onFsmTransition?: (rec: {+ agentId: string;+ event: string;+ status: string;+ turnActive: boolean;+ inbox: number;+ lastDeliverAt: number | null;+ lastProgressAt: number;+ idleSince: number | null;+ resetting: boolean;+ resettingSince: number | null;+ apmPhase: string;+ effects: string[];+ nowMs: number;+ /** `nowMs - lastProgressAt` — stalled/suspectedDeaf "no progress for how long". */+ sinceProgressMs: number;+ /** `nowMs - lastDeliverAt` (null if never delivered) — suspectedDeaf's half. */+ sinceDeliverMs: number | null;+ }) => void;/*** Optional context-timeline recorder. When provided, the manager logs each* spawn as a "running" row, fills in the session id on session_init, and closes@@ -549,6 +575,7 @@ export class AgentProcessManager {| "onAgentActivity"| "onBotAuditEvent"| "onAgentLocallyStopped"+ | "onFsmTransition"| "timeline"| "wakePromptFooter"| "onRuntimeSpawnFailed"@@ -566,6 +593,7 @@ export class AgentProcessManager {| "onAgentActivity"| "onBotAuditEvent"| "onAgentLocallyStopped"+ | "onFsmTransition"| "timeline"| "wakePromptFooter"| "onRuntimeSpawnFailed"@@ -906,6 +934,57 @@ export class AgentProcessManager {const before = this.deriveActivitySnapshot(this.state);const { state, effects } = reduceManager(this.state, event);this.state = state;+ // FSM transition trace (plans/daemon-fsm-desync.md): pure observability, no+ // behavior change. Emits one record per reduce so a wedge that leaves no+ // other log (the gated_hold/send branches are the only ones that log today,+ // and the wedge slips both) is reconstructable from the agent's FSM history+ // — the missing capability behind every "no log when it breaks" incident.+ // Guarded so it's a no-op unless a sink is wired (createDaemon opts).+ if (this.opts.onFsmTransition) {+ const nowMs = this.now();+ const emit = (agentId: string): void => {+ const a = this.state.agents[agentId];+ if (!a) return;+ // Effects this dispatch produced FOR THIS agent (effects carry agentId,+ // so a tick that terminates one wedged agent attributes correctly).+ const myEffects = effects.filter((e) => (e as { agentId?: string }).agentId === agentId).map((e) => e.type);+ this.opts.onFsmTransition!({+ agentId,+ event: event.type,+ status: a.status,+ turnActive: a.turnActive,+ inbox: a.inbox.length,+ lastDeliverAt: a.lastDeliverAt,+ lastProgressAt: a.lastProgressAt,+ idleSince: a.idleSince,+ resetting: a.resetting,+ resettingSince: a.resettingSince,+ apmPhase: a.apm.phase,+ effects: myEffects,+ nowMs,+ // Derived watchdog inputs — the ONLY way to judge, per wedge, WHY no+ // watchdog fired: `sinceProgressMs` is `stalled`/`suspectedDeaf`'s+ // "no progress for how long" (if it keeps getting reset small on a+ // wedged agent, lastProgressAt is being bumped by stray progress =+ // the anchor is unusable — the exit-1 decision). `sinceDeliverMs` is+ // suspectedDeaf's half (null when no deliver ever happened = its+ // blind spot). See plans/daemon-fsm-desync.md.+ sinceProgressMs: nowMs - a.lastProgressAt,+ sinceDeliverMs: a.lastDeliverAt === null ? null : nowMs - a.lastDeliverAt,+ });+ };+ const agentId = (event as { agentId?: string }).agentId;+ if (agentId) {+ // Agent-scoped event (wake / spawned / turn_end / …).+ emit(agentId);+ } else if (event.type === "tick") {+ // A tick carries no agentId but the reducer evaluates EVERY agent's+ // watchdogs — fan out so each agent's per-tick watchdog inputs + any+ // effect (or its ABSENCE) are on record. This is what makes "did the+ // tick run and why did stalled not fire" answerable (exit-3).+ for (const id of Object.keys(this.state.agents)) emit(id);+ }+ }for (const effect of effects) this.applyEffect(effect);if (this.opts.onAgentActivity) {const after = this.deriveActivitySnapshot(this.state);