Skip to content

armgit / alook

Public Imported from GitHub
0

Multi-Agent Collaboration for Team and Friends

feat(daemon): FSM transition trace for wedge diagnosis (observability, zero behavior change)

Gus4829ca58a96c
diff --git a/src/daemon/src/daemon/createDaemon.ts b/src/daemon/src/daemon/createDaemon.ts
index f11805e0e..e8522e641 100644
--- a/src/daemon/src/daemon/createDaemon.ts
+++ b/src/daemon/src/daemon/createDaemon.ts
@@ -26,6 +26,7 @@
* — the `driverFor` is INJECTED by the caller.
*/
import { homedir } from "os";
+import { appendFileSync } from "node:fs";
import { WsControlChannel } from "../server/wsControlChannel.js";
import { CredentialBroker, startCredentialProxy } from "../credentials/index.js";
import { AgentProcessManager, AgentRouter, createTypingScopeTracker } from "../manager/index.js";
@@ -586,6 +587,21 @@ export async function createDaemon(opts: CreateDaemonOptions): Promise<RunningDa
// `ready` frame's `runningAgents` reflects what's actually live and the
// server's reconciler safety net can flip stale pills to idle.
onAgentLocallyStopped: (info) => router?.markLocallyStopped(info.agentId),
+ // FSM transition trace → file, ONLY when ALOOK_FSM_TRACE is set (opt-in,
+ // zero cost otherwise). One JSON line per reduce so a wedge that logs
+ // nothing else is reconstructable from its FSM history. See
+ // plans/daemon-fsm-desync.md — the "no log when it breaks" fix.
+ ...(process.env.ALOOK_FSM_TRACE
+ ? {
+ onFsmTransition: (rec: Record<string, unknown>) => {
+ try {
+ appendFileSync(process.env.ALOOK_FSM_TRACE as string, JSON.stringify(rec) + "\n");
+ } catch {
+ /* never let tracing break the daemon */
+ }
+ },
+ }
+ : {}),
// Only the "pi" runtime declares `Driver.createSession` today (in-process
// SDK, no child process) — this is only ever consulted for that case.
sdkDriverDepsFor: (ctx) => createPiSdkDriverDeps(ctx),
diff --git a/src/daemon/src/manager/managerRuntime.test.ts b/src/daemon/src/manager/managerRuntime.test.ts
index 416a511a3..04bd376c1 100644
--- a/src/daemon/src/manager/managerRuntime.test.ts
+++ b/src/daemon/src/manager/managerRuntime.test.ts
@@ -1687,3 +1687,85 @@ describe("onBotAuditEvent — integration through onRuntimeEvent (T9/T10)", () =
expect(toolCalls).toHaveLength(0);
});
});
+
+// FSM transition trace (plans/daemon-fsm-desync.md): pure-observability hook
+// used to make a wedge that logs nothing else reconstructable. Two guarantees:
+// it fires per dispatch with the fields the wedge-triage needs, and it does NOT
+// change behavior (effects identical whether or not the hook is wired).
+describe("AgentProcessManager — onFsmTransition trace (observability, zero behavior change)", () => {
+ function makeWithTrace(trace?: (rec: Record<string, unknown>) => void) {
+ const session = fakeSession();
+ const mgr = new AgentProcessManager({
+ driverFor: () => fakeDriver("codex"),
+ baseContextFor: () => ({ workingDirectory: "/tmp", agentId: "a1", standingPrompt: "", config: {} as LaunchContext["config"], credentialProxy: {} as LaunchContext["credentialProxy"] }),
+ sessionFactory: () => session,
+ onFsmTransition: trace as never,
+ });
+ mgr.register("a1");
+ return { mgr, session };
+ }
+
+ it("fires once per agent-scoped dispatch with the wedge-triage fields", () => {
+ const recs: Record<string, unknown>[] = [];
+ const { mgr } = makeWithTrace((r) => recs.push(r));
+ mgr.deliver("a1", { seq: 1, text: "hello" });
+ expect(recs.length).toBeGreaterThan(0);
+ const wake = recs.find((r) => r.event === "wake");
+ expect(wake).toBeTruthy();
+ // Every field the triage needs to split the three "why no watchdog" exits.
+ for (const k of ["agentId", "event", "status", "turnActive", "inbox", "lastDeliverAt", "lastProgressAt", "resetting", "resettingSince", "apmPhase", "effects", "nowMs"]) {
+ expect(wake).toHaveProperty(k);
+ }
+ expect(wake!.agentId).toBe("a1");
+ expect(Array.isArray(wake!.effects)).toBe(true);
+ });
+
+ it("does NOT change behavior — the observed effect sequence is identical with and without the hook", () => {
+ // deliver() returns a boolean (produced-effect), so compare the effect
+ // SEQUENCE the trace observed instead: it reflects exactly what the reducer
+ // emitted. A wired hook must not perturb that sequence.
+ const seq: string[][] = [];
+ const withHook = makeWithTrace((r) => seq.push(r.effects as string[]));
+ const produced1 = withHook.mgr.deliver("a1", { seq: 1, text: "hi" });
+ const without = makeWithTrace(undefined);
+ const produced2 = without.mgr.deliver("a1", { seq: 1, text: "hi" });
+ // Same producedEffect result …
+ expect(produced1).toBe(produced2);
+ // … and the wake dispatch emitted a spawn (single-flight from idle),
+ // observed identically through the trace.
+ expect(seq.some((effs) => effs.includes("spawn"))).toBe(true);
+ });
+
+ it("fans out a per-agent record on every TICK with derived watchdog inputs (so 'why no watchdog fired' is answerable)", async () => {
+ vi.useFakeTimers();
+ try {
+ let now = 0;
+ const recs: Record<string, unknown>[] = [];
+ const session = fakeSession();
+ const mgr = new AgentProcessManager({
+ driverFor: () => fakeDriver("codex"),
+ baseContextFor: () => ({ workingDirectory: "/tmp", agentId: "a1", standingPrompt: "", config: {} as LaunchContext["config"], credentialProxy: {} as LaunchContext["credentialProxy"] }),
+ sessionFactory: () => session,
+ now: () => now,
+ tickIntervalMs: 5,
+ staleThresholdMs: 100,
+ onFsmTransition: ((r: Record<string, unknown>) => recs.push(r)) as never,
+ });
+ mgr.start(); // arm the tick timer
+ mgr.register("a1");
+ mgr.deliver("a1", { seq: 1, text: "hi" });
+ recs.length = 0;
+ now = 50;
+ await vi.advanceTimersByTimeAsync(10); // fires ticks
+ const tickRecs = recs.filter((r) => r.event === "tick" && r.agentId === "a1");
+ expect(tickRecs.length).toBeGreaterThan(0);
+ // The derived inputs that let triage judge WHY a watchdog didn't fire.
+ const t = tickRecs[0];
+ expect(t).toHaveProperty("sinceProgressMs");
+ expect(t).toHaveProperty("sinceDeliverMs");
+ expect(typeof t.sinceProgressMs).toBe("number");
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+});
diff --git a/src/daemon/src/manager/managerRuntime.ts b/src/daemon/src/manager/managerRuntime.ts
index eec9979ec..b8e38bcf1 100644
--- a/src/daemon/src/manager/managerRuntime.ts
+++ b/src/daemon/src/manager/managerRuntime.ts
@@ -153,6 +153,32 @@ export interface ManagerRuntimeOpts {
* so `ready.runningAgents` stays aligned with what's actually live.
*/
onAgentLocallyStopped?: (info: { agentId: string; reason: "stop" | "terminate_stalled" }) => void;
+ /**
+ * Pure-observability FSM transition trace. Called once per `dispatch` reduce
+ * (for events carrying an agentId) with the post-reduce key fields + the
+ * effect kinds produced. Wired in `createDaemon` to append to a file when
+ * `ALOOK_FSM_TRACE` is set — lets a wedge that produces no other log be
+ * reconstructed from its FSM history. No behavior change; omit ⇒ no-op.
+ */
+ onFsmTransition?: (rec: {
+ agentId: string;
+ event: string;
+ status: string;
+ turnActive: boolean;
+ inbox: number;
+ lastDeliverAt: number | null;
+ lastProgressAt: number;
+ idleSince: number | null;
+ resetting: boolean;
+ resettingSince: number | null;
+ apmPhase: string;
+ effects: string[];
+ nowMs: number;
+ /** `nowMs - lastProgressAt` — stalled/suspectedDeaf "no progress for how long". */
+ sinceProgressMs: number;
+ /** `nowMs - lastDeliverAt` (null if never delivered) — suspectedDeaf's half. */
+ sinceDeliverMs: number | null;
+ }) => void;
/**
* Optional context-timeline recorder. When provided, the manager logs each
* spawn as a "running" row, fills in the session id on session_init, and closes
@@ -549,6 +575,7 @@ export class AgentProcessManager {
| "onAgentActivity"
| "onBotAuditEvent"
| "onAgentLocallyStopped"
+ | "onFsmTransition"
| "timeline"
| "wakePromptFooter"
| "onRuntimeSpawnFailed"
@@ -566,6 +593,7 @@ export class AgentProcessManager {
| "onAgentActivity"
| "onBotAuditEvent"
| "onAgentLocallyStopped"
+ | "onFsmTransition"
| "timeline"
| "wakePromptFooter"
| "onRuntimeSpawnFailed"
@@ -906,6 +934,57 @@ export class AgentProcessManager {
const before = this.deriveActivitySnapshot(this.state);
const { state, effects } = reduceManager(this.state, event);
this.state = state;
+ // FSM transition trace (plans/daemon-fsm-desync.md): pure observability, no
+ // behavior change. Emits one record per reduce so a wedge that leaves no
+ // other log (the gated_hold/send branches are the only ones that log today,
+ // and the wedge slips both) is reconstructable from the agent's FSM history
+ // — the missing capability behind every "no log when it breaks" incident.
+ // Guarded so it's a no-op unless a sink is wired (createDaemon opts).
+ if (this.opts.onFsmTransition) {
+ const nowMs = this.now();
+ const emit = (agentId: string): void => {
+ const a = this.state.agents[agentId];
+ if (!a) return;
+ // Effects this dispatch produced FOR THIS agent (effects carry agentId,
+ // so a tick that terminates one wedged agent attributes correctly).
+ const myEffects = effects.filter((e) => (e as { agentId?: string }).agentId === agentId).map((e) => e.type);
+ this.opts.onFsmTransition!({
+ agentId,
+ event: event.type,
+ status: a.status,
+ turnActive: a.turnActive,
+ inbox: a.inbox.length,
+ lastDeliverAt: a.lastDeliverAt,
+ lastProgressAt: a.lastProgressAt,
+ idleSince: a.idleSince,
+ resetting: a.resetting,
+ resettingSince: a.resettingSince,
+ apmPhase: a.apm.phase,
+ effects: myEffects,
+ nowMs,
+ // Derived watchdog inputs — the ONLY way to judge, per wedge, WHY no
+ // watchdog fired: `sinceProgressMs` is `stalled`/`suspectedDeaf`'s
+ // "no progress for how long" (if it keeps getting reset small on a
+ // wedged agent, lastProgressAt is being bumped by stray progress =
+ // the anchor is unusable — the exit-1 decision). `sinceDeliverMs` is
+ // suspectedDeaf's half (null when no deliver ever happened = its
+ // blind spot). See plans/daemon-fsm-desync.md.
+ sinceProgressMs: nowMs - a.lastProgressAt,
+ sinceDeliverMs: a.lastDeliverAt === null ? null : nowMs - a.lastDeliverAt,
+ });
+ };
+ const agentId = (event as { agentId?: string }).agentId;
+ if (agentId) {
+ // Agent-scoped event (wake / spawned / turn_end / …).
+ emit(agentId);
+ } else if (event.type === "tick") {
+ // A tick carries no agentId but the reducer evaluates EVERY agent's
+ // watchdogs — fan out so each agent's per-tick watchdog inputs + any
+ // effect (or its ABSENCE) are on record. This is what makes "did the
+ // tick run and why did stalled not fire" answerable (exit-3).
+ for (const id of Object.keys(this.state.agents)) emit(id);
+ }
+ }
for (const effect of effects) this.applyEffect(effect);
if (this.opts.onAgentActivity) {
const after = this.deriveActivitySnapshot(this.state);