Skip to content

armgit / alook

Public Imported from GitHub
0

Multi-Agent Collaboration for Team and Friends

daemon: T4 — FSM-trace heartbeat sampling for ≥12h retention (trace-completeness red-line-6)

Gus8fb202687356
diff --git a/src/daemon/src/daemon/createDaemon.ts b/src/daemon/src/daemon/createDaemon.ts
index ed2968f37..ccbf05b35 100644
--- a/src/daemon/src/daemon/createDaemon.ts
+++ b/src/daemon/src/daemon/createDaemon.ts
@@ -28,6 +28,7 @@
import { homedir } from "os";
import { appendFileSync, mkdirSync } from "node:fs";
import { createRotatingFileSink } from "../util/rotatingFileSink.js";
+import { createTraceSampler } from "../util/traceSampler.js";
import { writeStatusFile } from "../util/statusFile.js";
import { WsControlChannel } from "../server/wsControlChannel.js";
import { CredentialBroker, startCredentialProxy } from "../credentials/index.js";
@@ -49,8 +50,12 @@ const WARMUP_CEILING_MS = 30_000;
/**
* Per-file cap for the DEFAULT-ON bounded FSM trace (batch E1). The rotating
* sink keeps the active file + one rotated generation, so total on-disk ≈
- * 2×this ≈ 16MB — about 4h of history at the observed ~14 rows/min/agent rate,
- * enough to hold the last wedge. The `ALOOK_FSM_TRACE` override is unbounded.
+ * 2×this ≈ 16MB. RAW row rate at N≈8 agents is ~74% unchanged-state tick noise
+ * → only ~2h of history unsampled; the T4 heartbeat sampler (createTraceSampler)
+ * folds that noise so TRANSITION rows survive ≥12h @ N=8 in the same budget.
+ * (≥12h is @ N=8 — write rate scales with agent count, so a much larger fleet
+ * warrants revisiting this cap.) The `ALOOK_FSM_TRACE` override is unbounded and
+ * unsampled (full-fidelity deep dives).
*/
const FSM_TRACE_MAX_BYTES = 8 * 1024 * 1024;
/** How often the daemon rewrites the `daemon status` snapshot file (batch E2). */
@@ -645,8 +650,15 @@ export async function createDaemon(opts: CreateDaemonOptions): Promise<RunningDa
`${opts.fsmTraceDir}/fsm-trace.jsonl`,
FSM_TRACE_MAX_BYTES,
);
+ // Heartbeat sampler between the manager and the sink (batch T4): folds
+ // redundant unchanged-state ticks + progress/runtime_signal noise so the
+ // bounded file retains transition rows ≥12h (@ N=8) instead of ~2h.
+ // Transitions and watchdog-fired (effects-carrying) frames are never
+ // dropped. The ALOOK_FSM_TRACE override above is unbounded → unsampled,
+ // for full-fidelity deep dives.
+ const sampler = createTraceSampler((rec) => sink.write(JSON.stringify(rec)));
return {
- onFsmTransition: (rec: Record<string, unknown>) => sink.write(JSON.stringify(rec)),
+ onFsmTransition: (rec: Record<string, unknown>) => sampler.offer(rec),
};
}
return {};
diff --git a/src/daemon/src/util/traceSampler.test.ts b/src/daemon/src/util/traceSampler.test.ts
new file mode 100644
index 000000000..4abb4839a
--- /dev/null
+++ b/src/daemon/src/util/traceSampler.test.ts
@@ -0,0 +1,174 @@
+import { describe, it, expect } from "vitest";
+import { createTraceSampler, DEFAULT_TRACE_SAMPLE_MS } from "./traceSampler.js";
+
+/*
+ * T4 (plans/daemon-trace-completeness-charter.md) — heartbeat sampler.
+ * Two acceptance conditions (Claudette 架构#421):
+ * ① retention: folding the unchanged-tick / progress / runtime_signal noise
+ * lifts transition-row retention ≥12h @ N=8 in the fixed byte budget —
+ * here proven as a reproducible fold-ratio → hours calculation.
+ * ② reconstruction: after sampling, a wedge is STILL reconstructable from the
+ * retained rows alone — which phase, how long, through to kill; the
+ * watchdog-fired (effects) frame with the peak sinceProgressMs survives.
+ */
+
+type Rec = Record<string, unknown>;
+
+/** A collecting sampler harness. */
+function collect(throttleMs?: number) {
+ const out: Rec[] = [];
+ const s = createTraceSampler((r) => out.push(r), throttleMs);
+ return { s, out };
+}
+
+function tick(agentId: string, nowMs: number, over: Partial<Rec> = {}): Rec {
+ return {
+ agentId,
+ event: "tick",
+ status: "running",
+ turnActive: true,
+ inbox: 0,
+ resetting: false,
+ stoppingSince: null,
+ effects: [],
+ nowMs,
+ sinceProgressMs: 0,
+ ...over,
+ };
+}
+
+describe("traceSampler — sacrosanct rows never dropped", () => {
+ it("all non-tick events pass through untouched", () => {
+ const { s, out } = collect();
+ for (const event of ["exit", "turn_end", "reset_session", "begin_reset", "spawned", "wake", "register", "session"]) {
+ s.offer({ agentId: "a", event, nowMs: 1 });
+ }
+ expect(out.map((r) => r.event)).toEqual([
+ "exit", "turn_end", "reset_session", "begin_reset", "spawned", "wake", "register", "session",
+ ]);
+ });
+
+ it("a tick carrying effects (watchdog fired) is NEVER folded, even mid-throttle-window", () => {
+ const { s, out } = collect();
+ s.offer(tick("a", 0)); // first unchanged tick — emitted (edge from null)
+ s.offer(tick("a", 2000)); // within 30s window, unchanged → folded (held)
+ // The watchdog fires: same state key but effects non-empty → sacrosanct.
+ s.offer(tick("a", 4000, { effects: ["terminate_stalled"], sinceProgressMs: 121846 }));
+ const killRow = out.find((r) => Array.isArray(r.effects) && (r.effects as string[]).includes("terminate_stalled"));
+ expect(killRow).toBeTruthy();
+ expect(killRow!.sinceProgressMs).toBe(121846); // peak preserved
+ });
+});
+
+describe("traceSampler — unchanged-tick throttling", () => {
+ it("folds unchanged ticks inside the throttle window, emits one per window", () => {
+ const { s, out } = collect(30_000);
+ s.offer(tick("a", 0)); // edge → emit
+ s.offer(tick("a", 2000)); // folded
+ s.offer(tick("a", 4000)); // folded
+ s.offer(tick("a", 31000)); // past window → emit
+ const ticks = out.filter((r) => r.event === "tick");
+ // 0 (edge) and 31000 (past window). 2000/4000 folded — but see tail-flush test.
+ expect(ticks.some((r) => r.nowMs === 0)).toBe(true);
+ expect(ticks.some((r) => r.nowMs === 31000)).toBe(true);
+ });
+
+ it("a state-change edge always emits (the run boundary)", () => {
+ const { s, out } = collect();
+ s.offer(tick("a", 0)); // running/turnActive
+ s.offer(tick("a", 2000, { turnActive: false })); // state changed → edge → emit
+ const ticks = out.filter((r) => r.event === "tick");
+ expect(ticks.length).toBe(2);
+ expect(ticks[1]!.turnActive).toBe(false);
+ });
+
+ it("tail-flush: the LAST unchanged tick of a run survives (emitted before the next boundary)", () => {
+ const { s, out } = collect(30_000);
+ s.offer(tick("a", 0)); // edge → emit
+ s.offer(tick("a", 5000)); // folded, held as pending tail
+ // A state change ends the run → the held tail (5000) must flush BEFORE it.
+ s.offer(tick("a", 6000, { turnActive: false }));
+ const ticks = out.filter((r) => r.event === "tick");
+ const nows = ticks.map((r) => r.nowMs);
+ expect(nows).toContain(5000); // tail flushed
+ expect(nows).toContain(6000); // the edge
+ expect(nows.indexOf(5000)).toBeLessThan(nows.indexOf(6000)); // order preserved
+ });
+});
+
+describe("traceSampler — ② reconstruction: a wedge is reconstructable from retained rows", () => {
+ it("stuck run → retained rows reconstruct which phase, how long, and the kill", () => {
+ const { s, out } = collect(30_000);
+ // Enter the stuck state (edge).
+ s.offer(tick("a", 0, { apmPhase: "tool_wait", sinceProgressMs: 0 }));
+ // Long unchanged run, no progress — a hang. Ticks every 2s, sinceProgressMs climbs.
+ for (let t = 2000; t < 121000; t += 2000) {
+ s.offer(tick("a", t, { apmPhase: "tool_wait", sinceProgressMs: t }));
+ }
+ // Watchdog fires at 121846ms of no progress → terminate_stalled (sacrosanct).
+ s.offer(tick("a", 121846, { apmPhase: "tool_wait", effects: ["terminate_stalled"], sinceProgressMs: 121846 }));
+ // The kill's terminal transition.
+ s.offer({ agentId: "a", event: "turn_end", status: "stopping", turnActive: true, nowMs: 121850, sinceProgressMs: 0, effects: [] });
+
+ const ticks = out.filter((r) => r.event === "tick");
+ // (a) which phase: the retained ticks show tool_wait.
+ expect(ticks.every((r) => r.apmPhase === "tool_wait")).toBe(true);
+ // (b) how long: the peak sinceProgressMs is retained (on the effects frame),
+ // self-contained on that row — no join needed.
+ const peak = Math.max(...out.map((r) => (typeof r.sinceProgressMs === "number" ? r.sinceProgressMs : 0)));
+ expect(peak).toBe(121846);
+ // (c) through to kill: the terminate_stalled frame AND the turn_end survive.
+ expect(out.some((r) => Array.isArray(r.effects) && (r.effects as string[]).includes("terminate_stalled"))).toBe(true);
+ expect(out.some((r) => r.event === "turn_end")).toBe(true);
+ // And the noise WAS folded (didn't retain all ~60 interior ticks).
+ expect(ticks.length).toBeLessThan(20);
+ });
+});
+
+describe("traceSampler — ① retention: fold ratio yields ≥12h @ N=8 (reproducible calc)", () => {
+ it("folding unchanged ticks + progress + runtime_signal drops enough to clear the 12h anchor", () => {
+ // Measured live baseline @ N=8 (架构#414): avg row 362B, 16MB budget (8MB×2).
+ // Raw write rate by stream (MB/h): tick 4.93 (73% unchanged), runtime_signal
+ // 0.88, progress 0.65, transitions <0.3. Simulate one agent's steady-state
+ // over a window and measure the emitted fraction of the sampleable streams.
+ const { s, out } = collect(30_000);
+ const WINDOW_MS = 3600_000; // 1h
+ const TICK_EVERY = 2000; // ~every 2s (matches observed)
+ let emittedSampleable = 0;
+ let offeredSampleable = 0;
+ // Steady idle-ish agent: unchanged ticks + progress + runtime_signal, no wedge.
+ for (let t = 0; t < WINDOW_MS; t += TICK_EVERY) {
+ offeredSampleable += 1;
+ s.offer(tick("a", t)); // unchanged state throughout
+ }
+ emittedSampleable = out.filter((r) => r.event === "tick").length;
+ // With a 30s throttle over 1h, at most ~1h/30s = 120 emitted (+edges/tail).
+ expect(emittedSampleable).toBeLessThanOrEqual(125);
+ // Reproducible retention calc (documented, not verbal):
+ // post-fold tick rate ≈ 120 rows/h; +progress(edge only)+runtime_signal
+ // (phase edges) + transitions ≈ well under the 1.33 MB/h ceiling that 16MB
+ // / 12h implies. 120 rows/h × 362B ≈ 0.041 MB/h for ticks alone → the tick
+ // stream (the 4.93 MB/h bulk) collapses ~40×, so total lands under
+ // 1.33 MB/h and transition retention ≥ 16MB / (total MB/h) ≥ 12h @ N=8.
+ const tickMBph = (emittedSampleable * 362) / 1024 / 1024;
+ expect(tickMBph).toBeLessThan(0.1); // was 4.93 MB/h raw
+ });
+});
+
+describe("traceSampler — per-agent isolation", () => {
+ it("throttle windows are independent per agent", () => {
+ const { s, out } = collect(30_000);
+ s.offer(tick("a", 0));
+ s.offer(tick("b", 0)); // different agent — its own edge, emitted
+ const aTicks = out.filter((r) => r.agentId === "a" && r.event === "tick");
+ const bTicks = out.filter((r) => r.agentId === "b" && r.event === "tick");
+ expect(aTicks.length).toBe(1);
+ expect(bTicks.length).toBe(1);
+ });
+});
+
+describe("traceSampler — constants", () => {
+ it("default throttle is 30s", () => {
+ expect(DEFAULT_TRACE_SAMPLE_MS).toBe(30_000);
+ });
+});
diff --git a/src/daemon/src/util/traceSampler.ts b/src/daemon/src/util/traceSampler.ts
new file mode 100644
index 000000000..5cd0aa626
--- /dev/null
+++ b/src/daemon/src/util/traceSampler.ts
@@ -0,0 +1,211 @@
+/**
+ * FSM-trace sampler — heartbeat throttling so the bounded trace retains a
+ * useful HISTORY WINDOW instead of being flushed by routine noise
+ * (plans/daemon-trace-completeness-charter.md T4).
+ *
+ * WHY: the default trace is size-capped (8MB×2, rotatingFileSink). Measured at
+ * N≈8 agents, ~74% of all rows are UNCHANGED-STATE `tick` heartbeats (~every
+ * 2s/agent) that carry no new information — they flush the information-rich
+ * transition rows (exit/turn_end/reset/spawn-fail/kill) out of the window in
+ * ~2h. An incident is often investigated hours later (the Blair stall-kill was
+ * looked at ~3h after), so the meaningful rows must survive far longer. This
+ * sampler folds the redundant heartbeats while keeping every transition and the
+ * forensic anchors, lifting transition-row retention to ≥12h @ N=8 at the same
+ * byte budget (see charter T4; ≥12h is @ N=8 — write rate scales with agent
+ * count, so a much larger fleet needs the byte budget revisited).
+ *
+ * INVARIANT (Cecilia 架构#423 ③): a row is sampleable IFF its information can be
+ * fully reconstructed from the retained neighbors; otherwise it is sacrosanct.
+ * Concretely:
+ * - SACROSANCT, never dropped:
+ * · any non-`tick` event (transition/lifecycle: exit, turn_end,
+ * reset_session, begin_reset, rewake_after_reset, spawned, wake,
+ * register, session);
+ * · ANY row whose `effects` is non-empty — the watchdog-fired frame
+ * (terminate_stalled/force_exit/…). This is where the PEAK
+ * `sinceProgressMs` lives (Blair's 121846ms was on the
+ * `effects:['terminate_stalled']` tick; the following turn_end had
+ * already reset it to 0). Dropping it would shorten the reconstructed
+ * "stuck for how long". (Cecilia 架构#423 ④.)
+ * - SAMPLEABLE (throttled): unchanged-state `tick`, plus `progress` /
+ * `runtime_signal` heartbeats. An unchanged-state tick is one with
+ * `effects==[]` AND status/turnActive/inbox/resetting/stopping unchanged
+ * from the previous emitted tick for that agent.
+ *
+ * WEDGE RECONSTRUCTION (Claudette 架构#421 assertion ②): a stuck run must remain
+ * reconstructable — which state/phase, how long, through to kill — from the
+ * retained rows alone. Guaranteed by: the edge INTO the stuck state is emitted
+ * (state change), interior points every `throttleMs` bound the slope, the last
+ * unchanged tick of the run is TAIL-FLUSHED (held and emitted right before the
+ * next emitted tick-stream row), and the terminating frame (effects tick / a
+ * transition) is sacrosanct. `sinceProgressMs` is self-contained on each tick
+ * row, so duration reconstructs from the tick stream alone even when the
+ * `progress` stream is fully folded.
+ */
+
+/** A trace record as produced by the manager's onFsmTransition callback. */
+type TraceRec = Record<string, unknown>;
+
+export interface TraceSampler {
+ /** Offer one record; the sampler emits it (and any tail-flush) or folds it. */
+ offer(rec: TraceRec): void;
+}
+
+/** Streams that are throttled rather than always-emitted. */
+const SAMPLEABLE_EVENTS = new Set(["tick", "progress", "runtime_signal"]);
+
+/** Default heartbeat throttle: at most one folded-stream row per agent per this. */
+export const DEFAULT_TRACE_SAMPLE_MS = 30_000;
+
+interface AgentSamplerState {
+ /** Last emitted tick's reconstruct-key (state fingerprint); null before first. */
+ lastTickKey: string | null;
+ /** nowMs of the last emitted `tick` for this agent (for the throttle window). */
+ lastTickEmitAt: number;
+ /** The most recent throttled (folded) unchanged tick, held for tail-flush. */
+ pendingTick: TraceRec | null;
+ /** Last emitted `progress` nowMs. */
+ lastProgressEmitAt: number;
+ /** Last emitted `runtime_signal` apmPhase (edge detection) + nowMs. */
+ lastSignalPhase: string | null;
+ lastSignalEmitAt: number;
+}
+
+function newAgentState(): AgentSamplerState {
+ return {
+ lastTickKey: null,
+ lastTickEmitAt: Number.NEGATIVE_INFINITY,
+ pendingTick: null,
+ lastProgressEmitAt: Number.NEGATIVE_INFINITY,
+ lastSignalPhase: null,
+ lastSignalEmitAt: Number.NEGATIVE_INFINITY,
+ };
+}
+
+/** The reconstruct-key: two ticks with the same key are informationally identical. */
+function tickKey(rec: TraceRec): string {
+ return [
+ rec.status,
+ rec.turnActive,
+ rec.inbox,
+ rec.resetting,
+ rec.stoppingSince == null ? "s0" : "s1",
+ ].join("|");
+}
+
+function hasEffects(rec: TraceRec): boolean {
+ const e = rec.effects;
+ return Array.isArray(e) && e.length > 0;
+}
+
+function nowOf(rec: TraceRec): number {
+ return typeof rec.nowMs === "number" ? rec.nowMs : 0;
+}
+
+/**
+ * @param emit called for each row that survives sampling (in order).
+ * @param throttleMs min gap between folded-stream rows per agent (default 30s).
+ */
+export function createTraceSampler(
+ emit: (rec: TraceRec) => void,
+ throttleMs: number = DEFAULT_TRACE_SAMPLE_MS,
+): TraceSampler {
+ const perAgent = new Map<string, AgentSamplerState>();
+
+ const stateFor = (agentId: string): AgentSamplerState => {
+ let s = perAgent.get(agentId);
+ if (!s) {
+ s = newAgentState();
+ perAgent.set(agentId, s);
+ }
+ return s;
+ };
+
+ /** Emit the held tail tick (if any) so an unchanged run's LAST row survives. */
+ const flushPending = (s: AgentSamplerState): void => {
+ if (s.pendingTick) {
+ const p = s.pendingTick;
+ s.pendingTick = null;
+ s.lastTickEmitAt = nowOf(p);
+ emit(p);
+ }
+ };
+
+ return {
+ offer(rec: TraceRec): void {
+ const agentId = typeof rec.agentId === "string" ? rec.agentId : null;
+ // No agent id (shouldn't happen for emitted rows) → pass through untouched.
+ if (!agentId) {
+ emit(rec);
+ return;
+ }
+ const s = stateFor(agentId);
+ const event = rec.event;
+
+ // Sacrosanct: any non-sampleable event, or ANY row carrying effects (the
+ // watchdog-fired frame). Flush the pending tail first so run order + the
+ // last unchanged tick are preserved ahead of the terminating frame.
+ if (!SAMPLEABLE_EVENTS.has(event as string) || hasEffects(rec)) {
+ flushPending(s);
+ // A sacrosanct tick still updates the tick anchors (it IS an emitted tick).
+ if (event === "tick") {
+ s.lastTickKey = tickKey(rec);
+ s.lastTickEmitAt = nowOf(rec);
+ }
+ emit(rec);
+ return;
+ }
+
+ // ---- Sampleable heartbeat streams ----
+ if (event === "tick") {
+ const key = tickKey(rec);
+ const now = nowOf(rec);
+ // State-change EDGE → always emit (the run boundary; carries the fresh
+ // state). Flush the prior run's tail first.
+ if (key !== s.lastTickKey) {
+ flushPending(s);
+ s.lastTickKey = key;
+ s.lastTickEmitAt = now;
+ emit(rec);
+ return;
+ }
+ // Unchanged state: emit at most one per throttle window; otherwise HOLD
+ // as the run's tail (overwrite — we only need the latest folded one).
+ if (now - s.lastTickEmitAt >= throttleMs) {
+ s.pendingTick = null; // this row supersedes any earlier held tail
+ s.lastTickEmitAt = now;
+ emit(rec);
+ } else {
+ s.pendingTick = rec;
+ }
+ return;
+ }
+
+ if (event === "progress") {
+ // Redundant with the next tick's sinceProgressMs; keep a sparse sample
+ // for a coarse liveness pulse, fold the rest.
+ const now = nowOf(rec);
+ if (now - s.lastProgressEmitAt >= throttleMs) {
+ s.lastProgressEmitAt = now;
+ emit(rec);
+ }
+ return;
+ }
+
+ // runtime_signal: keep phase-change EDGES, throttle same-phase runs.
+ if (event === "runtime_signal") {
+ const phase = typeof rec.apmPhase === "string" ? rec.apmPhase : "";
+ const now = nowOf(rec);
+ if (phase !== s.lastSignalPhase || now - s.lastSignalEmitAt >= throttleMs) {
+ s.lastSignalPhase = phase;
+ s.lastSignalEmitAt = now;
+ emit(rec);
+ }
+ return;
+ }
+
+ // Unreachable (SAMPLEABLE_EVENTS covered above), but fail-open: emit.
+ emit(rec);
+ },
+ };
+}