Multi-Agent Collaboration for Team and Friends
TypeScript agent-automationagent-orchestrationagent-workflowai-agentsai-collaborationai-memoryai-workforceautonomous-agentsclaude-codeclaude-code-skillscodexcoding-agentmulti-agentmultiple-agentone-person-businessone-person-companyone-person-teamopencodesolopreneur
daemon: T4 — FSM-trace heartbeat sampling for ≥12h retention (trace-completeness red-line-6)
Gus8fb202687356
diff --git a/src/daemon/src/daemon/createDaemon.ts b/src/daemon/src/daemon/createDaemon.tsindex ed2968f37..ccbf05b35 100644--- a/src/daemon/src/daemon/createDaemon.ts+++ b/src/daemon/src/daemon/createDaemon.ts@@ -28,6 +28,7 @@import { homedir } from "os";import { appendFileSync, mkdirSync } from "node:fs";import { createRotatingFileSink } from "../util/rotatingFileSink.js";+import { createTraceSampler } from "../util/traceSampler.js";import { writeStatusFile } from "../util/statusFile.js";import { WsControlChannel } from "../server/wsControlChannel.js";import { CredentialBroker, startCredentialProxy } from "../credentials/index.js";@@ -49,8 +50,12 @@ const WARMUP_CEILING_MS = 30_000;/*** Per-file cap for the DEFAULT-ON bounded FSM trace (batch E1). The rotating* sink keeps the active file + one rotated generation, so total on-disk ≈- * 2×this ≈ 16MB — about 4h of history at the observed ~14 rows/min/agent rate,- * enough to hold the last wedge. The `ALOOK_FSM_TRACE` override is unbounded.+ * 2×this ≈ 16MB. RAW row rate at N≈8 agents is ~74% unchanged-state tick noise+ * → only ~2h of history unsampled; the T4 heartbeat sampler (createTraceSampler)+ * folds that noise so TRANSITION rows survive ≥12h @ N=8 in the same budget.+ * (≥12h is @ N=8 — write rate scales with agent count, so a much larger fleet+ * warrants revisiting this cap.) The `ALOOK_FSM_TRACE` override is unbounded and+ * unsampled (full-fidelity deep dives).*/const FSM_TRACE_MAX_BYTES = 8 * 1024 * 1024;/** How often the daemon rewrites the `daemon status` snapshot file (batch E2). */@@ -645,8 +650,15 @@ export async function createDaemon(opts: CreateDaemonOptions): Promise<RunningDa`${opts.fsmTraceDir}/fsm-trace.jsonl`,FSM_TRACE_MAX_BYTES,);+ // Heartbeat sampler between the manager and the sink (batch T4): folds+ // redundant unchanged-state ticks + progress/runtime_signal noise so the+ // bounded file retains transition rows ≥12h (@ N=8) instead of ~2h.+ // Transitions and watchdog-fired (effects-carrying) frames are never+ // dropped. The ALOOK_FSM_TRACE override above is unbounded → unsampled,+ // for full-fidelity deep dives.+ const sampler = createTraceSampler((rec) => sink.write(JSON.stringify(rec)));return {- onFsmTransition: (rec: Record<string, unknown>) => sink.write(JSON.stringify(rec)),+ onFsmTransition: (rec: Record<string, unknown>) => sampler.offer(rec),};}return {};diff --git a/src/daemon/src/util/traceSampler.test.ts b/src/daemon/src/util/traceSampler.test.tsnew file mode 100644index 000000000..4abb4839a--- /dev/null+++ b/src/daemon/src/util/traceSampler.test.ts@@ -0,0 +1,174 @@+import { describe, it, expect } from "vitest";+import { createTraceSampler, DEFAULT_TRACE_SAMPLE_MS } from "./traceSampler.js";++/*+ * T4 (plans/daemon-trace-completeness-charter.md) — heartbeat sampler.+ * Two acceptance conditions (Claudette 架构#421):+ * ① retention: folding the unchanged-tick / progress / runtime_signal noise+ * lifts transition-row retention ≥12h @ N=8 in the fixed byte budget —+ * here proven as a reproducible fold-ratio → hours calculation.+ * ② reconstruction: after sampling, a wedge is STILL reconstructable from the+ * retained rows alone — which phase, how long, through to kill; the+ * watchdog-fired (effects) frame with the peak sinceProgressMs survives.+ */++type Rec = Record<string, unknown>;++/** A collecting sampler harness. */+function collect(throttleMs?: number) {+ const out: Rec[] = [];+ const s = createTraceSampler((r) => out.push(r), throttleMs);+ return { s, out };+}++function tick(agentId: string, nowMs: number, over: Partial<Rec> = {}): Rec {+ return {+ agentId,+ event: "tick",+ status: "running",+ turnActive: true,+ inbox: 0,+ resetting: false,+ stoppingSince: null,+ effects: [],+ nowMs,+ sinceProgressMs: 0,+ ...over,+ };+}++describe("traceSampler — sacrosanct rows never dropped", () => {+ it("all non-tick events pass through untouched", () => {+ const { s, out } = collect();+ for (const event of ["exit", "turn_end", "reset_session", "begin_reset", "spawned", "wake", "register", "session"]) {+ s.offer({ agentId: "a", event, nowMs: 1 });+ }+ expect(out.map((r) => r.event)).toEqual([+ "exit", "turn_end", "reset_session", "begin_reset", "spawned", "wake", "register", "session",+ ]);+ });++ it("a tick carrying effects (watchdog fired) is NEVER folded, even mid-throttle-window", () => {+ const { s, out } = collect();+ s.offer(tick("a", 0)); // first unchanged tick — emitted (edge from null)+ s.offer(tick("a", 2000)); // within 30s window, unchanged → folded (held)+ // The watchdog fires: same state key but effects non-empty → sacrosanct.+ s.offer(tick("a", 4000, { effects: ["terminate_stalled"], sinceProgressMs: 121846 }));+ const killRow = out.find((r) => Array.isArray(r.effects) && (r.effects as string[]).includes("terminate_stalled"));+ expect(killRow).toBeTruthy();+ expect(killRow!.sinceProgressMs).toBe(121846); // peak preserved+ });+});++describe("traceSampler — unchanged-tick throttling", () => {+ it("folds unchanged ticks inside the throttle window, emits one per window", () => {+ const { s, out } = collect(30_000);+ s.offer(tick("a", 0)); // edge → emit+ s.offer(tick("a", 2000)); // folded+ s.offer(tick("a", 4000)); // folded+ s.offer(tick("a", 31000)); // past window → emit+ const ticks = out.filter((r) => r.event === "tick");+ // 0 (edge) and 31000 (past window). 2000/4000 folded — but see tail-flush test.+ expect(ticks.some((r) => r.nowMs === 0)).toBe(true);+ expect(ticks.some((r) => r.nowMs === 31000)).toBe(true);+ });++ it("a state-change edge always emits (the run boundary)", () => {+ const { s, out } = collect();+ s.offer(tick("a", 0)); // running/turnActive+ s.offer(tick("a", 2000, { turnActive: false })); // state changed → edge → emit+ const ticks = out.filter((r) => r.event === "tick");+ expect(ticks.length).toBe(2);+ expect(ticks[1]!.turnActive).toBe(false);+ });++ it("tail-flush: the LAST unchanged tick of a run survives (emitted before the next boundary)", () => {+ const { s, out } = collect(30_000);+ s.offer(tick("a", 0)); // edge → emit+ s.offer(tick("a", 5000)); // folded, held as pending tail+ // A state change ends the run → the held tail (5000) must flush BEFORE it.+ s.offer(tick("a", 6000, { turnActive: false }));+ const ticks = out.filter((r) => r.event === "tick");+ const nows = ticks.map((r) => r.nowMs);+ expect(nows).toContain(5000); // tail flushed+ expect(nows).toContain(6000); // the edge+ expect(nows.indexOf(5000)).toBeLessThan(nows.indexOf(6000)); // order preserved+ });+});++describe("traceSampler — ② reconstruction: a wedge is reconstructable from retained rows", () => {+ it("stuck run → retained rows reconstruct which phase, how long, and the kill", () => {+ const { s, out } = collect(30_000);+ // Enter the stuck state (edge).+ s.offer(tick("a", 0, { apmPhase: "tool_wait", sinceProgressMs: 0 }));+ // Long unchanged run, no progress — a hang. Ticks every 2s, sinceProgressMs climbs.+ for (let t = 2000; t < 121000; t += 2000) {+ s.offer(tick("a", t, { apmPhase: "tool_wait", sinceProgressMs: t }));+ }+ // Watchdog fires at 121846ms of no progress → terminate_stalled (sacrosanct).+ s.offer(tick("a", 121846, { apmPhase: "tool_wait", effects: ["terminate_stalled"], sinceProgressMs: 121846 }));+ // The kill's terminal transition.+ s.offer({ agentId: "a", event: "turn_end", status: "stopping", turnActive: true, nowMs: 121850, sinceProgressMs: 0, effects: [] });++ const ticks = out.filter((r) => r.event === "tick");+ // (a) which phase: the retained ticks show tool_wait.+ expect(ticks.every((r) => r.apmPhase === "tool_wait")).toBe(true);+ // (b) how long: the peak sinceProgressMs is retained (on the effects frame),+ // self-contained on that row — no join needed.+ const peak = Math.max(...out.map((r) => (typeof r.sinceProgressMs === "number" ? r.sinceProgressMs : 0)));+ expect(peak).toBe(121846);+ // (c) through to kill: the terminate_stalled frame AND the turn_end survive.+ expect(out.some((r) => Array.isArray(r.effects) && (r.effects as string[]).includes("terminate_stalled"))).toBe(true);+ expect(out.some((r) => r.event === "turn_end")).toBe(true);+ // And the noise WAS folded (didn't retain all ~60 interior ticks).+ expect(ticks.length).toBeLessThan(20);+ });+});++describe("traceSampler — ① retention: fold ratio yields ≥12h @ N=8 (reproducible calc)", () => {+ it("folding unchanged ticks + progress + runtime_signal drops enough to clear the 12h anchor", () => {+ // Measured live baseline @ N=8 (架构#414): avg row 362B, 16MB budget (8MB×2).+ // Raw write rate by stream (MB/h): tick 4.93 (73% unchanged), runtime_signal+ // 0.88, progress 0.65, transitions <0.3. Simulate one agent's steady-state+ // over a window and measure the emitted fraction of the sampleable streams.+ const { s, out } = collect(30_000);+ const WINDOW_MS = 3600_000; // 1h+ const TICK_EVERY = 2000; // ~every 2s (matches observed)+ let emittedSampleable = 0;+ let offeredSampleable = 0;+ // Steady idle-ish agent: unchanged ticks + progress + runtime_signal, no wedge.+ for (let t = 0; t < WINDOW_MS; t += TICK_EVERY) {+ offeredSampleable += 1;+ s.offer(tick("a", t)); // unchanged state throughout+ }+ emittedSampleable = out.filter((r) => r.event === "tick").length;+ // With a 30s throttle over 1h, at most ~1h/30s = 120 emitted (+edges/tail).+ expect(emittedSampleable).toBeLessThanOrEqual(125);+ // Reproducible retention calc (documented, not verbal):+ // post-fold tick rate ≈ 120 rows/h; +progress(edge only)+runtime_signal+ // (phase edges) + transitions ≈ well under the 1.33 MB/h ceiling that 16MB+ // / 12h implies. 120 rows/h × 362B ≈ 0.041 MB/h for ticks alone → the tick+ // stream (the 4.93 MB/h bulk) collapses ~40×, so total lands under+ // 1.33 MB/h and transition retention ≥ 16MB / (total MB/h) ≥ 12h @ N=8.+ const tickMBph = (emittedSampleable * 362) / 1024 / 1024;+ expect(tickMBph).toBeLessThan(0.1); // was 4.93 MB/h raw+ });+});++describe("traceSampler — per-agent isolation", () => {+ it("throttle windows are independent per agent", () => {+ const { s, out } = collect(30_000);+ s.offer(tick("a", 0));+ s.offer(tick("b", 0)); // different agent — its own edge, emitted+ const aTicks = out.filter((r) => r.agentId === "a" && r.event === "tick");+ const bTicks = out.filter((r) => r.agentId === "b" && r.event === "tick");+ expect(aTicks.length).toBe(1);+ expect(bTicks.length).toBe(1);+ });+});++describe("traceSampler — constants", () => {+ it("default throttle is 30s", () => {+ expect(DEFAULT_TRACE_SAMPLE_MS).toBe(30_000);+ });+});diff --git a/src/daemon/src/util/traceSampler.ts b/src/daemon/src/util/traceSampler.tsnew file mode 100644index 000000000..5cd0aa626--- /dev/null+++ b/src/daemon/src/util/traceSampler.ts@@ -0,0 +1,211 @@+/**+ * FSM-trace sampler — heartbeat throttling so the bounded trace retains a+ * useful HISTORY WINDOW instead of being flushed by routine noise+ * (plans/daemon-trace-completeness-charter.md T4).+ *+ * WHY: the default trace is size-capped (8MB×2, rotatingFileSink). Measured at+ * N≈8 agents, ~74% of all rows are UNCHANGED-STATE `tick` heartbeats (~every+ * 2s/agent) that carry no new information — they flush the information-rich+ * transition rows (exit/turn_end/reset/spawn-fail/kill) out of the window in+ * ~2h. An incident is often investigated hours later (the Blair stall-kill was+ * looked at ~3h after), so the meaningful rows must survive far longer. This+ * sampler folds the redundant heartbeats while keeping every transition and the+ * forensic anchors, lifting transition-row retention to ≥12h @ N=8 at the same+ * byte budget (see charter T4; ≥12h is @ N=8 — write rate scales with agent+ * count, so a much larger fleet needs the byte budget revisited).+ *+ * INVARIANT (Cecilia 架构#423 ③): a row is sampleable IFF its information can be+ * fully reconstructed from the retained neighbors; otherwise it is sacrosanct.+ * Concretely:+ * - SACROSANCT, never dropped:+ * · any non-`tick` event (transition/lifecycle: exit, turn_end,+ * reset_session, begin_reset, rewake_after_reset, spawned, wake,+ * register, session);+ * · ANY row whose `effects` is non-empty — the watchdog-fired frame+ * (terminate_stalled/force_exit/…). This is where the PEAK+ * `sinceProgressMs` lives (Blair's 121846ms was on the+ * `effects:['terminate_stalled']` tick; the following turn_end had+ * already reset it to 0). Dropping it would shorten the reconstructed+ * "stuck for how long". (Cecilia 架构#423 ④.)+ * - SAMPLEABLE (throttled): unchanged-state `tick`, plus `progress` /+ * `runtime_signal` heartbeats. An unchanged-state tick is one with+ * `effects==[]` AND status/turnActive/inbox/resetting/stopping unchanged+ * from the previous emitted tick for that agent.+ *+ * WEDGE RECONSTRUCTION (Claudette 架构#421 assertion ②): a stuck run must remain+ * reconstructable — which state/phase, how long, through to kill — from the+ * retained rows alone. Guaranteed by: the edge INTO the stuck state is emitted+ * (state change), interior points every `throttleMs` bound the slope, the last+ * unchanged tick of the run is TAIL-FLUSHED (held and emitted right before the+ * next emitted tick-stream row), and the terminating frame (effects tick / a+ * transition) is sacrosanct. `sinceProgressMs` is self-contained on each tick+ * row, so duration reconstructs from the tick stream alone even when the+ * `progress` stream is fully folded.+ */++/** A trace record as produced by the manager's onFsmTransition callback. */+type TraceRec = Record<string, unknown>;++export interface TraceSampler {+ /** Offer one record; the sampler emits it (and any tail-flush) or folds it. */+ offer(rec: TraceRec): void;+}++/** Streams that are throttled rather than always-emitted. */+const SAMPLEABLE_EVENTS = new Set(["tick", "progress", "runtime_signal"]);++/** Default heartbeat throttle: at most one folded-stream row per agent per this. */+export const DEFAULT_TRACE_SAMPLE_MS = 30_000;++interface AgentSamplerState {+ /** Last emitted tick's reconstruct-key (state fingerprint); null before first. */+ lastTickKey: string | null;+ /** nowMs of the last emitted `tick` for this agent (for the throttle window). */+ lastTickEmitAt: number;+ /** The most recent throttled (folded) unchanged tick, held for tail-flush. */+ pendingTick: TraceRec | null;+ /** Last emitted `progress` nowMs. */+ lastProgressEmitAt: number;+ /** Last emitted `runtime_signal` apmPhase (edge detection) + nowMs. */+ lastSignalPhase: string | null;+ lastSignalEmitAt: number;+}++function newAgentState(): AgentSamplerState {+ return {+ lastTickKey: null,+ lastTickEmitAt: Number.NEGATIVE_INFINITY,+ pendingTick: null,+ lastProgressEmitAt: Number.NEGATIVE_INFINITY,+ lastSignalPhase: null,+ lastSignalEmitAt: Number.NEGATIVE_INFINITY,+ };+}++/** The reconstruct-key: two ticks with the same key are informationally identical. */+function tickKey(rec: TraceRec): string {+ return [+ rec.status,+ rec.turnActive,+ rec.inbox,+ rec.resetting,+ rec.stoppingSince == null ? "s0" : "s1",+ ].join("|");+}++function hasEffects(rec: TraceRec): boolean {+ const e = rec.effects;+ return Array.isArray(e) && e.length > 0;+}++function nowOf(rec: TraceRec): number {+ return typeof rec.nowMs === "number" ? rec.nowMs : 0;+}++/**+ * @param emit called for each row that survives sampling (in order).+ * @param throttleMs min gap between folded-stream rows per agent (default 30s).+ */+export function createTraceSampler(+ emit: (rec: TraceRec) => void,+ throttleMs: number = DEFAULT_TRACE_SAMPLE_MS,+): TraceSampler {+ const perAgent = new Map<string, AgentSamplerState>();++ const stateFor = (agentId: string): AgentSamplerState => {+ let s = perAgent.get(agentId);+ if (!s) {+ s = newAgentState();+ perAgent.set(agentId, s);+ }+ return s;+ };++ /** Emit the held tail tick (if any) so an unchanged run's LAST row survives. */+ const flushPending = (s: AgentSamplerState): void => {+ if (s.pendingTick) {+ const p = s.pendingTick;+ s.pendingTick = null;+ s.lastTickEmitAt = nowOf(p);+ emit(p);+ }+ };++ return {+ offer(rec: TraceRec): void {+ const agentId = typeof rec.agentId === "string" ? rec.agentId : null;+ // No agent id (shouldn't happen for emitted rows) → pass through untouched.+ if (!agentId) {+ emit(rec);+ return;+ }+ const s = stateFor(agentId);+ const event = rec.event;++ // Sacrosanct: any non-sampleable event, or ANY row carrying effects (the+ // watchdog-fired frame). Flush the pending tail first so run order + the+ // last unchanged tick are preserved ahead of the terminating frame.+ if (!SAMPLEABLE_EVENTS.has(event as string) || hasEffects(rec)) {+ flushPending(s);+ // A sacrosanct tick still updates the tick anchors (it IS an emitted tick).+ if (event === "tick") {+ s.lastTickKey = tickKey(rec);+ s.lastTickEmitAt = nowOf(rec);+ }+ emit(rec);+ return;+ }++ // ---- Sampleable heartbeat streams ----+ if (event === "tick") {+ const key = tickKey(rec);+ const now = nowOf(rec);+ // State-change EDGE → always emit (the run boundary; carries the fresh+ // state). Flush the prior run's tail first.+ if (key !== s.lastTickKey) {+ flushPending(s);+ s.lastTickKey = key;+ s.lastTickEmitAt = now;+ emit(rec);+ return;+ }+ // Unchanged state: emit at most one per throttle window; otherwise HOLD+ // as the run's tail (overwrite — we only need the latest folded one).+ if (now - s.lastTickEmitAt >= throttleMs) {+ s.pendingTick = null; // this row supersedes any earlier held tail+ s.lastTickEmitAt = now;+ emit(rec);+ } else {+ s.pendingTick = rec;+ }+ return;+ }++ if (event === "progress") {+ // Redundant with the next tick's sinceProgressMs; keep a sparse sample+ // for a coarse liveness pulse, fold the rest.+ const now = nowOf(rec);+ if (now - s.lastProgressEmitAt >= throttleMs) {+ s.lastProgressEmitAt = now;+ emit(rec);+ }+ return;+ }++ // runtime_signal: keep phase-change EDGES, throttle same-phase runs.+ if (event === "runtime_signal") {+ const phase = typeof rec.apmPhase === "string" ? rec.apmPhase : "";+ const now = nowOf(rec);+ if (phase !== s.lastSignalPhase || now - s.lastSignalEmitAt >= throttleMs) {+ s.lastSignalPhase = phase;+ s.lastSignalEmitAt = now;+ emit(rec);+ }+ return;+ }++ // Unreachable (SAMPLEABLE_EVENTS covered above), but fail-open: emit.+ emit(rec);+ },+ };+}