Skip to content

armgit / alook

Public Imported from GitHub
0

Multi-Agent Collaboration for Team and Friends

daemon: T1 — hard-exit physical fact into fsm-trace (trace-completeness charter)

Guseabe979d49b5
diff --git a/src/daemon/src/manager/managerPolicy.ts b/src/daemon/src/manager/managerPolicy.ts
index 536c7c427..1d6749d16 100644
--- a/src/daemon/src/manager/managerPolicy.ts
+++ b/src/daemon/src/manager/managerPolicy.ts
@@ -185,7 +185,18 @@ export type ManagerEvent =
terminationCause?: "runtime_error" | "killed_stalled";
errorDetail?: string;
}
- | { type: "exit"; agentId: string }
+ /**
+ * `exitCode`/`exitSignal`/`abnormal` are the RAW PHYSICAL termination fact of
+ * the process, recorded for fsm-trace forensics (T1,
+ * plans/daemon-trace-completeness-charter.md) — how the process died, for
+ * EVERY exit path, so a hard exit (segfault/OOM/external SIGKILL, which
+ * bypasses the normalizer and emits no turn_end) is distinguishable from a
+ * clean exit in the trace. OBSERVABILITY ONLY: `onExit` must NOT branch its
+ * respawn-vs-idle decision on these (red line — kept read-only). What the exit
+ * MEANS in FSM terms (deliberate kill vs crash) is a separate semantic layer
+ * left to T3, which may only LAYER fields on top, never overwrite these.
+ */
+ | { type: "exit"; agentId: string; exitCode?: number | null; exitSignal?: string | null; abnormal?: boolean }
| { type: "tick"; nowMs: number }
/**
* Owner-triggered "reset session". Nulls `AgentState.sessionId` so the next
diff --git a/src/daemon/src/manager/managerRuntime.test.ts b/src/daemon/src/manager/managerRuntime.test.ts
index b1b3e4e72..42bcb220d 100644
--- a/src/daemon/src/manager/managerRuntime.test.ts
+++ b/src/daemon/src/manager/managerRuntime.test.ts
@@ -2116,4 +2116,140 @@ describe("AgentProcessManager — onFsmTransition trace (observability, zero beh
vi.useRealTimers();
}
});
+
+ // ---- T1: hard-exit physical fact into trace (daemon-trace-completeness-charter) ----
+ // A hard exit (segfault/OOM/external SIGKILL) bypasses the normalizer, emits no
+ // turn_end, and would otherwise be indistinguishable from a clean exit in the
+ // trace. T1 threads the raw physical fact (exitCode/exitSignal/abnormal) onto
+ // the exit event → trace. Physical fact only; onExit behavior UNCHANGED.
+
+ it("T1 — an established session's exit on a signal records exitSignal + abnormal=true on the trace exit row", () => {
+ const recs: Record<string, unknown>[] = [];
+ const { mgr, session } = makeWithTrace((r) => recs.push(r));
+ mgr.deliver("a1", { seq: 1, text: "hi" });
+ session.fire("runtime_event", { kind: "session_init", sessionId: "s1" }); // hasEstablished
+ // Hard death on SIGKILL, NOT a requested stop → abnormal physical fact.
+ session.fire("exit", { signal: "SIGKILL", reason: "runtime_exit" });
+
+ const exitRec = recs.find((r) => r.event === "exit" && r.agentId === "a1");
+ expect(exitRec).toBeTruthy();
+ expect(exitRec!.exitSignal).toBe("SIGKILL");
+ expect(exitRec!.exitCode).toBeNull();
+ expect(exitRec!.abnormal).toBe(true);
+ });
+
+ it("T1 — an established session's non-zero code exit records exitCode + abnormal=true", () => {
+ const recs: Record<string, unknown>[] = [];
+ const { mgr, session } = makeWithTrace((r) => recs.push(r));
+ mgr.deliver("a1", { seq: 1, text: "hi" });
+ session.fire("runtime_event", { kind: "session_init", sessionId: "s1" });
+ session.fire("exit", { code: 137, reason: "runtime_exit" });
+
+ const exitRec = recs.find((r) => r.event === "exit" && r.agentId === "a1");
+ expect(exitRec).toBeTruthy();
+ expect(exitRec!.exitCode).toBe(137);
+ expect(exitRec!.exitSignal).toBeNull();
+ expect(exitRec!.abnormal).toBe(true);
+ });
+
+ it("T1 — a clean code-0 exit records abnormal=false (distinguishable from a crash in the trace)", () => {
+ const recs: Record<string, unknown>[] = [];
+ const { mgr, session } = makeWithTrace((r) => recs.push(r));
+ mgr.deliver("a1", { seq: 1, text: "hi" });
+ session.fire("runtime_event", { kind: "session_init", sessionId: "s1" });
+ session.fire("exit", { code: 0, reason: "runtime_exit" });
+
+ const exitRec = recs.find((r) => r.event === "exit" && r.agentId === "a1");
+ expect(exitRec).toBeTruthy();
+ expect(exitRec!.exitCode).toBe(0);
+ expect(exitRec!.abnormal).toBe(false);
+ });
+
+ it("T1 — a deliberate stop (reason=requested) records the physical fact but abnormal=false", () => {
+ // A requested stop that still died on a signal (SIGTERM grace → the process
+ // exits): the physical fact (signal) is recorded, but it's NOT abnormal —
+ // reason==="requested" gates that. So trace shows how it died without
+ // mislabeling an intentional stop as a crash.
+ const recs: Record<string, unknown>[] = [];
+ const { mgr, session } = makeWithTrace((r) => recs.push(r));
+ mgr.deliver("a1", { seq: 1, text: "hi" });
+ session.fire("runtime_event", { kind: "session_init", sessionId: "s1" });
+ session.fire("exit", { signal: "SIGTERM", reason: "requested" });
+
+ const exitRec = recs.find((r) => r.event === "exit" && r.agentId === "a1");
+ expect(exitRec).toBeTruthy();
+ expect(exitRec!.exitSignal).toBe("SIGTERM");
+ expect(exitRec!.abnormal).toBe(false);
+ });
+
+ it("T1 — the exit physical fact is READ-ONLY: abnormal exit and clean exit produce the SAME onExit respawn/idle decision (Claudette #382)", () => {
+ // With a queued inbox, onExit respawns (spawn effect); with empty inbox it
+ // settles idle. That decision keys on inbox.length ONLY — the new
+ // exitCode/exitSignal/abnormal fields must NOT change the branch.
+ function exitEffectsFor(exitInfo: Record<string, unknown>, queueBefore: boolean): string[] {
+ const recs: Record<string, unknown>[] = [];
+ const { mgr, session } = makeWithTrace((r) => recs.push(r));
+ mgr.deliver("a1", { seq: 1, text: "hi" });
+ session.fire("runtime_event", { kind: "session_init", sessionId: "s1" });
+ if (queueBefore) mgr.deliver("a1", { seq: 2, text: "queued" }); // inbox non-empty at exit
+ recs.length = 0;
+ session.fire("exit", exitInfo);
+ const exitRec = recs.find((r) => r.event === "exit" && r.agentId === "a1");
+ return (exitRec!.effects as string[]) ?? [];
+ }
+ // Empty inbox: idle (no spawn), identical for abnormal vs clean.
+ expect(exitEffectsFor({ signal: "SIGKILL", reason: "runtime_exit" }, false)).toEqual(
+ exitEffectsFor({ code: 0, reason: "runtime_exit" }, false),
+ );
+ // Queued inbox: respawn (spawn), identical for abnormal vs clean.
+ expect(exitEffectsFor({ signal: "SIGKILL", reason: "runtime_exit" }, true)).toEqual(
+ exitEffectsFor({ code: 0, reason: "runtime_exit" }, true),
+ );
+ });
+});
+
+describe("T1 — abnormal-exit user audit stays gated (no nap-noise on deliberate kill)", () => {
+ it("a deliberate stop (suppressExitLog) records the physical fact in trace but emits NO user-facing abnormal_exit audit", async () => {
+ vi.useFakeTimers();
+ try {
+ let now = 0;
+ const recs: Record<string, unknown>[] = [];
+ const onBotAuditEvent = vi.fn();
+ const session = fakeSession();
+ const mgr = new AgentProcessManager({
+ driverFor: () => fakeDriver("codex"),
+ baseContextFor: () => ({ workingDirectory: "/tmp", agentId: "a1", standingPrompt: "", config: {} as LaunchContext["config"], credentialProxy: {} as LaunchContext["credentialProxy"] }),
+ sessionFactory: () => session,
+ now: () => now,
+ tickIntervalMs: 5,
+ staleThresholdMs: 100,
+ onFsmTransition: ((r: Record<string, unknown>) => recs.push(r)) as never,
+ onBotAuditEvent: onBotAuditEvent as never,
+ });
+ mgr.start();
+ mgr.register("a1");
+ mgr.deliver("a1", { seq: 1, text: "hi" });
+ session.startResolver?.();
+ await Promise.resolve();
+ session.fire("runtime_event", { kind: "session_init", sessionId: "s1" });
+ // Stall → terminate_stalled sets suppressExitLog before the kill.
+ now = 200;
+ await vi.advanceTimersByTimeAsync(10);
+ // The killed process exits on a signal (a requested/deliberate stop path).
+ session.fire("exit", { signal: "SIGKILL", reason: "requested" });
+
+ // Trace: the physical fact IS recorded (forensics needs it).
+ const exitRec = recs.find((r) => r.event === "exit" && r.agentId === "a1");
+ expect(exitRec).toBeTruthy();
+ expect(exitRec!.exitSignal).toBe("SIGKILL");
+ // User audit: NO abnormal_exit row — a deliberate kill must not look like a
+ // fault in the user's activity (the nap-noise bug must not reappear here).
+ const abnormalAudits = onBotAuditEvent.mock.calls.filter(
+ ([, ev]) => (ev as { kind?: string; payload?: { code?: string } })?.payload?.code === "abnormal_exit",
+ );
+ expect(abnormalAudits).toHaveLength(0);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
});
diff --git a/src/daemon/src/manager/managerRuntime.ts b/src/daemon/src/manager/managerRuntime.ts
index bf3b31a61..5fd498950 100644
--- a/src/daemon/src/manager/managerRuntime.ts
+++ b/src/daemon/src/manager/managerRuntime.ts
@@ -197,6 +197,17 @@ export interface ManagerRuntimeOpts {
endReason?: "errored";
terminationCause?: "runtime_error" | "killed_stalled";
errorDetail?: string;
+ /**
+ * Present only on an `exit` event: the RAW PHYSICAL termination fact (T1,
+ * plans/daemon-trace-completeness-charter.md). `exitCode`/`exitSignal` = how
+ * the OS reported the death; `abnormal` = not a deliberate stop AND died on
+ * non-zero code or a signal. Makes a hard exit (segfault/OOM/external
+ * SIGKILL — no turn_end) distinguishable from a clean exit in the trace.
+ * Physical fact only; FSM semantics of the exit are a T3 layer on top.
+ */
+ exitCode?: number | null;
+ exitSignal?: string | null;
+ abnormal?: boolean;
}) => void;
/**
* Optional context-timeline recorder. When provided, the manager logs each
@@ -1080,6 +1091,16 @@ export class AgentProcessManager {
errorDetail: (event as { errorDetail?: string }).errorDetail,
}
: {}),
+ // Raw physical exit fact, carried on the `exit` event (not agent
+ // state). Spread on exit only so non-exit events omit the keys. T1 —
+ // makes a hard exit distinguishable from a clean one in the trace.
+ ...(event.type === "exit"
+ ? {
+ exitCode: (event as { exitCode?: number | null }).exitCode ?? null,
+ exitSignal: (event as { exitSignal?: string | null }).exitSignal ?? null,
+ abnormal: (event as { abnormal?: boolean }).abnormal ?? false,
+ }
+ : {}),
});
};
const agentId = (event as { agentId?: string }).agentId;
@@ -1428,21 +1449,35 @@ export class AgentProcessManager {
// Guarded by `hasReportedSpawnFailure` so an ENOENT (already reported
// via `error`) doesn't get overwritten with generic `pre_handshake_exit`.
reportSpawnFailure("pre_handshake_exit");
+ // `abnormal` = the RAW PHYSICAL FACT of how the process died: it wasn't a
+ // deliberate stop (`reason !== "requested"`) and it left a non-zero code or
+ // died on a signal. Computed UNCONDITIONALLY (not gated by
+ // `hasEstablished`/`suppressExitLog`) because it feeds the fsm-trace exit
+ // row (T1, plans/daemon-trace-completeness-charter.md): the trace is a
+ // forensic layer that must record how a process physically terminated for
+ // EVERY exit path — a hard exit (segfault/OOM/external SIGKILL) bypasses the
+ // normalizer, emits no turn_end, and would otherwise be indistinguishable
+ // from a clean exit in the trace. What this exit MEANS in FSM terms
+ // (deliberate kill vs crash) is a SEPARATE semantic layer left to T3; T1
+ // only records the physical fact and never reinterprets it.
+ const exitCode = typeof info?.code === "number" ? info.code : null;
+ const exitSignal = info?.signal ?? null;
+ const abnormal =
+ info?.reason !== "requested" && ((exitCode !== null && exitCode !== 0) || !!exitSignal);
// Only an ESTABLISHED session "ended" — a pre-handshake exit is
// already covered by the spawn-failed warning above. And only if this
// termination wasn't already logged under a more specific reason (see
// `suppressExitLog` above).
if (state.hasEstablished && !state.suppressExitLog) {
this.logSessionEnded(agentId, "exit");
- // An established session that dies with a non-zero code / on a signal
- // and WASN'T a deliberate stop (`reason !== "requested"`) is an
- // unexpected mid-run crash — surface it. A clean `code === 0` exit or
- // a requested stop is normal turn/lifecycle end, no error row.
- const abnormal =
- info?.reason !== "requested" &&
- ((typeof info?.code === "number" && info.code !== 0) || !!info?.signal);
+ // USER-FACING abnormal-exit audit. Stays STRICTLY inside the
+ // `!suppressExitLog` gate — a deliberate stop (reset/nap/terminate_stalled
+ // sets suppressExitLog) must NOT surface an `abnormal_exit` row in the
+ // user's activity, or it re-creates the nap-noise bug (a self-initiated
+ // kill showing as a fault). The `abnormal` physical fact above is shared,
+ // but its user-audit sink is gated while its trace sink (below) is not.
if (abnormal) {
- const detail = info?.signal ? `signal ${info.signal}` : `code ${info?.code}`;
+ const detail = exitSignal ? `signal ${exitSignal}` : `code ${exitCode}`;
this.emitErrorAudit(agentId, "exit", "abnormal_exit", `Session ended unexpectedly (${detail})`);
}
}
@@ -1458,7 +1493,7 @@ export class AgentProcessManager {
// from a stale/superseded spawn must NOT drop a fresh session's marker.
this.nonCleanEndMarker.delete(agentId);
}
- this.dispatch({ type: "exit", agentId });
+ this.dispatch({ type: "exit", agentId, exitCode, exitSignal, abnormal });
});
// Stamp the wake-prompt timestamp AT the last mile — right before the
diff --git a/src/daemon/src/runtime/runtimeSession.test.ts b/src/daemon/src/runtime/runtimeSession.test.ts
new file mode 100644
index 000000000..5eee4021d
--- /dev/null
+++ b/src/daemon/src/runtime/runtimeSession.test.ts
@@ -0,0 +1,94 @@
+import { describe, it, expect, afterEach } from "vitest";
+import { spawn, type ChildProcess } from "child_process";
+import { ChildProcessRuntimeSession } from "./runtimeSession.js";
+import type { Driver, LaunchContext } from "../types.js";
+
+/*
+ * Red-line-5(b) of plans/daemon-trace-completeness-charter.md (T1): the synthetic
+ * `session.fire("exit", {...})` tests in managerRuntime.test.ts prove the daemon
+ * THREADS exitCode/exitSignal/abnormal into the FSM/trace. They do NOT prove the
+ * SOURCE is real — that a genuinely killed subprocess actually fills `info.code`
+ * / `info.signal` on the runtime session's `exit` event, which is what the
+ * abnormal predicate (managerRuntime.ts) reads. A real subprocess dies here to
+ * confirm that contract end-to-end (Node fills signal=SIGKILL, code=null; a
+ * non-requested death → reason="runtime_exit"). Without this the "hardest to
+ * reconstruct" blind spot would rest on an unverified assumption.
+ */
+
+const spawned: ChildProcess[] = [];
+
+/** Minimal driver that spawns a real, long-lived child process. */
+function realSpawnDriver(): Driver {
+ return {
+ id: "test-real",
+ lifecycle: { kind: "persistent", start: "immediate", exit: "natural", inFlightWake: "queue" },
+ spawn: async () => {
+ const proc = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], {
+ stdio: ["pipe", "pipe", "pipe"],
+ });
+ spawned.push(proc);
+ return { process: proc };
+ },
+ parseLine: () => [],
+ encodeStdinMessage: () => null,
+ buildSystemPrompt: () => "",
+ } as unknown as Driver;
+}
+
+function minimalCtx(): LaunchContext {
+ return {
+ workingDirectory: process.cwd(),
+ agentId: "a1",
+ standingPrompt: "",
+ prompt: "",
+ config: {} as LaunchContext["config"],
+ credentialProxy: {} as LaunchContext["credentialProxy"],
+ } as LaunchContext;
+}
+
+afterEach(() => {
+ for (const p of spawned.splice(0)) {
+ try { p.kill("SIGKILL"); } catch { /* already dead */ }
+ }
+});
+
+describe("ChildProcessRuntimeSession — real subprocess exit fills the physical fact (T1 red-line-5b)", () => {
+ it("a real SIGKILLed subprocess emits exit with signal=SIGKILL, null code, reason=runtime_exit", async () => {
+ const session = new ChildProcessRuntimeSession(realSpawnDriver(), minimalCtx());
+ const exitInfo = await new Promise<{ code: number | null; signal: string | null; reason?: string }>(
+ (resolve) => {
+ session.on("exit", (...args: unknown[]) => resolve(args[0] as never));
+ void session.start({ text: "go" }).then(() => {
+ // Kill the real child directly (external kill, NOT session.stop()) so
+ // requestedStopReason stays unset → reason="runtime_exit", the crash
+ // shape the abnormal predicate must catch.
+ const pid = session.pid;
+ if (pid) process.kill(pid, "SIGKILL");
+ });
+ },
+ );
+ expect(exitInfo.signal).toBe("SIGKILL");
+ expect(exitInfo.code).toBeNull();
+ expect(exitInfo.reason).toBe("runtime_exit");
+ });
+
+ it("a real clean exit (code 0) emits code=0, null signal", async () => {
+ const cleanDriver = {
+ ...realSpawnDriver(),
+ spawn: async () => {
+ const proc = spawn(process.execPath, ["-e", "process.exit(0)"], { stdio: ["pipe", "pipe", "pipe"] });
+ spawned.push(proc);
+ return { process: proc };
+ },
+ } as unknown as Driver;
+ const session = new ChildProcessRuntimeSession(cleanDriver, minimalCtx());
+ const exitInfo = await new Promise<{ code: number | null; signal: string | null; reason?: string }>(
+ (resolve) => {
+ session.on("exit", (...args: unknown[]) => resolve(args[0] as never));
+ void session.start({ text: "go" });
+ },
+ );
+ expect(exitInfo.code).toBe(0);
+ expect(exitInfo.signal).toBeNull();
+ });
+});