an open-source alternative to the dots, bots and muses of the world, run an office of claude code/codex like agents on your laptop, sandboxes or anywhere, uses your existing subscriptions
HTML agent-harnessagent-orchestrationagentsai-agentsautonomous-agentsclaude-codecodexcoding-agentsdesktop-appdeveloper-toolselectrongemini-cliharnessharness-engineeringlocal-firstmemorymulti-agentopencodeorchestrationtypescript
fix(hive): an agent killed by a signal no longer dies without a trace
lavadmande85b5528af8
diff --git a/src/main/hive.ts b/src/main/hive.tsindex f3b9c03d..80255788 100644--- a/src/main/hive.ts+++ b/src/main/hive.ts@@ -546,7 +546,12 @@ export class HiveManager {// Keep the churny/ephemeral live files out of the hive git repo.const gitignore = join(root, '.gitignore');- const want = ['fleet.json', 'hooks.sock', 'cost-ledger.jsonl', '.DS_Store'];+ // `crashes/` holds raw PTY output from abnormal agent exits. It is+ // DELIBERATELY ignored: that output is whatever the provider printed, which+ // can include tokens, paths and prompt fragments, and the hive repo is+ // committed on every change — a secret written there would be permanent.+ // log.jsonl gets the structured, non-sensitive fields; the dump stays local.+ const want = ['fleet.json', 'hooks.sock', 'cost-ledger.jsonl', 'crashes/', '.DS_Store'];let lines: string[] = [];if (existsSync(gitignore)) { try { lines = readFileSync(gitignore, 'utf8').split('\n'); } catch { lines = []; } }const missing = want.filter((w) => !lines.includes(w));@@ -2228,6 +2233,66 @@ export class HiveManager {.filter((m): m is HiveMessage => m !== null);}+ /**+ * Record an agent's process exit so a death has a durable cause.+ *+ * Before this, an agent killed by its provider crashing was archived with+ * `{kind:'archive', agentId, archived:true}` and NOTHING else — no exit code,+ * no signal, no output. A two-second SIGILL death and a completed agent were+ * indistinguishable in the only record the hive keeps, and the crash banner+ * lived solely in a UI terminal pane. Observed live 2026-08-24: Michael's+ * `claude` CLI panicked 923ms into startup and left no trace on disk.+ *+ * Split by design:+ * - log.jsonl gets structured, non-sensitive fields (code, signal, path).+ * - crashes/ gets the raw tail, and is gitignored — see ensureHive.+ * A normal exit writes nothing at all; this is a diagnostic, not an audit log.+ */+ recordAgentExit(+ agentId: string,+ info: { exitCode?: number; signal?: number; tail?: string; command?: string }+ ): void {+ const root = this.root();+ if (!root) return;+ const { exitCode, signal, tail, command } = info;+ // A signal means killed (SIGILL/SIGSEGV/SIGKILL); node-pty reports exitCode 0+ // in that case, so signal must be checked independently of the code.+ const abnormal = (typeof signal === 'number' && signal !== 0) || (typeof exitCode === 'number' && exitCode !== 0);+ if (!abnormal) return;++ let tailPath: string | null = null;+ if (tail && tail.length) {+ try {+ const dir = join(root, 'crashes');+ mkdirSync(dir, { recursive: true });+ // Colons are illegal in filenames on Windows and awkward everywhere.+ const stamp = new Date().toISOString().replace(/[:.]/g, '-');+ const safeId = agentId.replace(/[^A-Za-z0-9._-]/g, '_');+ const p = join(dir, `${stamp}-${safeId}.log`);+ const header = [+ `agent: ${agentId}`,+ `exitCode: ${String(exitCode)}`,+ `signal: ${String(signal)}`,+ command ? `command: ${command}` : null,+ `captured: ${new Date().toISOString()}`,+ `--- last ${tail.length} bytes of pty output ---`,+ ''+ ].filter(Boolean).join('\n');+ writeFileSync(p, header + tail, 'utf8');+ tailPath = p;+ } catch { /* a diagnostic must never break teardown */ }+ }++ this.appendLog({+ kind: 'agent-exit',+ agentId,+ exitCode: exitCode ?? null,+ signal: signal ?? null,+ abnormal: true,+ tailPath+ });+ }+// — log —appendLog(event: Record<string, unknown>): void {const root = this.root();diff --git a/src/main/index.ts b/src/main/index.tsindex 5944af45..65968409 100644--- a/src/main/index.ts+++ b/src/main/index.ts@@ -575,7 +575,23 @@ function removeWorkerScratch(workerId: string): void {// SAME pty/window (no user click). Provider-agnostic. Idempotent by construction: the// relaunch carries `noAutoInstall`, so the installer can never fire (let alone loop) a// second time — a binary that's somehow still missing just spawns and exits normally.-ptyManager.setExitHandler((id, exitCode) => {+ptyManager.setExitHandler((id, exitCode, info) => {+ // Record an ABNORMAL death before teardown — teardownPty drops the+ // pty->agent mapping, so after it runs we can no longer say WHOSE process+ // died. Only abnormal exits are recorded (recordAgentExit returns early on a+ // clean one), so this adds no noise to a normal archive.+ try {+ const dyingAgent = ptyToAgent.get(id);+ if (dyingAgent) {+ hive.recordAgentExit(dyingAgent, {+ exitCode,+ signal: info?.signal,+ tail: info?.tail,+ command: info?.command+ });+ }+ } catch (e) { console.error('[pty] recordAgentExit failed:', e); }+const pending = pendingInstallRelaunch.get(id);if (pending) {pendingInstallRelaunch.delete(id);diff --git a/src/main/pty.ts b/src/main/pty.tsindex 35d557d6..421688c5 100644--- a/src/main/pty.ts+++ b/src/main/pty.ts@@ -29,6 +29,20 @@ export function withHiveRuntimeFallback(path: string, hiveRoot?: string): stringreturn [...entries, dir].join(delimiter);}+/** How much trailing PTY output to retain per session for crash diagnostics.+ * Big enough for a stack trace or a panic banner, small enough that N idle+ * agents cost kilobytes, not megabytes. */+const TAIL_MAX = 8192;++/** What the exit handler is told about a process that just died. Passed by+ * value because the session is deleted from the map before the handler runs. */+export interface PtyExitInfo {+ signal?: number;+ tail?: string;+ command?: string;+ cwd?: string;+}+interface PtySession {id: string;proc: pty.IPty;@@ -46,6 +60,13 @@ interface PtySession {* file) and the idle handshake that gates god's PTY nudge (never type into a* PTY that produced output in the last few seconds = mid-stream). */lastOutputAt: number;+ /** A bounded ring of the most recent output bytes, kept ONLY so an abnormal+ * exit can say what was on screen when the process died. A provider that+ * crashes on startup (Bun SIGILL, a missing shared library, an auth failure)+ * prints its explanation and vanishes; without this the explanation exists+ * nowhere but a terminal pane the operator may never have been looking at.+ * Capped at TAIL_MAX so a chatty agent cannot grow it without bound. */+ tail: string;/** True after the child has emitted at least one frame. Automation waits for* this before typing, so startup prompts cannot outrun the TUI subscription. */hasOutput: boolean;@@ -309,7 +330,8 @@ export class PtyManager {* externally), so the main process can run the SAME lifecycle teardown* (archive, worktree removal, map cleanup) that the explicit kill() path* runs. Best-effort — set once by the main process. */- private exitHandler: ((id: string, exitCode?: number) => void) | null = null;+ private exitHandler:+ ((id: string, exitCode?: number, info?: PtyExitInfo) => void) | null = null;/** The default/fallback output sink — set to the PRIMARY window. Used only for* sessions with no recorded owner; owned sessions route to their owner. */@@ -345,7 +367,9 @@ export class PtyManager {* onExit after the session is cleaned up. The exit code is forwarded so the* handler can distinguish a clean exit (e.g. a successful first-time CLI* install → auto restart-and-continue) from a crash. */- setExitHandler(handler: (id: string, exitCode?: number) => void): void {+ setExitHandler(+ handler: (id: string, exitCode?: number, info?: PtyExitInfo) => void+ ): void {this.exitHandler = handler;}@@ -661,6 +685,7 @@ export class PtyManager {command: resolved,lastOutputAt: Date.now(),hasOutput: false,+ tail: '',owner};this.sessions.set(opts.id, session);@@ -671,6 +696,9 @@ export class PtyManager {if (this.sessions.get(opts.id) !== session) return;session.hasOutput = true;session.lastOutputAt = Date.now();+ // Keep only the trailing window; slice AFTER appending so a single+ // oversized write still leaves us its end (the part that explains a death).+ session.tail = (session.tail + data).slice(-TAIL_MAX);// Route to the session's owner window (multi-window owner routing).this.safeSend(`pty:data:${opts.id}`, data, session.owner);});@@ -682,7 +710,21 @@ export class PtyManager {this.sessions.delete(opts.id);// Natural exit must run the same lifecycle teardown as an explicit kill.// Guarded so a teardown error can never crash node-pty's exit callback.- try { this.exitHandler?.(opts.id, exitCode); } catch { /* never throw out of onExit */ }+ // `signal` is forwarded, not dropped: a provider killed by SIGILL/SIGSEGV+ // exits with code 0 and a non-zero signal, so an exitCode-only handler+ // cannot tell a crash from a clean finish. `tail` carries whatever the+ // process printed on its way out.+ // The session is already out of `sessions` by now, so anything the+ // handler needs about the dead process must be handed to it here —+ // it cannot look the session up any more.+ try {+ this.exitHandler?.(opts.id, exitCode, {+ signal,+ tail: session.tail,+ command: session.command,+ cwd: session.cwd+ });+ } catch { /* never throw out of onExit */ }});return { ok: true };diff --git a/test/agent-exit-record.test.cjs b/test/agent-exit-record.test.cjsnew file mode 100644index 00000000..cf227355--- /dev/null+++ b/test/agent-exit-record.test.cjs@@ -0,0 +1,119 @@+'use strict';++/**+ * An agent killed by its provider crashing used to leave NO durable cause.+ *+ * Observed live on 2026-08-24: a `claude` CLI panicked 923 ms into startup+ * (Bun, SIGILL) and the hive recorded exactly+ * {"kind":"archive","agentId":"god","archived":true}+ * — no exit code, no signal, no output. The crash banner existed only in a UI+ * terminal pane. A two-second death and a completed agent were indistinguishable+ * in the only record the hive keeps.+ *+ * `recordAgentExit` fixes that. These tests pin the three properties that make+ * it useful and safe:+ * 1. a signal-death is recorded even though its exit code is 0;+ * 2. a CLEAN exit records nothing (a diagnostic, not an audit log);+ * 3. the raw output goes to a GITIGNORED file, never into log.jsonl —+ * the hive repo is committed on every change, so a token captured from+ * provider output would otherwise be permanent.+ */++const test = require('node:test');+const assert = require('node:assert/strict');+const fs = require('node:fs');+const os = require('node:os');+const path = require('node:path');+const loadTs = require('./load-ts.cjs');++const { HiveManager } = loadTs('src/main/hive.ts');++function tmpHome() {+ return fs.mkdtempSync(path.join(os.tmpdir(), 'md-exit-'));+}++function readLog(home) {+ const p = path.join(home, 'hive', 'log.jsonl');+ if (!fs.existsSync(p)) return [];+ return fs.readFileSync(p, 'utf8').split('\n').filter(Boolean).map((l) => JSON.parse(l));+}++async function freshHive(t) {+ const home = tmpHome();+ t.after(() => fs.rmSync(home, { recursive: true, force: true }));+ const hive = new HiveManager(() => home);+ await hive.ensureAgent({ id: 'a1', name: 'A', provider: 'claude', cwd: home });+ return { home, hive };+}++test('a signal death is recorded even though its exit code is 0', async (t) => {+ const { home, hive } = await freshHive(t);++ // SIGILL is exactly the shape that bit us: node-pty reports exitCode 0 and a+ // non-zero signal, so an exitCode-only check calls this a clean finish.+ hive.recordAgentExit('a1', { exitCode: 0, signal: 4, tail: 'panic: Illegal instruction\n' });++ const rows = readLog(home).filter((r) => r.kind === 'agent-exit');+ assert.equal(rows.length, 1, 'a signal death must be recorded');+ assert.equal(rows[0].agentId, 'a1');+ assert.equal(rows[0].signal, 4);+ assert.equal(rows[0].exitCode, 0);+ assert.equal(rows[0].abnormal, true);+});++test('a non-zero exit code is recorded', async (t) => {+ const { home, hive } = await freshHive(t);+ hive.recordAgentExit('a1', { exitCode: 127, tail: 'claude: command not found\n' });++ const rows = readLog(home).filter((r) => r.kind === 'agent-exit');+ assert.equal(rows.length, 1);+ assert.equal(rows[0].exitCode, 127);+ assert.equal(rows[0].signal, null);+});++test('a CLEAN exit records nothing — this is a diagnostic, not an audit log', async (t) => {+ const { home, hive } = await freshHive(t);++ hive.recordAgentExit('a1', { exitCode: 0, signal: 0, tail: 'goodbye\n' });+ hive.recordAgentExit('a1', { exitCode: 0, tail: 'goodbye\n' });+ hive.recordAgentExit('a1', {});++ assert.equal(readLog(home).filter((r) => r.kind === 'agent-exit').length, 0);+});++test('raw output goes to a gitignored crashes/ file, NOT into log.jsonl', async (t) => {+ const { home, hive } = await freshHive(t);++ // A plausible secret in provider output. It must reach the dump file and must+ // NOT reach log.jsonl, which the hive commits.+ const secret = 'sk-ant-EXAMPLE-NOT-A-REAL-KEY-000';+ hive.recordAgentExit('a1', { exitCode: 0, signal: 11, tail: `boom ${secret}\n`, command: '/usr/bin/claude --model x' });++ const rows = readLog(home).filter((r) => r.kind === 'agent-exit');+ assert.equal(rows.length, 1);++ const raw = fs.readFileSync(path.join(home, 'hive', 'log.jsonl'), 'utf8');+ assert.ok(!raw.includes(secret), 'log.jsonl must never carry raw provider output');++ const tailPath = rows[0].tailPath;+ assert.ok(tailPath, 'the row must point at the dump');+ const dump = fs.readFileSync(tailPath, 'utf8');+ assert.ok(dump.includes(secret), 'the dump keeps the output that explains the death');+ assert.ok(dump.includes('signal: 11'), 'the dump carries a header naming the signal');+ assert.ok(dump.includes('/usr/bin/claude --model x'), 'and the command that died');++ // The dump must live under crashes/, which ensureHive gitignores.+ assert.equal(path.basename(path.dirname(tailPath)), 'crashes');+ const ignore = fs.readFileSync(path.join(home, 'hive', '.gitignore'), 'utf8');+ assert.ok(ignore.split('\n').includes('crashes/'), 'crashes/ must be gitignored');+});++test('a death is still recorded when there is no output to dump', async (t) => {+ const { home, hive } = await freshHive(t);+ hive.recordAgentExit('a1', { exitCode: 0, signal: 9, tail: '' });++ const rows = readLog(home).filter((r) => r.kind === 'agent-exit');+ assert.equal(rows.length, 1, 'losing the output must not lose the fact of the death');+ assert.equal(rows[0].signal, 9);+ assert.equal(rows[0].tailPath, null);+});diff --git a/test/agent-exit-signal-e2e.test.cjs b/test/agent-exit-signal-e2e.test.cjsnew file mode 100644index 00000000..eb32f6ef--- /dev/null+++ b/test/agent-exit-signal-e2e.test.cjs@@ -0,0 +1,98 @@+'use strict';++/**+ * End-to-end: a PTY whose process dies by SIGNAL must reach the exit handler+ * with that signal AND with the output it printed on the way out.+ *+ * The unit tests for `recordAgentExit` call it directly. This one drives the+ * real `PtyManager`, because the original defect was NOT in the recording — it+ * was that `signal` never left `pty.ts`. `onExit` had `{ exitCode, signal }` in+ * hand and forwarded only `exitCode`, so every caller downstream was blind to+ * signal deaths by construction.+ *+ * The scenario is the one observed live on 2026-08-24: a provider that prints+ * a panic and dies of SIGILL, which node-pty reports as exitCode 0 + signal 4.+ * An exitCode-only handler sees "0" and calls that a clean finish.+ */++const test = require('node:test');+const assert = require('node:assert/strict');+const loadTs = require('./load-ts.cjs');++const { PtyManager } = loadTs('src/main/pty.ts');++const POSIX = process.platform !== 'win32';++function waitForExit(mgr) {+ return new Promise((resolve) => {+ mgr.setExitHandler((id, exitCode, info) => resolve({ id, exitCode, info }));+ });+}++test('a SIGILL death forwards the signal and the output that explains it', { skip: !POSIX }, async () => {+ const mgr = new PtyManager();+ const done = waitForExit(mgr);++ // Print a recognisable banner, flush it, then kill ourselves with SIGILL —+ // the exact shape of the crash that motivated this change.+ const res = mgr.spawn({+ id: 'e2e-sigill',+ cwd: process.cwd(),+ command: '/bin/sh',+ args: ['-c', 'echo "panic(main thread): Illegal instruction"; sleep 0.2; kill -ILL $$']+ });+ assert.equal(res.ok, true, `spawn failed: ${res.error}`);++ const { id, exitCode, info } = await done;+ assert.equal(id, 'e2e-sigill');++ // THE REGRESSION GUARD: signal must survive the trip out of pty.ts.+ assert.equal(info.signal, 4, 'SIGILL (4) must be forwarded, not dropped');++ // And the exit code alone must NOT be what a caller keys on — this asserts+ // the very trap the old handler fell into.+ assert.notEqual(+ typeof info.signal === 'number' && info.signal !== 0 ? 'abnormal' : 'clean',+ 'clean',+ 'a signal death must not classify as clean'+ );++ assert.match(info.tail ?? '', /Illegal instruction/, 'the dying output must be captured');+ assert.equal(typeof info.command, 'string', 'the command must be passed by value');+ assert.ok((info.command ?? '').length > 0, 'the session is deleted before the handler, so command cannot be looked up later');+});++test('a clean exit reports no signal', { skip: !POSIX }, async () => {+ const mgr = new PtyManager();+ const done = waitForExit(mgr);++ const res = mgr.spawn({+ id: 'e2e-clean',+ cwd: process.cwd(),+ command: '/bin/sh',+ args: ['-c', 'echo done; exit 0']+ });+ assert.equal(res.ok, true, `spawn failed: ${res.error}`);++ const { exitCode, info } = await done;+ assert.equal(exitCode, 0);+ assert.ok(!info.signal, 'a clean exit must not report a signal');+});++test('a non-zero exit is reported as a code, not a signal', { skip: !POSIX }, async () => {+ const mgr = new PtyManager();+ const done = waitForExit(mgr);++ const res = mgr.spawn({+ id: 'e2e-code',+ cwd: process.cwd(),+ command: '/bin/sh',+ args: ['-c', 'echo "command not found"; exit 127']+ });+ assert.equal(res.ok, true, `spawn failed: ${res.error}`);++ const { exitCode, info } = await done;+ assert.equal(exitCode, 127);+ assert.ok(!info.signal);+ assert.match(info.tail ?? '', /command not found/);+});