Skip to content

an open-source alternative to the dots, bots and muses of the world, run an office of claude code/codex like agents on your laptop, sandboxes or anywhere, uses your existing subscriptions

fix(hive): an agent killed by a signal no longer dies without a trace

lavadmande85b5528af8
diff --git a/src/main/hive.ts b/src/main/hive.ts
index f3b9c03d..80255788 100644
--- a/src/main/hive.ts
+++ b/src/main/hive.ts
@@ -546,7 +546,12 @@ export class HiveManager {
// Keep the churny/ephemeral live files out of the hive git repo.
const gitignore = join(root, '.gitignore');
- const want = ['fleet.json', 'hooks.sock', 'cost-ledger.jsonl', '.DS_Store'];
+ // `crashes/` holds raw PTY output from abnormal agent exits. It is
+ // DELIBERATELY ignored: that output is whatever the provider printed, which
+ // can include tokens, paths and prompt fragments, and the hive repo is
+ // committed on every change — a secret written there would be permanent.
+ // log.jsonl gets the structured, non-sensitive fields; the dump stays local.
+ const want = ['fleet.json', 'hooks.sock', 'cost-ledger.jsonl', 'crashes/', '.DS_Store'];
let lines: string[] = [];
if (existsSync(gitignore)) { try { lines = readFileSync(gitignore, 'utf8').split('\n'); } catch { lines = []; } }
const missing = want.filter((w) => !lines.includes(w));
@@ -2228,6 +2233,66 @@ export class HiveManager {
.filter((m): m is HiveMessage => m !== null);
}
+ /**
+ * Record an agent's process exit so a death has a durable cause.
+ *
+ * Before this, an agent killed by its provider crashing was archived with
+ * `{kind:'archive', agentId, archived:true}` and NOTHING else — no exit code,
+ * no signal, no output. A two-second SIGILL death and a completed agent were
+ * indistinguishable in the only record the hive keeps, and the crash banner
+ * lived solely in a UI terminal pane. Observed live 2026-08-24: Michael's
+ * `claude` CLI panicked 923ms into startup and left no trace on disk.
+ *
+ * Split by design:
+ * - log.jsonl gets structured, non-sensitive fields (code, signal, path).
+ * - crashes/ gets the raw tail, and is gitignored — see ensureHive.
+ * A normal exit writes nothing at all; this is a diagnostic, not an audit log.
+ */
+ recordAgentExit(
+ agentId: string,
+ info: { exitCode?: number; signal?: number; tail?: string; command?: string }
+ ): void {
+ const root = this.root();
+ if (!root) return;
+ const { exitCode, signal, tail, command } = info;
+ // A signal means killed (SIGILL/SIGSEGV/SIGKILL); node-pty reports exitCode 0
+ // in that case, so signal must be checked independently of the code.
+ const abnormal = (typeof signal === 'number' && signal !== 0) || (typeof exitCode === 'number' && exitCode !== 0);
+ if (!abnormal) return;
+
+ let tailPath: string | null = null;
+ if (tail && tail.length) {
+ try {
+ const dir = join(root, 'crashes');
+ mkdirSync(dir, { recursive: true });
+ // Colons are illegal in filenames on Windows and awkward everywhere.
+ const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+ const safeId = agentId.replace(/[^A-Za-z0-9._-]/g, '_');
+ const p = join(dir, `${stamp}-${safeId}.log`);
+ const header = [
+ `agent: ${agentId}`,
+ `exitCode: ${String(exitCode)}`,
+ `signal: ${String(signal)}`,
+ command ? `command: ${command}` : null,
+ `captured: ${new Date().toISOString()}`,
+ `--- last ${tail.length} bytes of pty output ---`,
+ ''
+ ].filter(Boolean).join('\n');
+ writeFileSync(p, header + tail, 'utf8');
+ tailPath = p;
+ } catch { /* a diagnostic must never break teardown */ }
+ }
+
+ this.appendLog({
+ kind: 'agent-exit',
+ agentId,
+ exitCode: exitCode ?? null,
+ signal: signal ?? null,
+ abnormal: true,
+ tailPath
+ });
+ }
+
// — log —
appendLog(event: Record<string, unknown>): void {
const root = this.root();
diff --git a/src/main/index.ts b/src/main/index.ts
index 5944af45..65968409 100644
--- a/src/main/index.ts
+++ b/src/main/index.ts
@@ -575,7 +575,23 @@ function removeWorkerScratch(workerId: string): void {
// SAME pty/window (no user click). Provider-agnostic. Idempotent by construction: the
// relaunch carries `noAutoInstall`, so the installer can never fire (let alone loop) a
// second time — a binary that's somehow still missing just spawns and exits normally.
-ptyManager.setExitHandler((id, exitCode) => {
+ptyManager.setExitHandler((id, exitCode, info) => {
+ // Record an ABNORMAL death before teardown — teardownPty drops the
+ // pty->agent mapping, so after it runs we can no longer say WHOSE process
+ // died. Only abnormal exits are recorded (recordAgentExit returns early on a
+ // clean one), so this adds no noise to a normal archive.
+ try {
+ const dyingAgent = ptyToAgent.get(id);
+ if (dyingAgent) {
+ hive.recordAgentExit(dyingAgent, {
+ exitCode,
+ signal: info?.signal,
+ tail: info?.tail,
+ command: info?.command
+ });
+ }
+ } catch (e) { console.error('[pty] recordAgentExit failed:', e); }
+
const pending = pendingInstallRelaunch.get(id);
if (pending) {
pendingInstallRelaunch.delete(id);
diff --git a/src/main/pty.ts b/src/main/pty.ts
index 35d557d6..421688c5 100644
--- a/src/main/pty.ts
+++ b/src/main/pty.ts
@@ -29,6 +29,20 @@ export function withHiveRuntimeFallback(path: string, hiveRoot?: string): string
return [...entries, dir].join(delimiter);
}
+/** How much trailing PTY output to retain per session for crash diagnostics.
+ * Big enough for a stack trace or a panic banner, small enough that N idle
+ * agents cost kilobytes, not megabytes. */
+const TAIL_MAX = 8192;
+
+/** What the exit handler is told about a process that just died. Passed by
+ * value because the session is deleted from the map before the handler runs. */
+export interface PtyExitInfo {
+ signal?: number;
+ tail?: string;
+ command?: string;
+ cwd?: string;
+}
+
interface PtySession {
id: string;
proc: pty.IPty;
@@ -46,6 +60,13 @@ interface PtySession {
* file) and the idle handshake that gates god's PTY nudge (never type into a
* PTY that produced output in the last few seconds = mid-stream). */
lastOutputAt: number;
+ /** A bounded ring of the most recent output bytes, kept ONLY so an abnormal
+ * exit can say what was on screen when the process died. A provider that
+ * crashes on startup (Bun SIGILL, a missing shared library, an auth failure)
+ * prints its explanation and vanishes; without this the explanation exists
+ * nowhere but a terminal pane the operator may never have been looking at.
+ * Capped at TAIL_MAX so a chatty agent cannot grow it without bound. */
+ tail: string;
/** True after the child has emitted at least one frame. Automation waits for
* this before typing, so startup prompts cannot outrun the TUI subscription. */
hasOutput: boolean;
@@ -309,7 +330,8 @@ export class PtyManager {
* externally), so the main process can run the SAME lifecycle teardown
* (archive, worktree removal, map cleanup) that the explicit kill() path
* runs. Best-effort — set once by the main process. */
- private exitHandler: ((id: string, exitCode?: number) => void) | null = null;
+ private exitHandler:
+ ((id: string, exitCode?: number, info?: PtyExitInfo) => void) | null = null;
/** The default/fallback output sink — set to the PRIMARY window. Used only for
* sessions with no recorded owner; owned sessions route to their owner. */
@@ -345,7 +367,9 @@ export class PtyManager {
* onExit after the session is cleaned up. The exit code is forwarded so the
* handler can distinguish a clean exit (e.g. a successful first-time CLI
* install → auto restart-and-continue) from a crash. */
- setExitHandler(handler: (id: string, exitCode?: number) => void): void {
+ setExitHandler(
+ handler: (id: string, exitCode?: number, info?: PtyExitInfo) => void
+ ): void {
this.exitHandler = handler;
}
@@ -661,6 +685,7 @@ export class PtyManager {
command: resolved,
lastOutputAt: Date.now(),
hasOutput: false,
+ tail: '',
owner
};
this.sessions.set(opts.id, session);
@@ -671,6 +696,9 @@ export class PtyManager {
if (this.sessions.get(opts.id) !== session) return;
session.hasOutput = true;
session.lastOutputAt = Date.now();
+ // Keep only the trailing window; slice AFTER appending so a single
+ // oversized write still leaves us its end (the part that explains a death).
+ session.tail = (session.tail + data).slice(-TAIL_MAX);
// Route to the session's owner window (multi-window owner routing).
this.safeSend(`pty:data:${opts.id}`, data, session.owner);
});
@@ -682,7 +710,21 @@ export class PtyManager {
this.sessions.delete(opts.id);
// Natural exit must run the same lifecycle teardown as an explicit kill.
// Guarded so a teardown error can never crash node-pty's exit callback.
- try { this.exitHandler?.(opts.id, exitCode); } catch { /* never throw out of onExit */ }
+ // `signal` is forwarded, not dropped: a provider killed by SIGILL/SIGSEGV
+ // exits with code 0 and a non-zero signal, so an exitCode-only handler
+ // cannot tell a crash from a clean finish. `tail` carries whatever the
+ // process printed on its way out.
+ // The session is already out of `sessions` by now, so anything the
+ // handler needs about the dead process must be handed to it here —
+ // it cannot look the session up any more.
+ try {
+ this.exitHandler?.(opts.id, exitCode, {
+ signal,
+ tail: session.tail,
+ command: session.command,
+ cwd: session.cwd
+ });
+ } catch { /* never throw out of onExit */ }
});
return { ok: true };
diff --git a/test/agent-exit-record.test.cjs b/test/agent-exit-record.test.cjs
new file mode 100644
index 00000000..cf227355
--- /dev/null
+++ b/test/agent-exit-record.test.cjs
@@ -0,0 +1,119 @@
+'use strict';
+
+/**
+ * An agent killed by its provider crashing used to leave NO durable cause.
+ *
+ * Observed live on 2026-08-24: a `claude` CLI panicked 923 ms into startup
+ * (Bun, SIGILL) and the hive recorded exactly
+ * {"kind":"archive","agentId":"god","archived":true}
+ * — no exit code, no signal, no output. The crash banner existed only in a UI
+ * terminal pane. A two-second death and a completed agent were indistinguishable
+ * in the only record the hive keeps.
+ *
+ * `recordAgentExit` fixes that. These tests pin the three properties that make
+ * it useful and safe:
+ * 1. a signal-death is recorded even though its exit code is 0;
+ * 2. a CLEAN exit records nothing (a diagnostic, not an audit log);
+ * 3. the raw output goes to a GITIGNORED file, never into log.jsonl —
+ * the hive repo is committed on every change, so a token captured from
+ * provider output would otherwise be permanent.
+ */
+
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const loadTs = require('./load-ts.cjs');
+
+const { HiveManager } = loadTs('src/main/hive.ts');
+
+function tmpHome() {
+ return fs.mkdtempSync(path.join(os.tmpdir(), 'md-exit-'));
+}
+
+function readLog(home) {
+ const p = path.join(home, 'hive', 'log.jsonl');
+ if (!fs.existsSync(p)) return [];
+ return fs.readFileSync(p, 'utf8').split('\n').filter(Boolean).map((l) => JSON.parse(l));
+}
+
+async function freshHive(t) {
+ const home = tmpHome();
+ t.after(() => fs.rmSync(home, { recursive: true, force: true }));
+ const hive = new HiveManager(() => home);
+ await hive.ensureAgent({ id: 'a1', name: 'A', provider: 'claude', cwd: home });
+ return { home, hive };
+}
+
+test('a signal death is recorded even though its exit code is 0', async (t) => {
+ const { home, hive } = await freshHive(t);
+
+ // SIGILL is exactly the shape that bit us: node-pty reports exitCode 0 and a
+ // non-zero signal, so an exitCode-only check calls this a clean finish.
+ hive.recordAgentExit('a1', { exitCode: 0, signal: 4, tail: 'panic: Illegal instruction\n' });
+
+ const rows = readLog(home).filter((r) => r.kind === 'agent-exit');
+ assert.equal(rows.length, 1, 'a signal death must be recorded');
+ assert.equal(rows[0].agentId, 'a1');
+ assert.equal(rows[0].signal, 4);
+ assert.equal(rows[0].exitCode, 0);
+ assert.equal(rows[0].abnormal, true);
+});
+
+test('a non-zero exit code is recorded', async (t) => {
+ const { home, hive } = await freshHive(t);
+ hive.recordAgentExit('a1', { exitCode: 127, tail: 'claude: command not found\n' });
+
+ const rows = readLog(home).filter((r) => r.kind === 'agent-exit');
+ assert.equal(rows.length, 1);
+ assert.equal(rows[0].exitCode, 127);
+ assert.equal(rows[0].signal, null);
+});
+
+test('a CLEAN exit records nothing — this is a diagnostic, not an audit log', async (t) => {
+ const { home, hive } = await freshHive(t);
+
+ hive.recordAgentExit('a1', { exitCode: 0, signal: 0, tail: 'goodbye\n' });
+ hive.recordAgentExit('a1', { exitCode: 0, tail: 'goodbye\n' });
+ hive.recordAgentExit('a1', {});
+
+ assert.equal(readLog(home).filter((r) => r.kind === 'agent-exit').length, 0);
+});
+
+test('raw output goes to a gitignored crashes/ file, NOT into log.jsonl', async (t) => {
+ const { home, hive } = await freshHive(t);
+
+ // A plausible secret in provider output. It must reach the dump file and must
+ // NOT reach log.jsonl, which the hive commits.
+ const secret = 'sk-ant-EXAMPLE-NOT-A-REAL-KEY-000';
+ hive.recordAgentExit('a1', { exitCode: 0, signal: 11, tail: `boom ${secret}\n`, command: '/usr/bin/claude --model x' });
+
+ const rows = readLog(home).filter((r) => r.kind === 'agent-exit');
+ assert.equal(rows.length, 1);
+
+ const raw = fs.readFileSync(path.join(home, 'hive', 'log.jsonl'), 'utf8');
+ assert.ok(!raw.includes(secret), 'log.jsonl must never carry raw provider output');
+
+ const tailPath = rows[0].tailPath;
+ assert.ok(tailPath, 'the row must point at the dump');
+ const dump = fs.readFileSync(tailPath, 'utf8');
+ assert.ok(dump.includes(secret), 'the dump keeps the output that explains the death');
+ assert.ok(dump.includes('signal: 11'), 'the dump carries a header naming the signal');
+ assert.ok(dump.includes('/usr/bin/claude --model x'), 'and the command that died');
+
+ // The dump must live under crashes/, which ensureHive gitignores.
+ assert.equal(path.basename(path.dirname(tailPath)), 'crashes');
+ const ignore = fs.readFileSync(path.join(home, 'hive', '.gitignore'), 'utf8');
+ assert.ok(ignore.split('\n').includes('crashes/'), 'crashes/ must be gitignored');
+});
+
+test('a death is still recorded when there is no output to dump', async (t) => {
+ const { home, hive } = await freshHive(t);
+ hive.recordAgentExit('a1', { exitCode: 0, signal: 9, tail: '' });
+
+ const rows = readLog(home).filter((r) => r.kind === 'agent-exit');
+ assert.equal(rows.length, 1, 'losing the output must not lose the fact of the death');
+ assert.equal(rows[0].signal, 9);
+ assert.equal(rows[0].tailPath, null);
+});
diff --git a/test/agent-exit-signal-e2e.test.cjs b/test/agent-exit-signal-e2e.test.cjs
new file mode 100644
index 00000000..eb32f6ef
--- /dev/null
+++ b/test/agent-exit-signal-e2e.test.cjs
@@ -0,0 +1,98 @@
+'use strict';
+
+/**
+ * End-to-end: a PTY whose process dies by SIGNAL must reach the exit handler
+ * with that signal AND with the output it printed on the way out.
+ *
+ * The unit tests for `recordAgentExit` call it directly. This one drives the
+ * real `PtyManager`, because the original defect was NOT in the recording — it
+ * was that `signal` never left `pty.ts`. `onExit` had `{ exitCode, signal }` in
+ * hand and forwarded only `exitCode`, so every caller downstream was blind to
+ * signal deaths by construction.
+ *
+ * The scenario is the one observed live on 2026-08-24: a provider that prints
+ * a panic and dies of SIGILL, which node-pty reports as exitCode 0 + signal 4.
+ * An exitCode-only handler sees "0" and calls that a clean finish.
+ */
+
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const loadTs = require('./load-ts.cjs');
+
+const { PtyManager } = loadTs('src/main/pty.ts');
+
+const POSIX = process.platform !== 'win32';
+
+function waitForExit(mgr) {
+ return new Promise((resolve) => {
+ mgr.setExitHandler((id, exitCode, info) => resolve({ id, exitCode, info }));
+ });
+}
+
+test('a SIGILL death forwards the signal and the output that explains it', { skip: !POSIX }, async () => {
+ const mgr = new PtyManager();
+ const done = waitForExit(mgr);
+
+ // Print a recognisable banner, flush it, then kill ourselves with SIGILL —
+ // the exact shape of the crash that motivated this change.
+ const res = mgr.spawn({
+ id: 'e2e-sigill',
+ cwd: process.cwd(),
+ command: '/bin/sh',
+ args: ['-c', 'echo "panic(main thread): Illegal instruction"; sleep 0.2; kill -ILL $$']
+ });
+ assert.equal(res.ok, true, `spawn failed: ${res.error}`);
+
+ const { id, exitCode, info } = await done;
+ assert.equal(id, 'e2e-sigill');
+
+ // THE REGRESSION GUARD: signal must survive the trip out of pty.ts.
+ assert.equal(info.signal, 4, 'SIGILL (4) must be forwarded, not dropped');
+
+ // And the exit code alone must NOT be what a caller keys on — this asserts
+ // the very trap the old handler fell into.
+ assert.notEqual(
+ typeof info.signal === 'number' && info.signal !== 0 ? 'abnormal' : 'clean',
+ 'clean',
+ 'a signal death must not classify as clean'
+ );
+
+ assert.match(info.tail ?? '', /Illegal instruction/, 'the dying output must be captured');
+ assert.equal(typeof info.command, 'string', 'the command must be passed by value');
+ assert.ok((info.command ?? '').length > 0, 'the session is deleted before the handler, so command cannot be looked up later');
+});
+
+test('a clean exit reports no signal', { skip: !POSIX }, async () => {
+ const mgr = new PtyManager();
+ const done = waitForExit(mgr);
+
+ const res = mgr.spawn({
+ id: 'e2e-clean',
+ cwd: process.cwd(),
+ command: '/bin/sh',
+ args: ['-c', 'echo done; exit 0']
+ });
+ assert.equal(res.ok, true, `spawn failed: ${res.error}`);
+
+ const { exitCode, info } = await done;
+ assert.equal(exitCode, 0);
+ assert.ok(!info.signal, 'a clean exit must not report a signal');
+});
+
+test('a non-zero exit is reported as a code, not a signal', { skip: !POSIX }, async () => {
+ const mgr = new PtyManager();
+ const done = waitForExit(mgr);
+
+ const res = mgr.spawn({
+ id: 'e2e-code',
+ cwd: process.cwd(),
+ command: '/bin/sh',
+ args: ['-c', 'echo "command not found"; exit 127']
+ });
+ assert.equal(res.ok, true, `spawn failed: ${res.error}`);
+
+ const { exitCode, info } = await done;
+ assert.equal(exitCode, 127);
+ assert.ok(!info.signal);
+ assert.match(info.tail ?? '', /command not found/);
+});