Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

feat: capture and propagate agent start-up failure logs by keeping tmux panes open on exit

0xken155017d31fe8ac
diff --git a/cli/src/cli.ts b/cli/src/cli.ts
index 3206c4824..d061e8186 100644
--- a/cli/src/cli.ts
+++ b/cli/src/cli.ts
@@ -46,6 +46,9 @@ import { clearDeleted, isRecentlyDeleted, markDeleted } from './lib/deletedSessi
import { terminateDeletedAgent } from './lib/deleteAgentFallback.js'
import { findLiveSession } from './lib/sessionRepair.js'
import { TmuxBackend } from './lib/tmuxBackend.js'
+import { basename } from 'node:path'
+import { captureTmuxPane, clearPaneRemainOnExit, tmuxPaneState } from './lib/tmux.js'
+import { describeAgentCreateFailure, summarizePaneOutput } from './lib/agentCreateDiagnosis.js'
import { HerdrBackend } from './lib/herdrBackend.js'
import {
discoverRunningHerdrSessions,
@@ -2670,25 +2673,58 @@ async function runForeground(session: AuthSession): Promise<void> {
detail: spawned.reason,
}
}
+ const createdAt = Date.now()
for (const delayMs of [150, 400, 800]) {
await new Promise((resolve) => setTimeout(resolve, delayMs))
await agentReconciler.triggerHint(spawned.runtime, engine)
const session = registry.byRuntimeEngine(spawned.runtime, engine)
- if (session) return { ok: true, session }
+ if (session) {
+ // A real agent now: give the pane back tmux's normal disposal so it vanishes when the engine
+ // exits, like every organically started one does.
+ await clearPaneRemainOnExit(spawned.runtime.paneId)
+ return { ok: true, session }
+ }
}
+
+ // Read the pane BEFORE the availability probe below — that probe spends up to 5s in the user's
+ // login shell, which is long enough for a dying engine's pane to change under us.
+ const paneState = await tmuxPaneState(spawned.runtime.paneId)
+ const diagnosis = describeAgentCreateFailure({
+ state: paneState,
+ output: summarizePaneOutput(
+ await captureTmuxPane(spawned.runtime.paneId, 40, { ansi: false }) ?? '',
+ ),
+ engineBin: command[0],
+ shellName: (() => {
+ const path = interactiveEngineShell()?.path
+ return path ? basename(path) : null
+ })(),
+ elapsedMs: Date.now() - createdAt,
+ })
+ // A dead pane exists only because `create` asked tmux to keep it, and its output has now been
+ // read. Dispose of it so a failed create leaves nothing behind, as it did before. A pane that is
+ // still ALIVE is deliberately left alone: it is usually a slow engine that the periodic
+ // reconcile picks up seconds later, and killing it would destroy a working agent.
+ if (paneState?.dead) await tmuxBackend.kill(spawned.runtime)
+ else await clearPaneRemainOnExit(spawned.runtime.paneId)
// A detached daemon and an already-running tmux server do not necessarily have the user's terminal
// PATH. The launch above therefore used the interactive shell; inspect that same context before
// claiming a binary is absent. Checking process.env here was a false "not installed" report on
// macOS and Ubuntu whenever the CLI lived in .zshrc/.bashrc (nvm, asdf, vendor installers).
const bin = command[0]
+ // `detail` rides back to the New Agent dialog, which already renders it. That is the whole point:
+ // the machine that knows why is frequently not the machine the person is sitting at, and the
+ // error code alone sent them to a log file they cannot open.
if (!await commandAvailableInInteractiveShell(bin)) {
const shell = interactiveEngineShell()
- console.warn(`[agent] create ${engine} failed · "${bin}" is unavailable from the user's ${shell?.label ?? 'daemon PATH'}`)
- return { ok: false, error: 'ENGINE_NOT_INSTALLED' }
+ const detail = `"${bin}" is not on PATH in the user's ${shell?.label ?? 'daemon PATH'} · ${diagnosis}`
+ console.warn(`[agent] create ${engine} failed · ${detail}`)
+ return { ok: false, error: 'ENGINE_NOT_INSTALLED', detail }
}
const shell = interactiveEngineShell()
- console.warn(`[agent] create ${engine} failed · "${bin}" resolved from the user's ${shell?.label ?? 'daemon PATH'} but no engine process appeared in the pane`)
- return { ok: false, error: 'ENGINE_DID_NOT_START' }
+ const detail = `"${bin}" resolves from the user's ${shell?.label ?? 'daemon PATH'} · ${diagnosis}`
+ console.warn(`[agent] create ${engine} failed · ${detail}`)
+ return { ok: false, error: 'ENGINE_DID_NOT_START', detail }
}
/**
diff --git a/cli/src/lib/agentCreateDiagnosis.spec.ts b/cli/src/lib/agentCreateDiagnosis.spec.ts
new file mode 100644
index 000000000..70321f184
--- /dev/null
+++ b/cli/src/lib/agentCreateDiagnosis.spec.ts
@@ -0,0 +1,86 @@
+import { describe, expect, it } from 'vitest'
+import {
+ describeAgentCreateFailure,
+ summarizePaneOutput,
+ type AgentCreatePaneFacts,
+} from './agentCreateDiagnosis.js'
+
+const base: AgentCreatePaneFacts = {
+ state: { dead: false, exitStatus: null, command: 'zsh' },
+ output: '',
+ engineBin: 'codex',
+ shellName: 'zsh',
+ elapsedMs: 1400,
+}
+
+describe('summarizePaneOutput', () => {
+ it('keeps the tail, which is where an engine prints why it quit', () => {
+ expect(summarizePaneOutput('welcome\nloading\n\nnot logged in. Run codex login.\n'))
+ .toBe('welcome · loading · not logged in. Run codex login.')
+ })
+
+ it('drops the dead-pane notice tmux paints in, since that is our own artifact', () => {
+ const captured = 'codex: config error\nPane is dead (status 3, Fri Aug 28 11:33:39 2026)'
+ expect(summarizePaneOutput(captured)).toBe('codex: config error')
+ })
+
+ it('strips escape sequences so no control byte reaches a log line or the dialog', () => {
+ const captured = '\u001b[31mfatal:\u001b[0m no auth\u001b]0;title\u0007'
+ expect(summarizePaneOutput(captured)).toBe('fatal: no auth')
+ })
+
+ it('is empty when the pane said nothing readable', () => {
+ expect(summarizePaneOutput('\n\n \n')).toBe('')
+ })
+
+ it('bounds its length so the dialog stays readable', () => {
+ expect(summarizePaneOutput('x'.repeat(500)).length).toBeLessThanOrEqual(180)
+ })
+})
+
+describe('describeAgentCreateFailure', () => {
+ it('names the slow-login-shell race, the case the retry window actually loses', () => {
+ const detail = describeAgentCreateFailure(base)
+ expect(detail).toContain('still running "zsh" startup files')
+ expect(detail).toContain('"codex" had not launched yet')
+ expect(detail).toContain('may still appear on its own')
+ })
+
+ it('reports an engine that started and exited, with its status and its own words', () => {
+ const detail = describeAgentCreateFailure({
+ ...base,
+ state: { dead: true, exitStatus: 3, command: 'codex' },
+ output: 'not logged in. Run codex login.',
+ })
+ expect(detail).toBe('"codex" started and exited with status 3 after 1.4s'
+ + ' · pane said: not logged in. Run codex login.')
+ })
+
+ it('distinguishes a running-but-unrecognised process from a missing one', () => {
+ const detail = describeAgentCreateFailure({
+ ...base,
+ state: { dead: false, exitStatus: null, command: 'node' },
+ })
+ expect(detail).toContain('was running "node"')
+ expect(detail).toContain('not recognised as a codex agent')
+ })
+
+ it('says so when tmux did not keep the pane, since the output is then unrecoverable', () => {
+ const detail = describeAgentCreateFailure({ ...base, state: null })
+ expect(detail).toContain('tmux did not keep it')
+ expect(detail).toContain('left no output to read')
+ })
+
+ it('omits the output clause rather than printing an empty one', () => {
+ expect(describeAgentCreateFailure(base)).not.toContain('pane said')
+ })
+
+ it('bounds the whole detail so it cannot overflow the dialog', () => {
+ const detail = describeAgentCreateFailure({
+ ...base,
+ state: { dead: true, exitStatus: 1, command: 'codex' },
+ output: 'e'.repeat(400),
+ })
+ expect(detail.length).toBeLessThanOrEqual(340)
+ })
+})
diff --git a/cli/src/lib/agentCreateDiagnosis.ts b/cli/src/lib/agentCreateDiagnosis.ts
new file mode 100644
index 000000000..eaaca4760
--- /dev/null
+++ b/cli/src/lib/agentCreateDiagnosis.ts
@@ -0,0 +1,113 @@
+/**
+ * Explain a `New agent` that opened a pane but never produced a discoverable engine.
+ *
+ * `ENGINE_DID_NOT_START` on its own is unactionable at a distance. Reaching it already proves tmux
+ * opened the pane AND the engine binary resolves from the user's own login shell, which leaves three
+ * causes that need three different fixes: the engine ran and exited, the login shell is still
+ * starting (its dotfiles are slower than the discovery window), or a process IS running that
+ * discovery did not recognise. The only machine that can tell them apart is the one that failed —
+ * and its `harness.log` is exactly what a person reporting this from another computer cannot reach.
+ * So the facts travel back in the error itself, via the `detail` the New Agent dialog already renders.
+ */
+
+/** What tmux knows about the created pane at the moment discovery gave up. */
+export interface AgentCreatePaneState {
+ dead: boolean
+ /** Exit status of the pane's process once it is dead; null while it is still running. */
+ exitStatus: number | null
+ /** `#{pane_current_command}` — the shell while startup files are still running, the engine after. */
+ command: string
+}
+
+export interface AgentCreatePaneFacts {
+ /** null when tmux no longer knows this pane, i.e. it exited and took its session with it. */
+ state: AgentCreatePaneState | null
+ /** Whatever the pane printed, ANSI already stripped. Empty when nothing could be read. */
+ output: string
+ /** The executable the launch was meant to `exec`, e.g. `codex`. */
+ engineBin: string
+ /** Process name of the wrapper login shell, e.g. `zsh`. Null when no interactive shell was used. */
+ shellName: string | null
+ elapsedMs: number
+}
+
+const MAX_OUTPUT_CHARS = 180
+const MAX_DETAIL_CHARS = 340
+
+/** tmux paints this into a pane it is keeping alive under `remain-on-exit`; it is our own artifact. */
+const DEAD_PANE_NOTICE = /^Pane is dead \(.*\)$/
+
+/**
+ * The last few things the pane said, as one line.
+ *
+ * Errors print last, so the tail is read rather than the head — a full-screen TUI that merely failed
+ * to be *discovered* would otherwise contribute a screenful of chrome and push its own message out.
+ */
+export function summarizePaneOutput(raw: string): string {
+ const lines = raw
+ // `capture-pane` without `-e` still lets stray control bytes through (a TUI's own writes, an
+ // OSC title). Strip them so a detail string can never carry escape bytes into a log line or a
+ // Flutter Text widget. Written as explicit \u escapes: literal ESC bytes in source do not
+ // survive copy/paste and review.
+ .replace(/\u001b\][^\u0007\u001b]*(?:\u0007|\u001b\\)/g, '')
+ .replace(/\u001b\[[0-9;?]*[ -\/]*[@-~]/g, '')
+ .replace(/\u001b[@-_]/g, '')
+ .replace(/[\u0000-\u0008\u000b-\u001f\u007f]/g, '')
+ .split('\n')
+ .map((line) => line.trim())
+ .filter((line) => line.length > 0 && !DEAD_PANE_NOTICE.test(line))
+ if (lines.length === 0) return ''
+ const tail = lines.slice(-3).join(' · ')
+ return tail.length > MAX_OUTPUT_CHARS ? `${tail.slice(0, MAX_OUTPUT_CHARS - 1)}…` : tail
+}
+
+function seconds(elapsedMs: number): string {
+ return `${(elapsedMs / 1000).toFixed(1)}s`
+}
+
+function withOutput(summary: string, output: string): string {
+ const detail = output ? `${summary} · pane said: ${output}` : summary
+ return detail.length > MAX_DETAIL_CHARS ? `${detail.slice(0, MAX_DETAIL_CHARS - 1)}…` : detail
+}
+
+/**
+ * One sentence naming which of the three causes happened, plus the pane's own words.
+ *
+ * Deliberately phrased for someone reading a dialog on a different computer than the one that
+ * failed: it says what the pane was doing, not what the code checked.
+ */
+export function describeAgentCreateFailure(facts: AgentCreatePaneFacts): string {
+ const { state, output, engineBin, shellName, elapsedMs } = facts
+ const waited = seconds(elapsedMs)
+
+ // tmux forgot the pane entirely. `create` asks tmux to keep dead panes, so reaching this means the
+ // option did not take (a tmux too old for it) and the process still exited — the output is gone
+ // with the session, which is itself the finding.
+ if (!state) {
+ return withOutput(
+ `the pane exited within ${waited} and tmux did not keep it, so "${engineBin}" left no output to read`,
+ output,
+ )
+ }
+
+ if (state.dead) {
+ const status = state.exitStatus === null ? '' : ` with status ${state.exitStatus}`
+ return withOutput(`"${engineBin}" started and exited${status} after ${waited}`, output)
+ }
+
+ // Still the login shell: the dotfiles (nvm, oh-my-zsh, asdf, corporate profiles) had not finished
+ // by the time discovery gave up, so `exec` had not replaced the shell with the engine yet. The
+ // agent normally still appears a few seconds later, once the periodic reconcile runs.
+ if (shellName && state.command === shellName) {
+ return withOutput(
+ `after ${waited} the pane was still running "${shellName}" startup files, so "${engineBin}" had not launched yet`
+ + ' — the agent may still appear on its own shortly',
+ output,
+ )
+ }
+
+ return withOutput(
+ `after ${waited} the pane was running "${state.command}" but it was not recognised as a ${engineBin} agent`,
+ output,
+ )
+}
diff --git a/cli/src/lib/tmux.ts b/cli/src/lib/tmux.ts
index d251243e3..37316b22f 100644
--- a/cli/src/lib/tmux.ts
+++ b/cli/src/lib/tmux.ts
@@ -609,6 +609,51 @@ export function setPaneMouseOn(pane: string): Promise<void> {
})
}
+/** What tmux knows about a pane right now. See `agentCreateDiagnosis.ts` for why this is read. */
+export interface TmuxPaneState {
+ dead: boolean
+ /** Exit status once the process is gone; null while it is still running. */
+ exitStatus: number | null
+ command: string
+}
+
+/**
+ * Pane liveness plus, for a pane kept alive by `remain-on-exit`, how its process ended.
+ *
+ * Returns null when tmux does not know the pane — which is itself the answer: the process exited and
+ * took its window (and, for a one-pane session, the session) with it.
+ */
+export function tmuxPaneState(pane: string): Promise<TmuxPaneState | null> {
+ return new Promise((resolve) => {
+ const format = '#{pane_dead}|#{pane_dead_status}|#{pane_current_command}'
+ execFile('tmux', ['display-message', '-p', '-t', pane, format], { timeout: 2_000 }, (err, stdout) => {
+ if (err) { resolve(null); return }
+ const fields = stdout.trim().split('|')
+ if (fields.length < 3) { resolve(null); return }
+ const status = Number(fields[1])
+ resolve({
+ dead: fields[0] === '1',
+ exitStatus: fields[1] !== '' && Number.isSafeInteger(status) ? status : null,
+ // A command name cannot contain `|`, but rejoining costs nothing and keeps a surprising
+ // one from silently truncating the field.
+ command: fields.slice(2).join('|'),
+ })
+ })
+ })
+}
+
+/**
+ * Hand a pane back to tmux's default disposal after `create()` asked tmux to keep it when dead.
+ *
+ * Called as soon as a created pane becomes a real agent: from then on it must vanish when its engine
+ * exits, exactly like a pane the user started themselves, rather than lingering as a dead pane.
+ */
+export function clearPaneRemainOnExit(pane: string): Promise<void> {
+ return new Promise((resolve) => {
+ execFile('tmux', ['set-option', '-w', '-t', pane, 'remain-on-exit', 'off'], { timeout: 2_000 }, () => resolve())
+ })
+}
+
function tmuxDeleteBuffer(name: string): Promise<void> {
return new Promise((resolve) => {
execFile('tmux', ['delete-buffer', '-b', name], { timeout: 2_000 }, () => resolve())
diff --git a/cli/src/lib/tmuxBackend.spec.ts b/cli/src/lib/tmuxBackend.spec.ts
index 448b90ccc..df1d1476f 100644
--- a/cli/src/lib/tmuxBackend.spec.ts
+++ b/cli/src/lib/tmuxBackend.spec.ts
@@ -38,7 +38,10 @@ esac
if (created.state !== 'succeeded') return
await expect(backend.kill(created.runtime)).resolves.toEqual({ state: 'succeeded', dispatch: 'executed' })
expect(readFileSync(calls, 'utf8').trim().split('\n')).toEqual([
- 'new-session -d -P -F #{pane_id} -c /tmp/work -s harness-test',
+ // `remain-on-exit` is chained into the SAME invocation, not sent after it: an engine that
+ // exits immediately would otherwise take its session down before a follow-up call landed,
+ // and its error text with it.
+ 'new-session -d -P -F #{pane_id} -c /tmp/work -s harness-test ; set-option -w remain-on-exit on',
'set-option -t %42 mouse on',
'display-message -p -t %42 #{session_id}',
'kill-session -t $7',
diff --git a/cli/src/lib/tmuxBackend.ts b/cli/src/lib/tmuxBackend.ts
index e09bd6340..c5db68663 100644
--- a/cli/src/lib/tmuxBackend.ts
+++ b/cli/src/lib/tmuxBackend.ts
@@ -79,6 +79,13 @@ export class TmuxBackend implements TerminalBackend<TmuxRuntimeRef> {
// Trailing args after this point become the session's shell-command. tmux execs them directly
// (no shell interposed) when given as separate argv elements, so no quoting/escaping is needed.
if (request.command?.length) args.push(...request.command)
+ // Keep the pane when its process dies, so an engine that exits immediately (not logged in, bad
+ // config) still has its error text readable afterwards instead of taking the whole session down
+ // with it. Chained into THIS tmux invocation on purpose: an engine can exit in under a
+ // millisecond, and a second `set-option` call loses that race — measured, the session was
+ // already gone before the follow-up command could reach the server.
+ // Whoever created the pane owns turning this back off; see `clearPaneRemainOnExit`.
+ args.push(';', 'set-option', '-w', 'remain-on-exit', 'on')
// `killed`/`signal` come from execFile's own error shape, which ErrnoException alone does not declare.
type ExecError = NodeJS.ErrnoException & { killed?: boolean; signal?: NodeJS.Signals | null }
const result = await new Promise<{ error: ExecError | null; stdout: string; stderr: string }>((resolve) => {