The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
feat: capture and propagate agent start-up failure logs by keeping tmux panes open on exit
0xken155017d31fe8ac
diff --git a/cli/src/cli.ts b/cli/src/cli.tsindex 3206c4824..d061e8186 100644--- a/cli/src/cli.ts+++ b/cli/src/cli.ts@@ -46,6 +46,9 @@ import { clearDeleted, isRecentlyDeleted, markDeleted } from './lib/deletedSessiimport { terminateDeletedAgent } from './lib/deleteAgentFallback.js'import { findLiveSession } from './lib/sessionRepair.js'import { TmuxBackend } from './lib/tmuxBackend.js'+import { basename } from 'node:path'+import { captureTmuxPane, clearPaneRemainOnExit, tmuxPaneState } from './lib/tmux.js'+import { describeAgentCreateFailure, summarizePaneOutput } from './lib/agentCreateDiagnosis.js'import { HerdrBackend } from './lib/herdrBackend.js'import {discoverRunningHerdrSessions,@@ -2670,25 +2673,58 @@ async function runForeground(session: AuthSession): Promise<void> {detail: spawned.reason,}}+ const createdAt = Date.now()for (const delayMs of [150, 400, 800]) {await new Promise((resolve) => setTimeout(resolve, delayMs))await agentReconciler.triggerHint(spawned.runtime, engine)const session = registry.byRuntimeEngine(spawned.runtime, engine)- if (session) return { ok: true, session }+ if (session) {+ // A real agent now: give the pane back tmux's normal disposal so it vanishes when the engine+ // exits, like every organically started one does.+ await clearPaneRemainOnExit(spawned.runtime.paneId)+ return { ok: true, session }+ }}++ // Read the pane BEFORE the availability probe below — that probe spends up to 5s in the user's+ // login shell, which is long enough for a dying engine's pane to change under us.+ const paneState = await tmuxPaneState(spawned.runtime.paneId)+ const diagnosis = describeAgentCreateFailure({+ state: paneState,+ output: summarizePaneOutput(+ await captureTmuxPane(spawned.runtime.paneId, 40, { ansi: false }) ?? '',+ ),+ engineBin: command[0],+ shellName: (() => {+ const path = interactiveEngineShell()?.path+ return path ? basename(path) : null+ })(),+ elapsedMs: Date.now() - createdAt,+ })+ // A dead pane exists only because `create` asked tmux to keep it, and its output has now been+ // read. Dispose of it so a failed create leaves nothing behind, as it did before. A pane that is+ // still ALIVE is deliberately left alone: it is usually a slow engine that the periodic+ // reconcile picks up seconds later, and killing it would destroy a working agent.+ if (paneState?.dead) await tmuxBackend.kill(spawned.runtime)+ else await clearPaneRemainOnExit(spawned.runtime.paneId)// A detached daemon and an already-running tmux server do not necessarily have the user's terminal// PATH. The launch above therefore used the interactive shell; inspect that same context before// claiming a binary is absent. Checking process.env here was a false "not installed" report on// macOS and Ubuntu whenever the CLI lived in .zshrc/.bashrc (nvm, asdf, vendor installers).const bin = command[0]+ // `detail` rides back to the New Agent dialog, which already renders it. That is the whole point:+ // the machine that knows why is frequently not the machine the person is sitting at, and the+ // error code alone sent them to a log file they cannot open.if (!await commandAvailableInInteractiveShell(bin)) {const shell = interactiveEngineShell()- console.warn(`[agent] create ${engine} failed · "${bin}" is unavailable from the user's ${shell?.label ?? 'daemon PATH'}`)- return { ok: false, error: 'ENGINE_NOT_INSTALLED' }+ const detail = `"${bin}" is not on PATH in the user's ${shell?.label ?? 'daemon PATH'} · ${diagnosis}`+ console.warn(`[agent] create ${engine} failed · ${detail}`)+ return { ok: false, error: 'ENGINE_NOT_INSTALLED', detail }}const shell = interactiveEngineShell()- console.warn(`[agent] create ${engine} failed · "${bin}" resolved from the user's ${shell?.label ?? 'daemon PATH'} but no engine process appeared in the pane`)- return { ok: false, error: 'ENGINE_DID_NOT_START' }+ const detail = `"${bin}" resolves from the user's ${shell?.label ?? 'daemon PATH'} · ${diagnosis}`+ console.warn(`[agent] create ${engine} failed · ${detail}`)+ return { ok: false, error: 'ENGINE_DID_NOT_START', detail }}/**diff --git a/cli/src/lib/agentCreateDiagnosis.spec.ts b/cli/src/lib/agentCreateDiagnosis.spec.tsnew file mode 100644index 000000000..70321f184--- /dev/null+++ b/cli/src/lib/agentCreateDiagnosis.spec.ts@@ -0,0 +1,86 @@+import { describe, expect, it } from 'vitest'+import {+ describeAgentCreateFailure,+ summarizePaneOutput,+ type AgentCreatePaneFacts,+} from './agentCreateDiagnosis.js'++const base: AgentCreatePaneFacts = {+ state: { dead: false, exitStatus: null, command: 'zsh' },+ output: '',+ engineBin: 'codex',+ shellName: 'zsh',+ elapsedMs: 1400,+}++describe('summarizePaneOutput', () => {+ it('keeps the tail, which is where an engine prints why it quit', () => {+ expect(summarizePaneOutput('welcome\nloading\n\nnot logged in. Run codex login.\n'))+ .toBe('welcome · loading · not logged in. Run codex login.')+ })++ it('drops the dead-pane notice tmux paints in, since that is our own artifact', () => {+ const captured = 'codex: config error\nPane is dead (status 3, Fri Aug 28 11:33:39 2026)'+ expect(summarizePaneOutput(captured)).toBe('codex: config error')+ })++ it('strips escape sequences so no control byte reaches a log line or the dialog', () => {+ const captured = '\u001b[31mfatal:\u001b[0m no auth\u001b]0;title\u0007'+ expect(summarizePaneOutput(captured)).toBe('fatal: no auth')+ })++ it('is empty when the pane said nothing readable', () => {+ expect(summarizePaneOutput('\n\n \n')).toBe('')+ })++ it('bounds its length so the dialog stays readable', () => {+ expect(summarizePaneOutput('x'.repeat(500)).length).toBeLessThanOrEqual(180)+ })+})++describe('describeAgentCreateFailure', () => {+ it('names the slow-login-shell race, the case the retry window actually loses', () => {+ const detail = describeAgentCreateFailure(base)+ expect(detail).toContain('still running "zsh" startup files')+ expect(detail).toContain('"codex" had not launched yet')+ expect(detail).toContain('may still appear on its own')+ })++ it('reports an engine that started and exited, with its status and its own words', () => {+ const detail = describeAgentCreateFailure({+ ...base,+ state: { dead: true, exitStatus: 3, command: 'codex' },+ output: 'not logged in. Run codex login.',+ })+ expect(detail).toBe('"codex" started and exited with status 3 after 1.4s'+ + ' · pane said: not logged in. Run codex login.')+ })++ it('distinguishes a running-but-unrecognised process from a missing one', () => {+ const detail = describeAgentCreateFailure({+ ...base,+ state: { dead: false, exitStatus: null, command: 'node' },+ })+ expect(detail).toContain('was running "node"')+ expect(detail).toContain('not recognised as a codex agent')+ })++ it('says so when tmux did not keep the pane, since the output is then unrecoverable', () => {+ const detail = describeAgentCreateFailure({ ...base, state: null })+ expect(detail).toContain('tmux did not keep it')+ expect(detail).toContain('left no output to read')+ })++ it('omits the output clause rather than printing an empty one', () => {+ expect(describeAgentCreateFailure(base)).not.toContain('pane said')+ })++ it('bounds the whole detail so it cannot overflow the dialog', () => {+ const detail = describeAgentCreateFailure({+ ...base,+ state: { dead: true, exitStatus: 1, command: 'codex' },+ output: 'e'.repeat(400),+ })+ expect(detail.length).toBeLessThanOrEqual(340)+ })+})diff --git a/cli/src/lib/agentCreateDiagnosis.ts b/cli/src/lib/agentCreateDiagnosis.tsnew file mode 100644index 000000000..eaaca4760--- /dev/null+++ b/cli/src/lib/agentCreateDiagnosis.ts@@ -0,0 +1,113 @@+/**+ * Explain a `New agent` that opened a pane but never produced a discoverable engine.+ *+ * `ENGINE_DID_NOT_START` on its own is unactionable at a distance. Reaching it already proves tmux+ * opened the pane AND the engine binary resolves from the user's own login shell, which leaves three+ * causes that need three different fixes: the engine ran and exited, the login shell is still+ * starting (its dotfiles are slower than the discovery window), or a process IS running that+ * discovery did not recognise. The only machine that can tell them apart is the one that failed —+ * and its `harness.log` is exactly what a person reporting this from another computer cannot reach.+ * So the facts travel back in the error itself, via the `detail` the New Agent dialog already renders.+ */++/** What tmux knows about the created pane at the moment discovery gave up. */+export interface AgentCreatePaneState {+ dead: boolean+ /** Exit status of the pane's process once it is dead; null while it is still running. */+ exitStatus: number | null+ /** `#{pane_current_command}` — the shell while startup files are still running, the engine after. */+ command: string+}++export interface AgentCreatePaneFacts {+ /** null when tmux no longer knows this pane, i.e. it exited and took its session with it. */+ state: AgentCreatePaneState | null+ /** Whatever the pane printed, ANSI already stripped. Empty when nothing could be read. */+ output: string+ /** The executable the launch was meant to `exec`, e.g. `codex`. */+ engineBin: string+ /** Process name of the wrapper login shell, e.g. `zsh`. Null when no interactive shell was used. */+ shellName: string | null+ elapsedMs: number+}++const MAX_OUTPUT_CHARS = 180+const MAX_DETAIL_CHARS = 340++/** tmux paints this into a pane it is keeping alive under `remain-on-exit`; it is our own artifact. */+const DEAD_PANE_NOTICE = /^Pane is dead \(.*\)$/++/**+ * The last few things the pane said, as one line.+ *+ * Errors print last, so the tail is read rather than the head — a full-screen TUI that merely failed+ * to be *discovered* would otherwise contribute a screenful of chrome and push its own message out.+ */+export function summarizePaneOutput(raw: string): string {+ const lines = raw+ // `capture-pane` without `-e` still lets stray control bytes through (a TUI's own writes, an+ // OSC title). Strip them so a detail string can never carry escape bytes into a log line or a+ // Flutter Text widget. Written as explicit \u escapes: literal ESC bytes in source do not+ // survive copy/paste and review.+ .replace(/\u001b\][^\u0007\u001b]*(?:\u0007|\u001b\\)/g, '')+ .replace(/\u001b\[[0-9;?]*[ -\/]*[@-~]/g, '')+ .replace(/\u001b[@-_]/g, '')+ .replace(/[\u0000-\u0008\u000b-\u001f\u007f]/g, '')+ .split('\n')+ .map((line) => line.trim())+ .filter((line) => line.length > 0 && !DEAD_PANE_NOTICE.test(line))+ if (lines.length === 0) return ''+ const tail = lines.slice(-3).join(' · ')+ return tail.length > MAX_OUTPUT_CHARS ? `${tail.slice(0, MAX_OUTPUT_CHARS - 1)}…` : tail+}++function seconds(elapsedMs: number): string {+ return `${(elapsedMs / 1000).toFixed(1)}s`+}++function withOutput(summary: string, output: string): string {+ const detail = output ? `${summary} · pane said: ${output}` : summary+ return detail.length > MAX_DETAIL_CHARS ? `${detail.slice(0, MAX_DETAIL_CHARS - 1)}…` : detail+}++/**+ * One sentence naming which of the three causes happened, plus the pane's own words.+ *+ * Deliberately phrased for someone reading a dialog on a different computer than the one that+ * failed: it says what the pane was doing, not what the code checked.+ */+export function describeAgentCreateFailure(facts: AgentCreatePaneFacts): string {+ const { state, output, engineBin, shellName, elapsedMs } = facts+ const waited = seconds(elapsedMs)++ // tmux forgot the pane entirely. `create` asks tmux to keep dead panes, so reaching this means the+ // option did not take (a tmux too old for it) and the process still exited — the output is gone+ // with the session, which is itself the finding.+ if (!state) {+ return withOutput(+ `the pane exited within ${waited} and tmux did not keep it, so "${engineBin}" left no output to read`,+ output,+ )+ }++ if (state.dead) {+ const status = state.exitStatus === null ? '' : ` with status ${state.exitStatus}`+ return withOutput(`"${engineBin}" started and exited${status} after ${waited}`, output)+ }++ // Still the login shell: the dotfiles (nvm, oh-my-zsh, asdf, corporate profiles) had not finished+ // by the time discovery gave up, so `exec` had not replaced the shell with the engine yet. The+ // agent normally still appears a few seconds later, once the periodic reconcile runs.+ if (shellName && state.command === shellName) {+ return withOutput(+ `after ${waited} the pane was still running "${shellName}" startup files, so "${engineBin}" had not launched yet`+ + ' — the agent may still appear on its own shortly',+ output,+ )+ }++ return withOutput(+ `after ${waited} the pane was running "${state.command}" but it was not recognised as a ${engineBin} agent`,+ output,+ )+}diff --git a/cli/src/lib/tmux.ts b/cli/src/lib/tmux.tsindex d251243e3..37316b22f 100644--- a/cli/src/lib/tmux.ts+++ b/cli/src/lib/tmux.ts@@ -609,6 +609,51 @@ export function setPaneMouseOn(pane: string): Promise<void> {})}+/** What tmux knows about a pane right now. See `agentCreateDiagnosis.ts` for why this is read. */+export interface TmuxPaneState {+ dead: boolean+ /** Exit status once the process is gone; null while it is still running. */+ exitStatus: number | null+ command: string+}++/**+ * Pane liveness plus, for a pane kept alive by `remain-on-exit`, how its process ended.+ *+ * Returns null when tmux does not know the pane — which is itself the answer: the process exited and+ * took its window (and, for a one-pane session, the session) with it.+ */+export function tmuxPaneState(pane: string): Promise<TmuxPaneState | null> {+ return new Promise((resolve) => {+ const format = '#{pane_dead}|#{pane_dead_status}|#{pane_current_command}'+ execFile('tmux', ['display-message', '-p', '-t', pane, format], { timeout: 2_000 }, (err, stdout) => {+ if (err) { resolve(null); return }+ const fields = stdout.trim().split('|')+ if (fields.length < 3) { resolve(null); return }+ const status = Number(fields[1])+ resolve({+ dead: fields[0] === '1',+ exitStatus: fields[1] !== '' && Number.isSafeInteger(status) ? status : null,+ // A command name cannot contain `|`, but rejoining costs nothing and keeps a surprising+ // one from silently truncating the field.+ command: fields.slice(2).join('|'),+ })+ })+ })+}++/**+ * Hand a pane back to tmux's default disposal after `create()` asked tmux to keep it when dead.+ *+ * Called as soon as a created pane becomes a real agent: from then on it must vanish when its engine+ * exits, exactly like a pane the user started themselves, rather than lingering as a dead pane.+ */+export function clearPaneRemainOnExit(pane: string): Promise<void> {+ return new Promise((resolve) => {+ execFile('tmux', ['set-option', '-w', '-t', pane, 'remain-on-exit', 'off'], { timeout: 2_000 }, () => resolve())+ })+}+function tmuxDeleteBuffer(name: string): Promise<void> {return new Promise((resolve) => {execFile('tmux', ['delete-buffer', '-b', name], { timeout: 2_000 }, () => resolve())diff --git a/cli/src/lib/tmuxBackend.spec.ts b/cli/src/lib/tmuxBackend.spec.tsindex 448b90ccc..df1d1476f 100644--- a/cli/src/lib/tmuxBackend.spec.ts+++ b/cli/src/lib/tmuxBackend.spec.ts@@ -38,7 +38,10 @@ esacif (created.state !== 'succeeded') returnawait expect(backend.kill(created.runtime)).resolves.toEqual({ state: 'succeeded', dispatch: 'executed' })expect(readFileSync(calls, 'utf8').trim().split('\n')).toEqual([- 'new-session -d -P -F #{pane_id} -c /tmp/work -s harness-test',+ // `remain-on-exit` is chained into the SAME invocation, not sent after it: an engine that+ // exits immediately would otherwise take its session down before a follow-up call landed,+ // and its error text with it.+ 'new-session -d -P -F #{pane_id} -c /tmp/work -s harness-test ; set-option -w remain-on-exit on','set-option -t %42 mouse on','display-message -p -t %42 #{session_id}','kill-session -t $7',diff --git a/cli/src/lib/tmuxBackend.ts b/cli/src/lib/tmuxBackend.tsindex e09bd6340..c5db68663 100644--- a/cli/src/lib/tmuxBackend.ts+++ b/cli/src/lib/tmuxBackend.ts@@ -79,6 +79,13 @@ export class TmuxBackend implements TerminalBackend<TmuxRuntimeRef> {// Trailing args after this point become the session's shell-command. tmux execs them directly// (no shell interposed) when given as separate argv elements, so no quoting/escaping is needed.if (request.command?.length) args.push(...request.command)+ // Keep the pane when its process dies, so an engine that exits immediately (not logged in, bad+ // config) still has its error text readable afterwards instead of taking the whole session down+ // with it. Chained into THIS tmux invocation on purpose: an engine can exit in under a+ // millisecond, and a second `set-option` call loses that race — measured, the session was+ // already gone before the follow-up command could reach the server.+ // Whoever created the pane owns turning this back off; see `clearPaneRemainOnExit`.+ args.push(';', 'set-option', '-w', 'remain-on-exit', 'on')// `killed`/`signal` come from execFile's own error shape, which ErrnoException alone does not declare.type ExecError = NodeJS.ErrnoException & { killed?: boolean; signal?: NodeJS.Signals | null }const result = await new Promise<{ error: ExecError | null; stdout: string; stderr: string }>((resolve) => {