Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

fix(cli): retry Codex account-routing startup timeouts (#559)

Deeee2f0fcf2707
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index cfc587af3..d53d53492 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -155,6 +155,11 @@ jobs:
env:
RUN_REAL_TMUX_STREAM: '1'
run: npx vitest run src/lib/tmuxStream.real.spec.ts
+ - name: Codex startup timeout recovery and cancellation
+ working-directory: cli
+ env:
+ RUN_REAL_TMUX_CODEX_STARTUP: '1'
+ run: npx vitest run src/lib/codexStartupRetry.real.spec.ts --maxWorkers=1
- name: Release unit tests
run: cargo test --locked --release
- name: Build static Linux binary
diff --git a/cli/src/lib/codexStartupRetry.real.spec.ts b/cli/src/lib/codexStartupRetry.real.spec.ts
new file mode 100644
index 000000000..bee6493ed
--- /dev/null
+++ b/cli/src/lib/codexStartupRetry.real.spec.ts
@@ -0,0 +1,100 @@
+import { execFile } from 'node:child_process'
+import { randomUUID } from 'node:crypto'
+import { mkdtemp, readFile, realpath, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { promisify } from 'node:util'
+import { afterEach, describe, expect, it } from 'vitest'
+import { resolveBinaryOnPath } from './binaryOnPath.js'
+import { buildEngineLaunchArgv } from './engineLaunch.js'
+
+const exec = promisify(execFile)
+const real = process.env.RUN_REAL_TMUX_CODEX_STARTUP === '1' ? describe : describe.skip
+const timeout = 'Error: account/read failed during TUI bootstrap: account/read failed: workspace routing discovery timed out (code -32603)'
+
+real('Codex startup recovery in an isolated real tmux server', () => {
+ const socket = `harness-startup-test-${randomUUID()}`
+ const tmuxBin = resolveBinaryOnPath('tmux')!
+ const dirs: string[] = []
+ const tmux = async (...args: string[]) => (await exec(tmuxBin, ['-L', socket, '-f', '/dev/null', ...args], { timeout: 3000 })).stdout.trim()
+ afterEach(async () => {
+ await tmux('kill-server').catch(() => {})
+ for (const dir of dirs.splice(0)) await rm(dir, { recursive: true, force: true })
+ })
+
+ async function eventually(check: () => Promise<boolean>, ms = 12_000) {
+ const deadline = Date.now() + ms
+ while (Date.now() < deadline) {
+ if (await check()) return
+ await new Promise(resolve => setTimeout(resolve, 30))
+ }
+ throw new Error(`condition not met; pane output: ${await tmux('capture-pane', '-p', '-J', '-t', '%0').catch(() => '')}`)
+ }
+
+ async function launch(failures: number, prompt?: string, exitStatus = 1, differentSecondError = false) {
+ const dir = await mkdtemp(join(tmpdir(), 'harness-codex-startup-real-'))
+ dirs.push(dir)
+ const state = join(dir, 'attempts.json')
+ await writeFile(state, '[]')
+ // A faithful terminal boundary: the engine receives three real TTYs and
+ // prints Codex's observed fatal error directly to stderr. No real account.
+ await writeFile(join(dir, 'codex'), `#!${process.execPath}\n`
+ + `const fs = require('node:fs');\n`
+ + `if (process.argv.includes('--help')) { console.log('--no-daemon'); process.exit(0); }\n`
+ + `const file = ${JSON.stringify(state)}; const attempts = JSON.parse(fs.readFileSync(file, 'utf8'));\n`
+ + `attempts.push({ args: process.argv.slice(2), tty: [process.stdin.isTTY, process.stdout.isTTY, process.stderr.isTTY], cwd: process.cwd() });\n`
+ + `fs.writeFileSync(file, JSON.stringify(attempts));\n`
+ + `if (attempts.length <= ${failures}) { console.error(${differentSecondError ? `attempts.length === 1 ? ${JSON.stringify(timeout)} : 'Error: invalid configuration'` : JSON.stringify(timeout)}); process.exitCode = ${exitStatus}; }\n`
+ + `else { console.log('STARTUP_RECOVERED'); process.stdin.resume(); }\n`, { mode: 0o755 })
+ const command = buildEngineLaunchArgv('codex', { cwd: dir, firstPrompt: prompt, permissionMode: 'auto' }, '/bin/sh', undefined, 'grid', tmuxBin)
+ // An interactive rc file may replace PATH. Never let a fixture resolve to
+ // the developer's installed Codex, even when their shell does that.
+ command[command.indexOf('codex')] = join(dir, 'codex')
+ const pane = await tmux('new-session', '-d', '-P', '-F', '#{pane_id}', '-x', '40', '-y', '12', '-s', 'test', '-e', `PATH=${dir}:${process.env.PATH}`, ...command)
+ const attempts = async () => JSON.parse(await readFile(state, 'utf8')) as Array<{ args: string[]; tty: boolean[]; cwd: string }>
+ const screen = () => tmux('capture-pane', '-p', '-J', '-t', pane)
+ const exit = () => tmux('show-option', '-p', '-v', '-t', pane, '@harness_engine_exit').catch(() => '')
+ return { pane, attempts, screen, exit, dir }
+ }
+
+ it.each([undefined, 'Fix spaces, "quotes", $(printf literal), `false`, ; & ! ^\nand a second line'])('recovers the same launch with first prompt %s', async prompt => {
+ const f = await launch(1, prompt)
+ await eventually(async () => (await f.screen()).includes('STARTUP_RECOVERED'))
+ const attempts = await f.attempts()
+ expect(attempts).toHaveLength(2)
+ for (const attempt of attempts) {
+ expect(attempt.args).toEqual(['--no-daemon', '--approve-for-me', ...(prompt ? [prompt] : [])])
+ expect(attempt.tty).toEqual([true, true, true])
+ expect(attempt.cwd).toBe(await realpath(f.dir))
+ }
+ expect(await f.exit()).toBe('')
+ }, 20_000)
+
+ it('stops after three failed attempts and returns the final status to the shell', async () => {
+ const f = await launch(100)
+ await eventually(async () => await f.exit() === '1')
+ expect(await f.attempts()).toHaveLength(3)
+ expect(await f.screen()).toContain('This pane is a shell now')
+ }, 20_000)
+
+ it('does not retry other errors after a transient first failure', async () => {
+ const f = await launch(100, undefined, 1, true)
+ await eventually(async () => await f.exit() === '1')
+ expect(await f.attempts()).toHaveLength(2)
+ }, 20_000)
+
+ it('does not retry a cancelled engine, even if timeout text remains visible', async () => {
+ const f = await launch(100, undefined, 130)
+ await eventually(async () => await f.exit() === '130')
+ expect(await f.attempts()).toHaveLength(1)
+ })
+
+ it('honors Ctrl-C during retry backoff', async () => {
+ const f = await launch(100)
+ await eventually(async () => (await f.screen()).includes('Retrying startup (2/3)'))
+ await tmux('send-keys', '-t', f.pane, 'C-c')
+ await eventually(async () => await f.exit() === '130')
+ expect(await f.attempts()).toHaveLength(1)
+ expect(await f.screen()).toContain('This pane is a shell now')
+ }, 15_000)
+})
diff --git a/cli/src/lib/codexStartupRetry.spec.ts b/cli/src/lib/codexStartupRetry.spec.ts
new file mode 100644
index 000000000..f1a0d056f
--- /dev/null
+++ b/cli/src/lib/codexStartupRetry.spec.ts
@@ -0,0 +1,65 @@
+import { execFileSync } from 'node:child_process'
+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterEach, describe, expect, it } from 'vitest'
+import { CODEX_STARTUP_RETRY_PROBE } from './codexStartupRetry.js'
+
+const timeout = 'Error: account/read failed during TUI bootstrap: account/read failed: workspace routing discovery timed out (code -32603)'
+const dirs: string[] = []
+afterEach(() => { for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }) })
+
+function fixture() {
+ const dir = mkdtempSync(join(tmpdir(), 'codex-startup-probe-'))
+ dirs.push(dir)
+ const screen = join(dir, 'screen')
+ const tmux = join(dir, 'tmux')
+ writeFileSync(screen, 'existing terminal output\n')
+ writeFileSync(tmux, '#!/bin/sh\ncat "$HARNESS_TEST_SCREEN"\n', { mode: 0o755 })
+ const probe = (mode: string, baseline = '', pane = '%7') => {
+ try {
+ return { status: 0, out: execFileSync(process.execPath, ['-e', CODEX_STARTUP_RETRY_PROBE, mode, tmux, pane, baseline], {
+ env: { ...process.env, HARNESS_TEST_SCREEN: screen }, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'],
+ }) }
+ } catch (error) { return { status: (error as { status: number }).status, out: '' } }
+ }
+ return { screen, tmux, probe }
+}
+
+describe('Codex startup timeout evidence', () => {
+ it('accepts only a newly printed, final bootstrap timeout', () => {
+ const f = fixture()
+ const before = f.probe('before').out
+ expect(before).not.toContain('existing terminal output')
+ writeFileSync(f.screen, `${timeout}\n\n\n`)
+ expect(f.probe('after', before).status).toBe(0)
+ // The same text already on screen before a later launch is not new evidence.
+ expect(f.probe('after', f.probe('before').out).status).toBe(1)
+ })
+
+ it.each([
+ 'Error: account/read failed during TUI bootstrap: workspace routing discovery unauthorized (401)',
+ 'Error: invalid config.toml',
+ 'Error: stream disconnected before completion: request timed out',
+ `${timeout}\n› Ask Codex to do anything`,
+ `quoted output: ${timeout}`,
+ '',
+ ])('does not replay a launch after unrelated output: %s', (screen) => {
+ const f = fixture()
+ const before = f.probe('before').out
+ writeFileSync(f.screen, screen)
+ expect(f.probe('after', before).status).toBe(1)
+ })
+
+ it('rejects old, invalid or unavailable evidence', () => {
+ const f = fixture()
+ const before = JSON.parse(f.probe('before').out)
+ writeFileSync(f.screen, timeout)
+ expect(f.probe('after', JSON.stringify({ ...before, at: Date.now() - 31_000 })).status).toBe(1)
+ expect(f.probe('after', JSON.stringify({ ...before, at: Date.now() + 10_000 })).status).toBe(1)
+ expect(f.probe('after', 'broken baseline').status).toBe(1)
+ expect(f.probe('before', '', 'some-session').status).toBe(1)
+ rmSync(f.tmux)
+ expect(f.probe('after', JSON.stringify(before)).status).toBe(1)
+ })
+})
diff --git a/cli/src/lib/codexStartupRetry.ts b/cli/src/lib/codexStartupRetry.ts
new file mode 100644
index 000000000..b43a9f275
--- /dev/null
+++ b/cli/src/lib/codexStartupRetry.ts
@@ -0,0 +1,34 @@
+/**
+ * Run inside the pane's shell, before/after a Codex process, using the daemon's Node.
+ *
+ * Codex 0.159.3 exits 1 when account/read's workspace-routing discovery times out,
+ * before thread/start or the first prompt. Retrying that exact bootstrap failure is
+ * safe; retrying an arbitrary exit could replay work. Inspect tmux AFTER exit so
+ * stdin, stdout and stderr stay real terminals (no tee, pipe or second PTY).
+ *
+ * The baseline rejects an old error left in the pane. Only a new, final error line
+ * within 30 seconds qualifies; normal exits, cancellation, auth/config errors,
+ * session failures and unavailable terminal evidence all fail closed. The baseline
+ * contains a hash, never terminal contents or the user's prompt.
+ */
+export const CODEX_STARTUP_RETRY_PROBE = String.raw`
+const { execFileSync } = require('node:child_process');
+const { createHash } = require('node:crypto');
+const [mode, tmux, pane, baseline] = process.argv.slice(1);
+try {
+ if (!/^%\d+$/.test(pane)) process.exit(1);
+ const screen = execFileSync(tmux, ['capture-pane', '-p', '-J', '-S', '-10', '-t', pane], {
+ timeout: 2000, maxBuffer: 1024 * 1024, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'],
+ });
+ const last = screen.trimEnd().split('\n').at(-1)?.trim() ?? '';
+ const hash = createHash('sha256').update(last).digest('hex');
+ if (mode === 'before') {
+ process.stdout.write(JSON.stringify({ at: Date.now(), hash }));
+ } else {
+ const before = JSON.parse(baseline);
+ const elapsed = Date.now() - before.at;
+ const error = 'Error: account/read failed during TUI bootstrap: account/read failed: workspace routing discovery timed out (code -32603)';
+ process.exit(elapsed >= 0 && elapsed <= 30000 && before.hash !== hash && last === error ? 0 : 1);
+ }
+} catch { process.exit(1); }
+`.trim()
diff --git a/cli/src/lib/engineLaunch.ts b/cli/src/lib/engineLaunch.ts
index 1044eeea8..51b558a18 100644
--- a/cli/src/lib/engineLaunch.ts
+++ b/cli/src/lib/engineLaunch.ts
@@ -10,6 +10,7 @@ import { GRID_NO_UPDATE_CHECK_VAR, gridBinaryPath } from './gridExec.js'
import { managedNodePath } from './nodeRuntime.js'
import { RAISE_OPEN_FILES_SH } from './openFiles.js'
import { ENGINE_EXIT_PANE_OPTION } from './tmux.js'
+import { CODEX_STARTUP_RETRY_PROBE } from './codexStartupRetry.js'
/**
* Best-effort "skip permission prompts" flag per engine, confirmed against each vendor's own docs.
@@ -499,8 +500,9 @@ export function engineFallbackPrelude(engine: AgentEngine, shellPath: string, tm
: ''
return 'harness_engine() {\n'
+ (engine === 'codex' ? codexOwnedLaunchPrelude() : '')
- + ' harness_status=0\n'
- + ' "$@" || harness_status=$?\n'
+ + (engine === 'codex' && tmuxBinary && isAbsolute(tmuxBinary)
+ ? codexStartupRetryScript(tmuxBinary)
+ : ' harness_status=0\n "$@" || harness_status=$?\n')
+ ' if [ "$harness_status" -eq 127 ]; then exit 127; fi\n'
+ mark
// Only a pane — something with a terminal on stdin — gets a shell to type into. Run without one
@@ -511,6 +513,31 @@ export function engineFallbackPrelude(engine: AgentEngine, shellPath: string, tm
+ '}\n'
}
+/** Keep a transient pre-session account lookup failure inside the original launch.
+ * Only the final exit gets the pane's engine-exit marker. The short backoff also
+ * keeps discovery from archiving the row between attempts. Never reparse "$@": it
+ * includes the original prompt, model, permissions and resume/fork arguments. */
+function codexStartupRetryScript(tmuxBinary: string): string {
+ const probe = `${shellSingleQuote(process.execPath)} -e ${shellSingleQuote(CODEX_STARTUP_RETRY_PROBE)}`
+ return ' harness_codex_attempt=1\n'
+ + ' while :; do\n'
+ + ' harness_codex_before=\n'
+ + ` if [ -n "\${TMUX_PANE:-}" ]; then harness_codex_before=$(${probe} before ${shellSingleQuote(tmuxBinary)} "$TMUX_PANE") || harness_codex_before=; fi\n`
+ + ' harness_status=0\n'
+ + ' "$@" || harness_status=$?\n'
+ + ' [ "$harness_status" -eq 1 ] && [ "$harness_codex_attempt" -lt 3 ] && [ -n "$harness_codex_before" ] || break\n'
+ + ` ${probe} after ${shellSingleQuote(tmuxBinary)} "$TMUX_PANE" "$harness_codex_before" || break\n`
+ + ' harness_codex_delay=$((harness_codex_attempt * 2))\n'
+ + ' harness_codex_attempt=$((harness_codex_attempt + 1))\n'
+ + ' harness_codex_cancelled=0\n'
+ + " trap 'harness_codex_cancelled=1' INT\n"
+ + ` printf '\\n%s\\n' "harness: Codex account lookup timed out. Retrying startup ($harness_codex_attempt/3) in \${harness_codex_delay}s; Ctrl-C cancels."\n`
+ + ' sleep "$harness_codex_delay" || harness_codex_cancelled=1\n'
+ + ' trap - INT\n'
+ + ' if [ "$harness_codex_cancelled" -eq 1 ]; then harness_status=130; break; fi\n'
+ + ' done\n'
+}
+
/** Codex 0.157+ otherwise puts the writer outside tmux in a shared server. Keep
* Harness-owned launches process-owned so Close, hook attribution, provider env
* and RAM accounting describe the same lifetime. Probe the binary AFTER any