The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
fix(cli): retry Codex account-routing startup timeouts (#559)
Deeee2f0fcf2707
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex cfc587af3..d53d53492 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -155,6 +155,11 @@ jobs:env:RUN_REAL_TMUX_STREAM: '1'run: npx vitest run src/lib/tmuxStream.real.spec.ts+ - name: Codex startup timeout recovery and cancellation+ working-directory: cli+ env:+ RUN_REAL_TMUX_CODEX_STARTUP: '1'+ run: npx vitest run src/lib/codexStartupRetry.real.spec.ts --maxWorkers=1- name: Release unit testsrun: cargo test --locked --release- name: Build static Linux binarydiff --git a/cli/src/lib/codexStartupRetry.real.spec.ts b/cli/src/lib/codexStartupRetry.real.spec.tsnew file mode 100644index 000000000..bee6493ed--- /dev/null+++ b/cli/src/lib/codexStartupRetry.real.spec.ts@@ -0,0 +1,100 @@+import { execFile } from 'node:child_process'+import { randomUUID } from 'node:crypto'+import { mkdtemp, readFile, realpath, rm, writeFile } from 'node:fs/promises'+import { tmpdir } from 'node:os'+import { join } from 'node:path'+import { promisify } from 'node:util'+import { afterEach, describe, expect, it } from 'vitest'+import { resolveBinaryOnPath } from './binaryOnPath.js'+import { buildEngineLaunchArgv } from './engineLaunch.js'++const exec = promisify(execFile)+const real = process.env.RUN_REAL_TMUX_CODEX_STARTUP === '1' ? describe : describe.skip+const timeout = 'Error: account/read failed during TUI bootstrap: account/read failed: workspace routing discovery timed out (code -32603)'++real('Codex startup recovery in an isolated real tmux server', () => {+ const socket = `harness-startup-test-${randomUUID()}`+ const tmuxBin = resolveBinaryOnPath('tmux')!+ const dirs: string[] = []+ const tmux = async (...args: string[]) => (await exec(tmuxBin, ['-L', socket, '-f', '/dev/null', ...args], { timeout: 3000 })).stdout.trim()+ afterEach(async () => {+ await tmux('kill-server').catch(() => {})+ for (const dir of dirs.splice(0)) await rm(dir, { recursive: true, force: true })+ })++ async function eventually(check: () => Promise<boolean>, ms = 12_000) {+ const deadline = Date.now() + ms+ while (Date.now() < deadline) {+ if (await check()) return+ await new Promise(resolve => setTimeout(resolve, 30))+ }+ throw new Error(`condition not met; pane output: ${await tmux('capture-pane', '-p', '-J', '-t', '%0').catch(() => '')}`)+ }++ async function launch(failures: number, prompt?: string, exitStatus = 1, differentSecondError = false) {+ const dir = await mkdtemp(join(tmpdir(), 'harness-codex-startup-real-'))+ dirs.push(dir)+ const state = join(dir, 'attempts.json')+ await writeFile(state, '[]')+ // A faithful terminal boundary: the engine receives three real TTYs and+ // prints Codex's observed fatal error directly to stderr. No real account.+ await writeFile(join(dir, 'codex'), `#!${process.execPath}\n`+ + `const fs = require('node:fs');\n`+ + `if (process.argv.includes('--help')) { console.log('--no-daemon'); process.exit(0); }\n`+ + `const file = ${JSON.stringify(state)}; const attempts = JSON.parse(fs.readFileSync(file, 'utf8'));\n`+ + `attempts.push({ args: process.argv.slice(2), tty: [process.stdin.isTTY, process.stdout.isTTY, process.stderr.isTTY], cwd: process.cwd() });\n`+ + `fs.writeFileSync(file, JSON.stringify(attempts));\n`+ + `if (attempts.length <= ${failures}) { console.error(${differentSecondError ? `attempts.length === 1 ? ${JSON.stringify(timeout)} : 'Error: invalid configuration'` : JSON.stringify(timeout)}); process.exitCode = ${exitStatus}; }\n`+ + `else { console.log('STARTUP_RECOVERED'); process.stdin.resume(); }\n`, { mode: 0o755 })+ const command = buildEngineLaunchArgv('codex', { cwd: dir, firstPrompt: prompt, permissionMode: 'auto' }, '/bin/sh', undefined, 'grid', tmuxBin)+ // An interactive rc file may replace PATH. Never let a fixture resolve to+ // the developer's installed Codex, even when their shell does that.+ command[command.indexOf('codex')] = join(dir, 'codex')+ const pane = await tmux('new-session', '-d', '-P', '-F', '#{pane_id}', '-x', '40', '-y', '12', '-s', 'test', '-e', `PATH=${dir}:${process.env.PATH}`, ...command)+ const attempts = async () => JSON.parse(await readFile(state, 'utf8')) as Array<{ args: string[]; tty: boolean[]; cwd: string }>+ const screen = () => tmux('capture-pane', '-p', '-J', '-t', pane)+ const exit = () => tmux('show-option', '-p', '-v', '-t', pane, '@harness_engine_exit').catch(() => '')+ return { pane, attempts, screen, exit, dir }+ }++ it.each([undefined, 'Fix spaces, "quotes", $(printf literal), `false`, ; & ! ^\nand a second line'])('recovers the same launch with first prompt %s', async prompt => {+ const f = await launch(1, prompt)+ await eventually(async () => (await f.screen()).includes('STARTUP_RECOVERED'))+ const attempts = await f.attempts()+ expect(attempts).toHaveLength(2)+ for (const attempt of attempts) {+ expect(attempt.args).toEqual(['--no-daemon', '--approve-for-me', ...(prompt ? [prompt] : [])])+ expect(attempt.tty).toEqual([true, true, true])+ expect(attempt.cwd).toBe(await realpath(f.dir))+ }+ expect(await f.exit()).toBe('')+ }, 20_000)++ it('stops after three failed attempts and returns the final status to the shell', async () => {+ const f = await launch(100)+ await eventually(async () => await f.exit() === '1')+ expect(await f.attempts()).toHaveLength(3)+ expect(await f.screen()).toContain('This pane is a shell now')+ }, 20_000)++ it('does not retry other errors after a transient first failure', async () => {+ const f = await launch(100, undefined, 1, true)+ await eventually(async () => await f.exit() === '1')+ expect(await f.attempts()).toHaveLength(2)+ }, 20_000)++ it('does not retry a cancelled engine, even if timeout text remains visible', async () => {+ const f = await launch(100, undefined, 130)+ await eventually(async () => await f.exit() === '130')+ expect(await f.attempts()).toHaveLength(1)+ })++ it('honors Ctrl-C during retry backoff', async () => {+ const f = await launch(100)+ await eventually(async () => (await f.screen()).includes('Retrying startup (2/3)'))+ await tmux('send-keys', '-t', f.pane, 'C-c')+ await eventually(async () => await f.exit() === '130')+ expect(await f.attempts()).toHaveLength(1)+ expect(await f.screen()).toContain('This pane is a shell now')+ }, 15_000)+})diff --git a/cli/src/lib/codexStartupRetry.spec.ts b/cli/src/lib/codexStartupRetry.spec.tsnew file mode 100644index 000000000..f1a0d056f--- /dev/null+++ b/cli/src/lib/codexStartupRetry.spec.ts@@ -0,0 +1,65 @@+import { execFileSync } from 'node:child_process'+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'+import { tmpdir } from 'node:os'+import { join } from 'node:path'+import { afterEach, describe, expect, it } from 'vitest'+import { CODEX_STARTUP_RETRY_PROBE } from './codexStartupRetry.js'++const timeout = 'Error: account/read failed during TUI bootstrap: account/read failed: workspace routing discovery timed out (code -32603)'+const dirs: string[] = []+afterEach(() => { for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }) })++function fixture() {+ const dir = mkdtempSync(join(tmpdir(), 'codex-startup-probe-'))+ dirs.push(dir)+ const screen = join(dir, 'screen')+ const tmux = join(dir, 'tmux')+ writeFileSync(screen, 'existing terminal output\n')+ writeFileSync(tmux, '#!/bin/sh\ncat "$HARNESS_TEST_SCREEN"\n', { mode: 0o755 })+ const probe = (mode: string, baseline = '', pane = '%7') => {+ try {+ return { status: 0, out: execFileSync(process.execPath, ['-e', CODEX_STARTUP_RETRY_PROBE, mode, tmux, pane, baseline], {+ env: { ...process.env, HARNESS_TEST_SCREEN: screen }, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'],+ }) }+ } catch (error) { return { status: (error as { status: number }).status, out: '' } }+ }+ return { screen, tmux, probe }+}++describe('Codex startup timeout evidence', () => {+ it('accepts only a newly printed, final bootstrap timeout', () => {+ const f = fixture()+ const before = f.probe('before').out+ expect(before).not.toContain('existing terminal output')+ writeFileSync(f.screen, `${timeout}\n\n\n`)+ expect(f.probe('after', before).status).toBe(0)+ // The same text already on screen before a later launch is not new evidence.+ expect(f.probe('after', f.probe('before').out).status).toBe(1)+ })++ it.each([+ 'Error: account/read failed during TUI bootstrap: workspace routing discovery unauthorized (401)',+ 'Error: invalid config.toml',+ 'Error: stream disconnected before completion: request timed out',+ `${timeout}\n› Ask Codex to do anything`,+ `quoted output: ${timeout}`,+ '',+ ])('does not replay a launch after unrelated output: %s', (screen) => {+ const f = fixture()+ const before = f.probe('before').out+ writeFileSync(f.screen, screen)+ expect(f.probe('after', before).status).toBe(1)+ })++ it('rejects old, invalid or unavailable evidence', () => {+ const f = fixture()+ const before = JSON.parse(f.probe('before').out)+ writeFileSync(f.screen, timeout)+ expect(f.probe('after', JSON.stringify({ ...before, at: Date.now() - 31_000 })).status).toBe(1)+ expect(f.probe('after', JSON.stringify({ ...before, at: Date.now() + 10_000 })).status).toBe(1)+ expect(f.probe('after', 'broken baseline').status).toBe(1)+ expect(f.probe('before', '', 'some-session').status).toBe(1)+ rmSync(f.tmux)+ expect(f.probe('after', JSON.stringify(before)).status).toBe(1)+ })+})diff --git a/cli/src/lib/codexStartupRetry.ts b/cli/src/lib/codexStartupRetry.tsnew file mode 100644index 000000000..b43a9f275--- /dev/null+++ b/cli/src/lib/codexStartupRetry.ts@@ -0,0 +1,34 @@+/**+ * Run inside the pane's shell, before/after a Codex process, using the daemon's Node.+ *+ * Codex 0.159.3 exits 1 when account/read's workspace-routing discovery times out,+ * before thread/start or the first prompt. Retrying that exact bootstrap failure is+ * safe; retrying an arbitrary exit could replay work. Inspect tmux AFTER exit so+ * stdin, stdout and stderr stay real terminals (no tee, pipe or second PTY).+ *+ * The baseline rejects an old error left in the pane. Only a new, final error line+ * within 30 seconds qualifies; normal exits, cancellation, auth/config errors,+ * session failures and unavailable terminal evidence all fail closed. The baseline+ * contains a hash, never terminal contents or the user's prompt.+ */+export const CODEX_STARTUP_RETRY_PROBE = String.raw`+const { execFileSync } = require('node:child_process');+const { createHash } = require('node:crypto');+const [mode, tmux, pane, baseline] = process.argv.slice(1);+try {+ if (!/^%\d+$/.test(pane)) process.exit(1);+ const screen = execFileSync(tmux, ['capture-pane', '-p', '-J', '-S', '-10', '-t', pane], {+ timeout: 2000, maxBuffer: 1024 * 1024, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'],+ });+ const last = screen.trimEnd().split('\n').at(-1)?.trim() ?? '';+ const hash = createHash('sha256').update(last).digest('hex');+ if (mode === 'before') {+ process.stdout.write(JSON.stringify({ at: Date.now(), hash }));+ } else {+ const before = JSON.parse(baseline);+ const elapsed = Date.now() - before.at;+ const error = 'Error: account/read failed during TUI bootstrap: account/read failed: workspace routing discovery timed out (code -32603)';+ process.exit(elapsed >= 0 && elapsed <= 30000 && before.hash !== hash && last === error ? 0 : 1);+ }+} catch { process.exit(1); }+`.trim()diff --git a/cli/src/lib/engineLaunch.ts b/cli/src/lib/engineLaunch.tsindex 1044eeea8..51b558a18 100644--- a/cli/src/lib/engineLaunch.ts+++ b/cli/src/lib/engineLaunch.ts@@ -10,6 +10,7 @@ import { GRID_NO_UPDATE_CHECK_VAR, gridBinaryPath } from './gridExec.js'import { managedNodePath } from './nodeRuntime.js'import { RAISE_OPEN_FILES_SH } from './openFiles.js'import { ENGINE_EXIT_PANE_OPTION } from './tmux.js'+import { CODEX_STARTUP_RETRY_PROBE } from './codexStartupRetry.js'/*** Best-effort "skip permission prompts" flag per engine, confirmed against each vendor's own docs.@@ -499,8 +500,9 @@ export function engineFallbackPrelude(engine: AgentEngine, shellPath: string, tm: ''return 'harness_engine() {\n'+ (engine === 'codex' ? codexOwnedLaunchPrelude() : '')- + ' harness_status=0\n'- + ' "$@" || harness_status=$?\n'+ + (engine === 'codex' && tmuxBinary && isAbsolute(tmuxBinary)+ ? codexStartupRetryScript(tmuxBinary)+ : ' harness_status=0\n "$@" || harness_status=$?\n')+ ' if [ "$harness_status" -eq 127 ]; then exit 127; fi\n'+ mark// Only a pane — something with a terminal on stdin — gets a shell to type into. Run without one@@ -511,6 +513,31 @@ export function engineFallbackPrelude(engine: AgentEngine, shellPath: string, tm+ '}\n'}+/** Keep a transient pre-session account lookup failure inside the original launch.+ * Only the final exit gets the pane's engine-exit marker. The short backoff also+ * keeps discovery from archiving the row between attempts. Never reparse "$@": it+ * includes the original prompt, model, permissions and resume/fork arguments. */+function codexStartupRetryScript(tmuxBinary: string): string {+ const probe = `${shellSingleQuote(process.execPath)} -e ${shellSingleQuote(CODEX_STARTUP_RETRY_PROBE)}`+ return ' harness_codex_attempt=1\n'+ + ' while :; do\n'+ + ' harness_codex_before=\n'+ + ` if [ -n "\${TMUX_PANE:-}" ]; then harness_codex_before=$(${probe} before ${shellSingleQuote(tmuxBinary)} "$TMUX_PANE") || harness_codex_before=; fi\n`+ + ' harness_status=0\n'+ + ' "$@" || harness_status=$?\n'+ + ' [ "$harness_status" -eq 1 ] && [ "$harness_codex_attempt" -lt 3 ] && [ -n "$harness_codex_before" ] || break\n'+ + ` ${probe} after ${shellSingleQuote(tmuxBinary)} "$TMUX_PANE" "$harness_codex_before" || break\n`+ + ' harness_codex_delay=$((harness_codex_attempt * 2))\n'+ + ' harness_codex_attempt=$((harness_codex_attempt + 1))\n'+ + ' harness_codex_cancelled=0\n'+ + " trap 'harness_codex_cancelled=1' INT\n"+ + ` printf '\\n%s\\n' "harness: Codex account lookup timed out. Retrying startup ($harness_codex_attempt/3) in \${harness_codex_delay}s; Ctrl-C cancels."\n`+ + ' sleep "$harness_codex_delay" || harness_codex_cancelled=1\n'+ + ' trap - INT\n'+ + ' if [ "$harness_codex_cancelled" -eq 1 ]; then harness_status=130; break; fi\n'+ + ' done\n'+}+/** Codex 0.157+ otherwise puts the writer outside tmux in a shared server. Keep* Harness-owned launches process-owned so Close, hook attribution, provider env* and RAM accounting describe the same lifetime. Probe the binary AFTER any