Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

14.6 KB
Raw Blame
/**
 * Codex's shared app-server, spoken to from the Codex worker: a real private daemon, tmux and supervised
 * workers, an older fake Codex that shares its server (no --no-daemon), and the server's real loopback
 * WebSocket reached through the fake CLI's raw-byte `app-server proxy` (harness/sharedCodex.ts).
 */
import { execFileSync } from 'node:child_process'
import { mkdirSync, writeFileSync } from 'node:fs'
import { join } from 'node:path'
import { pathToFileURL } from 'node:url'
import { afterEach, describe, expect, it, onTestFailed } from 'vitest'
import { LocalClient } from './harness/client.js'
import { CLI_ROOT, IsolatedDaemon, until, type DaemonOptions } from './harness/daemon.js'
import { alive, harnessdProcesses } from './harness/endurance.js'
import { sharedCodex } from './harness/sharedCodex.js'

type Row = Record<string, any>
const SUPERVISION = { HARNESSD_SERVICE_INITIAL_BACKOFF_MS: '200', HARNESSD_SERVICE_MAX_BACKOFF_MS: '1000',
  HARNESSD_SERVICE_HEARTBEAT_TIMEOUT_MS: '4000', HARNESSD_SERVICE_STOP_GRACE_MS: '100' }
const PARKING = { HARNESSD_SERVICE_PARK_CRASHES: '3', HARNESSD_SERVICE_PARK_RETRY_MS: '120000' }

describe('Codex\'s shared server, from the Codex worker', () => {
  let daemon: IsolatedDaemon | undefined, client: LocalClient | undefined, server: Awaited<ReturnType<typeof sharedCodex>> | undefined
  afterEach(async () => { client?.close(); await daemon?.close(); await server?.close(); client = undefined; daemon = undefined; server = undefined })

  /** A Codex agent in a held turn. `shared`: an older Codex with no --no-daemon, whose conversation lives on the
   *  fake shared server; otherwise today's, which Harness launches with --no-daemon to own its conversation. */
  async function fresh(env: DaemonOptions['env'] = {}, shared = true) {
    const d = daemon = await IsolatedDaemon.create({ env: { ...SUPERVISION, ...env } })
    onTestFailed(() => console.log(`---- daemon log\n${d.log().split('\n').slice(-200).join('\n')}`))
    if (shared) writeFileSync(join(d.root, 'bin', 'codex'), `#!${process.execPath}\nimport(${JSON.stringify(pathToFileURL(join(CLI_ROOT, 'e2e/harness/fakeEngine.mjs')).href)}).then(m => m.run('codex', ${JSON.stringify({ ...d.engineConfig, without: ['--no-daemon'] })}))\n`, { mode: 0o755 })
    const s = server = await sharedCodex(d.engineConfig.codexHome)
    await d.start()
    const c = client = await LocalClient.connect(d)
    const cwd = join(d.projectsDir, 'codex'); mkdirSync(cwd, { recursive: true })
    const created = await c.request('agent_create', { engine: 'codex', cwd, bypassPermission: true }, 90_000)
    expect(created.error, JSON.stringify(created)).toBeUndefined()
    const agent: Row = await until('the conversation to bind', async () =>
      (await c.request('agents_list', {})).agents.find((row: Row) => row.id === created.agent.id && row.sessionId) ?? null, 60_000, 200)
    s.state.threadId = agent.sessionId
    const started = c.next(frame => frame.type === 'turn_started' && frame.agentId === agent.id, 30_000, 'the held turn')
    c.send('message', { agentId: agent.id, content: '!hold' })
    await started
    const count = (method: string, since = 0) => s.state.requests.slice(since).filter(frame => frame.method === method).length
    const close = () => c.request('agent_close', { agentId: agent.id, sessionId: agent.sessionId, createdAt: agent.createdAt, mode: 'now' }, 90_000)
    const pane = async () => (await d.tmux.run('display-message', '-p', '-t', agent.tmuxPane, '#{pane_pid}')).trim()
    return { d, c, s, agent, count, close, pane }
  }

  /** This daemon's `codex app-server proxy` clients, as the process table shows them, with their parents. */
  const proxies = (d: IsolatedDaemon) => execFileSync('ps', ['-A', '-o', 'pid=,ppid=,command='], { encoding: 'utf8' }).split('\n')
    .map(line => /^\s*(\d+)\s+(\d+)\s+(.*)$/.exec(line)).filter((match): match is RegExpExecArray => !!match)
    .filter(match => match[3].includes(join(d.root, 'bin', 'codex')) && match[3].includes('app-server proxy'))
    .map(match => ({ pid: Number(match[1]), parent: Number(match[2]) }))
  // A replacement process, connected to the core: the worker is back, not merely started.
  const linked = (d: IsolatedDaemon) => d.log().split('[services] engine-codex connected').length - 1
  const replacement = async (d: IsolatedDaemon, before: number, connections: number) => until('a replacement Codex worker', () => {
    const pid = harnessdProcesses(d).get('engine-codex')
    return pid && pid !== before && alive(pid) && linked(d) > connections ? pid : null
  }, 30_000, 100)
  /** Killed until the master parks it, as a worker crashing on every start would be. */
  async function park(d: IsolatedDaemon) {
    for (let index = 0; index < 3; index++) {
      const before = harnessdProcesses(d).get('engine-codex')!
      process.kill(before, 'SIGKILL')
      if (index < 2) await until('a replacement Codex worker', () => {
        const pid = harnessdProcesses(d).get('engine-codex')
        return pid && pid !== before && alive(pid) ? pid : null
      }, 15_000, 100)
    }
    await until('the Codex worker parked', () => d.log().includes('service engine-codex ended 3 times') || null, 60_000, 200)
  }

  it('reads activity over the worker\'s connection; with that worker killed, a close sent at once still stops the conversation once', async () => {
    const { d, c, s, agent, count, close } = await fresh()
    await until('the activity read at the shared server', () => count('thread/read') > 0 || null, 45_000, 100)
    await until('verified working activity', async () => (await c.request('agents_list', {})).agents.find((row: Row) => row.id === agent.id)?.activity?.state === 'working' || null, 20_000, 200)
    const core = d.corePid(), worker = harnessdProcesses(d).get('engine-codex')!
    // The connection is the worker's: its proxy client is the worker's child, never the core's.
    expect(proxies(d).map(proxy => proxy.parent)).toEqual([worker])
    expect(s.state.open).toBe(1)
    process.kill(worker, 'SIGKILL')
    // Sent before the worker is back: the close's read refused by the restart waits for it and reads again.
    const before = s.state.requests.length
    const result = await close()
    expect(result.error, JSON.stringify(result)).toBeUndefined()
    for (const method of ['thread/goal/set', 'turn/interrupt', 'thread/archive', 'thread/unarchive']) expect(count(method, before), method).toBe(1)
    expect(s.state.loaded).toBe(false)
    expect(s.state.archived).toBe(false)
    await until('no client of the killed worker left', () => proxies(d).every(proxy => proxy.parent === harnessdProcesses(d).get('engine-codex')) || null, 10_000, 100)
    expect(d.corePid()).toBe(core)
    expect(d.coresStarted()).toBe(1)
  })

  it('restarts a failed shared conversation after a Codex update without leaving its writer locked', async () => {
    const { d, c, s, agent, count } = await fresh()
    // The old client shares its server; the installed update supports --no-daemon. The failed turn
    // has no active goal, but its server still owns the thread, exactly as Codex 0.163.0 did.
    s.state.failed = true
    s.holdWriter()
    writeFileSync(join(d.root, 'bin', 'codex'), `#!${process.execPath}\nimport(${JSON.stringify(pathToFileURL(join(CLI_ROOT, 'e2e/harness/fakeEngine.mjs')).href)}).then(m => m.run('codex', ${JSON.stringify(d.engineConfig)}))\n`, { mode: 0o755 })
    const before = s.state.requests.length
    const restarted = await c.request('agent_restart', { agentId: agent.id }, 90_000)
    expect(restarted.error, JSON.stringify(restarted)).toBeUndefined()
    expect(s.state.loaded, 'the old server must release the conversation before the replacement opens it').toBe(false)
    expect(s.state.archived).toBe(false)
    expect(count('thread/archive', before)).toBe(1)
    expect(count('thread/unarchive', before)).toBe(1)
    expect(count('turn/interrupt', before)).toBe(0)
    expect(count('thread/goal/set', before)).toBe(0)
    const screen = await d.tmux.run('capture-pane', '-p', '-t', agent.tmuxPane)
    expect(screen).not.toContain('This conversation is open in another app')
    const ended = c.next(frame => frame.type === 'turn_ended' && frame.agentId === agent.id, 30_000, 'a turn in the resumed conversation')
    c.send('message', { agentId: agent.id, content: 'continue after the failed turn' })
    await ended
    const row = (await c.request('agents_list', {})).agents.find((row: Row) => row.id === agent.id)
    expect(row.sessionId).toBe(agent.sessionId)
    expect(d.coresStarted()).toBe(1)
  })

  it('keeps the original client when the Codex worker is lost while restart releases its shared conversation', async () => {
    const { d, c, s, agent, count, pane } = await fresh()
    const panePid = await pane()
    s.state.hold = 'thread/turns/list'
    const restarting = c.request('agent_restart', { agentId: agent.id }, 90_000)
    await until('restart waiting on the server', () => count('thread/turns/list') > 0 || null, 30_000, 50)
    process.kill(harnessdProcesses(d).get('engine-codex')!, 'SIGKILL')
    const refused = await restarting
    expect(refused.error, JSON.stringify(refused)).toBe('RESTART_FAILED')
    expect(refused.detail).toContain('could not stop')
    s.release()
    expect(await pane()).toBe(panePid)
    expect(s.state.loaded).toBe(true)
    expect(count('thread/archive')).toBe(0)
    expect(d.coresStarted()).toBe(1)
  })

  it('restarts a process-owned conversation without unloading an unrelated shared server', async () => {
    const { d, c, s, agent, count } = await fresh({}, false)
    const before = s.state.requests.length
    const restarted = await c.request('agent_restart', { agentId: agent.id }, 90_000)
    expect(restarted.error, JSON.stringify(restarted)).toBeUndefined()
    expect(count('thread/read', before)).toBe(0)
    expect(count('thread/archive', before)).toBe(0)
    expect(s.state.loaded).toBe(true)
    expect(s.state.archived).toBe(false)
    const ended = c.next(frame => frame.type === 'turn_ended' && frame.agentId === agent.id, 30_000, 'a turn after process-owned restart')
    c.send('message', { agentId: agent.id, content: 'continue in this terminal' })
    await ended
    expect(d.coresStarted()).toBe(1)
  })

  it('a worker killed while a stop waits on the server sends nothing more, leaves the pane, and a second close stops it once', async () => {
    const { d, s, count, close, pane } = await fresh()
    const core = d.corePid(), worker = harnessdProcesses(d).get('engine-codex')!, panePid = await pane()
    s.state.hold = 'thread/turns/list'
    const first = close()
    await until('the stop waiting on the server', () => count('thread/turns/list') > 0 || null, 45_000, 50)
    const connections = linked(d)
    process.kill(worker, 'SIGKILL')
    const refused = await first
    expect(refused.error, JSON.stringify(refused)).toBeDefined()
    const killedAt = s.state.requests.length
    s.release()
    await replacement(d, worker, connections)
    // Nothing of the stopped attempt reaches the server after its worker went: no interrupt, no archive.
    for (const method of ['turn/interrupt', 'thread/archive', 'thread/unarchive']) expect(count(method), method).toBe(0)
    expect(await pane()).toBe(panePid)
    const result = await close()
    expect(result.error, JSON.stringify(result)).toBeUndefined()
    for (const method of ['turn/interrupt', 'thread/archive', 'thread/unarchive']) expect(count(method), method).toBe(1)
    expect(count('thread/goal/set', killedAt)).toBe(1)
    expect(s.state.loaded).toBe(false)
    expect(d.corePid()).toBe(core)
    expect(d.coresStarted()).toBe(1)
  })

  it('a worker killed between archive and unarchive: the next worker returns the thread\'s history, once', async () => {
    const { d, s, count, close, pane } = await fresh()
    const worker = harnessdProcesses(d).get('engine-codex')!, panePid = await pane()
    s.state.hold = 'thread/archive'
    const first = close()
    await until('the archive sent', () => s.state.archived || null, 45_000, 50)
    const connections = linked(d)
    process.kill(worker, 'SIGKILL')
    const refused = await first
    expect(refused.error, JSON.stringify(refused)).toBeDefined()
    expect(await pane()).toBe(panePid)
    await replacement(d, worker, connections)
    await until('the archived history returned', () => !s.state.archived || null, 30_000, 100)
    expect(count('thread/unarchive')).toBe(1)
    await until('the repair logged', () => d.log().includes('restored the history of codex conversation') || null, 10_000, 100)
    // Forgotten once done: another restart repairs nothing more.
    const again = harnessdProcesses(d).get('engine-codex')!, more = linked(d)
    process.kill(again, 'SIGKILL')
    await replacement(d, again, more)
    await new Promise(resolve => setTimeout(resolve, 500))
    expect(count('thread/unarchive')).toBe(1)
  })

  it('with the Codex worker parked, Stop still stops an agent that owns its conversation, needing no worker', async () => {
    const { d, c, agent, pane } = await fresh(PARKING, false)
    const core = d.corePid(), panePid = Number(await pane())
    await park(d)
    const stopped = await c.request('agent_delete', { agentId: agent.id }, 90_000)
    expect(stopped.error, JSON.stringify(stopped)).toBeUndefined()
    await until('the agent gone', async () => !(await c.request('agents_list', {})).agents.some((row: Row) => row.id === agent.id) || null, 30_000, 200)
    await until('its process ended', () => !alive(panePid) || null, 30_000, 100)
    expect(d.corePid()).toBe(core)
    expect(d.coresStarted()).toBe(1)
  })

  it('with the Codex worker parked, a conversation on the shared server is not reported stopped: the pane stays, and it is logged', async () => {
    const { d, c, s, agent, count, pane } = await fresh(PARKING)
    const panePid = await pane()
    await park(d)
    const before = s.state.requests.length
    const refused = await c.request('agent_delete', { agentId: agent.id }, 90_000)
    expect(refused.error, JSON.stringify(refused)).toBeDefined()
    expect(d.log()).toContain('the codex worker did not answer the stop')
    expect(await pane()).toBe(panePid)
    expect((await c.request('agents_list', {})).agents.some((row: Row) => row.id === agent.id)).toBe(true)
    expect(count('thread/archive', before)).toBe(0)
    expect(d.coresStarted()).toBe(1)
  })

  it('closes with the control in the core\'s own process in explicit inline mode', async () => {
    const { d, s, count, close } = await fresh({ HARNESSD_SERVICES: 'none' })
    const result = await close()
    expect(result.error, JSON.stringify(result)).toBeUndefined()
    for (const method of ['thread/goal/set', 'turn/interrupt', 'thread/archive', 'thread/unarchive']) expect(count(method), method).toBe(1)
    expect(s.state.loaded).toBe(false)
    expect(harnessdProcesses(d).get('engine-codex')).toBeUndefined()
  })
})