Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

6.4 KB
Raw Blame
/**
 * Coming back, for Claude Code and Codex, on the real daemon: restarted again and again while agents
 * work, its master killed outright, an engine missing from this machine, and a registry lock left by
 * a daemon that crashed holding it. Agents live in tmux, so every way back must find them again.
 */
import { mkdirSync, renameSync, writeFileSync } from 'node:fs'
import { join } from 'node:path'
import { afterEach, describe, expect, it, onTestFailed } from 'vitest'
import { LocalClient, type Frame } from './harness/client.js'
import { IsolatedDaemon, until } from './harness/daemon.js'

type Engine = 'claude' | 'codex'

const row = async (client: LocalClient, agentId: string) =>
  ((await client.request<{ agents: Array<Record<string, any>> }>('agents_list', { includeStopped: true }, 30_000)).agents)
    .find((agent) => agent.id === agentId)
async function create(daemon: IsolatedDaemon, client: LocalClient, engine: Engine, folder: string): Promise<Record<string, any>> {
  const cwd = join(daemon.projectsDir, folder)
  mkdirSync(cwd, { recursive: true })
  const created = await client.request('agent_create', { engine, cwd, bypassPermission: true }, 90_000)
  expect(created.error, `${folder}: ${JSON.stringify(created)}`).toBeUndefined()
  return until(`${folder} to bind its conversation`, async () => {
    const agent = await row(client, created.agent.id)
    return agent?.sessionId && agent.status === 'active' ? agent : null
  }, 60_000, 500)
}
const back = (client: LocalClient, agent: Record<string, any>) =>
  until(`${agent.id.slice(0, 8)} to be back`, async () => {
    const now = await row(client, agent.id)
    return now?.status === 'active' && now.sessionId === agent.sessionId ? now : null
  }, 60_000, 500)
const isTurn = (type: string, agentId: string) => (frame: Frame) => frame.type === type && frame.agentId === agentId
async function turn(client: LocalClient, agentId: string, content: string): Promise<void> {
  const ended = client.next(isTurn('turn_ended', agentId), 45_000, `turn_ended (${content})`)
  client.send('message', { agentId, content })
  await ended
}

describe('coming back', () => {
  let daemon: IsolatedDaemon | undefined
  afterEach(async () => { await daemon?.close(); daemon = undefined })
  const fresh = async () => {
    const d = await IsolatedDaemon.create()
    daemon = d
    onTestFailed(() => { console.log(`---- daemon log\n${d.log().split('\n').slice(-120).join('\n')}`) })
    return d
  }

  it('restarted five times in a row while agents work: they are back and take a turn every time', async () => {
    const d = await fresh()
    await d.start()
    let client = await LocalClient.connect(d)
    const agents = [await create(d, client, 'claude', 'restarts-claude'), await create(d, client, 'codex', 'restarts-codex')]
    for (let restart = 1; restart <= 5; restart++) {
      // A turn in flight as the daemon goes, every other time.
      if (restart % 2) client.send('message', { agentId: agents[0].id, content: '!slow 3000' })
      client.close()
      await d.restart()
      client = await LocalClient.connect(d)
      for (const agent of agents) await back(client, agent)
      await Promise.all(agents.map((agent) => turn(client, agent.id, `after restart ${restart}`)))
    }
    client.close()
  })

  it('its master killed outright: the core stops too, and the next start finds every agent', async () => {
    const d = await fresh()
    await d.start()
    const client = await LocalClient.connect(d)
    const agents = [await create(d, client, 'claude', 'killed-claude'), await create(d, client, 'codex', 'killed-codex')]
    const core = d.corePid()
    client.close()
    await d.kill()
    await until('the core to stop once its master is gone', () => !IsolatedDaemon.alive(core) || null, 30_000, 250)
    await d.start()
    const again = await LocalClient.connect(d)
    for (const agent of agents) await back(again, agent)
    await Promise.all(agents.map((agent) => turn(again, agent.id, 'after the master was killed')))
    again.close()
  })

  it('an unavailable engine holds its saved launch while another engine works, then recovers the same request', async () => {
    const d = await fresh()
    const executable = join(d.root, 'bin', 'codex')
    renameSync(executable, executable + '.unavailable')
    await d.start()
    const client = await LocalClient.connect(d)
    const cwd = join(d.projectsDir, 'no-codex')
    mkdirSync(cwd, { recursive: true })
    const payload = { creationId: 'missing-codex-recovery', engine: 'codex', cwd, bypassPermission: true }
    const created = await client.request('agent_create', payload, 90_000)
    expect(created).toMatchObject({ state: 'pending', held: { service: 'native' } })
    expect(created.held.detail).toContain('executable')
    expect((await client.request('agents_list', { includeStopped: true })).agents).toEqual([])
    const claude = await create(d, client, 'claude', 'still-claude')
    await turn(client, claude.id, 'claude is still here')
    renameSync(executable + '.unavailable', executable)
    const recovered = await until('the original missing-engine request to recover', async () => {
      const reply = await client.request('agent_create_status', { creationId: payload.creationId })
      return reply.state === 'created' ? reply : null
    }, 60_000, 250)
    await until('the recovered Codex conversation to bind', async () => (await row(client, recovered.agent.id))?.sessionId || null)
    expect(await client.request('agent_create', payload)).toMatchObject({ state: 'created', agent: { id: recovered.agent.id } })
    await turn(client, recovered.agent.id, 'the held launch recovered')
    expect(d.coresStarted()).toBe(1)
    client.close()
  })

  it('a registry lock left by a daemon that crashed holding it does not stop the next one', async () => {
    const d = await fresh()
    // A lock whose owner is a process that no longer exists, as a crash leaves it.
    const lock = join(d.dataDir, 'registry.json.lock')
    mkdirSync(lock, { recursive: true, mode: 0o700 })
    writeFileSync(join(lock, 'owner.json'), JSON.stringify({ pid: 999_999, token: 'crashed', at: Date.now() - 60_000 }), { mode: 0o600 })
    await d.start()
    const client = await LocalClient.connect(d)
    const agent = await create(d, client, 'codex', 'after-a-stale-lock')
    await turn(client, agent.id, 'the registry took the write')
    await d.restart()
    const again = await LocalClient.connect(d)
    await back(again, agent)
    again.close()
    client.close()
  })
})