Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

13.7 KB
Raw Blame
/**
 * The core's event loop held still for seconds at a time while agents work, for Claude Code and Codex.
 * A synchronous read of a huge file, a heap paged out under memory pressure, a laptop deep in swap: each
 * holds every timer, socket and child of the core at once. When the hold ends, Node runs the timers that
 * came due before it reads the answers that arrived meanwhile, so a probe in flight (`ps`, `tmux`) times
 * out with its answer sitting unread. That says nothing about any agent. A timeout is "could not tell",
 * and only "known gone" may unbind, retire, stop, relaunch or forget.
 *
 * The core is held 2-8 s at a time (core/stall.ts), each hold landing as it starts a child process, so
 * a probe is always in flight: at random, and on purpose while it binds new agents, takes turns,
 * restarts an agent, resumes one, adopts an engine typed into a terminal, reconciles, and restores and
 * attaches every agent after a daemon restart. Every agent must stay bound and active and take its next
 * turn, no engine may run twice, and the master must never take the held core for hung (every hold is
 * far inside its 40 s).
 */
import { execFileSync } from 'node:child_process'
import { mkdirSync } from 'node:fs'
import { join } from 'node:path'
import { afterEach, describe, expect, it, onTestFailed } from 'vitest'
import { LocalClient, type Frame } from './harness/client.js'
import { IsolatedDaemon, until } from './harness/daemon.js'

type Engine = 'claude' | 'codex'
type Row = Record<string, any>

const STALLS = 'core.stall:2000-8000@12000/spawn'
const sleep = (ms: number) => new Promise((done) => setTimeout(done, ms))
const rows = async (client: LocalClient): Promise<Row[]> =>
  (await client.request<{ agents: Row[] }>('agents_list', { includeStopped: true }, 60_000)).agents
const row = async (client: LocalClient, agentId: string) => (await rows(client)).find((agent) => agent.id === agentId)
const isTurn = (type: string, agentId: string) => (frame: Frame) => frame.type === type && frame.agentId === agentId

/** Hold the core at the next child process it starts, as a test chooses the moment. */
function hold(d: IsolatedDaemon): void {
  const core = d.corePid()
  if (core) process.kill(core, 'SIGUSR2')
}

async function folder(d: IsolatedDaemon, name: string): Promise<string> {
  const cwd = join(d.projectsDir, name)
  mkdirSync(cwd, { recursive: true })
  return cwd
}

const bound = (client: LocalClient, agentId: string, sessionId?: string) =>
  until(`${agentId.slice(0, 8)} to be bound and active`, async () => {
    const now = await row(client, agentId)
    return now?.status === 'active' && now.sessionId && (!sessionId || now.sessionId === sessionId) ? now : null
  }, 120_000, 500)

/** A turn that starts with what was sent and ends once. */
async function turn(client: LocalClient, agentId: string, content: string, during?: () => void): Promise<void> {
  const from = client.frames.length
  const started = client.next(isTurn('turn_started', agentId), 90_000, `turn_started (${content})`)
  const ended = client.next(isTurn('turn_ended', agentId), 120_000, `turn_ended (${content})`)
  client.send('message', { agentId, content })
  during?.()
  expect((await started).payload?.userMessage).toBe(content)
  await ended
  // A turn the held core could not see end must not end again when it looks.
  await sleep(1_000)
  expect(client.frames.slice(from).filter(isTurn('turn_ended', agentId)), `${content}: ended once`).toHaveLength(1)
}

/** The engine processes in each tmux pane, by pid, from tmux's own pane list and the process table. The
 *  fake engines' process title is their engine's name, then their arguments, as a CLI's is. */
async function enginePids(d: IsolatedDaemon): Promise<Map<string, number[]>> {
  const panes = (await d.tmux.run('list-panes', '-a', '-F', '#{pane_id} #{pane_pid}')).trim().split('\n')
    .map((line) => line.split(' ')).map(([id, pid]) => ({ id, pid: Number(pid) }))
  const table = execFileSync('ps', ['-A', '-o', 'pid=,ppid=,command=']).toString().trim().split('\n')
    .map((line) => line.trim().match(/^(\d+)\s+(\d+)\s+(.*)$/)).filter(Boolean) as RegExpMatchArray[]
  const children = new Map<number, number[]>()
  for (const [, pid, ppid] of table) children.set(Number(ppid), [...(children.get(Number(ppid)) ?? []), Number(pid)])
  const result = new Map<string, number[]>()
  for (const pane of panes) {
    const tree = new Set<number>([pane.pid])
    for (const pid of tree) for (const child of children.get(pid) ?? []) tree.add(child)
    result.set(pane.id, table.filter(([, pid, , command]) => tree.has(Number(pid)) && /^(claude|codex)(?:\s|$)/.test(command.trim()))
      .map(([, pid]) => Number(pid)))
  }
  return result
}
const enginesByPane = async (d: IsolatedDaemon): Promise<Map<string, number>> =>
  new Map([...(await enginePids(d))].map(([pane, pids]) => [pane, pids.length]))

/** Every agent in `expected` active, on the conversation it had, with one engine in its pane, and no
 *  engine anywhere else. */
async function intact(d: IsolatedDaemon, client: LocalClient, expected: Row[]): Promise<void> {
  const all = await rows(client)
  for (const agent of expected) {
    const now = all.find((one) => one.id === agent.id)
    expect(now?.status, `${agent.id.slice(0, 8)} (${agent.engine}) is active`).toBe('active')
    expect(now?.sessionId, `${agent.id.slice(0, 8)} (${agent.engine}) is on its conversation`).toBe(agent.sessionId)
  }
  // Nothing was minted beside them for a pane or an engine they already own.
  expect(all.filter((one) => one.status === 'active').map((one) => one.id).sort()).toEqual(expected.map((one) => one.id).sort())
  const engines = await until('one engine per agent', async () => {
    const counts = await enginesByPane(d)
    const ok = expected.every((agent) => counts.get(agent.tmuxPane) === 1)
      && [...counts.values()].reduce((sum, n) => sum + n, 0) === expected.length
    return ok ? counts : null
  }, 20_000, 500).catch(async () => enginesByPane(d))
  for (const agent of expected) expect(engines.get(agent.tmuxPane), `${agent.id.slice(0, 8)}: engines in its pane`).toBe(1)
  expect([...engines.values()].reduce((sum, n) => sum + n, 0), 'engines on the tmux server').toBe(expected.length)
}

/** The holds the core logged: there were some, and each was far inside the master's patience. */
function holds(d: IsolatedDaemon): number[] {
  return [...d.log().matchAll(/\[stall\] holding the event loop for (\d+) ms/g)].map((match) => Number(match[1]))
}

/** Nothing on the way was decided from a probe that could not answer. A core held as it is stopped
 *  outlives its grace and is killed, which is a stop doing its job; one killed for silence is not. */
function nothingRetired(d: IsolatedDaemon): void {
  const log = d.log()
  expect(log).not.toMatch(/\[harnessd\] core (sent no heartbeat|.*hung)/)
  expect(log).not.toMatch(/\[discovery\] \S+ (retained|dormant|removed|kept) · /)
  expect(log).not.toMatch(/ENGINE_DID_NOT_START|START_TIMEOUT|restored pane disappeared/)
}

describe('the core held still while agents work', () => {
  let daemon: IsolatedDaemon | undefined
  afterEach(async () => { await daemon?.close(); daemon = undefined })
  const fresh = async () => {
    const d = await IsolatedDaemon.create({ env: { HARNESSD_TEST_FAULTS: STALLS } })
    daemon = d
    onTestFailed(() => { console.log(`---- daemon log\n${d.log().split('\n').slice(-250).join('\n')}`) })
    await d.start()
    await until('the stalls to be armed', () => /\[stall\] armed/.test(d.log()) || null, 30_000, 100)
    return d
  }

  it('creating, turns, a restart, a resume and an engine typed into a terminal: nothing is lost or retired', async () => {
    const d = await fresh()
    const client = await LocalClient.connect(d)

    // Two agents created at once, the core held while their engines start and bind.
    const asked = (['claude', 'codex'] as const).map(async (engine: Engine) =>
      client.request('agent_create', { engine, cwd: await folder(d, `held-${engine}`), bypassPermission: true }, 120_000))
    await sleep(300)
    hold(d)
    const created = await Promise.all(asked)
    for (const answer of created) expect(answer.error, JSON.stringify(answer)).toBeUndefined()
    const agents = await Promise.all(created.map((answer) => bound(client, answer.agent.id)))
    const [claude, codex] = agents

    // Turns, held while they run.
    for (const agent of agents) await turn(client, agent.id, `held during a turn (${agent.engine})`, () => hold(d))
    await turn(client, claude.id, '!slow 6000', () => setTimeout(() => hold(d), 1_500))

    // A restart, held the moment its new engine is up, and a message sent right then by a client that
    // sees the agent still active. The restart may not have recorded the new engine yet: such a message
    // used to be dropped as "no longer running", and an attach then unbound the agent (e2e/races.e2e.ts,
    // a restart nobody waited for).
    for (const agent of agents) {
      const before = (await enginePids(d)).get(agent.tmuxPane) ?? []
      const restarting = client.request('agent_restart', { agentId: agent.id }, 120_000)
      await until('the new engine to start', async () =>
        ((await enginePids(d)).get(agent.tmuxPane) ?? []).some((pid) => !before.includes(pid)) || null, 60_000, 50)
      hold(d)
      await turn(client, agent.id, `while a held restart finishes (${agent.engine})`)
      const restarted = await restarting
      expect(restarted.error, JSON.stringify(restarted)).toBeUndefined()
      await bound(client, agent.id, agent.sessionId)
      await turn(client, agent.id, `after a held restart (${agent.engine})`)
    }

    // A stop and a resume, held while the resumed engine starts.
    expect((await client.request('agent_delete', { agentId: codex.id }, 120_000)).error).toBeUndefined()
    await until('codex to stop', async () => (await row(client, codex.id))?.status === 'stopped' || null, 90_000, 500)
    const resuming = client.request('agent_resume', { agentId: codex.id }, 120_000)
    await sleep(200)
    hold(d)
    const resumed = await resuming
    expect(resumed.error, JSON.stringify(resumed)).toBeUndefined()
    const back = await bound(client, codex.id, codex.sessionId)
    await turn(client, codex.id, 'after a held resume')

    // An engine somebody typed into a terminal, adopted while the core is held.
    const tile = await client.request('agent_create', { engine: 'terminal', cwd: await folder(d, 'held-terminal') }, 120_000)
    expect(tile.error, JSON.stringify(tile)).toBeUndefined()
    const shell = await until('the terminal to have its pane', async () => {
      const now = await row(client, tile.agent.id)
      return now?.tmuxPane && now.status === 'active' ? now : null
    }, 60_000, 500)
    await sleep(2_000)
    await d.tmux.run('send-keys', '-t', shell.tmuxPane, `${join(d.root, 'bin', 'claude')} --dangerously-skip-permissions`, 'Enter')
    await sleep(300)
    hold(d)
    const adopted = await until('the typed engine to be adopted and bound', async () => {
      const now = await row(client, tile.agent.id)
      return now?.engine === 'claude' && now.sessionId && now.status === 'active' ? now : null
    }, 120_000, 500)
    await turn(client, adopted.id, 'typed into a terminal, then held')

    // Reconciles go on under the random holds; nothing may be retired by one that could not answer.
    const everyone = [{ ...claude }, { ...back }, { ...adopted }]
    for (const agent of everyone) agent.tmuxPane = (await row(client, agent.id))!.tmuxPane
    await sleep(40_000)
    await intact(d, client, everyone)
    for (const agent of everyone) await turn(client, agent.id, `still here (${agent.engine})`)

    const held = holds(d)
    expect(held.length, 'the core was held, again and again').toBeGreaterThanOrEqual(8)
    for (const ms of held) expect(ms).toBeLessThanOrEqual(8_000)
    expect(d.coresStarted()).toBe(1)
    nothingRetired(d)
    client.close()
  }, 900_000)

  it('restarted while held: restore and attach find every agent, bound and active, and each takes a turn', async () => {
    const d = await fresh()
    let client = await LocalClient.connect(d)
    const created = await Promise.all((['claude', 'codex'] as const).map(async (engine: Engine) =>
      client.request('agent_create', { engine, cwd: await folder(d, `restart-${engine}`), bypassPermission: true }, 120_000)))
    for (const answer of created) expect(answer.error, JSON.stringify(answer)).toBeUndefined()
    const agents = await Promise.all(created.map((answer) => bound(client, answer.agent.id)))
    for (const agent of agents) await turn(client, agent.id, `before the restarts (${agent.engine})`)

    for (let restart = 1; restart <= 3; restart++) {
      // A turn in flight as the daemon goes, every other time.
      if (restart % 2) client.send('message', { agentId: agents[restart % 2].id, content: '!slow 4000' })
      client.close()
      await d.stop()
      const from = d.log().length
      await d.start({ ready: 'none' })
      // Held as soon as the new core can be (before it listens for the signal, SIGUSR2 would end it), and
      // again while it restores and attaches.
      await until('the new core to arm its stalls', () => /\[stall\] armed/.test(d.log().slice(from)) || null, 60_000, 50)
      hold(d)
      await until('the daemon to be ready', () => /\[cli\] ready/.test(d.log().slice(from)) || null, 120_000, 100)
      hold(d)
      client = await LocalClient.connect(d)
      for (const agent of agents) await bound(client, agent.id, agent.sessionId)
      for (const agent of agents) await turn(client, agent.id, `after held restart ${restart} (${agent.engine})`, () => hold(d))
      await intact(d, client, agents)
    }

    const held = holds(d)
    expect(held.length).toBeGreaterThanOrEqual(6)
    for (const ms of held) expect(ms).toBeLessThanOrEqual(8_000)
    expect(d.coresStarted()).toBe(4)
    nothingRetired(d)
    client.close()
  }, 900_000)
})