Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

11.1 KB
Raw Blame
/**
 * harnessd's master over the real daemon: whatever happens to the core, the daemon comes back on its
 * own, the agents keep running in tmux, and a client picks up where it was. Whatever happens to the
 * master, nothing is left behind holding the port.
 */
import { execFileSync } from 'node:child_process'
import { existsSync, mkdirSync, readFileSync } from 'node:fs'
import { join } from 'node:path'
import { afterEach, describe, expect, it, onTestFailed } from 'vitest'
import { HARNESSD_PROTOCOL } from '../src/harnessd/protocol.js'
import { LocalClient, type Frame } from './harness/client.js'
import { IsolatedDaemon, until } from './harness/daemon.js'

const ready = /\[cli\] ready/g
const wiredCount = (daemon: IsolatedDaemon) => [...daemon.log().matchAll(ready)].length

async function withAgent(daemon: IsolatedDaemon) {
  const client = await LocalClient.connect(daemon)
  const cwd = join(daemon.projectsDir, 'work')
  mkdirSync(cwd, { recursive: true })
  const agentId: string = (await client.request('agent_create', { engine: 'claude', cwd, bypassPermission: true }, 60_000)).agent.id
  await until('the conversation to bind', async () => rowOf(client, agentId).then((row) => row?.sessionId), 45_000, 250)
  return { client, agentId }
}

const rowOf = async (client: LocalClient, agentId: string) =>
  ((await client.request('agents_list', { includeStopped: true })).agents as Array<Record<string, any>>).find((agent) => agent.id === agentId)

/** A full turn on a fresh connection: proof the restarted daemon serves the same agent. */
async function turnWorks(daemon: IsolatedDaemon, agentId: string) {
  const client = await LocalClient.connect(daemon)
  await until('the agent to be live again', async () => {
    const row = await rowOf(client, agentId)
    return row && row.status !== 'stopped' ? row : null
  }, 60_000, 250)
  const ended = client.next((frame: Frame) => frame.type === 'turn_ended' && frame.agentId === agentId, 30_000, 'a turn after the restart')
  client.send('message', { agentId, content: 'are you still there?' })
  await ended
  client.close()
}

describe('harnessd', () => {
  let daemon: IsolatedDaemon | undefined
  afterEach(async () => { await daemon?.close(); daemon = undefined })
  const make = async (env: Record<string, string> = {}, { start = true } = {}) => {
    daemon = await IsolatedDaemon.create({ env: { HARNESSD_INITIAL_BACKOFF_MS: '100', ...env } })
    const d = daemon
    onTestFailed(() => { console.log(`---- daemon log\n${d.log().split('\n').slice(-80).join('\n')}`) })
    if (start) await d.start()
    return d
  }

  it('runs the core under a master that claims the pid file for itself', async () => {
    const d = await make()
    expect(d.corePid()).not.toBe(d.pid)
    expect(readFileSync(join(d.dataDir, 'adapter.pid'), 'utf8').trim()).toBe(String(d.pid))
    // The app judges who owns the daemon by this pid: the master's, as in the pid file, not the core's.
    const status = await (await fetch(`http://127.0.0.1:${d.port}/api/status`, { headers: d.localHeaders() })).json() as Record<string, any>
    expect(status.pid).toBe(d.pid)
    expect(status.corePid).toBe(d.corePid())
    expect(status.harnessd.masterPid).toBe(d.pid)
  })

  it('restarts a core that crashes; the agent and its client carry on', async () => {
    const d = await make()
    const { client, agentId } = await withAgent(d)
    const first = d.corePid()!
    const wired = wiredCount(d)
    process.kill(first, 'SIGKILL')
    await until('a new core to finish starting', () => wiredCount(d) > wired, 60_000)
    expect(d.corePid()).not.toBe(first)
    expect(IsolatedDaemon.alive(d.pid)).toBe(true)
    await until('the old connection to close', () => client.closed, 10_000)
    await turnWorks(d, agentId)
  })

  it('kills and restarts a core that hangs, and only one that hangs', async () => {
    // A timeout shorter than the core's default beat (5 s): the core is told it and beats inside it.
    // Before, every healthy core here was "hung" three seconds after binding.
    const d = await make({ HARNESSD_HEARTBEAT_TIMEOUT_MS: '3000' })
    const { agentId } = await withAgent(d)
    const hung = d.corePid()!
    const hangs = () => [...d.log().matchAll(/it is hung/g)].length
    await new Promise((done) => setTimeout(done, 6_000))
    expect(hangs(), 'a healthy core outlives twice the timeout').toBe(0)
    expect(d.corePid()).toBe(hung)
    const wired = wiredCount(d)
    process.kill(hung, 'SIGSTOP')
    await until('the master to notice', () => hangs() > 0, 20_000)
    await until('a new core to finish starting', () => wiredCount(d) > wired, 60_000)
    expect(IsolatedDaemon.alive(hung)).toBe(false)
    await turnWorks(d, agentId)
    const next = d.corePid()
    await new Promise((done) => setTimeout(done, 6_000))
    expect(d.corePid(), 'the new core stays up').toBe(next)
    expect(hangs()).toBe(1)
  })

  it('takes the core down with it when the master is killed, and starts clean again', async () => {
    const d = await make()
    const { agentId } = await withAgent(d)
    const core = d.corePid()!
    await d.kill()
    await until('the core to follow its master', () => !IsolatedDaemon.alive(core), 10_000)
    await d.start()
    await turnWorks(d, agentId)
  })

  it('stops core and master within the grace harness stop gives them, removing the pid file', async () => {
    const d = await make()
    const core = d.corePid()!
    const master = d.pid!
    const started = Date.now()
    process.kill(master, 'SIGTERM')
    await until('both to exit', () => !IsolatedDaemon.alive(master) && !IsolatedDaemon.alive(core), 10_000, 25)
    expect(Date.now() - started).toBeLessThan(3_000)
    expect(existsSync(join(d.dataDir, 'adapter.pid'))).toBe(false)
  })

  it('`harness start -f` runs the master in the foreground, as launchd does: the core its child, a stop ends both', async () => {
    // It ran the core alone, with no master: every update then had to be handed over by the core itself.
    daemon = await IsolatedDaemon.create({ foreground: true, env: { HARNESSD_INITIAL_BACKOFF_MS: '100' } })
    const d = daemon
    onTestFailed(() => { console.log(`---- daemon log\n${d.log().split('\n').slice(-80).join('\n')}`) })
    await d.start()
    const master = d.pid!
    const core = d.corePid()!
    expect(core).not.toBe(master)
    expect(Number(execFileSync('ps', ['-o', 'ppid=', '-p', String(core)], { encoding: 'utf8' }).trim())).toBe(master)
    const status = await (await fetch(`http://127.0.0.1:${d.port}/api/status`, { headers: d.localHeaders() })).json() as Record<string, any>
    expect(status.harnessd.masterPid).toBe(master)
    expect(readFileSync(join(d.dataDir, 'adapter.pid'), 'utf8').trim()).toBe(String(master))
    // A crashed core is restarted under it, and the agent goes on.
    const { agentId } = await withAgent(d)
    const wired = wiredCount(d)
    process.kill(core, 'SIGKILL')
    await until('a new core to finish starting', () => wiredCount(d) > wired, 60_000)
    await turnWorks(d, agentId)
    const second = d.corePid()!
    process.kill(master, 'SIGTERM')
    await until('both to exit', () => !IsolatedDaemon.alive(master) && !IsolatedDaemon.alive(second), 10_000, 25)
    expect(existsSync(join(d.dataDir, 'adapter.pid'))).toBe(false)
  })

  it('restarts a core that outgrows its memory budget, backing off while it keeps doing so', async () => {
    daemon = await IsolatedDaemon.create({ env: { HARNESSD_INITIAL_BACKOFF_MS: '200', HARNESSD_RSS_LIMIT_MIB: '1' } })
    const d = daemon
    await d.start({ ready: 'none' })
    await until('the master to restart it for its memory', () => /over its 1 MiB budget — restarting it/.test(d.log()), 30_000)
    await until('another core', () => d.coresStarted() >= 2, 30_000)
    // 200, 400, 800, 1600 ms… — never as fast as a core can bind.
    await new Promise((resolve) => setTimeout(resolve, 3_000))
    expect(d.coresStarted()).toBeLessThanOrEqual(6)
  })

  it('restarts a core that something other than its master stops — it does not take the daemon down', async () => {
    const d = await make()
    const { agentId } = await withAgent(d)
    const first = d.corePid()!
    const wired = wiredCount(d)
    process.kill(first, 'SIGTERM')
    await until('a new core to finish starting', () => wiredCount(d) > wired, 60_000)
    expect(IsolatedDaemon.alive(d.pid)).toBe(true)
    expect(d.log()).toMatch(/core exited \(code 0, crashed\) — restarting/)
    await turnWorks(d, agentId)
  })

  it('records what the master knows where `harness status` reads it, the core alive or not', async () => {
    const d = await make()
    const statusFile = join(d.dataDir, 'harnessd-status.json')
    const read = () => JSON.parse(readFileSync(statusFile, 'utf8'))
    await until('the status file to say running', () => existsSync(statusFile) && read().state === 'running', 10_000)
    expect(read()).toMatchObject({ masterPid: d.pid, corePid: d.corePid(), restarts: 0, safeMode: null, protocol: HARNESSD_PROTOCOL })
    const core = d.corePid()!
    process.kill(core, 'SIGKILL')
    await until('the status file to name the crash', () => read().restarts === 1 && read().lastExitReason === 'crashed', 10_000)
    await until('the status file to say running again', () => read().state === 'running', 60_000)
  })

  it('puts a core that keeps crashing in safe mode, and tries a normal one once safe mode runs out', async () => {
    const d = await make({ HARNESSD_CRASH_LOOP_WINDOW_MS: '120000', ADAPTER_SAFE_MODE_MS: '6000' })
    const statusFile = join(d.dataDir, 'harnessd-status.json')
    const read = () => JSON.parse(readFileSync(statusFile, 'utf8'))
    for (let crash = 1; crash <= 3; crash++) {
      const core = d.corePid()!
      process.kill(core, 'SIGKILL')
      await until(`core ${crash + 1} to start`, () => d.coresStarted() > crash, 30_000)
      await until(`core ${crash + 1} to bind`, () => d.corePid() !== core && read().state !== 'restarting', 60_000)
    }
    expect(d.log()).toMatch(/core crashed 3 times in 2 min — starting it in safe mode/)
    await until('the safe-mode core to say so', () => read().safeMode !== null && read().state === 'running', 30_000)
    expect(read().safeMode).toMatch(/crash again and again/)
    // Its time runs out with no fix: a normal core starts and finishes starting.
    const wired = wiredCount(d)
    await until('a normal core again', () => wiredCount(d) > wired, 60_000)
    await until('the status file to drop safe mode', () => read().safeMode === null && read().state === 'running', 30_000)
  })

  it('kills a core that binds but never finishes starting, and keeps doing so into safe mode', async () => {
    const d = await make({ HARNESSD_TEST_HOLD_READY: '1', HARNESSD_READY_TIMEOUT_MS: '3000', HARNESSD_CRASH_LOOP_WINDOW_MS: '120000' }, { start: false })
    await d.start({ ready: 'none' })
    await until('the master to give up on a start-up', () => /core bound but not ready within 3000 ms — killing it/.test(d.log()), 30_000)
    await until('safe mode', () => /starting it in safe mode/.test(d.log()), 60_000)
    const statusFile = join(d.dataDir, 'harnessd-status.json')
    await until('the safe-mode core to be up', () => {
      const status = JSON.parse(readFileSync(statusFile, 'utf8'))
      return status.state === 'running' && status.safeMode !== null
    }, 30_000)
  })
})