Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

21.5 KB
Raw Blame
/**
 * The master re-executing itself on an update, on real release bundles (harnessd/reexec.ts): the core
 * stages a newer build and exits 75, and the master, before it starts a core on it, replaces itself
 * with that build in its own process. The pid stays, the core comes back on the new build under the
 * new master, and the agent at work in tmux carries on.
 *
 * A build whose master does not answer its probe is rolled back and remembered, like one whose core
 * fails. One whose master dies after it re-executed leaves its marker, and the next start (the
 * desktop app's `harness start`, or launchd's restart) rolls it back and moves onto the build before.
 * Nothing is left orphaned at any step: every core is the master's child.
 *
 * The bundles are built from this checkout at made-up versions and served from a local manifest, as in
 * e2e/update.e2e.ts, so nothing here reaches beyond the machine.
 */
import { execFileSync } from 'node:child_process'
import { createHash } from 'node:crypto'
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
import { createServer, type Server } from 'node:http'
import { tmpdir } from 'node:os'
import { dirname, join } from 'node:path'
import { afterAll, beforeAll, describe, expect, it, onTestFailed } from 'vitest'
import { LocalClient, type Frame } from './harness/client.js'
import { CLI_ROOT, IsolatedDaemon, until } from './harness/daemon.js'
import { atVersion, withFault } from './harness/release.js'
import { PROBE_COMMAND, runProbe } from '../src/harnessd/reexec.js'

const FIRST = '42.0.1'
const sha = (bytes: Buffer): string => createHash('sha256').update(bytes).digest('hex')
const count = (text: string, part: string): number => text.split(part).length - 1
const sleep = (ms: number) => new Promise((done) => setTimeout(done, ms))

describe('the master re-executing itself on an update', () => {
  let scratch = ''
  let server: Server | undefined
  let daemon: IsolatedDaemon | undefined
  let master: number | null = null
  let agent: Record<string, any> = {}
  const releases = new Map<string, Buffer>()
  let notify = Buffer.alloc(0)
  let offered = FIRST
  const cliDir = () => join(scratch, 'cli')
  const installed = () => readFileSync(join(cliDir(), 'cli.js'))
  const rejected = () => JSON.parse(readFileSync(join(cliDir(), 'update-rejected.json'), 'utf8')) as string[]
  const marker = () => join(daemon!.dataDir, 'harnessd-reexec.json')
  const status = async (): Promise<Record<string, any> | null> =>
    fetch(`http://127.0.0.1:${daemon!.port}/api/status`, { headers: daemon!.localHeaders() }).then((response) => response.json()).catch(() => null)
  /** The build the core runs, and the one the master runs, as the daemon reports them. */
  const builds = async (): Promise<{ core: string | null; master: string | null; masterPid: number | null; reexecs: number | null }> => {
    const now = await status()
    return { core: now?.version ?? null, master: now?.harnessd?.masterVersion ?? null, masterPid: now?.harnessd?.masterPid ?? null, reexecs: now?.harnessd?.reexecs ?? null }
  }
  const on = (version: string) => until(`the core and the master to run ${version}`, async () => {
    const now = await builds()
    return now.core === version && now.master === version ? now : null
  }, 90_000, 250)

  const rows = async (client: LocalClient) =>
    (await client.request<{ agents: Array<Record<string, any>> }>('agents_list', { includeStopped: true }, 30_000)).agents
  /** The agent is back, the same conversation, and takes a turn: the core under the new master serves it. */
  const agentCarriesOn = async () => {
    const client = await LocalClient.connect(daemon!)
    try {
      await until('the agent to be back', async () => {
        const row = (await rows(client)).find((candidate) => candidate.id === agent.id)
        return row?.status === 'active' && row.sessionId === agent.sessionId ? row : null
      }, 60_000, 500)
      const ended = client.next((frame: Frame) => frame.type === 'turn_ended' && frame.agentId === agent.id, 45_000, 'a turn')
      client.send('message', { agentId: agent.id, content: 'still there?' })
      await ended
    } finally { client.close() }
  }

  /**
   * Every process running a build of this machine, from the process table. Each core must be the
   * master's child: one left behind by a master that replaced itself would be an orphan holding the
   * port, re-parented to launchd or init.
   */
  const daemonProcesses = (): Array<{ pid: number; ppid: number; command: string }> =>
    execFileSync('ps', ['-axo', 'pid=,ppid=,command='], { encoding: 'utf8' }).split('\n')
      .map((line) => /^\s*(\d+)\s+(\d+)\s+(.*)$/.exec(line))
      // The core and the services from the lean bundle in the data folder, or from cli.js.
      .filter((match): match is RegExpExecArray => !!match && (match[3].includes(join(cliDir(), 'cli.js')) || (!!daemon && match[3].includes(join(daemon.dataDir, 'lean')))))
      .map((match) => ({ pid: Number(match[1]), ppid: Number(match[2]), command: match[3] }))
  const nothingOrphaned = async (masterPid: number) => {
    // A probe is over within the update; give one a moment to end.
    await until('only the master\'s own children', () => daemonProcesses().every((process) => process.ppid === masterPid && process.command.endsWith(' __run')), 10_000, 200)
    expect(daemonProcesses().filter((process) => process.command.endsWith(' __run'))).toHaveLength(1)
  }

  beforeAll(async () => {
    scratch = mkdtempSync(join(tmpdir(), 'harnessd-reexec-'))
    const out = join(scratch, 'build')
    execFileSync(process.execPath, ['build-bundle.mjs'], {
      cwd: CLI_ROOT, env: { ...process.env, ADAPTER_VERSION: FIRST, BUNDLE_OUT_DIR: out }, stdio: 'pipe',
    })
    const first = readFileSync(join(out, 'cli.js'), 'utf8')
    notify = readFileSync(join(out, 'notify.mjs'))
    expect(count(first, FIRST)).toBeGreaterThanOrEqual(2)
    const release = (version: string, inject = ''): void => {
      // The version is baked in at build time; the others are the same bytes with it swapped, and a
      // fault put in at the top where a test needs one.
      const source = withFault(atVersion(first, FIRST, version), inject)
      const file = join(out, `cli-${version}.js`)
      writeFileSync(file, source)
      expect(execFileSync(process.execPath, [file, 'version'], { encoding: 'utf8' }).trim()).toBe(version)
      releases.set(version, Buffer.from(source))
    }
    release(FIRST)
    release('42.0.2')
    // Passes the updater's canary (`cli.js version`), and its master does not answer its probe.
    release('42.0.3', 'if(process.argv[2]==="__harnessd-probe")process.exit(3);')
    release('42.0.4')
    // Answers its probe, and dies as soon as a master re-executes on it.
    release('42.0.5', 'if(process.argv[2]==="__harnessd"&&process.env.HARNESSD_RESUME)process.exit(4);')

    server = createServer((request, response) => {
      const url = request.url ?? ''
      const origin = `http://127.0.0.1:${(server!.address() as { port: number }).port}`
      if (url === '/metadata.json') {
        const cli = releases.get(offered)!
        response.end(JSON.stringify({ cli: {
          version: offered,
          cli: { url: `${origin}/cli-${offered}.js`, sha256: sha(cli), size: cli.length },
          notify: { url: `${origin}/notify.mjs`, sha256: sha(notify), size: notify.length },
        } }))
        return
      }
      const asked = /^\/cli-(.+)\.js$/.exec(url)?.[1]
      if (asked && releases.has(asked)) { response.end(releases.get(asked)); return }
      if (url === '/notify.mjs') { response.end(notify); return }
      response.statusCode = 404
      response.end()
    })
    await new Promise<void>((done) => server!.listen(0, '127.0.0.1', done))
    const local = `http://127.0.0.1:${(server.address() as { port: number }).port}`

    mkdirSync(cliDir())
    writeFileSync(join(cliDir(), 'cli.js'), releases.get(FIRST)!)
    writeFileSync(join(cliDir(), 'notify.mjs'), notify)
    writeFileSync(join(cliDir(), 'package.json'), '{"type":"module"}\n')
    daemon = await IsolatedDaemon.create({
      scriptPath: join(cliDir(), 'cli.js'),
      env: {
        ADAPTER_CLI_DIR: cliDir(),
        ADAPTER_UPDATE_DISABLE: 'false',
        ADAPTER_UPDATE_URL: `${local}/metadata.json`,
        ADAPTER_UPDATE_CHECK_MS: '1000',
        ADAPTER_UPDATE_SLOT_SEC: '-1',
        HARNESSD_UPDATE_PROBATION_MS: '3000',
        HARNESSD_INITIAL_BACKOFF_MS: '100',
        HARNESS_TUI_MANIFEST_URL: `${local}/tui/metadata.json`,
        ADAPTER_RUNTIME_METADATA_URL: `${local}/runtime/metadata.json`,
        ADAPTER_GRID_RUNTIME_METADATA_URL: `${local}/grid/metadata.json`,
      },
    })
    await daemon.start()
    master = daemon.pid
    expect(await builds()).toMatchObject({ core: FIRST, master: FIRST, masterPid: master, reexecs: 0 })

    // An agent at work, whose pane lives in tmux through every swap below.
    const client = await LocalClient.connect(daemon)
    const cwd = join(daemon.projectsDir, 'reexec')
    mkdirSync(cwd, { recursive: true })
    const created = await client.request('agent_create', { engine: 'claude', cwd, bypassPermission: true }, 90_000)
    agent = await until('the agent to bind its conversation', async () => {
      const row = (await rows(client)).find((candidate) => candidate.id === created.agent.id)
      return row?.sessionId && row.status === 'active' ? row : null
    }, 60_000, 500)
    client.close()
  })

  afterAll(async () => {
    await daemon?.close()
    await new Promise<void>((done) => server ? server.close(() => done()) : done())
    if (scratch) rmSync(scratch, { recursive: true, force: true })
  })

  const showLog = () => onTestFailed(() => { console.log(`---- daemon log\n${daemon?.log().split('\n').slice(-150).join('\n')}`) })

  it('replaces the master with the new build in its own process: same pid, and the core and the agent carry on', async () => {
    showLog()
    offered = '42.0.2'
    const now = await on('42.0.2')
    expect(now).toMatchObject({ masterPid: master, reexecs: 1 })
    expect(daemon!.pid).toBe(master)
    expect(IsolatedDaemon.alive(master)).toBe(true)
    expect(daemon!.log()).toContain(`[harnessd] master re-executed (pid ${master}) · now v42.0.2`)
    await until('the master to keep the update', () => daemon!.log().includes('the update stayed up — keeping it'), 30_000)
    expect(installed()).toEqual(releases.get('42.0.2'))
    expect(existsSync(marker())).toBe(false)
    await agentCarriesOn()
    await nothingOrphaned(master!)
  })

  it('rolls back a build whose master does not answer its probe, and remembers it; the master stays as it was', async () => {
    showLog()
    const from = daemon!.log().length
    offered = '42.0.3'
    await until('the rollback', () => daemon!.log().slice(from).includes('the new bundle\'s master would not start — rolled back to the previous bundle'), 90_000)
    expect(daemon!.log().slice(from)).toContain('did not answer its probe')
    expect(await on('42.0.2')).toMatchObject({ masterPid: master, reexecs: 1 })
    expect(installed()).toEqual(releases.get('42.0.2'))
    expect(rejected()).toEqual(['42.0.3'])
    expect(existsSync(join(cliDir(), 'update-pending.json'))).toBe(false)
    // A check every second: a few more of them, and the rejected build is not staged again.
    await until('the core to say it waits for a newer build', () => daemon!.log().slice(from).includes('42.0.3 was rolled back on this machine'), 30_000)
    const cores = daemon!.coresStarted()
    await sleep(4_000)
    expect(daemon!.coresStarted()).toBe(cores)
    await agentCarriesOn()
    await nothingOrphaned(master!)
  })

  it('moves on when a newer build is published', async () => {
    showLog()
    offered = '42.0.4'
    expect(await on('42.0.4')).toMatchObject({ masterPid: master, reexecs: 2 })
    await until('the master to keep the update', () => count(daemon!.log(), 'the update stayed up — keeping it') === 2, 30_000)
    expect(rejected()).toEqual(['42.0.3'])
    await agentCarriesOn()
    await nothingOrphaned(master!)
  })

  it('rolls back, at the next start, a build whose master died after re-executing on it', async () => {
    showLog()
    offered = '42.0.5'
    await until('the master that re-executed on 42.0.5 to die', () => !IsolatedDaemon.alive(master), 90_000, 200)
    expect(readFileSync(marker(), 'utf8')).toContain(`"pid":${master}`)
    expect(installed()).toEqual(releases.get('42.0.5'))
    // The core it replaced had exited for the update; nothing was left running in its name.
    expect(daemonProcesses()).toEqual([])

    // What the desktop app's `harness start`, or launchd restarting the master, would do.
    const from = daemon!.log().length
    await daemon!.start()
    const restarted = daemon!.pid!
    expect(restarted).not.toBe(master)
    expect(daemon!.log().slice(from)).toContain(`the master that re-executed on the new bundle (pid ${master}) never brought a core up — rolled back to the previous bundle`)
    expect(await on('42.0.4')).toMatchObject({ masterPid: restarted, reexecs: 1 })
    expect(installed()).toEqual(releases.get('42.0.4'))
    expect(rejected()).toEqual(['42.0.3', '42.0.5'])
    await until('the marker to be gone', () => !existsSync(marker()), 30_000)
    await agentCarriesOn()
    await nothingOrphaned(restarted)
  })
})

/**
 * A released master and this one, each over the other's core (protocol.ts only grows). Run when
 * MIGRATION_FROM names a released bundle, as in e2e/migration.e2e.ts.
 *
 * An older master cannot re-execute: after an update it supervises the newer core until it is
 * restarted, as every machine will once this ships. A newer master that finds the older bundle back on
 * disk does not re-execute on it either: the older bundle does not know the probe, and is never run as
 * a master to find out. Its core runs under the newer master. A release that answers the probe must
 * re-execute in both directions, with the same pid and the version of the bundle now on disk.
 */
const RELEASED = process.env.MIGRATION_FROM
describe.skipIf(!RELEASED)('a released master and this one, each over the other\'s core', () => {
  const NEXT = '0.99.0'
  let scratch = ''
  let server: Server | undefined
  let daemon: IsolatedDaemon | undefined
  let offered: 'released' | 'next' | 'later' = 'released'
  let releasedVersion = ''
  let releasedCanReexec = false
  const bundles = new Map<'released' | 'next' | 'later', { cli: Buffer; notify: Buffer; version: string }>()
  const LATER = '0.99.1'
  /** The newest build this machine has kept: NEXT, or LATER once a released master's core updated to it. */
  let newest = NEXT
  const cliDir = () => join(scratch, 'cli')
  const status = async (): Promise<Record<string, any> | null> =>
    fetch(`http://127.0.0.1:${daemon!.port}/api/status`, { headers: daemon!.localHeaders() }).then((response) => response.json()).catch(() => null)

  beforeAll(async () => {
    scratch = mkdtempSync(join(tmpdir(), 'harnessd-reexec-compat-'))
    const out = join(scratch, 'build')
    execFileSync(process.execPath, ['build-bundle.mjs'], { cwd: CLI_ROOT, env: { ...process.env, ADAPTER_VERSION: NEXT, BUNDLE_OUT_DIR: out }, stdio: 'pipe' })
    bundles.set('next', { cli: readFileSync(join(out, 'cli.js')), notify: readFileSync(join(out, 'notify.mjs')), version: NEXT })
    bundles.set('later', { cli: Buffer.from(atVersion(readFileSync(join(out, 'cli.js'), 'utf8'), NEXT, LATER)), notify: readFileSync(join(out, 'notify.mjs')), version: LATER })
    releasedVersion = execFileSync(process.execPath, [RELEASED!, 'version'], { encoding: 'utf8' }).trim()
    bundles.set('released', { cli: readFileSync(RELEASED!), notify: readFileSync(join(dirname(RELEASED!), 'notify.mjs')), version: releasedVersion })
    server = createServer((request, response) => {
      const origin = `http://127.0.0.1:${(server!.address() as { port: number }).port}`
      const bundle = bundles.get(offered)!
      if (request.url === '/metadata.json') {
        response.end(JSON.stringify({ cli: {
          version: bundle.version,
          cli: { url: `${origin}/cli.js`, sha256: sha(bundle.cli), size: bundle.cli.length },
          notify: { url: `${origin}/notify.mjs`, sha256: sha(bundle.notify), size: bundle.notify.length },
        } }))
        return
      }
      if (request.url === '/cli.js') { response.end(bundle.cli); return }
      if (request.url === '/notify.mjs') { response.end(bundle.notify); return }
      response.statusCode = 404
      response.end()
    })
    await new Promise<void>((done) => server!.listen(0, '127.0.0.1', done))
    const local = `http://127.0.0.1:${(server.address() as { port: number }).port}`
    mkdirSync(cliDir())
    writeFileSync(join(cliDir(), 'cli.js'), bundles.get('released')!.cli)
    writeFileSync(join(cliDir(), 'notify.mjs'), bundles.get('released')!.notify)
    writeFileSync(join(cliDir(), 'package.json'), '{"type":"module"}\n')
    daemon = await IsolatedDaemon.create({
      scriptPath: join(cliDir(), 'cli.js'),
      env: {
        ADAPTER_CLI_DIR: cliDir(), ADAPTER_UPDATE_DISABLE: 'false', ADAPTER_UPDATE_URL: `${local}/metadata.json`,
        ADAPTER_UPDATE_CHECK_MS: '1000', ADAPTER_UPDATE_SLOT_SEC: '-1', HARNESSD_UPDATE_PROBATION_MS: '3000', HARNESSD_INITIAL_BACKOFF_MS: '100',
        HARNESS_TUI_MANIFEST_URL: `${local}/tui/metadata.json`, ADAPTER_RUNTIME_METADATA_URL: `${local}/runtime/metadata.json`,
        ADAPTER_GRID_RUNTIME_METADATA_URL: `${local}/grid/metadata.json`,
      },
    })
    // Found by QA on a quiet machine: 0.3.60 re-executes, but this fixture assumed every published
    // release predated it. Probe the actual bundle; a timeout or broken probe must not select the old
    // expectations and hide a regression. Old releases explicitly reject the private command.
    const probe = await runProbe(process.execPath, [join(cliDir(), 'cli.js'), PROBE_COMMAND], daemon.env).result
    if (!probe.ok) expect(probe.detail).toBe(`Unknown command: ${PROBE_COMMAND}`)
    releasedCanReexec = probe.ok
    await daemon.start()
  })

  afterAll(async () => {
    await daemon?.close()
    await new Promise<void>((done) => server ? server.close(() => done()) : done())
    if (scratch) rmSync(scratch, { recursive: true, force: true })
  })

  it('a released master adopts the update, re-executing when supported, and restarts on the new build', async () => {
    onTestFailed(() => { console.log(`---- daemon log\n${daemon?.log().split('\n').slice(-120).join('\n')}`) })
    const master = daemon!.pid
    expect((await status())?.version).toBe(releasedVersion)
    offered = 'next'
    const now = await until('the newer core under the released master', async () => {
      const current = await status()
      return current?.version === NEXT ? current : null
    }, 90_000, 250)
    expect(now.harnessd.masterPid).toBe(master)
    if (releasedCanReexec) {
      expect(now.harnessd).toMatchObject({ masterVersion: NEXT, reexecs: 1 })
      expect(daemon!.log()).toContain(`now v${NEXT}`)
    } else {
      expect(now.harnessd.masterVersion).toBeUndefined()
      expect(daemon!.log()).not.toContain('re-executing')
    }
    await until('the released master to keep the update', () => daemon!.log().includes('the update stayed up — keeping it'), 30_000)
    if (!releasedCanReexec) {
      // A master from before the updater left the core runs none, and could not become this build's: the
      // core runs it beside itself (core/updaterBeside.ts), so the next build still arrives.
      expect(daemon!.log()).toContain('[update] this core\'s master runs no updater — running it beside this core')
      offered = 'later'
      await until(`the released master to run ${LATER}`, async () => (await status())?.version === LATER || null, 90_000, 250)
      expect((await status())?.harnessd.masterPid).toBe(master)
      await until('the released master to keep it too', () => daemon!.log().split('the update stayed up — keeping it').length > 2 || null, 30_000)
      newest = LATER
    }
    await daemon!.restart()
    expect((await status())?.harnessd).toMatchObject({ masterVersion: newest, masterPid: daemon!.pid })
  })

  it('this master returns to the released bundle, re-executing only when it answers the probe', async () => {
    onTestFailed(() => { console.log(`---- daemon log\n${daemon?.log().split('\n').slice(-120).join('\n')}`) })
    offered = 'released'   // not newer: the older core's updater leaves it alone
    const master = daemon!.pid
    const from = daemon!.log().length
    writeFileSync(join(cliDir(), 'cli.js'), bundles.get('released')!.cli)
    process.kill((await status())!.corePid as number, 'SIGKILL')
    const now = await until('the older core under this master', async () => {
      const current = await status()
      return current?.version === releasedVersion ? current : null
    }, 60_000, 250)
    expect(now.harnessd).toMatchObject({
      masterVersion: releasedCanReexec ? releasedVersion : newest,
      masterPid: master,
      reexecs: releasedCanReexec ? 1 : 0,
    })
    expect(daemon!.log().slice(from)).toContain(releasedCanReexec
      ? `now v${releasedVersion}`
      : 'did not answer its probe (Unknown command: __harnessd-probe) — keeping this master')
    // This master over a core too old to ask for a process on demand starts each as that core binds
    // (harnessd/services.ts `unasked`): models, which a core of protocol 4 asks for only once grid is in use,
    // the gateway, which it asks for only signed in or with something paired, and the viewers, which a core of
    // protocol 5 asks for only once something needs them.
    for (const name of releasedCanReexec ? [] : ['models', 'gateway', 'viewers']) {
      await until(`this master to start ${name} for the older core`, () => daemon!.log().slice(from).includes(`[harnessd] service ${name} started`) || null, 30_000, 200)
    }
  })
})