Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

19.6 KB
Raw Blame
import { execFileSync } from 'node:child_process'
import { afterEach, describe, expect, it, vi } from 'vitest'
import type { AgentLaunch, RegisteredSession } from '../../lib/registry.js'
import { heldLaunch, type RestoreSummary } from '../../lib/restoreAgents.js'
import { createHeldLaunches, heldFor, heldPaneArgv, heldRetryMs } from './heldLaunches.js'

const running: ReturnType<typeof createHeldLaunches>[] = []
afterEach(() => { for (const held of running.splice(0)) held.close() })
const HELD = heldLaunch('models')
const row = (agentId: string, launch: AgentLaunch, over: Partial<RegisteredSession> = {}) =>
  ({ agentId, sessionId: `s-${agentId}`, launch, ...over }) as RegisteredSession
const summary = (over: Partial<RestoreSummary> = {}): RestoreSummary => ({ restored: [], skipped: [], failed: [], held: [], unsurveyed: [], ...over })

/** A registry of rows and a restore that does to them what `answer` says, recording what it was asked. */
function setup(rows: RegisteredSession[], answer: (only: ReadonlySet<string>) => RestoreSummary = (only) => summary({ restored: [...only] })) {
  const registry = new Map(rows.map((r) => [r.agentId, r]))
  const asked: string[][] = []
  const restore = vi.fn(async (only: ReadonlySet<string>, _current: () => boolean) => {
    asked.push([...only])
    const result = answer(only)
    for (const id of result.restored) { const r = registry.get(id); if (r) r.launch = { state: 'starting' } }
    return result
  })
  const log = vi.fn()
  const held = createHeldLaunches({ registry: { list: () => [...registry.values()], byAgent: (id: string) => registry.get(id) } as never, restore, log })
  running.push(held)
  return { registry, asked, restore, log, held }
}

describe('agents held until a service is ready', () => {
  it('says once why an agent stays held, and does not repeat a pass that changed nothing', async () => {
    let reason = 'the pty host has gone'
    const { held, log } = setup([row('a', HELD)], (only) => summary({ held: [...only], failed: [...only].map(agentId => ({ agentId, reason })) }))
    await held.boot(async () => {})
    await held.restoreHeld(); await held.restoreHeld(); await held.restoreHeld()
    reason = 'no such folder'
    await held.restoreHeld()
    expect(log.mock.calls.map(([line]) => line)).toEqual([
      '[restore] a · could not launch from held · the pty host has gone',
      '[restore] held · every service · launched 0 of 1 · 1 still waiting',
      '[restore] a · could not launch from held · no such folder',
    ])
  })

  it.each(['removed', 'closed'] as const)('does not restore a queued row that was %s before its preparation', async state => {
    const h = setup([row('a', HELD)])
    await h.held.boot(async () => {})
    if (state === 'removed') vi.spyOn(h.registry, 'get').mockReturnValueOnce(undefined)
    const running = h.held.restoreHeld()
    if (state === 'closed') queueMicrotask(() => h.held.close())
    expect(await running).toMatchObject({ restored: [] })
    expect(h.restore).not.toHaveBeenCalled()
  })

  it('revokes the boot and every active restore authority immediately on close', async () => {
    const h = setup([row('a', HELD)])
    let bootCurrent!: () => boolean, flightCurrent!: () => boolean, finish!: () => void
    await h.held.boot(async current => { bootCurrent = current; expect(current()).toBe(true) })
    const restore = vi.fn(async (_only: ReadonlySet<string>, current: () => boolean) => {
      flightCurrent = current
      expect(current()).toBe(true)
      await new Promise<void>(done => { finish = done })
      return summary()
    })
    h.restore.mockImplementation(restore)
    const pending = h.held.restoreHeld()
    await vi.waitFor(() => expect(restore).toHaveBeenCalledOnce())
    h.held.close()
    expect(bootCurrent()).toBe(false)
    expect(flightCurrent()).toBe(false)
    finish(); await pending
  })

  it('bounds actual preparation flights and lets a later ready agent progress after one slot settles', async () => {
    vi.useFakeTimers()
    try {
      const h = setup(Array.from({ length: 5 }, (_, index) => row(String(index), HELD)))
      const complete: Array<() => void> = []
      h.restore.mockImplementation(only => new Promise(resolve => complete.push(() => {
        for (const id of only) h.registry.get(id)!.launch = { state: 'starting' }
        resolve(summary({ restored: [...only] }))
      })))
      await h.held.boot(async () => {})
      const first = h.held.restoreHeld()
      await vi.advanceTimersByTimeAsync(0)
      expect(h.restore).toHaveBeenCalledTimes(4)
      expect(await h.held.restartHeld('4')).toMatchObject({ error: 'SERVICE_UNAVAILABLE' })
      complete.shift()!()
      await vi.advanceTimersByTimeAsync(500)
      expect(h.restore).toHaveBeenCalledTimes(5)
      complete.splice(0).forEach(done => done())
      await first
      expect(h.registry.get('4')?.launch).toEqual({ state: 'starting' })
    } finally { vi.useRealTimers() }
  })

  it('names the service a row is held for, and none for any other', () => {
    expect(heldFor(row('a', HELD))).toBe('models')
    expect(heldFor(row('a', { state: 'ready' }))).toBeNull()
    expect(heldFor(row('a', { state: 'failed', error: 'X' }))).toBeNull()
    expect(heldFor(undefined)).toBeNull()
  })

  it("its pane says why, survives an interrupt, and idles: the agent's launch takes it over", () => {
    const argv = heldPaneArgv('Waiting for the models service.', undefined, 'linux')
    expect(argv.slice(0, 2)).toEqual(['/bin/sh', '-c'])
    if (process.platform === 'win32') return
    expect(argv[2]).toContain('trap "" INT QUIT TSTP')
    // The script itself, run with its loop cut short: it prints the reason as given, and nothing else.
    const once = argv[2]!.replace('while :; do sleep 3600; done', 'exit 0')
    expect(execFileSync('/bin/sh', ['-c', once, argv[3]!, argv[4]!], { encoding: 'utf8' })).toBe('Waiting for the models service.\n')
  })

  it('waits in the daemon\'s Node on Windows, which has no /bin/sh, still told by its start command', () => {
    const argv = heldPaneArgv('Waiting for the Store.', 'tok-1', 'win32')
    expect(argv.slice(1, 2)).toEqual(['-e'])
    expect(argv.slice(3)).toEqual(['harness-held', 'Waiting for the Store.', 'tok-1'])
    expect(argv).not.toContain('/bin/sh')
    expect(heldPaneArgv('Waiting for the Store.', undefined, 'linux').slice(0, 2)).toEqual(['/bin/sh', '-c'])
  })

  it('runs no pass before the boot has restored, whatever it asked meanwhile', async () => {
    const { held, asked } = setup([row('a', HELD)])
    const pass = held.restoreHeld()
    await Promise.resolve()
    expect(asked).toEqual([])
    expect(await held.boot(async () => 'booted')).toBe('booted')
    expect(await pass).toMatchObject({ restored: ['a'] })
    // A boot that throws opens them all the same.
    const failing = setup([row('b', HELD)])
    const later = failing.held.restoreHeld()
    await expect(failing.held.boot(async () => { throw new Error('restore broke') })).rejects.toThrow('restore broke')
    expect(await later).toMatchObject({ restored: ['b'] })
  })

  it('launches every held agent once the core is ready, or those held for one service when it connects', async () => {
    const { held, asked, log } = setup([row('a', HELD), row('b', heldLaunch('store')), row('c', { state: 'ready' })])
    await held.boot(async () => {})
    expect(await held.restoreHeld('store')).toMatchObject({ restored: ['b'] })
    expect(await held.restoreHeld()).toMatchObject({ restored: ['a'] })
    expect(asked).toEqual([['b'], ['a']])
    // Nothing held: no pass at all.
    expect(await held.restoreHeld()).toBeNull()
    expect(asked).toHaveLength(2)
    expect(log.mock.calls.map(([line]) => line)).toEqual([
      '[restore] held · store · launched 1 of 1',
      '[restore] held · every service · launched 1 of 1',
    ])
  })

  it('says what is still waiting after a pass that could not ask', async () => {
    const { held, log } = setup([row('a', HELD), row('b', HELD)], (only) => summary({ held: [...only] }))
    await held.boot(async () => {})
    await held.restoreHeld('models')
    expect(log).toHaveBeenLastCalledWith('[restore] held · models · launched 0 of 2 · 2 still waiting')
  })

  it.each(['prepared', 'restart'] as const)('retries transient core contention after the only %s notice, coalescing retries', async trigger => {
    vi.useFakeTimers()
    try {
      const h = setup([row('a', HELD)])
      h.restore.mockResolvedValueOnce(summary({ retry: true, held: ['a'] }))
        .mockResolvedValueOnce(summary({ retry: true, held: ['a'] }))
      await h.held.boot(async () => {})
      if (trigger === 'prepared') await h.held.restoreHeld('models')
      else await h.held.restartHeld('a')
      await h.held.restoreHeld('models')
      expect(h.restore).toHaveBeenCalledTimes(2)
      await vi.advanceTimersByTimeAsync(500)
      expect(h.restore).toHaveBeenCalledTimes(3)
      expect(h.registry.get('a')?.launch?.state).toBe('starting')
    } finally { vi.useRealTimers() }
  })

  it('logs a failed core-contention retry without an unhandled rejection', async () => {
    vi.useFakeTimers()
    try {
      const h = setup([row('a', HELD)])
      h.restore.mockResolvedValueOnce(summary({ retry: true })).mockRejectedValueOnce(new Error('test retry failure'))
      await h.held.boot(async () => {})
      await h.held.restoreHeld()
      await vi.advanceTimersByTimeAsync(500)
      expect(h.log).toHaveBeenLastCalledWith(expect.stringContaining('test retry failure'))
    } finally { vi.useRealTimers() }
  })

  it('joins a flight for the same agent, and a failed flight permits the next attempt', async () => {
    let release!: () => void
    const order: string[] = []
    const { held, restore } = setup([row('a', HELD)])
    restore.mockImplementationOnce(async () => { order.push('first'); await new Promise<void>((done) => { release = done }); throw new Error('models hung up') })
    await held.boot(async () => {})
    const first = held.restoreHeld().catch(error => error)
    const second = held.restoreHeld().catch(error => error)
    await vi.waitFor(() => expect(order).toEqual(['first']))
    expect(restore).toHaveBeenCalledTimes(1)
    release()
    expect((await first).message).toBe('models hung up')
    expect((await second).message).toBe('models hung up')
    expect(await held.restoreHeld()).toMatchObject({ restored: ['a'] })
  })

  describe("a person's restart of a held agent", () => {
    it('is not one for an agent that is not held: it restarts as it always has', () => {
      const { held } = setup([row('a', { state: 'ready' })])
      expect(held.restartHeld('a')).toBeNull()
      expect(held.restartHeld('nobody')).toBeNull()
    })

    it('launches it, and answers it as a restart would', async () => {
      const { held, log } = setup([row('a', HELD)])
      await held.boot(async () => {})
      expect(await held.restartHeld('a')).toMatchObject({ ok: true, session: { agentId: 'a' }, resumed: true })
      expect(log).toHaveBeenCalledWith('[restart] a launched from held')
      const fresh = setup([row('b', HELD, { sessionId: '' })])
      await fresh.held.boot(async () => {})
      expect(await fresh.held.restartHeld('b')).toMatchObject({ ok: true, resumed: false })
    })

    it('says it still waits, or why its launch failed, or that it could not be launched', async () => {
      const waiting = setup([row('a', HELD)], (only) => summary({ held: [...only] }))
      await waiting.held.boot(async () => {})
      expect(await waiting.held.restartHeld('a')).toEqual({ ok: false, error: 'SERVICE_UNAVAILABLE', detail: HELD.state === 'held' ? HELD.detail : '' })
      const refused = setup([row('b', HELD)], (only) => summary({ failed: [...only].map((agentId) => ({ agentId, reason: 'no way there' })) }))
      refused.restore.mockImplementationOnce(async (only) => {
        refused.registry.get('b')!.launch = { state: 'failed', error: 'GRID_ENGINE_UNSUPPORTED', detail: 'no way there' }
        return summary({ failed: [...only].map((agentId) => ({ agentId, reason: 'no way there' })) })
      })
      await refused.held.boot(async () => {})
      expect(await refused.held.restartHeld('b')).toEqual({ ok: false, error: 'GRID_ENGINE_UNSUPPORTED', detail: 'no way there' })
      const bare = setup([row('c', HELD)])
      bare.restore.mockImplementationOnce(async () => { bare.registry.get('c')!.launch = { state: 'failed', error: 'START_TIMEOUT' }; return summary() })
      await bare.held.boot(async () => {})
      expect(await bare.held.restartHeld('c')).toEqual({ ok: false, error: 'START_TIMEOUT' })
      const lost = setup([row('d', HELD)], (only) => summary({ failed: [...only].map((agentId) => ({ agentId, reason: 'tmux said no' })) }))
      lost.restore.mockImplementationOnce(async (only) => { lost.registry.get('d')!.launch = { state: 'ready' }; return summary({ failed: [...only].map((agentId) => ({ agentId, reason: 'tmux said no' })) }) })
      await lost.held.boot(async () => {})
      expect(await lost.held.restartHeld('d')).toEqual({ ok: false, error: 'RESTART_FAILED', detail: 'tmux said no' })
      const none = setup([row('e', HELD)], () => summary())
      none.restore.mockImplementationOnce(async () => { none.registry.get('e')!.launch = { state: 'ready' }; return summary() })
      await none.held.boot(async () => {})
      expect(await none.held.restartHeld('e')).toEqual({ ok: false, error: 'RESTART_FAILED', detail: 'The harness could not be launched.' })
    })

    it('leaves an agent a pass launched meanwhile, or that was stopped or closed meanwhile, as it is', async () => {
      const { held, registry, restore } = setup([row('a', HELD), row('b', HELD)])
      const launchedMeanwhile = held.restartHeld('a')!
      const closedMeanwhile = held.restartHeld('b')!
      registry.get('a')!.launch = { state: 'ready' }
      registry.delete('b')
      await held.boot(async () => {})
      expect(await launchedMeanwhile).toMatchObject({ ok: true, session: { agentId: 'a' } })
      expect(await closedMeanwhile).toEqual({ ok: false, error: 'AGENT_NOT_FOUND' })
      expect(restore).not.toHaveBeenCalled()
      // Closed while its own pass ran.
      const gone = setup([row('c', HELD)])
      gone.restore.mockImplementationOnce(async () => { gone.registry.delete('c'); return summary({ restored: ['c'] }) })
      await gone.held.boot(async () => {})
      expect(await gone.held.restartHeld('c')).toEqual({ ok: false, error: 'AGENT_NOT_FOUND' })
    })
  })

  it('restores only rows still held when the pass runs: one stopped or closed since is left alone', async () => {
    const { held, registry, asked } = setup([row('a', HELD), row('b', HELD)])
    const pass = held.restoreHeld()
    registry.delete('b')
    await held.boot(async () => {})
    await pass
    expect(asked).toEqual([['a']])
  })
})


it('restores a sibling while another native preparation is stalled', async () => {
  const h = setup([row('a', heldLaunch('native')), row('b', heldLaunch('native'))])
  let finish!: (result: RestoreSummary) => void
  h.restore.mockImplementationOnce(() => new Promise(resolve => { finish = resolve }))
  await h.held.boot(async () => {})
  const pass = h.held.restoreHeld()
  await vi.waitFor(() => expect(h.registry.get('b')?.launch?.state).toBe('starting'))
  expect(h.registry.get('a')?.launch?.state).toBe('held')
  finish(summary({ held: ['a'] }))
  expect(await pass).toMatchObject({ restored: ['b'], held: ['a'] })
})

it('retries a native hold without a service reconnect and stops scheduling at close', async () => {
  vi.useFakeTimers()
  try {
    const h = setup([row('a', heldLaunch('native'))])
    h.restore.mockResolvedValueOnce(summary({ held: ['a'] }))
    await h.held.boot(async () => {})
    await h.held.restoreHeld()
    await vi.advanceTimersByTimeAsync(2_000)
    expect(h.restore).toHaveBeenCalledTimes(2)
    h.registry.get('a')!.launch = heldLaunch('native')
    h.held.close()
    expect(await h.held.restoreHeld()).toBeNull()
    await vi.advanceTimersByTimeAsync(10_000)
    expect(h.restore).toHaveBeenCalledTimes(2)
  } finally { vi.useRealTimers() }
})

it('retries a held sibling while another agent’s observation has not settled', async () => {
  vi.useFakeTimers()
  try {
    const h = setup([row('a', heldLaunch('native')), row('b', heldLaunch('native'))])
    let finish!: (result: RestoreSummary) => void
    h.restore.mockImplementationOnce(() => new Promise(resolve => { finish = resolve }))
      .mockResolvedValueOnce(summary({ held: ['b'] }))
    await h.held.boot(async () => {})
    const pass = h.held.restoreHeld()
    await vi.advanceTimersByTimeAsync(2_001)
    expect(h.registry.get('b')?.launch?.state).toBe('starting')
    expect(h.restore.mock.calls.map(([only]) => [...only])).toEqual([['a'], ['b'], ['b']])
    finish(summary({ held: ['a'] }))
    await pass
  } finally { vi.useRealTimers() }
})

it('waits longer after each timer pass that launched nothing, up to half a minute', () => {
  expect([0, 1, 2, 3, 4, 5, 40].map(heldRetryMs)).toEqual([2_000, 4_000, 8_000, 16_000, 30_000, 30_000, 30_000])
})

it('backs off a hold nothing else wakes, and climbs again from the start once something changes', async () => {
  vi.useFakeTimers()
  try {
    const h = setup([row('a', heldLaunch('native'))], () => summary({ held: ['a'] }))
    await h.held.boot(async () => {})
    await h.held.restoreHeld()
    const passesAfter = async (ms: number) => { await vi.advanceTimersByTimeAsync(ms); return h.restore.mock.calls.length }
    // 2 s, 4 s, 8 s, 16 s, then every 30 s: a day of it is about 2,900 passes, not 43,000.
    expect(await passesAfter(1_999)).toBe(1)
    expect(await passesAfter(1)).toBe(2)
    expect(await passesAfter(4_000)).toBe(3)
    expect(await passesAfter(8_000)).toBe(4)
    expect(await passesAfter(16_000)).toBe(5)
    expect(await passesAfter(29_999)).toBe(5)
    expect(await passesAfter(1)).toBe(6)
    expect(await passesAfter(30_000)).toBe(7)
    // A service connecting is news: it is asked at once, the waiting pass keeps its time, and the one after
    // that is 4 s later again rather than 30.
    await h.held.restoreHeld('native')
    expect(h.restore).toHaveBeenCalledTimes(8)
    expect(await passesAfter(30_000)).toBe(9)
    expect(await passesAfter(3_999)).toBe(9)
    expect(await passesAfter(1)).toBe(10)
  } finally { vi.useRealTimers() }
})

it('climbs again from the start once a pass launches an agent, and when a person restarts a held one', async () => {
  vi.useFakeTimers()
  try {
    let launch = false
    const h = setup([row('a', heldLaunch('native')), row('b', heldLaunch('native'))],
      only => only.has('b') && launch ? summary({ restored: ['b'] }) : summary({ held: [...only] }))
    await h.held.boot(async () => {})
    await h.held.restoreHeld()
    await vi.advanceTimersByTimeAsync(2_000 + 4_000 + 8_000 + 16_000 + 30_000)
    // How long until the next pass, timer to timer. Stepping 1 ms at a time took 30,000 event-loop turns a call: at
    // load ~85 (MacBook-Pro-19, 2026-10-30) it ran 17 s and timed the test out.
    const nextPass = async () => {
      const before = h.restore.mock.calls.length, start = Date.now()
      while (vi.getTimerCount() > 0) {
        await vi.advanceTimersToNextTimerAsync()
        const waited = Date.now() - start
        if (waited > 30_000) return null
        if (h.restore.mock.calls.length > before) return waited
      }
      return null
    }
    launch = true
    expect(await nextPass()).toBe(30_000)
    expect(h.registry.get('b')?.launch?.state).toBe('starting')
    // The pass after the one that launched waits 4 s, not 30.
    expect(await nextPass()).not.toBeNull()
    expect(await nextPass()).toBe(4_000)
    await vi.advanceTimersByTimeAsync(8_000 + 16_000 + 30_000)
    expect(await h.held.restartHeld('a')).toMatchObject({ ok: false, error: 'SERVICE_UNAVAILABLE' })
    expect(await nextPass()).not.toBeNull()
    expect(await nextPass()).toBe(4_000)
  } finally { vi.useRealTimers() }
})