The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
import { execFileSync } from 'node:child_process'
import { afterEach, describe, expect, it, vi } from 'vitest'
import type { AgentLaunch, RegisteredSession } from '../../lib/registry.js'
import { heldLaunch, type RestoreSummary } from '../../lib/restoreAgents.js'
import { createHeldLaunches, heldFor, heldPaneArgv, heldRetryMs } from './heldLaunches.js'
const running: ReturnType<typeof createHeldLaunches>[] = []
afterEach(() => { for (const held of running.splice(0)) held.close() })
const HELD = heldLaunch('models')
const row = (agentId: string, launch: AgentLaunch, over: Partial<RegisteredSession> = {}) =>
({ agentId, sessionId: `s-${agentId}`, launch, ...over }) as RegisteredSession
const summary = (over: Partial<RestoreSummary> = {}): RestoreSummary => ({ restored: [], skipped: [], failed: [], held: [], unsurveyed: [], ...over })
/** A registry of rows and a restore that does to them what `answer` says, recording what it was asked. */
function setup(rows: RegisteredSession[], answer: (only: ReadonlySet<string>) => RestoreSummary = (only) => summary({ restored: [...only] })) {
const registry = new Map(rows.map((r) => [r.agentId, r]))
const asked: string[][] = []
const restore = vi.fn(async (only: ReadonlySet<string>, _current: () => boolean) => {
asked.push([...only])
const result = answer(only)
for (const id of result.restored) { const r = registry.get(id); if (r) r.launch = { state: 'starting' } }
return result
})
const log = vi.fn()
const held = createHeldLaunches({ registry: { list: () => [...registry.values()], byAgent: (id: string) => registry.get(id) } as never, restore, log })
running.push(held)
return { registry, asked, restore, log, held }
}
describe('agents held until a service is ready', () => {
it('says once why an agent stays held, and does not repeat a pass that changed nothing', async () => {
let reason = 'the pty host has gone'
const { held, log } = setup([row('a', HELD)], (only) => summary({ held: [...only], failed: [...only].map(agentId => ({ agentId, reason })) }))
await held.boot(async () => {})
await held.restoreHeld(); await held.restoreHeld(); await held.restoreHeld()
reason = 'no such folder'
await held.restoreHeld()
expect(log.mock.calls.map(([line]) => line)).toEqual([
'[restore] a · could not launch from held · the pty host has gone',
'[restore] held · every service · launched 0 of 1 · 1 still waiting',
'[restore] a · could not launch from held · no such folder',
])
})
it.each(['removed', 'closed'] as const)('does not restore a queued row that was %s before its preparation', async state => {
const h = setup([row('a', HELD)])
await h.held.boot(async () => {})
if (state === 'removed') vi.spyOn(h.registry, 'get').mockReturnValueOnce(undefined)
const running = h.held.restoreHeld()
if (state === 'closed') queueMicrotask(() => h.held.close())
expect(await running).toMatchObject({ restored: [] })
expect(h.restore).not.toHaveBeenCalled()
})
it('revokes the boot and every active restore authority immediately on close', async () => {
const h = setup([row('a', HELD)])
let bootCurrent!: () => boolean, flightCurrent!: () => boolean, finish!: () => void
await h.held.boot(async current => { bootCurrent = current; expect(current()).toBe(true) })
const restore = vi.fn(async (_only: ReadonlySet<string>, current: () => boolean) => {
flightCurrent = current
expect(current()).toBe(true)
await new Promise<void>(done => { finish = done })
return summary()
})
h.restore.mockImplementation(restore)
const pending = h.held.restoreHeld()
await vi.waitFor(() => expect(restore).toHaveBeenCalledOnce())
h.held.close()
expect(bootCurrent()).toBe(false)
expect(flightCurrent()).toBe(false)
finish(); await pending
})
it('bounds actual preparation flights and lets a later ready agent progress after one slot settles', async () => {
vi.useFakeTimers()
try {
const h = setup(Array.from({ length: 5 }, (_, index) => row(String(index), HELD)))
const complete: Array<() => void> = []
h.restore.mockImplementation(only => new Promise(resolve => complete.push(() => {
for (const id of only) h.registry.get(id)!.launch = { state: 'starting' }
resolve(summary({ restored: [...only] }))
})))
await h.held.boot(async () => {})
const first = h.held.restoreHeld()
await vi.advanceTimersByTimeAsync(0)
expect(h.restore).toHaveBeenCalledTimes(4)
expect(await h.held.restartHeld('4')).toMatchObject({ error: 'SERVICE_UNAVAILABLE' })
complete.shift()!()
await vi.advanceTimersByTimeAsync(500)
expect(h.restore).toHaveBeenCalledTimes(5)
complete.splice(0).forEach(done => done())
await first
expect(h.registry.get('4')?.launch).toEqual({ state: 'starting' })
} finally { vi.useRealTimers() }
})
it('names the service a row is held for, and none for any other', () => {
expect(heldFor(row('a', HELD))).toBe('models')
expect(heldFor(row('a', { state: 'ready' }))).toBeNull()
expect(heldFor(row('a', { state: 'failed', error: 'X' }))).toBeNull()
expect(heldFor(undefined)).toBeNull()
})
it("its pane says why, survives an interrupt, and idles: the agent's launch takes it over", () => {
const argv = heldPaneArgv('Waiting for the models service.', undefined, 'linux')
expect(argv.slice(0, 2)).toEqual(['/bin/sh', '-c'])
if (process.platform === 'win32') return
expect(argv[2]).toContain('trap "" INT QUIT TSTP')
// The script itself, run with its loop cut short: it prints the reason as given, and nothing else.
const once = argv[2]!.replace('while :; do sleep 3600; done', 'exit 0')
expect(execFileSync('/bin/sh', ['-c', once, argv[3]!, argv[4]!], { encoding: 'utf8' })).toBe('Waiting for the models service.\n')
})
it('waits in the daemon\'s Node on Windows, which has no /bin/sh, still told by its start command', () => {
const argv = heldPaneArgv('Waiting for the Store.', 'tok-1', 'win32')
expect(argv.slice(1, 2)).toEqual(['-e'])
expect(argv.slice(3)).toEqual(['harness-held', 'Waiting for the Store.', 'tok-1'])
expect(argv).not.toContain('/bin/sh')
expect(heldPaneArgv('Waiting for the Store.', undefined, 'linux').slice(0, 2)).toEqual(['/bin/sh', '-c'])
})
it('runs no pass before the boot has restored, whatever it asked meanwhile', async () => {
const { held, asked } = setup([row('a', HELD)])
const pass = held.restoreHeld()
await Promise.resolve()
expect(asked).toEqual([])
expect(await held.boot(async () => 'booted')).toBe('booted')
expect(await pass).toMatchObject({ restored: ['a'] })
// A boot that throws opens them all the same.
const failing = setup([row('b', HELD)])
const later = failing.held.restoreHeld()
await expect(failing.held.boot(async () => { throw new Error('restore broke') })).rejects.toThrow('restore broke')
expect(await later).toMatchObject({ restored: ['b'] })
})
it('launches every held agent once the core is ready, or those held for one service when it connects', async () => {
const { held, asked, log } = setup([row('a', HELD), row('b', heldLaunch('store')), row('c', { state: 'ready' })])
await held.boot(async () => {})
expect(await held.restoreHeld('store')).toMatchObject({ restored: ['b'] })
expect(await held.restoreHeld()).toMatchObject({ restored: ['a'] })
expect(asked).toEqual([['b'], ['a']])
// Nothing held: no pass at all.
expect(await held.restoreHeld()).toBeNull()
expect(asked).toHaveLength(2)
expect(log.mock.calls.map(([line]) => line)).toEqual([
'[restore] held · store · launched 1 of 1',
'[restore] held · every service · launched 1 of 1',
])
})
it('says what is still waiting after a pass that could not ask', async () => {
const { held, log } = setup([row('a', HELD), row('b', HELD)], (only) => summary({ held: [...only] }))
await held.boot(async () => {})
await held.restoreHeld('models')
expect(log).toHaveBeenLastCalledWith('[restore] held · models · launched 0 of 2 · 2 still waiting')
})
it.each(['prepared', 'restart'] as const)('retries transient core contention after the only %s notice, coalescing retries', async trigger => {
vi.useFakeTimers()
try {
const h = setup([row('a', HELD)])
h.restore.mockResolvedValueOnce(summary({ retry: true, held: ['a'] }))
.mockResolvedValueOnce(summary({ retry: true, held: ['a'] }))
await h.held.boot(async () => {})
if (trigger === 'prepared') await h.held.restoreHeld('models')
else await h.held.restartHeld('a')
await h.held.restoreHeld('models')
expect(h.restore).toHaveBeenCalledTimes(2)
await vi.advanceTimersByTimeAsync(500)
expect(h.restore).toHaveBeenCalledTimes(3)
expect(h.registry.get('a')?.launch?.state).toBe('starting')
} finally { vi.useRealTimers() }
})
it('logs a failed core-contention retry without an unhandled rejection', async () => {
vi.useFakeTimers()
try {
const h = setup([row('a', HELD)])
h.restore.mockResolvedValueOnce(summary({ retry: true })).mockRejectedValueOnce(new Error('test retry failure'))
await h.held.boot(async () => {})
await h.held.restoreHeld()
await vi.advanceTimersByTimeAsync(500)
expect(h.log).toHaveBeenLastCalledWith(expect.stringContaining('test retry failure'))
} finally { vi.useRealTimers() }
})
it('joins a flight for the same agent, and a failed flight permits the next attempt', async () => {
let release!: () => void
const order: string[] = []
const { held, restore } = setup([row('a', HELD)])
restore.mockImplementationOnce(async () => { order.push('first'); await new Promise<void>((done) => { release = done }); throw new Error('models hung up') })
await held.boot(async () => {})
const first = held.restoreHeld().catch(error => error)
const second = held.restoreHeld().catch(error => error)
await vi.waitFor(() => expect(order).toEqual(['first']))
expect(restore).toHaveBeenCalledTimes(1)
release()
expect((await first).message).toBe('models hung up')
expect((await second).message).toBe('models hung up')
expect(await held.restoreHeld()).toMatchObject({ restored: ['a'] })
})
describe("a person's restart of a held agent", () => {
it('is not one for an agent that is not held: it restarts as it always has', () => {
const { held } = setup([row('a', { state: 'ready' })])
expect(held.restartHeld('a')).toBeNull()
expect(held.restartHeld('nobody')).toBeNull()
})
it('launches it, and answers it as a restart would', async () => {
const { held, log } = setup([row('a', HELD)])
await held.boot(async () => {})
expect(await held.restartHeld('a')).toMatchObject({ ok: true, session: { agentId: 'a' }, resumed: true })
expect(log).toHaveBeenCalledWith('[restart] a launched from held')
const fresh = setup([row('b', HELD, { sessionId: '' })])
await fresh.held.boot(async () => {})
expect(await fresh.held.restartHeld('b')).toMatchObject({ ok: true, resumed: false })
})
it('says it still waits, or why its launch failed, or that it could not be launched', async () => {
const waiting = setup([row('a', HELD)], (only) => summary({ held: [...only] }))
await waiting.held.boot(async () => {})
expect(await waiting.held.restartHeld('a')).toEqual({ ok: false, error: 'SERVICE_UNAVAILABLE', detail: HELD.state === 'held' ? HELD.detail : '' })
const refused = setup([row('b', HELD)], (only) => summary({ failed: [...only].map((agentId) => ({ agentId, reason: 'no way there' })) }))
refused.restore.mockImplementationOnce(async (only) => {
refused.registry.get('b')!.launch = { state: 'failed', error: 'GRID_ENGINE_UNSUPPORTED', detail: 'no way there' }
return summary({ failed: [...only].map((agentId) => ({ agentId, reason: 'no way there' })) })
})
await refused.held.boot(async () => {})
expect(await refused.held.restartHeld('b')).toEqual({ ok: false, error: 'GRID_ENGINE_UNSUPPORTED', detail: 'no way there' })
const bare = setup([row('c', HELD)])
bare.restore.mockImplementationOnce(async () => { bare.registry.get('c')!.launch = { state: 'failed', error: 'START_TIMEOUT' }; return summary() })
await bare.held.boot(async () => {})
expect(await bare.held.restartHeld('c')).toEqual({ ok: false, error: 'START_TIMEOUT' })
const lost = setup([row('d', HELD)], (only) => summary({ failed: [...only].map((agentId) => ({ agentId, reason: 'tmux said no' })) }))
lost.restore.mockImplementationOnce(async (only) => { lost.registry.get('d')!.launch = { state: 'ready' }; return summary({ failed: [...only].map((agentId) => ({ agentId, reason: 'tmux said no' })) }) })
await lost.held.boot(async () => {})
expect(await lost.held.restartHeld('d')).toEqual({ ok: false, error: 'RESTART_FAILED', detail: 'tmux said no' })
const none = setup([row('e', HELD)], () => summary())
none.restore.mockImplementationOnce(async () => { none.registry.get('e')!.launch = { state: 'ready' }; return summary() })
await none.held.boot(async () => {})
expect(await none.held.restartHeld('e')).toEqual({ ok: false, error: 'RESTART_FAILED', detail: 'The harness could not be launched.' })
})
it('leaves an agent a pass launched meanwhile, or that was stopped or closed meanwhile, as it is', async () => {
const { held, registry, restore } = setup([row('a', HELD), row('b', HELD)])
const launchedMeanwhile = held.restartHeld('a')!
const closedMeanwhile = held.restartHeld('b')!
registry.get('a')!.launch = { state: 'ready' }
registry.delete('b')
await held.boot(async () => {})
expect(await launchedMeanwhile).toMatchObject({ ok: true, session: { agentId: 'a' } })
expect(await closedMeanwhile).toEqual({ ok: false, error: 'AGENT_NOT_FOUND' })
expect(restore).not.toHaveBeenCalled()
// Closed while its own pass ran.
const gone = setup([row('c', HELD)])
gone.restore.mockImplementationOnce(async () => { gone.registry.delete('c'); return summary({ restored: ['c'] }) })
await gone.held.boot(async () => {})
expect(await gone.held.restartHeld('c')).toEqual({ ok: false, error: 'AGENT_NOT_FOUND' })
})
})
it('restores only rows still held when the pass runs: one stopped or closed since is left alone', async () => {
const { held, registry, asked } = setup([row('a', HELD), row('b', HELD)])
const pass = held.restoreHeld()
registry.delete('b')
await held.boot(async () => {})
await pass
expect(asked).toEqual([['a']])
})
})
it('restores a sibling while another native preparation is stalled', async () => {
const h = setup([row('a', heldLaunch('native')), row('b', heldLaunch('native'))])
let finish!: (result: RestoreSummary) => void
h.restore.mockImplementationOnce(() => new Promise(resolve => { finish = resolve }))
await h.held.boot(async () => {})
const pass = h.held.restoreHeld()
await vi.waitFor(() => expect(h.registry.get('b')?.launch?.state).toBe('starting'))
expect(h.registry.get('a')?.launch?.state).toBe('held')
finish(summary({ held: ['a'] }))
expect(await pass).toMatchObject({ restored: ['b'], held: ['a'] })
})
it('retries a native hold without a service reconnect and stops scheduling at close', async () => {
vi.useFakeTimers()
try {
const h = setup([row('a', heldLaunch('native'))])
h.restore.mockResolvedValueOnce(summary({ held: ['a'] }))
await h.held.boot(async () => {})
await h.held.restoreHeld()
await vi.advanceTimersByTimeAsync(2_000)
expect(h.restore).toHaveBeenCalledTimes(2)
h.registry.get('a')!.launch = heldLaunch('native')
h.held.close()
expect(await h.held.restoreHeld()).toBeNull()
await vi.advanceTimersByTimeAsync(10_000)
expect(h.restore).toHaveBeenCalledTimes(2)
} finally { vi.useRealTimers() }
})
it('retries a held sibling while another agent’s observation has not settled', async () => {
vi.useFakeTimers()
try {
const h = setup([row('a', heldLaunch('native')), row('b', heldLaunch('native'))])
let finish!: (result: RestoreSummary) => void
h.restore.mockImplementationOnce(() => new Promise(resolve => { finish = resolve }))
.mockResolvedValueOnce(summary({ held: ['b'] }))
await h.held.boot(async () => {})
const pass = h.held.restoreHeld()
await vi.advanceTimersByTimeAsync(2_001)
expect(h.registry.get('b')?.launch?.state).toBe('starting')
expect(h.restore.mock.calls.map(([only]) => [...only])).toEqual([['a'], ['b'], ['b']])
finish(summary({ held: ['a'] }))
await pass
} finally { vi.useRealTimers() }
})
it('waits longer after each timer pass that launched nothing, up to half a minute', () => {
expect([0, 1, 2, 3, 4, 5, 40].map(heldRetryMs)).toEqual([2_000, 4_000, 8_000, 16_000, 30_000, 30_000, 30_000])
})
it('backs off a hold nothing else wakes, and climbs again from the start once something changes', async () => {
vi.useFakeTimers()
try {
const h = setup([row('a', heldLaunch('native'))], () => summary({ held: ['a'] }))
await h.held.boot(async () => {})
await h.held.restoreHeld()
const passesAfter = async (ms: number) => { await vi.advanceTimersByTimeAsync(ms); return h.restore.mock.calls.length }
// 2 s, 4 s, 8 s, 16 s, then every 30 s: a day of it is about 2,900 passes, not 43,000.
expect(await passesAfter(1_999)).toBe(1)
expect(await passesAfter(1)).toBe(2)
expect(await passesAfter(4_000)).toBe(3)
expect(await passesAfter(8_000)).toBe(4)
expect(await passesAfter(16_000)).toBe(5)
expect(await passesAfter(29_999)).toBe(5)
expect(await passesAfter(1)).toBe(6)
expect(await passesAfter(30_000)).toBe(7)
// A service connecting is news: it is asked at once, the waiting pass keeps its time, and the one after
// that is 4 s later again rather than 30.
await h.held.restoreHeld('native')
expect(h.restore).toHaveBeenCalledTimes(8)
expect(await passesAfter(30_000)).toBe(9)
expect(await passesAfter(3_999)).toBe(9)
expect(await passesAfter(1)).toBe(10)
} finally { vi.useRealTimers() }
})
it('climbs again from the start once a pass launches an agent, and when a person restarts a held one', async () => {
vi.useFakeTimers()
try {
let launch = false
const h = setup([row('a', heldLaunch('native')), row('b', heldLaunch('native'))],
only => only.has('b') && launch ? summary({ restored: ['b'] }) : summary({ held: [...only] }))
await h.held.boot(async () => {})
await h.held.restoreHeld()
await vi.advanceTimersByTimeAsync(2_000 + 4_000 + 8_000 + 16_000 + 30_000)
// How long until the next pass, timer to timer. Stepping 1 ms at a time took 30,000 event-loop turns a call: at
// load ~85 (MacBook-Pro-19, 2026-10-30) it ran 17 s and timed the test out.
const nextPass = async () => {
const before = h.restore.mock.calls.length, start = Date.now()
while (vi.getTimerCount() > 0) {
await vi.advanceTimersToNextTimerAsync()
const waited = Date.now() - start
if (waited > 30_000) return null
if (h.restore.mock.calls.length > before) return waited
}
return null
}
launch = true
expect(await nextPass()).toBe(30_000)
expect(h.registry.get('b')?.launch?.state).toBe('starting')
// The pass after the one that launched waits 4 s, not 30.
expect(await nextPass()).not.toBeNull()
expect(await nextPass()).toBe(4_000)
await vi.advanceTimersByTimeAsync(8_000 + 16_000 + 30_000)
expect(await h.held.restartHeld('a')).toMatchObject({ ok: false, error: 'SERVICE_UNAVAILABLE' })
expect(await nextPass()).not.toBeNull()
expect(await nextPass()).toBe(4_000)
} finally { vi.useRealTimers() }
})