Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

test: expand orchestration coverage and add bounded live harness projects

Deee0aa396ac057
diff --git a/cli/scripts/orchestrator-e2e-peer.ts b/cli/scripts/orchestrator-e2e-peer.ts
index 188295716..7b354259d 100644
--- a/cli/scripts/orchestrator-e2e-peer.ts
+++ b/cli/scripts/orchestrator-e2e-peer.ts
@@ -1,14 +1,17 @@
-/** Isolated daemon adapter + real tmux + real viewers. No live Harness state. */
+/** Isolated daemon adapter + real tmux + real viewers. No live Harness state.
+ * Deterministic by default; real models require the separate LIVE opt-in. */
import { execFile } from 'node:child_process'
import { createServer } from 'node:http'
-import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises'
-import { tmpdir } from 'node:os'
+import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'
+import { tmpdir, homedir } from 'node:os'
import { join, dirname, resolve } from 'node:path'
import { fileURLToPath } from 'node:url'
import { randomBytes } from 'node:crypto'
import { promisify } from 'node:util'
-if (process.env.HARNESS_ORCHESTRATOR_E2E !== '1') throw new Error('Opt in with HARNESS_ORCHESTRATOR_E2E=1')
+const live = process.env.HARNESS_ORCHESTRATOR_LIVE === '1'
+if (!live && process.env.HARNESS_ORCHESTRATOR_E2E !== '1') throw new Error('Opt in with HARNESS_ORCHESTRATOR_E2E=1, or explicitly authorize model usage with HARNESS_ORCHESTRATOR_LIVE=1')
+const installedRoot = process.env.DSH_DIR ?? join(homedir(), '.harness', 'dsh')
const exec = promisify(execFile)
const root = await mkdtemp(join(tmpdir(), 'orchestrator-e2e-'))
const cliRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..')
@@ -22,27 +25,31 @@ Object.assign(process.env, {
DISABLE_HOOK_INSTALL: 'true', DISABLE_GRID_INSTALL: 'true',
})
const [{ BackendSocket }, { attachLocalWsServer }, { registry }, { createAndRegisterPane },
- { installedDsh, upsertInstalledRecord }, { materializeWorkspace }, { DshViewerManager }, { shellQuote }] = await Promise.all([
+ { installedDsh, upsertInstalledRecord }, { materializeWorkspace }, { DshViewerManager }, { shellQuote }, { dshLaunch }] = await Promise.all([
import('../src/backendSocket.js'), import('../src/localWsServer.js'), import('../src/lib/registry.js'), import('../src/lib/createAgentPane.js'),
- import('../src/dsh/installed.js'), import('../src/dsh/materialize.js'), import('../src/dsh/viewer.js'), import('../src/orchestrator/prompts.js'),
+ import('../src/dsh/installed.js'), import('../src/dsh/materialize.js'), import('../src/dsh/viewer.js'), import('../src/orchestrator/prompts.js'), import('../src/dsh/launch.js'),
])
const backend = new BackendSocket(machineId)
const urls = new Map<string, string | null>()
const viewers = new DshViewerManager({ onUrl: (id, url) => urls.set(id, url), log: line => console.error(line) })
-let created = 0, stopped = false
+let created = 0, modelTurns = 0, stopped = false
const inputs: string[] = []
const pending = new Set<Promise<unknown>>()
const track = (work: Promise<unknown>) => { pending.add(work); void work.catch(e => console.error(e)).finally(() => pending.delete(work)) }
const server = createServer(async (req, res) => {
- if (req.url === '/fixture-stats') { res.setHeader('content-type', 'application/json'); res.end(JSON.stringify({ created, inputs })); return }
+ if (req.url === '/fixture-stats') { res.setHeader('content-type', 'application/json'); res.end(JSON.stringify({ created, modelTurns, inputs })); return }
+ if (live && req.url === '/live-turn' && req.method === 'POST') {
+ if (modelTurns >= 12 || stopped) { res.writeHead(429); res.end('Live test turn limit reached'); return }
+ modelTurns++; res.end('ok'); return
+ }
if (req.url !== '/fixture-event' || req.method !== 'POST') { res.writeHead(404); res.end(); return }
let body = ''
for await (const chunk of req) { body += chunk; if (body.length > 32_000) { res.writeHead(413); res.end(); return } }
try {
const event = JSON.parse(body)
const agent = registry.list().find(a => a.cwd === event.cwd)
- if (!agent || !['text_delta', 'turn_started', 'turn_ended'].includes(event.type)) { res.writeHead(400); res.end(); return }
- backend.send({ type: event.type, agentId: agent.agentId, payload: { content: event.content } })
+ if (!agent || !['text_delta', 'turn_started', 'turn_ended', 'error'].includes(event.type)) { res.writeHead(400); res.end(); return }
+ backend.send({ type: event.type, agentId: agent.agentId, payload: { content: event.content, message: event.content } })
res.end('ok')
} catch { res.writeHead(400); res.end() }
})
@@ -50,9 +57,13 @@ const local = attachLocalWsServer(server, { machineId, backend })
await new Promise<void>(resolve => server.listen(0, '127.0.0.1', resolve))
const port = (server.address() as { port: number }).port
backend.orchestratorCommand = `${[process.execPath, '--import', loader, join(cliRoot, 'src/cli.ts')].map(shellQuote).join(' ')} orchestrator --port ${port} --machine ${machineId}`
-backend.engineProbeProvider = async (engines = ['claude']) => engines.map(engine => ({ engine, installed: true, command: process.execPath, installable: false }))
+if (!live) backend.engineProbeProvider = async (engines = ['claude']) => engines.map(engine => ({ engine, installed: true, command: process.execPath, installable: false }))
backend.dshFrameProvider = agent => ({ id: agent.dsh ?? null, name: agent.dsh ?? null, viewerUrl: urls.get(agent.agentId) ?? null, viewerName: 'Fixture viewer', verdict: null })
-for (const name of ['shape', 'research', 'scene', 'film']) {
+if (live) {
+ // Read-only links to ALREADY installed harnesses; never change the real index.
+ const records = JSON.parse(await readFile(join(installedRoot, 'installed.json'), 'utf8'))
+ for (const record of records) if (['autonomous/solid', 'autonomous/autonomous-workshop', 'autonomous/blender', 'autonomous/model-viewer', 'autonomous/cad-viewer'].includes(record.id)) upsertInstalledRecord(record)
+} else for (const name of ['shape', 'research', 'scene', 'film']) {
const dir = join(root, 'dsh', 'fixture', name)
await mkdir(dir, { recursive: true, mode: 0o700 })
await writeFile(join(dir, 'AGENTS.md'), `This is the ${name} integration-test harness. Produce deliverable.txt and verify its inputs.\n`)
@@ -65,12 +76,14 @@ for (const name of ['shape', 'research', 'scene', 'film']) {
await mkdir(join(root, 'workspace'), { mode: 0o700 })
await exec('tmux', ['-S', socket, '-f', '/dev/null', 'new-session', '-d', '-s', 'fixture-keeper', '/bin/sleep', '3600'])
backend.onCreateAgent = async input => {
+ if (live && created >= 6) return { ok: false, error: 'LIVE_TEST_LIMIT', detail: 'The live validation is limited to six agents total. Report the result instead of creating more.' }
const dsh = input.dsh ? installedDsh(input.dsh) : undefined
if (dsh) await materializeWorkspace(dsh, input.cwd)
const result = await createAndRegisterPane({
tmuxBackend: {
create: async request => {
- const result = await exec('tmux', ['-S', socket, 'new-session', '-d', '-P', '-F', '#{pane_id}', '-s', request.label!, '-c', input.cwd, ...request.command!])
+ const environment = Object.entries(request.env ?? {}).flatMap(([key, value]) => ['-e', `${key}=${value}`])
+ const result = await exec('tmux', ['-S', socket, 'new-session', '-d', '-P', '-F', '#{pane_id}', '-s', request.label!, '-c', input.cwd, ...environment, ...request.command!])
return { state: 'succeeded', dispatch: 'executed', runtime: { backend: 'tmux', paneId: result.stdout.trim() } }
},
kill: async runtime => {
@@ -79,8 +92,11 @@ backend.onCreateAgent = async input => {
},
},
registry, engine: input.engine, cwd: input.cwd, dsh: input.dsh, defaultName: input.name,
+ ...(dsh ? { env: dshLaunch(dsh, input.cwd).env } : {}),
sessionLabel: `harness-fixture-${++created}`,
- argv: [process.execPath, '--import', loader, join(cliRoot, 'scripts/orchestrator-fixture-engine.ts'), input.cwd, dsh ? 'worker' : 'director', String(port), machineId, join(cliRoot, 'src/cli.ts'), loader],
+ argv: live
+ ? [process.execPath, '--import', loader, join(cliRoot, 'scripts/orchestrator-live-engine.ts'), input.cwd, input.engine, String(port), input.prompt]
+ : [process.execPath, '--import', loader, join(cliRoot, 'scripts/orchestrator-fixture-engine.ts'), input.cwd, dsh ? 'worker' : 'director', String(port), machineId, join(cliRoot, 'src/cli.ts'), loader],
})
if (!result.ok) return result
if (dsh?.manifest.viewer) track(viewers.start(result.pending.agentId, dsh, input.cwd))
@@ -102,11 +118,12 @@ backend.onCancel = id => {
async function cleanup(code: number) {
if (stopped) return
stopped = true
+ if (live) for (const agent of registry.list()) backend.onCancel?.(agent.agentId)
await local.close(); await backend.stop(); await viewers.stopAll()
await Promise.allSettled([...pending])
await exec('tmux', ['-S', socket, 'kill-server']).catch(() => {})
server.close()
- if (code === 0 && process.env.HARNESS_ORCHESTRATOR_KEEP !== '1') await rm(root, { recursive: true, force: true })
+ if (!live && code === 0 && process.env.HARNESS_ORCHESTRATOR_KEEP !== '1') await rm(root, { recursive: true, force: true })
else console.error(`Fixture files retained at ${root}`)
process.exit(code)
}
@@ -114,4 +131,5 @@ process.stdin.resume(); process.stdin.on('end', () => { void cleanup(0) })
process.on('SIGTERM', () => { void cleanup(0) }); process.on('SIGINT', () => { void cleanup(0) })
process.on('uncaughtException', e => { console.error(e); void cleanup(1) })
process.on('unhandledRejection', e => { console.error(e); void cleanup(1) })
+if (live) setTimeout(() => { console.error('Live validation reached its 12-minute limit.'); void cleanup(1) }, 12 * 60_000).unref()
console.log(JSON.stringify({ port, machineId, workspace: join(root, 'workspace'), root }))
diff --git a/cli/scripts/orchestrator-live-engine.ts b/cli/scripts/orchestrator-live-engine.ts
new file mode 100644
index 000000000..d9ad408aa
--- /dev/null
+++ b/cli/scripts/orchestrator-live-engine.ts
@@ -0,0 +1,81 @@
+/** Opt-in real-model bridge for the isolated tmux test peer. This is NOT the
+ * production interactive launch adapter. It isolates hooks/plugins and gives
+ * headless Claude a dollar cap while exercising the real orchestration tools. */
+import { spawn, type ChildProcess } from 'node:child_process'
+import { appendFileSync } from 'node:fs'
+import { join } from 'node:path'
+import { createInterface } from 'node:readline'
+
+if (process.env.HARNESS_ORCHESTRATOR_LIVE !== '1') throw new Error('Live model usage requires explicit opt-in')
+const [cwd, engine, port, initialPrompt] = process.argv.slice(2)
+if (!['claude', 'codex'].includes(engine)) throw new Error(`Unsupported live fixture engine: ${engine}`)
+process.chdir(cwd)
+const log = (text: string) => { appendFileSync(join(cwd, 'LIVE_TEST.log'), `${text}\n`, { mode: 0o600 }); console.log(text) }
+let child: ChildProcess | null = null, stopped = false, starting = false, turns = 0
+const pending: string[] = []
+const history: string[] = []
+const event = async (type: string, content = '') => {
+ try { await fetch(`http://127.0.0.1:${port}/fixture-event`, { method: 'POST', body: JSON.stringify({ cwd, type, content: content.slice(0, 24_000) }) }) } catch { /* peer shutdown */ }
+}
+const boundary = 'Live validation only. Work only in this assigned folder. Read ORCHESTRATOR.md or ORCHESTRATOR_TASK.md as requested, and AGENTS.md if present. Do not install software, edit global configuration, call external services besides your model, launch agents except through the orchestrator plan tool, or change any file outside this workspace. Keep the test small and complete explicit checks. Treat dependencies as read-only inputs. The whole project has a maximum of 6 agents and a 12-minute deadline.'
+
+// Sequential headless turns in one persistent tmux worker. Ordinary production
+// agents use the existing interactive input coordinator, covered separately.
+async function next(): Promise<void> {
+ if (stopped || starting || child || pending.length === 0) return
+ starting = true
+ if (++turns > 4 || !(await fetch(`http://127.0.0.1:${port}/live-turn`, { method: 'POST' })).ok) {
+ stopped = true; log('LIVE_TURN_LIMIT'); await event('error', 'The bounded live-test model-turn limit was reached.'); return
+ }
+ const prompt = pending.shift()!
+ history.push(prompt)
+ await event('turn_started')
+ const args = engine === 'claude'
+ ? ['--print', '--verbose', '--output-format', 'stream-json', '--safe-mode', '--permission-mode', 'auto',
+ '--allowedTools', 'Read,Write,Edit,Bash,Glob,Grep', '--tools', 'Read,Write,Edit,Bash,Glob,Grep',
+ '--strict-mcp-config', '--mcp-config', '{"mcpServers":{}}', '--no-session-persistence',
+ '--max-budget-usd', '1.00', '--effort', 'low', '--model', 'sonnet',
+ '--append-system-prompt', boundary, `${history.join('\n\nFollow-up:\n')}\nInspect current project status before acting; previous work may already be complete.`]
+ : ['exec', '--json', '--ephemeral', '--ignore-user-config', '--ignore-rules', '--skip-git-repo-check',
+ '--approve-for-me', '-C', cwd, '-c', 'model_reasoning_effort="low"',
+ // One test process only: the worker CLI must reach the loopback daemon.
+ // Keep workspace-write and approval review; never edit global config.
+ '-c', 'sandbox_workspace_write.network_access=true',
+ `${boundary}\n${history.join('\n\nFollow-up:\n')}`]
+ child = spawn(engine, args, { cwd, env: process.env, stdio: ['ignore', 'pipe', 'pipe'] })
+ starting = false
+ const running = child
+ let buffer = ''
+ running.stdout!.on('data', data => {
+ buffer += String(data)
+ while (buffer.includes('\n')) {
+ const at = buffer.indexOf('\n'), line = buffer.slice(0, at); buffer = buffer.slice(at + 1)
+ try {
+ const value = JSON.parse(line)
+ const texts = engine === 'claude'
+ ? value.type === 'assistant' ? (value.message?.content ?? []).filter((c: { type: string }) => c.type === 'text').map((c: { text: string }) => c.text) : []
+ : value.type === 'item.completed' && value.item?.type === 'agent_message' ? [value.item.text] : []
+ for (const text of texts) { log(text); void event('text_delta', text) }
+ if (value.type === 'result') log(`MODEL_RESULT ${JSON.stringify({ subtype: value.subtype, error: value.is_error, costUsd: value.total_cost_usd, turns: value.num_turns })}`)
+ if (value.type === 'error') { log(`MODEL_ERROR ${JSON.stringify(value)}`); void event('error', value.message ?? 'Model error') }
+ } catch { if (line.trim()) log(line.slice(0, 1000)) }
+ }
+ })
+ running.stderr!.on('data', data => log(`stderr: ${String(data).slice(0, 2000)}`))
+ running.on('error', error => { log(`PROCESS_ERROR ${error.message}`); void event('error', error.message) })
+ running.on('close', code => {
+ log(`MODEL_EXIT ${code}`)
+ if (code !== 0) void event('error', `Live ${engine} process exited with code ${code}. Inspect LIVE_TEST.log; no automatic retry.`)
+ child = null
+ void event('turn_ended').then(() => next())
+ })
+}
+createInterface({ input: process.stdin }).on('line', line => {
+ if (stopped) return
+ pending.push(Buffer.from(line.trim(), 'base64').toString('utf8'))
+ void next()
+})
+process.on('SIGINT', () => { stopped = true; pending.length = 0; child?.kill('SIGTERM'); log('LIVE_TURN_CANCELLED') })
+process.on('SIGTERM', () => { stopped = true; child?.kill('SIGTERM'); process.exit(0) })
+setInterval(() => {}, 1000)
+setTimeout(() => { pending.push(initialPrompt); void next() }, 500)
diff --git a/cli/scripts/orchestrator-live-project.ts b/cli/scripts/orchestrator-live-project.ts
new file mode 100644
index 000000000..3971b910c
--- /dev/null
+++ b/cli/scripts/orchestrator-live-project.ts
@@ -0,0 +1,83 @@
+/** Explicitly opt-in paid/account-backed smoke test. Does not touch the user's daemon. */
+import { spawn } from 'node:child_process'
+import { createInterface } from 'node:readline'
+import { writeFile, readFile } from 'node:fs/promises'
+import { randomBytes, createHash } from 'node:crypto'
+import { join } from 'node:path'
+import { localOrchestratorRequest } from '../src/orchestrator/command.js'
+
+if (process.env.HARNESS_ORCHESTRATOR_LIVE !== '1') throw new Error('This test uses model accounts. Set HARNESS_ORCHESTRATOR_LIVE=1 only with permission.')
+const peer = spawn(process.execPath, ['--import', 'tsx', 'scripts/orchestrator-e2e-peer.ts'], { stdio: ['pipe', 'pipe', 'inherit'] })
+const config = await new Promise<{ port: number; machineId: string; root: string; workspace: string }>((resolve, reject) => {
+ const lines = createInterface({ input: peer.stdout })
+ const timer = setTimeout(() => reject(new Error('Live peer startup timed out')), 30_000)
+ lines.on('line', line => {
+ try { const value = JSON.parse(line); if (value.port) { clearTimeout(timer); resolve(value) } else console.log(line) } catch { console.log(line) }
+ })
+ peer.once('exit', code => { clearTimeout(timer); reject(new Error(`Live peer exited: ${code}`)) })
+})
+console.log(`LIVE_WORKSPACE ${config.root}`)
+const request = (payload: Record<string, unknown>) => localOrchestratorRequest(config.port, config.machineId, payload, 60_000)
+const id = randomBytes(16).toString('hex')
+const cadFit = process.env.HARNESS_ORCHESTRATOR_LIVE_SCENARIO === 'cad-fit'
+let project: any, previous = ''
+const checks: Record<string, boolean> = {}
+const started = Date.now()
+try {
+ const reply = await request({ action: 'start', id, engine: cadFit ? 'codex' : 'claude', cwd: config.workspace, parallelism: 2, bypassPermission: false,
+ prompt: cadFit ? `Build a SMALL real two-harness mechanical-fit project using autonomous/solid then autonomous/autonomous-workshop. Both use the installed Codex account. Delegate exactly TWO tasks and inspect the checks before completing.
+Solid: create a circular desk token 24 mm diameter and 5 mm thick with a 4 mm through-hole offset 6 mm from center. Output model.step, model.stl, dimensions.json (units mm, positive volume_mm3, bounds_mm) and source. Use the actual installed CAD tools, not fake STEP/STL.
+Autonomous Workshop: consume that exact pinned model.step and dimensions.json. Design a simple circular holder with 32 mm outer diameter, a 24.6 mm diameter recess, 2 mm base and 5 mm recess depth. Place the imported token in the recess with 0.3 mm radial clearance. Do not redraw the upstream token. Output model.step containing the holder and actual imported token as an assembly, model.stl, fit.json reporting solids, dimensions, radial_clearance_mm, units and explicit geometric validity checks, plus source. The holder must not intersect the token; verify intersection volume is zero (within numerical tolerance) and both solids have positive volume.
+Keep source and outputs in each assigned folder. Each worker must call the exact finish command with deliverables after checks pass. No installation, downloads, video or rendering. No extra tasks and no automatic retry if blocked: explain the obstacle. Finish the whole project explicitly when both real harness tasks succeed.` : `Build a SMALL real CAD-to-studio-render project with the installed Solid and Blender harnesses. You are the director; delegate creation, then inspect and complete.
+Design a simple circular desk token: 24 mm diameter, 5 mm thick, one 4 mm through-hole offset 6 mm from its center. The CAD worker must create model.step, model.stl and dimensions.json with positive volume_mm3, bounds_mm and units "mm", plus its source and checks. Use the actual Solid harness and its installed tools, not a hand-written fake file.
+The dependent Blender worker must import that exact pinned STL (do not redraw the token), explicitly convert millimeters to meters, give it a warm coral material, and create one small 512x512 studio render render.png and scene.glb, plus validation.json describing the source input, scale, geometry checks and image. Use installed bpy only and a cheap render (at most 16 samples). No video, installation or downloads.
+Use only TWO specialist tasks: Solid (Codex) followed by Blender (Claude). Do not use a generic engine instead of the installed harnesses. Each worker must report finish with all deliverable paths. Do not retry automatically or create extra tasks if blocked: explain the obstacle. Inspect final artifacts and complete only after acceptance checks pass. Keep this a bounded smoke test, not a production design.` })
+ if (reply.error) throw new Error(JSON.stringify(reply))
+ project = reply.project
+ console.log(`LIVE_PROJECT ${id}`)
+ while (Date.now() - started < 11 * 60_000) {
+ const reply = await request({ action: 'status', id })
+ if (reply.error) throw new Error(JSON.stringify(reply))
+ project = reply.project
+ const status = `${project.state}: ${project.tasks.map((t: any) => `${t.id}=${t.state}`).join(', ')}`
+ if (status !== previous) { console.log(status); previous = status }
+ if (project.error && !project.directorWorking) { console.error(`LIVE_DIRECTOR_ERROR ${project.error}`); break }
+ if (!project.directorWorking && project.tasks.length === 0 && Date.now() - started > 15_000) {
+ console.error('LIVE_NO_PLAN: director ended without delegating. Inspect its transcript; no blind retry.'); break
+ }
+ if (['completed', 'failed', 'cancelled'].includes(project.state) || project.tasks.some((t: any) => ['failed', 'blocked'].includes(t.state))) break
+ await new Promise(resolve => setTimeout(resolve, 2000))
+ }
+ checks.directorCompleted = project.state === 'completed'
+ const cad = project.tasks.find((t: any) => t.harness === 'autonomous/solid')
+ const scene = project.tasks.find((t: any) => t.harness === (cadFit ? 'autonomous/autonomous-workshop' : 'autonomous/blender'))
+ checks.realHarnessDag = Boolean(cad?.state === 'succeeded' && scene?.state === 'succeeded' && scene.dependsOn.includes(cad.id))
+ if (cad && scene) {
+ const step = await readFile(join(cad.cwd, 'model.step')).catch(() => Buffer.alloc(0))
+ const stl = await readFile(join(cad.cwd, 'model.stl')).catch(() => Buffer.alloc(0))
+ const pinned = await readFile(join(scene.cwd, 'inputs', cad.id, 'model.stl')).catch(() => Buffer.alloc(0))
+ const png = await readFile(join(scene.cwd, 'render.png')).catch(() => Buffer.alloc(0))
+ const glb = await readFile(join(scene.cwd, 'scene.glb')).catch(() => Buffer.alloc(0))
+ checks.stepHeader = step.subarray(0, 100).toString().includes('ISO-10303-21')
+ checks.pinnedMesh = stl.length > 84 && stl.equals(pinned)
+ checks.meshReceiptHash = cad.artifacts.some((a: any) => a.path === 'model.stl' && a.sha256 === createHash('sha256').update(stl).digest('hex'))
+ if (cadFit) {
+ const assembly = await readFile(join(scene.cwd, 'model.step')).catch(() => Buffer.alloc(0))
+ const fit = JSON.parse(await readFile(join(scene.cwd, 'fit.json'), 'utf8').catch(() => '{}'))
+ checks.assemblyHeader = assembly.subarray(0, 100).toString().includes('ISO-10303-21')
+ checks.clearanceReport = Math.abs(Number(fit.radial_clearance_mm) - 0.3) < 1e-6
+ } else {
+ checks.pngDimensions = png.length > 24 && png.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])) && png.readUInt32BE(16) === 512 && png.readUInt32BE(20) === 512
+ checks.glbHeader = glb.length > 12 && glb.subarray(0, 4).toString() === 'glTF' && glb.readUInt32LE(4) === 2 && glb.readUInt32LE(8) === glb.length
+ }
+ }
+ console.log(`LIVE_CHECKS ${JSON.stringify(checks)}`)
+ if (Object.values(checks).some(value => !value)) process.exitCode = 1
+} catch (error) {
+ console.error(error); process.exitCode = 1
+} finally {
+ if (project && !['completed', 'failed', 'cancelled'].includes(project.state)) await request({ action: 'cancel', id }).catch(() => {})
+ await writeFile(join(config.root, 'live-result.json'), JSON.stringify({ id, scenario: cadFit ? 'cad-fit' : 'cad-blender', started, durationMs: Date.now() - started, checks, project }, null, 2), { mode: 0o600 })
+ peer.stdin.end()
+ await new Promise<void>(resolve => { if (peer.exitCode !== null) resolve(); else peer.once('exit', () => resolve()) })
+}
diff --git a/cli/src/orchestrator/command.spec.ts b/cli/src/orchestrator/command.spec.ts
index 8a0dd74ed..a8226e993 100644
--- a/cli/src/orchestrator/command.spec.ts
+++ b/cli/src/orchestrator/command.spec.ts
@@ -1,5 +1,6 @@
-import { describe, expect, it } from 'vitest'
-import { summarizeOrchestratorReply, parseOrchestratorArgs } from './command.js'
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { WebSocketServer } from 'ws'
+import { summarizeOrchestratorReply, parseOrchestratorArgs, localOrchestratorRequest, orchestratorCommand } from './command.js'
describe('orchestrator tool output', () => {
it('keeps large projects readable without discarding worker results or mutating UI state', () => {
@@ -21,4 +22,86 @@ describe('orchestrator tool output', () => {
action: 'steer', taskId: 'shape', attempt: 2, text: 'Use millimeters', messageId: 'b'.repeat(32),
})
})
+ it('handles sparse and non-project replies without inventing a transcript', () => {
+ for (const project of [null, 3, [], 'no project']) {
+ const reply = { project }; expect(summarizeOrchestratorReply(reply)).toBe(reply)
+ }
+ expect(summarizeOrchestratorReply({ project: {} })).toEqual({ project: { tasks: [], deliveries: [] } })
+ expect(summarizeOrchestratorReply({ project: { messages: [{ text: 'No receipt' }] } })).toEqual({ project: { tasks: [], deliveries: [] } })
+ })
+})
+
+describe('orchestrator argument validation', () => {
+ const parse = (...args: string[]) => parseOrchestratorArgs(['--port', '1234', '--machine', 'local-test', ...args]).payload
+ it.each(['list', 'catalog', 'status', 'resume'])('parses %s without changing its identity', action => {
+ expect(parse(action, 'project')).toEqual({ action, id: 'project' })
+ })
+ it('parses plan, both result outcomes, scoped cancel/retry, completion and chat', () => {
+ expect(parse('plan', 'project', '[{"id":"x"}]')).toEqual({ action: 'plan', id: 'project', tasks: [{ id: 'x' }] })
+ expect(parse('plan', 'project').tasks).toBeNull()
+ for (const action of ['finish', 'fail']) expect(parse(action, 'project', 'task', '2', 'verified', 'a.step', 'b.png')).toEqual({ action, id: 'project', taskId: 'task', attempt: 2, summary: 'verified', artifacts: ['a.step', 'b.png'] })
+ for (const action of ['retry', 'cancel']) {
+ expect(parse(action, 'project', 'task')).toEqual({ action, id: 'project', taskId: 'task' })
+ expect(parse(action, 'project')).toEqual({ action, id: 'project' })
+ }
+ expect(parse('complete', 'project', 'verified').summary).toBe('verified')
+ expect(parse('message', 'project', 'hello').messageId).toMatch(/^[a-f0-9]{32}$/)
+ expect(parse('message', 'project', 'hello', 'fixed').messageId).toBe('fixed')
+ expect(parse('steer', 'project', 'task', '1', 'hello').messageId).toMatch(/^[a-f0-9]{32}$/)
+ })
+ it('rejects malformed JSON, unknown commands, missing identity and invalid ports', () => {
+ expect(() => parse('plan', 'project', '{')).toThrow()
+ expect(() => parse('install')).toThrow(/Usage/)
+ for (const port of ['0', '65536', '1.1', 'no']) expect(() => parseOrchestratorArgs(['--port', port, '--machine', 'local', 'list'])).toThrow(/running local daemon/)
+ expect(() => parseOrchestratorArgs(['--port', '1234', '--machine'])).toThrow(/identity/)
+ })
+})
+
+describe('real loopback orchestrator transport', () => {
+ const servers: WebSocketServer[] = []
+ afterEach(async () => {
+ vi.restoreAllMocks()
+ await Promise.all(servers.splice(0).map(server => new Promise<void>(resolve => { for (const client of server.clients) client.terminate(); server.close(() => resolve()) })))
+ })
+ async function server(mode: 'ok' | 'refusal' | 'disconnect' | 'invalid' | 'silent' = 'ok') {
+ const ws = new WebSocketServer({ port: 0, host: '127.0.0.1' }); servers.push(ws)
+ await new Promise<void>(resolve => ws.once('listening', resolve))
+ const received: any[] = []
+ ws.on('connection', socket => socket.on('message', data => {
+ const frame = JSON.parse(data.toString()); received.push(frame)
+ if (mode === 'disconnect') { socket.close(); return }
+ if (mode === 'invalid') { socket.send('{'); return }
+ if (mode === 'silent') return
+ if (frame.type === 'machine_select') socket.send(JSON.stringify({ type: 'connected' }))
+ else {
+ socket.send(JSON.stringify({ type: 'orchestrator_changed', payload: { requestId: 'unrelated' } }))
+ socket.send(JSON.stringify({ type: 'orchestrator_result', payload: { requestId: frame.payload.requestId, ...(mode === 'refusal' ? { error: 'PROJECT_INACTIVE' } : { projects: [] }) } }))
+ }
+ }))
+ return { port: (ws.address() as { port: number }).port, received, ws }
+ }
+ it('selects the machine and correlates replies by receipt rather than unsolicited events', async () => {
+ const peer = await server()
+ expect(await localOrchestratorRequest(peer.port, 'test-machine', { action: 'list' })).toMatchObject({ projects: [] })
+ expect(peer.received[0]).toEqual({ type: 'machine_select', payload: { machineId: 'test-machine', localProtocolVersion: 1 } })
+ expect(peer.received[1].payload).toMatchObject({ action: 'list', requestId: expect.stringMatching(/^[a-f0-9]{32}$/) })
+ })
+ it.each([['disconnect', /disconnected/], ['invalid', /invalid response/], ['silent', /did not confirm/]] as const)('fails safely on %s without retrying', async (mode, message) => {
+ const peer = await server(mode)
+ await expect(localOrchestratorRequest(peer.port, 'machine', { action: 'list' }, 50)).rejects.toThrow(message)
+ expect(peer.received.filter(frame => frame.type === 'machine_select')).toHaveLength(1)
+ })
+ it('surfaces connection errors and command exit codes', async () => {
+ const peer = await server()
+ const output = vi.spyOn(console, 'log').mockImplementation(() => {})
+ const error = vi.spyOn(console, 'error').mockImplementation(() => {})
+ expect(await orchestratorCommand(['--port', String(peer.port), '--machine', 'machine', 'list'])).toBe(0)
+ expect(JSON.parse(output.mock.calls[0][0]).projects).toEqual([])
+ const refusal = await server('refusal')
+ expect(await orchestratorCommand(['--port', String(refusal.port), '--machine', 'machine', 'list'])).toBe(1)
+ expect(await orchestratorCommand(['--port', '0', '--machine', 'machine', 'list'])).toBe(1)
+ expect(error).toHaveBeenCalled()
+ await new Promise<void>(resolve => peer.ws.close(() => resolve()))
+ await expect(localOrchestratorRequest(peer.port, 'machine', { action: 'list' })).rejects.toThrow(/ECONNREFUSED/)
+ })
})
diff --git a/cli/src/orchestrator/wire.spec.ts b/cli/src/orchestrator/wire.spec.ts
new file mode 100644
index 000000000..24aa114eb
--- /dev/null
+++ b/cli/src/orchestrator/wire.spec.ts
@@ -0,0 +1,47 @@
+import { describe, expect, it, vi } from 'vitest'
+import { orchestratorRequest } from './wire.js'
+import { OrchestratorError } from './model.js'
+import type { OrchestratorService } from './service.js'
+
+const id = 'a'.repeat(32), messageId = 'b'.repeat(32)
+const makeService = () => Object.fromEntries(['list', 'catalog', 'start', 'snapshot', 'plan', 'finish', 'retry', 'cancel', 'resume', 'complete', 'chat', 'steer'].map(key => [key, vi.fn(() => ({ id }))]))
+describe('orchestrator RPC boundary', () => {
+ it.each(['list', 'catalog', 'start'])('returns %s data under the stable wire key', async action => {
+ const service = makeService()
+ expect(await orchestratorRequest(service as unknown as OrchestratorService, { action, id })).toEqual({ [action === 'list' ? 'projects' : action === 'catalog' ? 'harnesses' : 'project']: { id } })
+ expect(service[action]).toHaveBeenCalledTimes(1)
+ expect(service.snapshot).not.toHaveBeenCalled()
+ })
+ it.each([
+ ['status', 'snapshot', [id]],
+ ['plan', 'plan', [id, []]],
+ ['finish', 'finish', [id, 'task', 1, 'verified', ['file.step'], false]],
+ ['fail', 'finish', [id, 'task', 1, 'verified', [], true]],
+ ['retry', 'retry', [id, 'task']], ['cancel', 'cancel', [id, 'task']],
+ ['resume', 'resume', [id]], ['complete', 'complete', [id, 'verified']],
+ ['message', 'chat', [id, messageId, 'hello']],
+ ['steer', 'steer', [id, 'task', 1, messageId, 'hello']],
+ ])('validates and dispatches %s', async (action, method, args) => {
+ const service = makeService()
+ const reply = await orchestratorRequest(service as unknown as OrchestratorService, { action, id, tasks: [], taskId: 'task', attempt: 1, summary: 'verified', ...(action === 'finish' ? { artifacts: ['file.step'] } : {}), messageId, text: 'hello' })
+ expect(reply).toEqual({ project: { id } })
+ expect(service[method as string]).toHaveBeenCalledWith(...args as unknown[])
+ })
+ it('supports whole-project cancellation and reports validation errors without dispatch', async () => {
+ const service = makeService(), wire = (payload: Record<string, unknown>) => orchestratorRequest(service as unknown as OrchestratorService, payload)
+ await wire({ action: 'cancel', id }); expect(service.cancel).toHaveBeenCalledWith(id, undefined)
+ for (const payload of [{ action: 'start' }, { action: 'finish', attempt: 0 }, { action: 'steer', attempt: 1, messageId: 'bad' }]) {
+ if (payload.action === 'start') continue // the service validates the creation spec
+ expect(await wire({ id, taskId: 'task', summary: 'verified', ...payload })).toMatchObject({ error: 'INVALID_REQUEST' })
+ }
+ expect(service.finish).not.toHaveBeenCalled(); expect(service.steer).not.toHaveBeenCalled()
+ })
+ it.each([
+ [new OrchestratorError('KNOWN_ERROR', 'Actionable detail'), 'KNOWN_ERROR', 'Actionable detail'],
+ [new Error('Disk full'), 'ORCHESTRATOR_FAILED', 'Disk full'],
+ ['untyped failure', 'ORCHESTRATOR_FAILED', 'Orchestration request failed.'],
+ ])('normalizes failures without losing known codes', async (failure, code, detail) => {
+ const service = makeService(); service.list.mockImplementation(() => { throw failure })
+ expect(await orchestratorRequest(service as unknown as OrchestratorService, { action: 'list' })).toEqual({ error: code, detail })
+ })
+})
diff --git a/desktop/test/dsh_install_panel_test.dart b/desktop/test/dsh_install_panel_test.dart
index 9cbbf82b9..e0ee4abd8 100644
--- a/desktop/test/dsh_install_panel_test.dart
+++ b/desktop/test/dsh_install_panel_test.dart
@@ -80,7 +80,9 @@ void main() {
testWidgets('a run nobody has heard from yet is fetching', (tester) async {
await _show(tester, DshInstallRun('autonomous/marp'));
expect(find.text('Installing Marp on Studio'), findsOneWidget);
- expect(find.text('0:00'), findsOneWidget);
+ // A loaded test host can spend seconds between construction and first
+ // paint. Assert the clock format, not sub-second wall-clock scheduling.
+ expect(find.textContaining(RegExp(r'^\d+:\d{2}$')), findsOneWidget);
expect(_markOf(tester, 'Fetch Marp'), _Mark.active);
expect(find.text('Fetching…'), findsOneWidget);
expect(_markOf(tester, 'Set up the toolchain'), _Mark.pending);