Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

5.7 KB
Raw Blame
// The transcript scorer (score.mts). Run: npx tsx --test scripts/agent-eval/score.test.mts
import { test } from 'node:test'
import assert from 'node:assert/strict'
import { readFileSync } from 'node:fs'
import { callsFromClaude, callsFromCodex, evalEnv, harnessVerb, score, type Case } from './score.mts'

const claude = (calls: Array<[string, string, boolean]>, answer: string): string[] => [
  ...calls.flatMap(([command, output, isError], i) => [
    JSON.stringify({ type: 'assistant', message: { content: [{ type: 'tool_use', id: `t${i}`, name: 'Bash', input: { command } }] } }),
    JSON.stringify({ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: `t${i}`, content: output, is_error: isError }] } }),
  ]),
  JSON.stringify({ type: 'result', result: answer }),
]

test('finds the harness verb in a command, also by full path and inside bash -lc', () => {
  assert.equal(harnessVerb('harness ps'), 'ps')
  assert.equal(harnessVerb('~/.local/bin/harness wait reviewer --timeout 20m'), 'wait')
  assert.equal(harnessVerb("bash -lc 'harness new codex --name r -- go'"), 'new')
  assert.equal(harnessVerb('harness --help'), '--help')
  assert.equal(harnessVerb('ls harnesses/'), null)
})

test('reads Claude Code stream-json: commands, exit codes and the final answer', () => {
  const { calls, answer } = callsFromClaude(claude([['harness ps', 'rows', false], ['harness show x', 'Exit code 1\nNo harness named "x"', true]], 'done'))
  assert.deepEqual(calls.map((c) => [c.command, c.exit]), [['harness ps', 0], ['harness show x', 1]])
  assert.equal(answer, 'done')
})

test('reads Codex exec --json: command executions and the last agent message', () => {
  const { calls, answer } = callsFromCodex([
    JSON.stringify({ type: 'item.completed', item: { type: 'command_execution', command: "bash -lc 'harness ps'", exit_code: 0, aggregated_output: 'rows' } }),
    JSON.stringify({ type: 'item.completed', item: { type: 'agent_message', text: 'Codex found two bugs.' } }),
  ])
  assert.deepEqual(calls.map((c) => [c.command, c.exit]), [["bash -lc 'harness ps'", 0]])
  assert.equal(answer, 'Codex found two bugs.')
})

const review: Case = { id: 'second-opinion', prompt: 'p', setup: 's', expect: { run: ['new', 'show'], answer: 'bug' } }

test('a clean run: one look, start, wait, read, close', () => {
  const { calls, answer } = callsFromClaude(claude([
    ['harness ps', 'rows', false], ['harness new codex --name r -- review', 'Created r', false],
    ['harness wait r', 'r: idle', false], ['harness show r', 'answer', false], ['harness close r', 'Closed r.', false],
  ], 'One bug in ledger.ts'))
  assert.deepEqual(score(review, 'claude', calls, answer), {
    id: 'second-opinion', agent: 'claude', calls: 5, firstAction: 1, orientation: 1, wrong: 0, retriesAfterRefusal: 0,
    leftOpen: 0, ranExpected: true, ranForbidden: [], answerOk: true, done: true,
  })
})

test('counts wrong options, retries after a refusal, and harnesses left open', () => {
  const { calls, answer } = callsFromClaude(claude([
    ['harness --help', 'Usage', false], ['harness version', '0.4.4', false],
    ['harness wait r --timout 5m', 'Exit code 2\nUnknown option --timout.', true],
    ['harness new codex --name r -- go', 'Exit code 1\nAgents may not start harnesses: the person turns on …', true],
    ['harness new claude --name r -- go', 'Exit code 1\nAgents may not start harnesses: the person turns on …', true],
  ], 'I could not'))
  const s = score({ ...review, expect: { notRun: ['new'], answer: 'person|turn on' } }, 'claude', calls, answer)
  assert.equal(s.orientation, 2)
  assert.equal(s.wrong, 1)
  assert.equal(s.retriesAfterRefusal, 1)
  assert.equal(s.answerOk, false)
  assert.equal(s.done, false)
})

test('starting again after closing one, as the refusal says, is not a retry', () => {
  const { calls, answer } = callsFromClaude(claude([
    ...[1, 2, 3, 4].map((i): [string, string, boolean] => [`harness new codex --name r${i} -- go`, 'Created', false]),
    ['harness new codex --name r5 -- go', 'Exit code 1\nYou have 4 harnesses running. Close one you are done with first (harness close <name>).', true],
    ['harness close r1', 'Closed r1.', false],
    ['harness new codex --name r5 -- go', 'Created', false],
    ...[2, 3, 4, 5].map((i): [string, string, boolean] => [`harness close r${i}`, 'Closed', false]),
  ], 'all five'))
  const s = score({ id: 'five-at-once', prompt: 'p', setup: 's', expect: { run: ['new', 'close'] } }, 'claude', calls, answer)
  assert.equal(s.retriesAfterRefusal, 0)
  assert.equal(s.leftOpen, 0)
  assert.equal(s.done, true)
})

test('sees every harness command joined with &&', () => {
  const { calls, answer } = callsFromClaude(claude([
    ['harness new codex --name r -- go && harness wait r && harness show r', 'answer', false],
    ['harness close r', 'Closed r.', false],
  ], 'One bug in ledger.ts'))
  const s = score({ id: 'x', prompt: 'p', setup: 's', expect: { run: ['new', 'wait', 'show'] } }, 'claude', calls, answer)
  assert.equal(s.ranExpected, true)
  assert.equal(s.calls, 4)
  assert.equal(s.leftOpen, 0)
})

test('the cases accept an ordinary right answer', () => {
  const cases = JSON.parse(readFileSync(new URL('./cases.json', import.meta.url), 'utf8')) as Case[]
  const answer = cases.find((c) => c.id === 'second-opinion')!.expect.answer!
  assert.match('childrenOf() drops rows after the ledger file rotates, so limits undercount.', new RegExp(answer, 'i'))
})

test('the eval agent does not inherit the operator\'s session', () => {
  const env = evalEnv({ PATH: '/bin', HOME: '/h', TMUX_PANE: '%3', CLAUDECODE: '1', CLAUDE_CODE_SESSION_ID: 's', CODEX_THREAD_ID: 't', AI_AGENT: 'x', HARNESS_AGENT_SESSION: 'y' })
  assert.deepEqual(env, { PATH: '/bin', HOME: '/h' })
})