Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

6.3 KB
Raw Blame
// How lost an agent got with Harness, read from its transcript (README.md). Pure: no process, no network.

export interface Call { command: string; exit: number | null; output: string }
export interface Case {
  id: string
  /** What the agent is asked, in a person's words. `{machine}` and `{offline}` are filled in by run.mts. */
  prompt: string
  /** What the operator sets up before the run, said to them by run.mts. */
  setup: string
  /** run: verbs that must have succeeded; notRun: verbs that must not have succeeded; answer: a regex the final answer matches. */
  expect: { run?: string[]; notRun?: string[]; answer?: string }
}
export interface Score {
  id: string; agent: string; calls: number
  /** Index, among harness calls, of the first that acts (new, show, wait, close, send), or null. */
  firstAction: number | null
  /** Harness calls before the first action that only look around (ps, whoami, settings, machines, version, help). */
  orientation: number
  /** Calls the CLI rejected as written: exit 2, or an unknown option or command. */
  wrong: number
  /** `new` run again after a `new` was refused. Should be 0: a refusal is not retried. */
  retriesAfterRefusal: number
  /** Harnesses started and not closed at the end. Should be 0. */
  leftOpen: number
  ranExpected: boolean; ranForbidden: string[]; answerOk: boolean; done: boolean
}

/** What would make the eval agent someone else to Harness: the operator's pane, or the operator's own agent session. */
const SESSION_KEYS = ['TMUX_PANE', 'CLAUDECODE', 'CLAUDE_CODE_SESSION_ID', 'CODEX_THREAD_ID', 'AI_AGENT', 'HARNESS_AGENT_SESSION']

/** The environment an eval agent starts with: the operator's, without who the operator is. Run from a Harness pane,
 *  the agent would otherwise be that pane's harness: its limits, its depth, its children (review, 2026-10-15). */
export function evalEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
  return Object.fromEntries(Object.entries(env).filter(([key]) => !SESSION_KEYS.includes(key)))
}

const ACTIONS = new Set(['new', 'show', 'wait', 'close', 'send'])
const LOOKS = new Set(['ps', 'whoami', 'settings', 'machines', 'version', 'help', '--help', 'status'])
const REFUSED = /may not start|Close one you are done with|the most allowed|in 10 minutes|turns on "Agents can start/i

/** The verb of each `harness` in a shell command, in order, by any path, inside quotes too: agents join them with &&. */
export function harnessVerbs(command: string): string[] {
  return [...command.matchAll(/(?:^|[\s;&|('"])(?:[^\s;&|('"]*\/)?harness\s+(--help|[a-z][a-z-]*)/g)].map((m) => m[1])
}

/** The verb of the first `harness` in a shell command, or null. */
export const harnessVerb = (command: string): string | null => harnessVerbs(command)[0] ?? null

const parse = (line: string): any => { try { return JSON.parse(line) } catch { return null } }

/** Claude Code `-p --output-format stream-json --verbose`: Bash tool uses, their results, and the final result. */
export function callsFromClaude(lines: readonly string[]): { calls: Call[]; answer: string } {
  const calls: Call[] = [], byId = new Map<string, Call>()
  let answer = ''
  for (const e of lines.map(parse)) {
    if (!e) continue
    if (e.type === 'assistant') for (const c of e.message?.content ?? []) {
      if (c.type === 'tool_use' && c.name === 'Bash' && typeof c.input?.command === 'string') {
        const call: Call = { command: c.input.command, exit: null, output: '' }
        calls.push(call); byId.set(c.id, call)
      }
    }
    if (e.type === 'user') for (const c of e.message?.content ?? []) {
      const call = c.type === 'tool_result' ? byId.get(c.tool_use_id) : undefined
      if (!call) continue
      call.output = typeof c.content === 'string' ? c.content : (c.content ?? []).map((p: { text?: string }) => p.text ?? '').join('')
      const code = call.output.match(/^Exit code (\d+)/)
      call.exit = c.is_error ? (code ? Number(code[1]) : 1) : 0
    }
    if (e.type === 'result' && typeof e.result === 'string') answer = e.result
  }
  return { calls, answer }
}

/** Codex `exec --json`: completed command executions, and the last agent message. */
export function callsFromCodex(lines: readonly string[]): { calls: Call[]; answer: string } {
  const calls: Call[] = []
  let answer = ''
  for (const e of lines.map(parse)) {
    if (e?.type !== 'item.completed') continue
    if (e.item?.type === 'command_execution') {
      calls.push({ command: String(e.item.command ?? ''), exit: typeof e.item.exit_code === 'number' ? e.item.exit_code : null, output: String(e.item.aggregated_output ?? '') })
    }
    if (e.item?.type === 'agent_message' && typeof e.item.text === 'string') answer = e.item.text
  }
  return { calls, answer }
}

export function score(c: Case, agent: string, calls: readonly Call[], answer: string): Score {
  // One entry a harness command; commands joined in one call share its exit and output.
  const harness = calls.flatMap((call) => harnessVerbs(call.command).map((verb) => ({ ...call, verb })))
  const first = harness.findIndex((h) => ACTIONS.has(h.verb))
  const orientation = (first === -1 ? harness : harness.slice(0, first)).filter((h) => LOOKS.has(h.verb)).length
  const wrong = harness.filter((h) => h.exit === 2 || /Unknown (option|command)/i.test(h.output)).length
  let retries = 0, refused = false
  for (const h of harness) {
    if (h.verb === 'new' && refused) retries++
    if (h.verb === 'new' && h.exit !== 0 && REFUSED.test(h.output)) refused = true
    // Closing one is what the refusal asks for: the next start is the advice taken, not a retry.
    if (h.verb === 'close' && h.exit === 0) refused = false
  }
  const ok = (verb: string) => harness.some((h) => h.verb === verb && h.exit === 0)
  const count = (verb: string) => harness.filter((h) => h.verb === verb && h.exit === 0).length
  const leftOpen = Math.max(0, count('new') - count('close'))
  const ranExpected = (c.expect.run ?? []).every(ok)
  const ranForbidden = (c.expect.notRun ?? []).filter(ok)
  const answerOk = c.expect.answer ? new RegExp(c.expect.answer, 'i').test(answer) : true
  return {
    id: c.id, agent, calls: harness.length, firstAction: first === -1 ? null : first, orientation, wrong,
    retriesAfterRefusal: retries, leftOpen, ranExpected, ranForbidden, answerOk,
    done: ranExpected && !ranForbidden.length && answerOk && retries === 0 && leftOpen === 0,
  }
}