Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

7.1 KB
Raw Blame
/** The pure half of the hand-run refresh (scripts/refresh-benchmark-priors.ts): reads the three benchmarks' public boards from
 * text the script fetched, maps their model names to ours and rewrites the checked-in table. No network and no files here. */
import { nameKey, type PriorsTable, type PriorScore } from './priors.js'

export interface BoardRow { benchmarkName: string; score: number; harness: string; date: string }
/** One benchmark's board as read, or why it could not be. */
export type Board = { benchmark: string; source: string; rows: BoardRow[] } | { benchmark: string; error: string }
type Read = Extract<Board, { rows: BoardRow[] }>

const finite = (value: unknown, what: string): number => {
  if (typeof value !== 'number' || !Number.isFinite(value)) throw new Error(`${what}: accuracy is not a number`)
  return value
}

/** The JSON array at the start of [text], ignoring what follows it (the rest of the page): found by bracket depth outside strings. */
function leadingArray(text: string): unknown[] {
  let depth = 0
  let inString = false
  for (let i = 0; i < text.length; i++) {
    const c = text[i]
    if (inString) { if (c === '\\') i++; else if (c === '"') inString = false; continue }
    if (c === '"') inString = true
    else if (c === '[' || c === '{') depth++
    else if ((c === ']' || c === '}') && --depth === 0) return JSON.parse(text.slice(0, i + 1))
  }
  throw new Error('unterminated')
}

/** tbench.ai's leaderboard page, which carries its rows as an escaped JSON string. Only the 4.0 board is read: scores of
 * different Terminal-Bench versions do not compare. */
export function parseTerminalBench(html: string): Read {
  const text = html.replace(/\\"/g, '"')
  const at = text.indexOf('"rows":[')
  let rows: Array<{ metadata: { date: string; model_display: { label: string }; agent_display: { label: string }; reasoning_effort?: string }; metrics: { accuracy: unknown } }> = []
  try { if (at >= 0) rows = leadingArray(text.slice(at + 7)) as typeof rows } catch { /* reported below */ }
  if (rows.length === 0) throw new Error('The Terminal-Bench page has no rows')
  if (!text.includes('"title":"Terminal-Bench 4.0"')) throw new Error('The page is not the Terminal-Bench 4.0 board')
  return { benchmark: 'terminal-bench-4.0', source: 'https://www.tbench.ai/leaderboard', rows: rows.map(r => ({
    benchmarkName: r.metadata.model_display.label,
    score: finite(r.metrics.accuracy, r.metadata.model_display.label),
    harness: r.metadata.reasoning_effort ? `${r.metadata.agent_display.label}, ${r.metadata.reasoning_effort} effort` : r.metadata.agent_display.label,
    date: r.metadata.date,
  })) }
}

/** swebench.com's leaderboards file. Rows of the Verified board that run one model; the agent is the harness. */
export function parseSweBench(file: unknown): Read {
  const boards = (file as { leaderboards?: Array<{ name: string; results?: unknown[] }> } | null)?.leaderboards
  const verified = Array.isArray(boards) ? boards.find(b => b.name === 'Verified') : undefined
  if (!verified?.results) throw new Error('The SWE-bench file has no Verified board')
  const rows: BoardRow[] = []
  for (const entry of verified.results as Array<{ agent?: string; name: string; resolved: number; date: string; tags?: string[] }>) {
    const models = (entry.tags ?? []).filter(t => t.startsWith('Model: ')).map(t => t.slice('Model: '.length))
    if (models.length !== 1 || typeof entry.resolved !== 'number') continue
    rows.push({ benchmarkName: models[0], score: entry.resolved, harness: entry.agent ?? entry.name, date: entry.date })
  }
  return { benchmark: 'swe-bench-verified', source: 'https://www.swebench.com', rows }
}

/** Aider's polyglot leaderboard file (a flat list of `- dirname:` entries with `key: value` lines). The second-try pass rate. */
export function parseAider(yaml: string): Read {
  const rows: BoardRow[] = []
  for (const block of yaml.split(/^- dirname:/m).slice(1)) {
    const field = (key: string) => block.match(new RegExp(`^  ${key}: (.+)$`, 'm'))?.[1].trim()
    const model = field('model')
    const rate = Number(field('pass_rate_2'))
    const date = field('date')
    if (!model || !date || !Number.isFinite(rate)) continue
    rows.push({ benchmarkName: model, score: rate, harness: `aider, ${field('edit_format') ?? 'unknown'} edit format`, date })
  }
  if (rows.length === 0) throw new Error('The Aider file has no entries')
  return { benchmark: 'aider-polyglot', source: 'https://aider.chat/docs/leaderboards/', rows }
}

/** The table with the boards applied: for each benchmark read, every `leaderboard` score is replaced by each of our models'
 * best row under its benchmark names (efforts merged, as on the leaderboards). A benchmark whose board failed keeps its rows,
 * and `model-card` scores are never touched. Returns a new table. */
export function applyBoards(table: PriorsTable, boards: Board[], today: string): { table: PriorsTable; warnings: string[] } {
  const warnings: string[] = []
  const read = new Map<string, Read>()
  for (const board of boards) {
    if ('error' in board) warnings.push(`${board.benchmark} was not refreshed (${board.error}); its previous rows stay.`)
    else read.set(board.benchmark, board)
  }
  const order = table.benchmarks.map(b => b.id)
  const models = table.models.map(model => {
    const kept = model.scores.filter(s => s.via === 'model-card' || !read.has(s.benchmark))
    const added: PriorScore[] = []
    for (const [benchmark, board] of read) {
      const names = new Set(model.benchmarkNames.map(nameKey))
      const best = board.rows.filter(r => names.has(nameKey(r.benchmarkName))).sort((a, b) => b.score - a.score)[0]
      if (best) added.push({ benchmark, score: best.score, harness: best.harness, source: board.source, date: best.date, via: 'leaderboard' })
    }
    const scores = [...kept, ...added].sort((a, b) => order.indexOf(a.benchmark) - order.indexOf(b.benchmark))
    return { ...model, scores }
  })
  return { table: { ...table, refreshed: read.size > 0 ? today : table.refreshed, models }, warnings }
}

const ident = /^[A-Za-z_$][\w$]*$/
const quote = (s: string) => `'${s.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'`
function literal(value: unknown, indent: string): string {
  if (Array.isArray(value)) {
    if (value.length === 0) return '[]'
    const inner = value.map(v => `${indent}  ${literal(v, indent + '  ')},`).join('\n')
    return `[\n${inner}\n${indent}]`
  }
  if (value && typeof value === 'object') {
    const inner = Object.entries(value).map(([k, v]) => `${indent}  ${ident.test(k) ? k : quote(k)}: ${literal(v, indent + '  ')},`).join('\n')
    return `{\n${inner}\n${indent}}`
  }
  return typeof value === 'string' ? quote(value) : String(value)
}

/** The text of data.ts for [table]. */
export function renderData(table: PriorsTable): string {
  return `/** Generated by scripts/refresh-benchmark-priors.ts; rows with via: 'model-card' are typed in by hand and kept on refresh.
 * Each score names the board or card it came from, the day it was posted and the harness it ran in (docs/benchmark-priors.md). */
import type { PriorsTable } from './priors.js'

export const BENCHMARK_PRIORS: PriorsTable = ${literal(table, '')}
`
}