Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

4.8 KB
Raw Blame
/** Generated by scripts/refresh-benchmark-priors.ts; rows with via: 'model-card' are typed in by hand and kept on refresh.
 * Each score names the board or card it came from, the day it was posted and the harness it ran in (docs/benchmark-priors.md). */
import type { PriorsTable } from './priors.js'

export const BENCHMARK_PRIORS: PriorsTable = {
  refreshed: '2026-10-29',
  note: 'Public benchmark scores, each with its source, date and the harness it ran in. A score depends on the harness (Claude Code, Codex, Kimi Code, Aider ...) as much as on the model, many are reported by the model vendor, and none is our codebase: they set a starting point, never a verdict. A model with no score keeps its hand rating.',
  benchmarks: [
    {
      id: 'terminal-bench-4.0',
      name: 'Terminal-Bench 4.0',
      url: 'https://www.tbench.ai/leaderboard',
    },
    {
      id: 'swe-bench-verified',
      name: 'SWE-bench Verified',
      url: 'https://www.swebench.com',
    },
    {
      id: 'aider-polyglot',
      name: 'Aider polyglot',
      url: 'https://aider.chat/docs/leaderboards/',
    },
  ],
  models: [
    {
      engine: 'claude',
      model: 'claude-opus-5-5',
      names: [
        'opus',
      ],
      benchmarkNames: [
        'Opus 5.5',
        'claude-opus-5-5',
      ],
      scores: [
        {
          benchmark: 'terminal-bench-4.0',
          score: 64.85,
          harness: 'Claude Code, max effort',
          source: 'https://www.tbench.ai/leaderboard',
          date: '2026-09-22',
          via: 'leaderboard',
        },
      ],
    },
    {
      engine: 'claude',
      model: 'claude-sonnet-5-5',
      names: [
        'sonnet',
      ],
      benchmarkNames: [
        'Sonnet 5.5',
        'claude-sonnet-5-5',
      ],
      scores: [
        {
          benchmark: 'terminal-bench-4.0',
          score: 61.82,
          harness: 'Claude Code, max effort',
          source: 'https://www.tbench.ai/leaderboard',
          date: '2026-09-28',
          via: 'leaderboard',
        },
      ],
    },
    {
      engine: 'claude',
      model: 'claude-fable-5-1',
      names: [
        'fable',
      ],
      benchmarkNames: [
        'Fable 5.1',
        'claude-fable-5-1',
      ],
      scores: [
        {
          benchmark: 'terminal-bench-4.0',
          score: 57.88,
          harness: 'Claude Code, max effort',
          source: 'https://www.tbench.ai/leaderboard',
          date: '2026-09-01',
          via: 'leaderboard',
        },
      ],
    },
    {
      engine: 'claude',
      model: 'claude-haiku-5-5',
      names: [
        'haiku',
      ],
      benchmarkNames: [
        'Haiku 5.5',
        'claude-haiku-5-5',
      ],
      scores: [],
      noData: 'On none of the three boards (Terminal-Bench 4.0, SWE-bench Verified, Aider polyglot) on 2026-10-29, and no model card reports one.',
    },
    {
      engine: 'codex',
      model: 'gpt-6.1-sol',
      names: [
        'default',
      ],
      benchmarkNames: [
        'GPT-6.1 Sol',
        'gpt-6.1-sol',
      ],
      scores: [
        {
          benchmark: 'terminal-bench-4.0',
          score: 58.18,
          harness: 'Codex, max effort',
          source: 'https://www.tbench.ai/leaderboard',
          date: '2026-09-29',
          via: 'leaderboard',
        },
      ],
    },
    {
      engine: 'grid',
      model: 'moonshotai/kimi-k3',
      names: [
        'kimi-k3',
      ],
      benchmarkNames: [
        'Kimi K3',
        'K3',
        'kimi-k3',
      ],
      scores: [
        {
          benchmark: 'terminal-bench-4.0',
          score: 12.6,
          harness: 'Claude Code',
          source: 'https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash-1017',
          date: '2026-10-17',
          via: 'model-card',
          note: 'Reported in DeepSeek\'s model card, not Moonshot\'s. Moonshot runs K3 in its own Kimi Code harness, so this may undersell it.',
        },
      ],
    },
    {
      engine: 'grid',
      model: 'deepseek/deepseek-v4.1-flash-1017',
      names: [
        'deepseek-v4.1-flash-1017',
      ],
      benchmarkNames: [
        'DS-V4.1-Flash-1017',
        'DeepSeek-V4.1-Flash-1017',
      ],
      scores: [
        {
          benchmark: 'terminal-bench-4.0',
          score: 42.3,
          harness: 'Claude Code',
          source: 'https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash-1017',
          date: '2026-10-17',
          via: 'model-card',
          note: 'Reported by DeepSeek in its own model card.',
        },
      ],
    },
    {
      engine: 'grid',
      model: 'qwen3:4b',
      names: [
        'qwen3-4b',
      ],
      benchmarkNames: [
        'Qwen3 4B',
      ],
      scores: [],
      noData: 'On none of the three boards (Terminal-Bench 4.0, SWE-bench Verified, Aider polyglot) on 2026-10-29, and no model card reports one.',
    },
  ],
}