Skip to content

The only tool you need to know what is happening and how to fix it.

more models

narrow-adventure9d99551b2d24
diff --git a/cli/test/evals/models.yaml b/cli/test/evals/models.yaml
index 85bc28dd..4d630118 100644
--- a/cli/test/evals/models.yaml
+++ b/cli/test/evals/models.yaml
@@ -80,6 +80,84 @@ agents:
output_tokens: [5000, 15000]
fallback_price: {prompt: 0.15, completion: 0.60, cache_read: 0.003, cache_write: 0.15}
+ glm:
+ # Z.AI first party (fp8). Third-party fp4 hosts list far lower prices;
+ # the benchmark pins the vendor so the quantization is known.
+ id: z-ai/glm-5.3
+ label: GLM-5.3
+ provider:
+ order: [z-ai]
+ allow_fallbacks: true
+ caching: automatic
+ reasoning_effort: medium
+ max_tokens_per_turn: 16000
+ estimate:
+ input_tokens: [100000, 400000]
+ cached_fraction: 0.7
+ output_tokens: [5000, 15000]
+ fallback_price: {prompt: 1.40, completion: 4.40, cache_read: 0.26, cache_write: 1.40}
+
+ qwen-27b:
+ # Alibaba first party; the Apache-licensed distilled Qwen3.8.
+ id: qwen/qwen3.8-27b
+ label: Qwen3.8 27B
+ provider:
+ order: [alibaba]
+ allow_fallbacks: true
+ caching: automatic
+ reasoning_effort: medium
+ max_tokens_per_turn: 16000
+ estimate:
+ input_tokens: [100000, 400000]
+ cached_fraction: 0.7
+ output_tokens: [5000, 15000]
+ fallback_price: {prompt: 0.425, completion: 2.55, cache_read: 0.085, cache_write: 0.425}
+
+ minimax-flash:
+ id: minimax/minimax-m3.1-flash
+ label: MiniMax M3.1 Flash
+ provider:
+ order: [minimax]
+ allow_fallbacks: true
+ caching: automatic
+ reasoning_effort: medium
+ max_tokens_per_turn: 16000
+ estimate:
+ input_tokens: [100000, 400000]
+ cached_fraction: 0.7
+ output_tokens: [5000, 15000]
+ fallback_price: {prompt: 0.15, completion: 0.60, cache_read: 0.015, cache_write: 0.15}
+
+ mimo-pro:
+ id: xiaomi/mimo-v2.6-pro
+ label: MiMo-V2.6 Pro
+ provider:
+ order: [xiaomi]
+ allow_fallbacks: true
+ caching: automatic
+ reasoning_effort: medium
+ max_tokens_per_turn: 16000
+ estimate:
+ input_tokens: [100000, 400000]
+ cached_fraction: 0.7
+ output_tokens: [5000, 15000]
+ fallback_price: {prompt: 0.435, completion: 0.87, cache_read: 0.0036, cache_write: 0.435}
+
+ luna:
+ id: openai/gpt-6-luna
+ label: GPT-6 Luna
+ provider:
+ order: [openai]
+ allow_fallbacks: true
+ caching: automatic
+ reasoning_effort: medium
+ max_tokens_per_turn: 16000
+ estimate:
+ input_tokens: [100000, 400000]
+ cached_fraction: 0.7
+ output_tokens: [5000, 15000]
+ fallback_price: {prompt: 0.10, completion: 0.50, cache_read: 0.01, cache_write: 0.10}
+
judge:
id: anthropic/claude-haiku-5.5
label: Claude Haiku 5.5 (judge)
diff --git a/website/app/globals.css b/website/app/globals.css
index f25206a5..869b3938 100644
--- a/website/app/globals.css
+++ b/website/app/globals.css
@@ -35,6 +35,17 @@
--ok: #22e0a8;
--crit: #ff5a5f;
+ /* Benchmark series: one fixed slot per model, dark steps of the
+ categorical palette (validated for CVD separation and contrast). */
+ --series-1: #3987e5;
+ --series-2: #d95926;
+ --series-3: #199e70;
+ --series-4: #c98500;
+ --series-5: #d55181;
+ --series-6: #008300;
+ --series-7: #9085e9;
+ --series-8: #e66767;
+
/* Dashboard blue family, shared with the docs restyle. Used for links
site-wide and as the blog band's accent. */
--blue: oklch(0.623 0.214 259.815);
@@ -289,6 +300,14 @@
--a4: #9a7b1a;
--ok: #0c9d6e;
--crit: #d63b41;
+ --series-1: #2a78d6;
+ --series-2: #eb6834;
+ --series-3: #1baf7a;
+ --series-4: #eda100;
+ --series-5: #e87ba4;
+ --series-6: #008300;
+ --series-7: #6250d6;
+ --series-8: #e34948;
background: var(--ink-0);
color: var(--fg-1);
color-scheme: light;
diff --git a/website/lib/benchmark-format.ts b/website/lib/benchmark-format.ts
index 20a15e57..d0f36e3b 100644
--- a/website/lib/benchmark-format.ts
+++ b/website/lib/benchmark-format.ts
@@ -1,9 +1,16 @@
import type { ModelKey, RunKind } from "@/lib/benchmarks";
+// One palette slot per model, in a fixed order: a model keeps its color
+// whatever else is on the chart.
export const MODEL_COLOR: Record<ModelKey, string> = {
- haiku: "var(--a3)",
- "deepseek-flash": "var(--a4)",
- opus: "var(--a1)",
+ haiku: "var(--series-1)",
+ "deepseek-flash": "var(--series-2)",
+ opus: "var(--series-3)",
+ luna: "var(--series-4)",
+ "minimax-flash": "var(--series-5)",
+ "mimo-pro": "var(--series-6)",
+ "qwen-27b": "var(--series-7)",
+ glm: "var(--series-8)",
};
export const KIND_LABEL: Record<RunKind, string> = {
diff --git a/website/lib/benchmarks.ts b/website/lib/benchmarks.ts
index 0698882c..0f1a3e2f 100644
--- a/website/lib/benchmarks.ts
+++ b/website/lib/benchmarks.ts
@@ -1,6 +1,14 @@
import matter from "gray-matter";
-export type ModelKey = "haiku" | "deepseek-flash" | "opus";
+export type ModelKey =
+ | "haiku"
+ | "deepseek-flash"
+ | "opus"
+ | "luna"
+ | "minimax-flash"
+ | "mimo-pro"
+ | "qwen-27b"
+ | "glm";
export type Totals = {
runs: number;
@@ -90,9 +98,16 @@ export type Group = {
results: Record<ModelKey, { runs: number; pass: number }>;
};
-// One batch of runs: which suites ran on which day at which commit. A page
+// One batch of runs: which suites ran on which day at which commit, for
+// which models when the batch did not cover every model on the page. A page
// built from one run has one part.
-export type RunPart = { suites: string[]; arms?: string[]; commit: string; date: string };
+export type RunPart = {
+ suites: string[];
+ arms?: string[];
+ models?: ModelKey[];
+ commit: string;
+ date: string;
+};
export type BenchmarkData = {
run: {
@@ -403,11 +418,12 @@ export function parseBenchmark(source: string): BenchmarkData {
}));
const totalRuns = models.reduce((n, m) => n + m.totals.runs, 0);
- const fmParts = (fm.parts ?? []) as { suites: string; arms?: string; commit: string; date: string }[];
+ const fmParts = (fm.parts ?? []) as { suites: string; arms?: string; models?: string; commit: string; date: string }[];
const parts: RunPart[] = fmParts.length
? fmParts.map((p) => ({
suites: String(p.suites).split(","),
...(p.arms ? { arms: String(p.arms).split(",") } : {}),
+ ...(p.models ? { models: String(p.models).split(",") as ModelKey[] } : {}),
commit: String(p.commit),
date: String(p.date),
}))
diff --git a/website/scripts/build-benchmarks-data.mjs b/website/scripts/build-benchmarks-data.mjs
index fac0d5c8..04915c9a 100644
--- a/website/scripts/build-benchmarks-data.mjs
+++ b/website/scripts/build-benchmarks-data.mjs
@@ -28,10 +28,39 @@ const HARNESS_URL =
const METHODOLOGY_URL = "https://docs.tracewayapp.com/learn/agent-benchmarks";
const PAGE_URL = "https://tracewayapp.com/benchmarks";
-const MODEL_ORDER = ["haiku", "deepseek-flash", "opus"];
-const MODEL_SHORT = { haiku: "Haiku", "deepseek-flash": "DeepSeek", opus: "Opus" };
-const MODEL_VENDOR = { haiku: "Anthropic", "deepseek-flash": "DeepSeek", opus: "Anthropic" };
-const MODEL_WEIGHTS = { haiku: "proprietary", "deepseek-flash": "open", opus: "proprietary" };
+// Display order. A model's palette slot (lib/benchmark-format.ts) is fixed
+// to its key, so adding a model never recolors the others.
+const MODEL_ORDER = ["haiku", "deepseek-flash", "opus", "luna", "minimax-flash", "mimo-pro", "qwen-27b", "glm"];
+const MODEL_SHORT = {
+ haiku: "Haiku",
+ "deepseek-flash": "DeepSeek",
+ opus: "Opus",
+ luna: "Luna",
+ "minimax-flash": "MiniMax",
+ "mimo-pro": "MiMo",
+ "qwen-27b": "Qwen 27B",
+ glm: "GLM",
+};
+const MODEL_VENDOR = {
+ haiku: "Anthropic",
+ "deepseek-flash": "DeepSeek",
+ opus: "Anthropic",
+ luna: "OpenAI",
+ "minimax-flash": "MiniMax",
+ "mimo-pro": "Xiaomi",
+ "qwen-27b": "Alibaba",
+ glm: "Z.ai",
+};
+const MODEL_WEIGHTS = {
+ haiku: "proprietary",
+ "deepseek-flash": "open",
+ opus: "proprietary",
+ luna: "proprietary",
+ "minimax-flash": "open",
+ "mimo-pro": "open",
+ "qwen-27b": "open",
+ glm: "open",
+};
// Every suite the harness has, in display order. Only the suites with runs in
// the source directory are rendered.
@@ -316,39 +345,52 @@ function parseArgs(argv) {
// may be narrowed to the setups that were run again after a fix:
// `--part debug@cli+skill:1a2b3c4d:2026-10-22` says the debug suite's
// cli+skill runs come from that later batch, and the suite's other setups
-// from the part that names the suite alone.
+// from the part that names the suite alone. A part may also be narrowed to
+// the models it ran, when models were added to the page in a later batch:
+// `--part perf,debug,write/glm,luna:5e6f7a8b:2026-10-22`. Every suite needs
+// exactly one unnarrowed part per model.
function parseParts(values, suites, header, runs) {
if (!values) return [{ suites: suites.map((s) => s.id), commit: header.gitSha.slice(0, 8), date: header.started.slice(0, 10) }];
const parts = values.map((v) => {
- const m = v.match(/^([\w,-]+)(?:@([\w+,-]+))?:([0-9a-f]{7,40}):(\d{4}-\d{2}-\d{2})$/);
- if (!m) throw new Error(`--part ${v}: want <suites>[@<arms>]:<commit>:<YYYY-MM-DD>`);
+ const m = v.match(/^([\w,-]+)(?:@([\w+,-]+))?(?:\/([\w,-]+))?:([0-9a-f]{7,40}):(\d{4}-\d{2}-\d{2})$/);
+ if (!m) throw new Error(`--part ${v}: want <suites>[@<arms>][/<models>]:<commit>:<YYYY-MM-DD>`);
const part = { suites: m[1].split(",") };
if (m[2]) part.arms = m[2].split(",");
- part.commit = m[3].slice(0, 8);
- part.date = m[4];
+ if (m[3]) part.models = m[3].split(",");
+ part.commit = m[4].slice(0, 8);
+ part.date = m[5];
return part;
});
- for (const s of suites) {
- const owners = parts.filter((p) => !p.arms && p.suites.includes(s.id));
- if (owners.length !== 1) throw new Error(`suite ${s.id} must be in exactly one --part without arms, found ${owners.length}`);
- }
- for (const p of parts)
+ const covers = (p, model) => !p.models || p.models.includes(model);
+ for (const s of suites)
+ for (const model of MODEL_ORDER) {
+ const owners = parts.filter((p) => !p.arms && p.suites.includes(s.id) && covers(p, model));
+ if (owners.length !== 1) throw new Error(`suite ${s.id} for ${model} must be in exactly one --part without arms, found ${owners.length}`);
+ }
+ for (const p of parts) {
+ for (const model of p.models ?? [])
+ if (!MODEL_ORDER.includes(model)) throw new Error(`--part names model ${model}, which is not on the page`);
for (const id of p.suites) {
if (!suites.some((s) => s.id === id)) throw new Error(`--part names suite ${id}, which has no runs`);
for (const arm of p.arms ?? [])
- if (!runs.some((r) => r.suite === id && r.arm === arm)) throw new Error(`--part names ${id}@${arm}, which has no runs`);
+ for (const model of p.models ?? MODEL_ORDER)
+ if (!runs.some((r) => r.suite === id && r.arm === arm && r.model === model))
+ throw new Error(`--part names ${id}@${arm} for ${model}, which has no runs`);
}
+ }
return parts;
}
// partText names a part in prose: "debug on 2026-10-22 at commit 21bbe026",
// or for a part narrowed to the setups run again after a skill fix, "the
// debug CLI + skill runs again on 2026-10-22, with the skill fix on top of
-// commit 21bbe026" (the fix is what the rerun measures, so it is named).
-function partText(p, setups) {
- if (!p.arms) return `${joinWords(p.suites)} on ${p.date} at commit \`${p.commit}\``;
+// commit 21bbe026" (the fix is what the rerun measures, so it is named). A
+// part narrowed to some models names them: "... for Haiku, DeepSeek and Opus".
+function partText(p, setups, models) {
+ const forModels = p.models ? ` for ${joinWords(p.models.map((k) => models.find((m) => m.key === k).short))}` : "";
+ if (!p.arms) return `${joinWords(p.suites)}${forModels} on ${p.date} at commit \`${p.commit}\``;
const labels = p.arms.map((a) => setups.find((x) => x.suite === p.suites[0] && x.arm === a)?.label ?? a);
- return `the ${joinWords(p.suites)} ${joinWords(labels)} runs again on ${p.date}, with the skill fix of commit \`${p.commit}\``;
+ return `the ${joinWords(p.suites)} ${joinWords(labels)} runs${forModels} again on ${p.date}, with the skill fix of commit \`${p.commit}\``;
}
function parseCsv(text) {
@@ -848,7 +890,7 @@ function crossCheck(data, summaryMd, statsMd) {
const usd = (x, digits) => `$${x.toFixed(digits)}`;
const frac = (pass, runs) => `${pass}/${runs} (${pct(pass, runs)}%)`;
const signedPp = (pp) => `${pp >= 0 ? "+" : "-"}${Math.abs(pp)}pp`;
-const words = ["zero", "one", "two", "three", "four", "five"];
+const words = ["zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten"];
const joinWords = (xs) => (xs.length < 2 ? xs.join("") : `${xs.slice(0, -1).join(", ")} and ${xs[xs.length - 1]}`);
function yamlScalar(v) {
@@ -888,7 +930,13 @@ function renderMarkdown(data) {
title: run.title,
date: run.date,
commit: run.commit,
- parts: run.parts.map((p) => ({ suites: p.suites.join(","), ...(p.arms ? { arms: p.arms.join(",") } : {}), commit: p.commit, date: p.date })),
+ parts: run.parts.map((p) => ({
+ suites: p.suites.join(","),
+ ...(p.arms ? { arms: p.arms.join(",") } : {}),
+ ...(p.models ? { models: p.models.join(",") } : {}),
+ commit: p.commit,
+ date: p.date,
+ })),
...(run.regraded ? { regraded: run.regraded } : {}),
cliVersion: run.cliVersion,
backendVersion: run.backendVersion,
@@ -936,7 +984,7 @@ function renderMarkdown(data) {
const partsText =
run.parts.length > 1
- ? ` in ${words[run.parts.length]} batches: ${run.parts.map((p) => partText(p, setups)).join(", ")}`
+ ? ` in ${words[run.parts.length]} batches: ${run.parts.map((p) => partText(p, setups, models)).join(", ")}`
: ` on ${run.date}, at commit \`${run.commit}\``;
const regradedText = run.regraded
? ` Every saved transcript was scored on ${run.regraded} by the same graders: after a grader fix the transcripts are scored again, so no number here mixes two versions of a grader.`