The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
more models
narrow-adventure9d99551b2d24
diff --git a/cli/test/evals/models.yaml b/cli/test/evals/models.yamlindex 85bc28dd..4d630118 100644--- a/cli/test/evals/models.yaml+++ b/cli/test/evals/models.yaml@@ -80,6 +80,84 @@ agents:output_tokens: [5000, 15000]fallback_price: {prompt: 0.15, completion: 0.60, cache_read: 0.003, cache_write: 0.15}+ glm:+ # Z.AI first party (fp8). Third-party fp4 hosts list far lower prices;+ # the benchmark pins the vendor so the quantization is known.+ id: z-ai/glm-5.3+ label: GLM-5.3+ provider:+ order: [z-ai]+ allow_fallbacks: true+ caching: automatic+ reasoning_effort: medium+ max_tokens_per_turn: 16000+ estimate:+ input_tokens: [100000, 400000]+ cached_fraction: 0.7+ output_tokens: [5000, 15000]+ fallback_price: {prompt: 1.40, completion: 4.40, cache_read: 0.26, cache_write: 1.40}++ qwen-27b:+ # Alibaba first party; the Apache-licensed distilled Qwen3.8.+ id: qwen/qwen3.8-27b+ label: Qwen3.8 27B+ provider:+ order: [alibaba]+ allow_fallbacks: true+ caching: automatic+ reasoning_effort: medium+ max_tokens_per_turn: 16000+ estimate:+ input_tokens: [100000, 400000]+ cached_fraction: 0.7+ output_tokens: [5000, 15000]+ fallback_price: {prompt: 0.425, completion: 2.55, cache_read: 0.085, cache_write: 0.425}++ minimax-flash:+ id: minimax/minimax-m3.1-flash+ label: MiniMax M3.1 Flash+ provider:+ order: [minimax]+ allow_fallbacks: true+ caching: automatic+ reasoning_effort: medium+ max_tokens_per_turn: 16000+ estimate:+ input_tokens: [100000, 400000]+ cached_fraction: 0.7+ output_tokens: [5000, 15000]+ fallback_price: {prompt: 0.15, completion: 0.60, cache_read: 0.015, cache_write: 0.15}++ mimo-pro:+ id: xiaomi/mimo-v2.6-pro+ label: MiMo-V2.6 Pro+ provider:+ order: [xiaomi]+ allow_fallbacks: true+ caching: automatic+ reasoning_effort: medium+ max_tokens_per_turn: 16000+ estimate:+ input_tokens: [100000, 400000]+ cached_fraction: 0.7+ output_tokens: [5000, 15000]+ fallback_price: {prompt: 0.435, completion: 0.87, cache_read: 0.0036, cache_write: 0.435}++ luna:+ id: openai/gpt-6-luna+ label: GPT-6 Luna+ provider:+ order: [openai]+ allow_fallbacks: true+ caching: automatic+ reasoning_effort: medium+ max_tokens_per_turn: 16000+ estimate:+ input_tokens: [100000, 400000]+ cached_fraction: 0.7+ output_tokens: [5000, 15000]+ fallback_price: {prompt: 0.10, completion: 0.50, cache_read: 0.01, cache_write: 0.10}+judge:id: anthropic/claude-haiku-5.5label: Claude Haiku 5.5 (judge)diff --git a/website/app/globals.css b/website/app/globals.cssindex f25206a5..869b3938 100644--- a/website/app/globals.css+++ b/website/app/globals.css@@ -35,6 +35,17 @@--ok: #22e0a8;--crit: #ff5a5f;+ /* Benchmark series: one fixed slot per model, dark steps of the+ categorical palette (validated for CVD separation and contrast). */+ --series-1: #3987e5;+ --series-2: #d95926;+ --series-3: #199e70;+ --series-4: #c98500;+ --series-5: #d55181;+ --series-6: #008300;+ --series-7: #9085e9;+ --series-8: #e66767;+/* Dashboard blue family, shared with the docs restyle. Used for linkssite-wide and as the blog band's accent. */--blue: oklch(0.623 0.214 259.815);@@ -289,6 +300,14 @@--a4: #9a7b1a;--ok: #0c9d6e;--crit: #d63b41;+ --series-1: #2a78d6;+ --series-2: #eb6834;+ --series-3: #1baf7a;+ --series-4: #eda100;+ --series-5: #e87ba4;+ --series-6: #008300;+ --series-7: #6250d6;+ --series-8: #e34948;background: var(--ink-0);color: var(--fg-1);color-scheme: light;diff --git a/website/lib/benchmark-format.ts b/website/lib/benchmark-format.tsindex 20a15e57..d0f36e3b 100644--- a/website/lib/benchmark-format.ts+++ b/website/lib/benchmark-format.ts@@ -1,9 +1,16 @@import type { ModelKey, RunKind } from "@/lib/benchmarks";+// One palette slot per model, in a fixed order: a model keeps its color+// whatever else is on the chart.export const MODEL_COLOR: Record<ModelKey, string> = {- haiku: "var(--a3)",- "deepseek-flash": "var(--a4)",- opus: "var(--a1)",+ haiku: "var(--series-1)",+ "deepseek-flash": "var(--series-2)",+ opus: "var(--series-3)",+ luna: "var(--series-4)",+ "minimax-flash": "var(--series-5)",+ "mimo-pro": "var(--series-6)",+ "qwen-27b": "var(--series-7)",+ glm: "var(--series-8)",};export const KIND_LABEL: Record<RunKind, string> = {diff --git a/website/lib/benchmarks.ts b/website/lib/benchmarks.tsindex 0698882c..0f1a3e2f 100644--- a/website/lib/benchmarks.ts+++ b/website/lib/benchmarks.ts@@ -1,6 +1,14 @@import matter from "gray-matter";-export type ModelKey = "haiku" | "deepseek-flash" | "opus";+export type ModelKey =+ | "haiku"+ | "deepseek-flash"+ | "opus"+ | "luna"+ | "minimax-flash"+ | "mimo-pro"+ | "qwen-27b"+ | "glm";export type Totals = {runs: number;@@ -90,9 +98,16 @@ export type Group = {results: Record<ModelKey, { runs: number; pass: number }>;};-// One batch of runs: which suites ran on which day at which commit. A page+// One batch of runs: which suites ran on which day at which commit, for+// which models when the batch did not cover every model on the page. A page// built from one run has one part.-export type RunPart = { suites: string[]; arms?: string[]; commit: string; date: string };+export type RunPart = {+ suites: string[];+ arms?: string[];+ models?: ModelKey[];+ commit: string;+ date: string;+};export type BenchmarkData = {run: {@@ -403,11 +418,12 @@ export function parseBenchmark(source: string): BenchmarkData {}));const totalRuns = models.reduce((n, m) => n + m.totals.runs, 0);- const fmParts = (fm.parts ?? []) as { suites: string; arms?: string; commit: string; date: string }[];+ const fmParts = (fm.parts ?? []) as { suites: string; arms?: string; models?: string; commit: string; date: string }[];const parts: RunPart[] = fmParts.length? fmParts.map((p) => ({suites: String(p.suites).split(","),...(p.arms ? { arms: String(p.arms).split(",") } : {}),+ ...(p.models ? { models: String(p.models).split(",") as ModelKey[] } : {}),commit: String(p.commit),date: String(p.date),}))diff --git a/website/scripts/build-benchmarks-data.mjs b/website/scripts/build-benchmarks-data.mjsindex fac0d5c8..04915c9a 100644--- a/website/scripts/build-benchmarks-data.mjs+++ b/website/scripts/build-benchmarks-data.mjs@@ -28,10 +28,39 @@ const HARNESS_URL =const METHODOLOGY_URL = "https://docs.tracewayapp.com/learn/agent-benchmarks";const PAGE_URL = "https://tracewayapp.com/benchmarks";-const MODEL_ORDER = ["haiku", "deepseek-flash", "opus"];-const MODEL_SHORT = { haiku: "Haiku", "deepseek-flash": "DeepSeek", opus: "Opus" };-const MODEL_VENDOR = { haiku: "Anthropic", "deepseek-flash": "DeepSeek", opus: "Anthropic" };-const MODEL_WEIGHTS = { haiku: "proprietary", "deepseek-flash": "open", opus: "proprietary" };+// Display order. A model's palette slot (lib/benchmark-format.ts) is fixed+// to its key, so adding a model never recolors the others.+const MODEL_ORDER = ["haiku", "deepseek-flash", "opus", "luna", "minimax-flash", "mimo-pro", "qwen-27b", "glm"];+const MODEL_SHORT = {+ haiku: "Haiku",+ "deepseek-flash": "DeepSeek",+ opus: "Opus",+ luna: "Luna",+ "minimax-flash": "MiniMax",+ "mimo-pro": "MiMo",+ "qwen-27b": "Qwen 27B",+ glm: "GLM",+};+const MODEL_VENDOR = {+ haiku: "Anthropic",+ "deepseek-flash": "DeepSeek",+ opus: "Anthropic",+ luna: "OpenAI",+ "minimax-flash": "MiniMax",+ "mimo-pro": "Xiaomi",+ "qwen-27b": "Alibaba",+ glm: "Z.ai",+};+const MODEL_WEIGHTS = {+ haiku: "proprietary",+ "deepseek-flash": "open",+ opus: "proprietary",+ luna: "proprietary",+ "minimax-flash": "open",+ "mimo-pro": "open",+ "qwen-27b": "open",+ glm: "open",+};// Every suite the harness has, in display order. Only the suites with runs in// the source directory are rendered.@@ -316,39 +345,52 @@ function parseArgs(argv) {// may be narrowed to the setups that were run again after a fix:// `--part debug@cli+skill:1a2b3c4d:2026-10-22` says the debug suite's// cli+skill runs come from that later batch, and the suite's other setups-// from the part that names the suite alone.+// from the part that names the suite alone. A part may also be narrowed to+// the models it ran, when models were added to the page in a later batch:+// `--part perf,debug,write/glm,luna:5e6f7a8b:2026-10-22`. Every suite needs+// exactly one unnarrowed part per model.function parseParts(values, suites, header, runs) {if (!values) return [{ suites: suites.map((s) => s.id), commit: header.gitSha.slice(0, 8), date: header.started.slice(0, 10) }];const parts = values.map((v) => {- const m = v.match(/^([\w,-]+)(?:@([\w+,-]+))?:([0-9a-f]{7,40}):(\d{4}-\d{2}-\d{2})$/);- if (!m) throw new Error(`--part ${v}: want <suites>[@<arms>]:<commit>:<YYYY-MM-DD>`);+ const m = v.match(/^([\w,-]+)(?:@([\w+,-]+))?(?:\/([\w,-]+))?:([0-9a-f]{7,40}):(\d{4}-\d{2}-\d{2})$/);+ if (!m) throw new Error(`--part ${v}: want <suites>[@<arms>][/<models>]:<commit>:<YYYY-MM-DD>`);const part = { suites: m[1].split(",") };if (m[2]) part.arms = m[2].split(",");- part.commit = m[3].slice(0, 8);- part.date = m[4];+ if (m[3]) part.models = m[3].split(",");+ part.commit = m[4].slice(0, 8);+ part.date = m[5];return part;});- for (const s of suites) {- const owners = parts.filter((p) => !p.arms && p.suites.includes(s.id));- if (owners.length !== 1) throw new Error(`suite ${s.id} must be in exactly one --part without arms, found ${owners.length}`);- }- for (const p of parts)+ const covers = (p, model) => !p.models || p.models.includes(model);+ for (const s of suites)+ for (const model of MODEL_ORDER) {+ const owners = parts.filter((p) => !p.arms && p.suites.includes(s.id) && covers(p, model));+ if (owners.length !== 1) throw new Error(`suite ${s.id} for ${model} must be in exactly one --part without arms, found ${owners.length}`);+ }+ for (const p of parts) {+ for (const model of p.models ?? [])+ if (!MODEL_ORDER.includes(model)) throw new Error(`--part names model ${model}, which is not on the page`);for (const id of p.suites) {if (!suites.some((s) => s.id === id)) throw new Error(`--part names suite ${id}, which has no runs`);for (const arm of p.arms ?? [])- if (!runs.some((r) => r.suite === id && r.arm === arm)) throw new Error(`--part names ${id}@${arm}, which has no runs`);+ for (const model of p.models ?? MODEL_ORDER)+ if (!runs.some((r) => r.suite === id && r.arm === arm && r.model === model))+ throw new Error(`--part names ${id}@${arm} for ${model}, which has no runs`);}+ }return parts;}// partText names a part in prose: "debug on 2026-10-22 at commit 21bbe026",// or for a part narrowed to the setups run again after a skill fix, "the// debug CLI + skill runs again on 2026-10-22, with the skill fix on top of-// commit 21bbe026" (the fix is what the rerun measures, so it is named).-function partText(p, setups) {- if (!p.arms) return `${joinWords(p.suites)} on ${p.date} at commit \`${p.commit}\``;+// commit 21bbe026" (the fix is what the rerun measures, so it is named). A+// part narrowed to some models names them: "... for Haiku, DeepSeek and Opus".+function partText(p, setups, models) {+ const forModels = p.models ? ` for ${joinWords(p.models.map((k) => models.find((m) => m.key === k).short))}` : "";+ if (!p.arms) return `${joinWords(p.suites)}${forModels} on ${p.date} at commit \`${p.commit}\``;const labels = p.arms.map((a) => setups.find((x) => x.suite === p.suites[0] && x.arm === a)?.label ?? a);- return `the ${joinWords(p.suites)} ${joinWords(labels)} runs again on ${p.date}, with the skill fix of commit \`${p.commit}\``;+ return `the ${joinWords(p.suites)} ${joinWords(labels)} runs${forModels} again on ${p.date}, with the skill fix of commit \`${p.commit}\``;}function parseCsv(text) {@@ -848,7 +890,7 @@ function crossCheck(data, summaryMd, statsMd) {const usd = (x, digits) => `$${x.toFixed(digits)}`;const frac = (pass, runs) => `${pass}/${runs} (${pct(pass, runs)}%)`;const signedPp = (pp) => `${pp >= 0 ? "+" : "-"}${Math.abs(pp)}pp`;-const words = ["zero", "one", "two", "three", "four", "five"];+const words = ["zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten"];const joinWords = (xs) => (xs.length < 2 ? xs.join("") : `${xs.slice(0, -1).join(", ")} and ${xs[xs.length - 1]}`);function yamlScalar(v) {@@ -888,7 +930,13 @@ function renderMarkdown(data) {title: run.title,date: run.date,commit: run.commit,- parts: run.parts.map((p) => ({ suites: p.suites.join(","), ...(p.arms ? { arms: p.arms.join(",") } : {}), commit: p.commit, date: p.date })),+ parts: run.parts.map((p) => ({+ suites: p.suites.join(","),+ ...(p.arms ? { arms: p.arms.join(",") } : {}),+ ...(p.models ? { models: p.models.join(",") } : {}),+ commit: p.commit,+ date: p.date,+ })),...(run.regraded ? { regraded: run.regraded } : {}),cliVersion: run.cliVersion,backendVersion: run.backendVersion,@@ -936,7 +984,7 @@ function renderMarkdown(data) {const partsText =run.parts.length > 1- ? ` in ${words[run.parts.length]} batches: ${run.parts.map((p) => partText(p, setups)).join(", ")}`+ ? ` in ${words[run.parts.length]} batches: ${run.parts.map((p) => partText(p, setups, models)).join(", ")}`: ` on ${run.date}, at commit \`${run.commit}\``;const regradedText = run.regraded? ` Every saved transcript was scored on ${run.regraded} by the same graders: after a grader fix the transcripts are scored again, so no number here mixes two versions of a grader.`