The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
import Link from "next/link";
import type { Metadata } from "next";
import { ArrowRight, Github } from "lucide-react";
import { SectionHead } from "@/components/section-head";
import { Eyebrow } from "@/components/eyebrow";
import { SectionShareLink } from "@/components/section-share-link";
import { StatsStrip } from "@/components/stats-strip";
import { FinalCTA } from "@/components/final-cta";
import { AuroraBackground } from "@/components/aurora-background";
import { BenchmarkCaseMatrix } from "@/components/benchmark-case-matrix";
import {
BenchmarkComparisonTable,
type ComparisonRow,
type ComparisonValue,
} from "@/components/benchmark-comparison-table";
import {
BenchmarkIndexChart,
type IndexView,
} from "@/components/benchmark-index-chart";
import {
BenchmarkCostChart,
type CostPoint,
} from "@/components/benchmark-cost-chart";
import { BENCHMARK } from "@/lib/benchmark-data";
import {
MODEL_COLOR,
formatPp,
formatUsd,
numberWord,
rate,
} from "@/lib/benchmark-format";
import type { Model, ModelKey } from "@/lib/benchmarks";
import { GITHUB_URL } from "@/lib/links";
const { run, models, suites, setups, comparisons, groups, knownIssues, cases } =
BENCHMARK;
const modelByKey = Object.fromEntries(models.map((m) => [m.key, m])) as Record<
ModelKey,
Model
>;
const setupLabels = Object.fromEntries(setups.map((s) => [s.arm, s.label]));
const suiteLabel = Object.fromEntries(suites.map((s) => [s.id, s.label]));
const groupById = Object.fromEntries(groups.map((g) => [g.id, g]));
const capitalize = (s: string) => s[0].toUpperCase() + s.slice(1);
// One batch as a line of the methodology list: "debug for Haiku, DeepSeek and
// Opus", or for a batch narrowed to the setups run again after a fix, "debug
// CLI + skill for ..., run again with the skill fix".
const partLine = (p: (typeof run.parts)[number]) => {
const forModels = p.models ? ` for ${joinList(p.models.map((k) => modelByKey[k].short))}` : "";
if (!p.arms) return `${joinList(p.suites)}${forModels}`;
const labels = p.arms.map((a) => setupLabels[a] ?? a);
return `${joinList(p.suites)} ${joinList(labels)}${forModels}, run again with the skill fix`;
};
const regradedText = run.regraded
? `Every transcript was scored on ${run.regraded} by the same graders. After a grader fix, every transcript is scored again, so no number here mixes two versions of a grader.`
: "";
const runDates = [...new Set(run.parts.map((p) => p.date))].sort();
const totalCases = suites.reduce((n, s) => n + s.cases, 0);
const joinList = (items: string[], conj = "and") =>
items.length < 2
? items.join("")
: `${items.slice(0, -1).join(", ")} ${conj} ${items[items.length - 1]}`;
const runsPerCaseText = run.runsPerCase === 1 ? "one run" : `${run.runsPerCase} runs`;
const modelCount = numberWord(models.length);
const title = "AI SRE BENCH · An open benchmark by Traceway";
const description = `An open benchmark for AI site reliability engineering: ${totalCases} cases, ${run.runsPerModel} runs per model, ${modelCount} models. Compare incident diagnosis, code fixes and safe operations, graded by deterministic code.`;
const shareImage = {
url: "/benchmarks/ai-sre-bench.png",
width: 1200,
height: 630,
alt: "AI SRE BENCH by Traceway. An open benchmark for AI site reliability engineering.",
};
export const metadata: Metadata = {
title,
description,
alternates: { canonical: "/benchmarks" },
openGraph: {
title,
description,
url: "/benchmarks",
siteName: "Traceway",
type: "website",
images: [shareImage],
},
twitter: {
card: "summary_large_image",
title,
description,
images: [shareImage],
},
};
const CARD_STYLE = {
background: "var(--ink-0)",
border: "1px solid var(--hair-2)",
boxShadow:
"0 1px 2px rgba(10,14,24,0.04), 0 12px 30px -18px rgba(10,14,24,0.14)",
};
function totalsOf(key: ModelKey) {
return modelByKey[key].totals;
}
function setupResult(suite: string, arm: string, key: ModelKey) {
return setups.find((s) => s.suite === suite && s.arm === arm)!.results[key];
}
function pickBy(score: (m: Model) => number, best: "max" | "min") {
return models.reduce((a, b) =>
best === "max"
? score(b) > score(a)
? b
: a
: score(b) < score(a)
? b
: a,
);
}
const best = pickBy((m) => m.totals.pass, "max");
const costPerPass = (m: Model) => m.totals.costUsd / m.totals.pass;
const value = pickBy(costPerPass, "min");
// The next cheapest model per passing run, which is what the value pick is
// measured against.
const runnerUp = models
.filter((m) => m.key !== value.key)
.sort((a, b) => costPerPass(a) - costPerPass(b))[0];
const distinctSetups = new Set(setups.map((s) => s.arm)).size;
const hasSuite = (id: string) => suites.some((s) => s.id === id);
const upliftOf = (key: ModelKey, suite = "perf") =>
comparisons.skillUplift.find((u) => u.suite === suite && u.model === key)!;
const upliftSuites = [...new Set(comparisons.skillUplift.map((u) => u.suite))];
// One row per (suite, CLI arm) pair the harness compares. A pair carried by a
// single case (debug-no-writes on the write arms) is kept in the data but not
// shown as a difference: one run is not a rate.
const cliMcpPairs = [
...new Map(comparisons.cliVsMcp.map((c) => [`${c.suite} ${c.cliArm}`, c])).values(),
].filter(
(c) =>
setups.find((s) => s.suite === c.suite && s.arm === c.cliArm)!.results[models[0].key].runs >= 3,
);
// The uplift badge goes to a model only when it leads on every suite that
// has a control arm; the finding below gives the per-suite numbers.
const upliftLeaders = upliftSuites.map((suite) =>
pickBy((m) => upliftOf(m.key, suite).pp, "max"),
);
const biggestUplift =
upliftLeaders.length && upliftLeaders.every((m) => m.key === upliftLeaders[0].key)
? upliftLeaders[0]
: null;
const cheapest = pickBy((m) => m.totals.costUsd, "min");
const priciest = pickBy((m) => m.totals.costUsd, "max");
const BADGES: Partial<Record<ModelKey, string[]>> = {};
const badgeOf: [Model | null, string][] = [
[best, "Highest pass rate"],
[value, "Lowest cost per pass"],
[biggestUplift, "Biggest skill uplift"],
];
for (const [m, label] of badgeOf) if (m) (BADGES[m.key] ??= []).push(label);
const controls = groupById["perf-ctl"];
const byControls = [...models].sort(
(a, b) => controls.results[b.key].pass - controls.results[a.key].pass,
);
const mcpPerf = (key: ModelKey) =>
comparisons.cliVsMcp.find((c) => c.suite === "perf" && c.model === key)!;
const mcpFavoured = models.filter((m) => mcpPerf(m.key).pp < 0);
const mcpGapText = (a: ModelKey, b: ModelKey) => {
const gap = mcpPerf(b).mcp - mcpPerf(a).mcp;
if (gap === 0) return `level with ${modelByKey[b].short}`;
return `${Math.abs(gap)}pp ${gap > 0 ? "behind" : "ahead of"} ${modelByKey[b].short}`;
};
const upliftOrder = [...models].sort(
(a, b) => upliftOf(b.key).pp - upliftOf(a.key).pp,
);
const upliftList = (suite: string) =>
joinList(
[...models]
.sort((a, b) => upliftOf(b.key, suite).pp - upliftOf(a.key, suite).pp)
.map((m) => `${m.short} ${formatPp(upliftOf(m.key, suite).pp)}`),
);
// The model with the lowest perf pass rate on the CLI without the skill.
const weakestWithoutSkill = pickBy(
(m) => rate(setupResult("perf", "cli-noskill", m.key).pass, setupResult("perf", "cli-noskill", m.key).runs),
"min",
);
const controlsClean = (m: Model) => controls.results[m.key].pass === controls.results[m.key].runs;
const suitePasses = (suiteId: string, key: ModelKey) =>
setups
.filter((s) => s.suite === suiteId)
.reduce((n, s) => n + s.results[key].pass, 0);
const suiteRuns = (suiteId: string, key: ModelKey) =>
setups
.filter((s) => s.suite === suiteId)
.reduce((n, s) => n + s.results[key].runs, 0);
const byDebug = hasSuite("debug")
? [...models].sort(
(a, b) => suitePasses("debug", b.key) - suitePasses("debug", a.key),
)
: [];
const debugMargin =
byDebug.length > 1
? suitePasses("debug", byDebug[0].key) - suitePasses("debug", byDebug[1].key)
: 0;
// "Of 41 runs each, Opus passed 35, DeepSeek 34 and Luna 19."
const debugTotalsText = !hasSuite("debug")
? ""
: new Set(models.map((m) => suiteRuns("debug", m.key))).size === 1
? `Of ${suiteRuns("debug", models[0].key)} runs each, ${joinList(
byDebug.map((m, i) => `${m.short}${i === 0 ? " passed" : ""} ${suitePasses("debug", m.key)}`),
)}.`
: `${joinList(
byDebug.map((m) => `${m.short} passed ${suitePasses("debug", m.key)} of ${suiteRuns("debug", m.key)}`),
)}.`;
// "DeepSeek leads with 26 of 27; then Opus 23, MiniMax 21 and Haiku 10."
const groupStandings = (id: string) => {
const g = groupById[id];
const sorted = [...models].sort((a, b) => g.results[b.key].pass - g.results[a.key].pass);
const top = g.results[sorted[0].key];
const leaders = sorted.filter((m) => g.results[m.key].pass === top.pass);
const rest = sorted.slice(leaders.length);
if (!rest.length) return `every model passed ${top.pass} of ${top.runs}`;
return `${joinList(leaders.map((m) => m.short))} ${leaders.length === 1 ? "leads" : "lead"} with ${top.pass} of ${top.runs}; then ${joinList(
rest.map((m) => `${m.short} ${g.results[m.key].pass}`),
)}`;
};
const noWrites = cases.find((c) => c.id === "debug-no-writes");
const noWritesRuns = noWrites
? models.flatMap((m) =>
Object.entries(noWrites.runs[m.key]).map(([arm, r]) => ({ model: m, arm, run: r })),
)
: [];
const noWritesPassed = noWritesRuns.filter((x) => x.run.status === "pass");
const noWritesTouched = noWritesRuns.filter((x) => x.run.graders.includes("state"));
const GRADER_NAME: Record<string, string> = {
answer: "answer",
grounding: "grounding",
trajectory: "trajectory",
report: "report",
state: "state",
hidden_tests: "hidden-test",
};
const noWritesFailedOn = [
...new Set(
noWritesRuns
.filter((x) => x.run.status !== "pass")
.flatMap((x) => x.run.graders.map((g) => GRADER_NAME[g] ?? g)),
),
];
const armText = (arm: string) => (setupLabels[arm] ?? arm).replace(/^Write /, "write ");
const whereRan = (xs: typeof noWritesRuns) =>
joinList(xs.map((x) => `${x.model.short} on the ${armText(x.arm)}`));
const whereRanCompact = (xs: typeof noWritesRuns) => {
const arms = new Set(xs.map((x) => x.arm));
if (arms.size !== 1 || xs.length < 2) return whereRan(xs);
return `${joinList(xs.map((x) => x.model.short))} (${xs.length === 2 ? "both" : "all"} on the ${armText(xs[0].arm)})`;
};
// A setup with one or two cases is a coin flip, not a rate: it stays in the
// tables and out of the charts.
const chartSetups = setups.filter((s) => s.results[models[0].key].runs >= 3);
const unchartedSetups = setups.length - chartSetups.length;
const costPoints: CostPoint[] = [
...chartSetups.flatMap((s) =>
models.map((m) => ({
id: `${s.suite}-${s.arm}-${m.key}`,
model: m.key,
modelLabel: m.label,
modelShort: m.short,
label: `${suiteLabel[s.suite]} · ${s.label}`,
pass: s.results[m.key].pass,
runs: s.results[m.key].runs,
costPerRunUsd: s.results[m.key].costPerRunUsd,
overall: false,
})),
),
...models.map((m) => ({
id: `all-${m.key}`,
model: m.key,
modelLabel: m.label,
modelShort: m.short,
label: `All ${m.totals.runs} runs`,
pass: m.totals.pass,
runs: m.totals.runs,
costPerRunUsd: m.totals.costPerRunUsd,
overall: true,
})),
];
const maxCost = Math.max(...models.map((m) => m.totals.costUsd));
const issueRuns = (key: ModelKey) =>
cases.flatMap((c) => Object.values(c.runs[key])).filter((r) => r.knownIssue).length;
const KIND_WORD: Record<string, string> = { case: "cases", skill: "skill", harness: "harness" };
const issueKinds = Object.keys(KIND_WORD)
.filter((k) => knownIssues.some((i) => i.kind === k))
.map((k) => KIND_WORD[k]);
const issueGroups = groups
.filter((g) => knownIssues.some((i) => cases.find((c) => c.id === i.case)?.group === g.id))
.map((g) => g.label);
const groupRightAnswer = (groupId: string, key: ModelKey) =>
cases
.filter((c) => c.group === groupId)
.flatMap((c) => Object.values(c.runs[key]))
.filter((r) => r.kind === "pass" || r.kind === "trajectory").length;
const indexViews: IndexView[] = [
{
id: "overall",
tab: "All runs",
title: `by model, all ${run.runsPerModel} runs`,
bars: models.map((m) => ({
id: m.key,
model: m.key,
weights: m.weights,
label: m.label,
detail: m.vendor,
pass: m.totals.pass,
rightAnswerEvidence: m.totals.rightAnswerEvidence,
runs: m.totals.runs,
})),
},
{
id: "setup",
tab: "By suite and setup",
title: "by suite and setup",
bars: chartSetups.flatMap((s) =>
models.map((m) => ({
id: `${s.suite}-${s.arm}-${m.key}`,
model: m.key,
weights: m.weights,
label: m.short,
detail: `${suiteLabel[s.suite]} · ${s.label}`,
pass: s.results[m.key].pass,
rightAnswerEvidence: s.results[m.key].rightAnswerEvidence,
runs: s.results[m.key].runs,
})),
),
},
{
id: "group",
tab: "By case group",
title: "by case group",
bars: groups.flatMap((g) =>
models.map((m) => ({
id: `${g.id}-${m.key}`,
model: m.key,
weights: m.weights,
label: m.short,
detail: g.label,
pass: g.results[m.key].pass,
rightAnswerEvidence: groupRightAnswer(g.id, m.key),
runs: g.results[m.key].runs,
})),
),
},
];
const partlyJudged = models.filter((m) => m.totals.judged < m.totals.runs);
const judgedText = !partlyJudged.length
? ""
: new Set(partlyJudged.map((m) => m.totals.runs)).size === 1
? ` The judge scored ${joinList(partlyJudged.map((m) => `${m.totals.judged} of ${m.short}'s`))} ${partlyJudged[0].totals.runs} runs.`
: ` ${partlyJudged.map((m) => `The judge scored ${m.totals.judged} of ${m.short}'s ${m.totals.runs} runs.`).join(" ")}`;
const indexSubtitle = `Run on ${joinList(runDates)}: ${joinList(
suites.map((s, i) => `${s.label} (${s.cases}${i === 0 ? " cases" : ""})`),
)} on ${distinctSetups} setups, ${runsPerCaseText} per case, graded by deterministic code.${
unchartedSetups
? ` The ${numberWord(unchartedSetups)} setups with fewer than three cases are in the tables, not the charts.`
: ""
}`;
const byPass = [...models].sort((a, b) => b.totals.pass - a.totals.pass);
const pctText = (pass: number, runs: number) =>
`${((100 * pass) / runs).toFixed(1)}%`;
const passValue = (
r: { pass: number; runs: number },
note?: string,
): ComparisonValue => ({
value: r.pass / r.runs,
text: pctText(r.pass, r.runs),
note,
title: `${r.pass} of ${r.runs} runs`,
});
const perModelRow = (
label: string,
sub: string,
f: (key: ModelKey) => ComparisonValue[],
better: "higher" | "lower" = "higher",
): ComparisonRow => ({
label,
sub,
better,
values: Object.fromEntries(models.map((m) => [m.key, f(m.key)])) as Record<
ModelKey,
ComparisonValue[]
>,
});
const suiteCases = (id: string) => suites.find((s) => s.id === id)!.cases;
const armCases = (suite: string, arm: string) =>
setupResult(suite, arm, models[0].key).runs;
// "13 of 14 cases" when a setup does not run every case of its suite.
const casesText = (suite: string, arm: string) => {
const n = armCases(suite, arm);
const all = suiteCases(suite);
return n === all ? `${n} cases` : `${n} of ${all} cases`;
};
const GROUP_ORDER = [
"perf-ctl",
"perf-db",
"perf-app",
"perf-net",
"perf-cache",
"perf-infra",
"perf-xs",
"debug-backend",
"debug-mobile",
"debug-ctl",
"setup",
].filter((id) => groupById[id]);
const smallestGroup = Math.min(
...groups.map((g) => g.results[models[0].key].runs),
);
const comparisonRows: ComparisonRow[] = [
perModelRow(
"Performance diagnosis",
`perf · ${suiteCases("perf")} cases, CLI`,
(k) => [
passValue(setupResult("perf", "cli+skill", k), "with the skill"),
passValue(setupResult("perf", "cli-noskill", k), "without the skill"),
],
),
perModelRow(
"Performance diagnosis over MCP",
`perf · ${suiteCases("perf")} cases, MCP server`,
(k) => [passValue(setupResult("perf", "mcp", k))],
),
...(hasSuite("debug")
? [
perModelRow(
"Root-cause debugging",
`debug · ${casesText("debug", "cli+skill")}, CLI`,
(k) => [
passValue(setupResult("debug", "cli+skill", k), "with the skill"),
passValue(setupResult("debug", "cli-noskill", k), "without the skill"),
],
),
perModelRow(
"Root-cause debugging over MCP",
`debug · ${casesText("debug", "mcp")}, MCP server`,
(k) => [passValue(setupResult("debug", "mcp", k))],
),
]
: []),
perModelRow(
"Safe write actions",
`write · ${suiteCases("write")} cases`,
(k) => [
passValue(setupResult("write", "cli-write+skill", k), "write CLI + skill"),
passValue(setupResult("write", "mcp-write", k), "write MCP server"),
],
),
...GROUP_ORDER.map((id) => {
const g = groupById[id];
return perModelRow(
g.label,
`${g.code} · ${g.cases} cases, ${g.results[models[0].key].runs} runs`,
(k) => [passValue(g.results[k])],
);
}),
perModelRow("Overall pass rate", `all ${run.runsPerModel} runs`, (k) => [
passValue(totalsOf(k)),
]),
perModelRow(
"Right answer and evidence",
"trajectory check not counted",
(k) => [
passValue({
pass: totalsOf(k).rightAnswerEvidence,
runs: totalsOf(k).runs,
}),
],
),
perModelRow(
"Agent spend",
`all ${run.runsPerModel} runs, lower is better`,
(k) => [
{ value: totalsOf(k).costUsd, text: formatUsd(totalsOf(k).costUsd) },
],
"lower",
),
perModelRow(
"Cost per passing run",
"agent spend ÷ passed runs, lower is better",
(k) => {
const v = totalsOf(k).costUsd / totalsOf(k).pass;
return [{ value: v, text: formatUsd(v) }];
},
"lower",
),
];
export default function BenchmarksPage() {
return (
<main className="relative">
<section className="hero hero-product relative">
<AuroraBackground variant="hero" />
<div className="wrap relative z-10">
<div className="flex flex-col items-center text-center">
<p className="mb-5 font-mono text-[0.6875rem] uppercase tracking-[0.18em] text-fg-2">
An open benchmark by Traceway
</p>
<h1 style={{ fontSize: "clamp(34px, 7vw, 80px)", letterSpacing: "-0.05em" }}>
AI SRE <span className="text-a2">BENCH</span>
</h1>
<p className="hero-sub text-pretty" style={{ maxWidth: 720 }}>
How well can AI agents do the work of a site reliability engineer?
Compare models on incident diagnosis, code fixes and safe operations
using seeded production scenarios and real Traceway tools.
Open-source cases. Deterministic grading. Every result published.
</p>
<div className="hero-cta-row justify-center">
<Link href="#results" className="btn btn-accent">
View results
<ArrowRight className="h-4 w-4" />
</Link>
<Link
href={run.methodologyUrl}
target="_blank"
rel="noopener noreferrer"
className="btn btn-ghost"
>
How we measure
<ArrowRight className="h-4 w-4" />
</Link>
</div>
</div>
<StatsStrip
className="mx-auto max-w-4xl"
stats={[
{ num: `<em>${run.totalRuns}</em>`, label: "Graded runs" },
{
num: `<em>${suites.reduce((n, s) => n + s.cases, 0)}</em>`,
label: "Cases",
},
{ num: `<em>${distinctSetups}</em>`, label: "Setups" },
{ num: `<em>${models.length}</em>`, label: "Models" },
]}
/>
</div>
</section>
<div className="band-light">
<section id="results" className="wrap scroll-mt-20 pt-16 pb-10 md:pt-20">
<SectionHead
eyebrow="AI SRE BENCH results"
shareId="results"
title={
<>
Pass rate, <em>model by model.</em>
</>
}
description={`Every model ran the same ${run.runsPerModel} runs. A run passes when its answer is right, its evidence is grounded, its writes are safe and, where it fixed code, the hidden tests pass. Tool-call counts are reported but never decide a pass.`}
/>
<BenchmarkIndexChart views={indexViews} subtitle={indexSubtitle} />
<div className="mt-8 grid gap-5 sm:grid-cols-2 lg:grid-cols-4">
{byPass.map((m) => (
<ModelCard key={m.key} model={m} badges={BADGES[m.key] ?? []} />
))}
</div>
</section>
<section id="compare" className="wrap scroll-mt-20 py-16 md:py-20">
<SectionHead
eyebrow="At a glance"
shareId="compare"
title={
<>
Every benchmark, <em>side by side.</em>
</>
}
description="Pass rates per suite, setup and case group, then what the runs cost. The best result in each row is in bold, and a shaded cell means that model led every number in the row."
/>
<BenchmarkComparisonTable
models={byPass.map((m) => ({
key: m.key,
label: m.label,
short: m.short,
note: m.weights === "open" ? "open weights" : m.vendor,
}))}
rows={comparisonRows}
/>
<p className="mt-4 text-[0.8125rem] leading-relaxed text-fg-3">
One run per case and setup, so a single run moves the smallest
groups ({smallestGroup} runs) by {Math.round(100 / smallestGroup)}{" "}
points. Open a{" "}
<Link
href="https://github.com/tracewayapp/traceway/pulls"
target="_blank"
rel="noopener noreferrer"
className="text-blue-bright hover:underline"
>
PR
</Link>{" "}
or email{" "}
<a
href="mailto:[email protected]"
className="text-blue-bright hover:underline"
>
[email protected]
</a>{" "}
if you'd like to see your model featured.
</p>
</section>
<section id="findings" className="wrap scroll-mt-20 py-16 md:py-20">
<SectionHead
eyebrow="Key findings"
shareId="findings"
title={
<>
What the runs say, <em>and what they don't.</em>
</>
}
/>
<div className="grid gap-12 md:grid-cols-2">
<Finding
label="01 · Judgment"
title={
controlsClean(byControls[0])
? `${joinList(byControls.filter(controlsClean).map((m) => m.short))} ${byControls.filter(controlsClean).length === 1 ? "never false-alarms" : "never false-alarm"}.`
: `${byControls[0].short} false-alarms least.`
}
>
The control cases are traps: a healthy endpoint, endpoints marked
slow on purpose, a misleading p95 metric, a pasted alert.{" "}
{byControls[0].short} passed {controls.results[byControls[0].key].pass}{" "}
of {controls.results[byControls[0].key].runs} control runs;{" "}
{joinList(
byControls
.slice(1)
.map((m) => `${m.short} ${controls.results[m.key].pass}`),
)}
. The gap between the models is in judgment, not in reading
telemetry.
</Finding>
<Finding label="02 · Value" title={`${value.short} is the value pick.`}>
{rate(value.totals.pass, value.totals.runs)}% of its runs passed for{" "}
{formatUsd(value.totals.costUsd)} of agent spend,{" "}
{formatUsd(costPerPass(value))} per passing run. {runnerUp.short} is
next at {formatUsd(costPerPass(runnerUp))} per passing run:{" "}
{rate(runnerUp.totals.pass, runnerUp.totals.runs)}% passed for{" "}
{formatUsd(runnerUp.totals.costUsd)},{" "}
{Math.abs(runnerUp.totals.pass - value.totals.pass)}{" "}
{runnerUp.totals.pass >= value.totals.pass ? "more" : "fewer"} passing
runs.
{best.key !== value.key
? ` ${best.short} passed ${best.totals.pass - value.totals.pass} more runs than ${value.short}, for ${Math.round(best.totals.costUsd / value.totals.costUsd)}× its spend. On performance cases over MCP, ${value.short} is ${mcpGapText(value.key, best.key)}.`
: null}
</Finding>
<Finding
label="03 · The skill"
title={
upliftOrder[0].key === weakestWithoutSkill.key
? "The skill helps the weakest model most."
: `The skill helps ${upliftOrder[0].short} most.`
}
>
On the performance suite, the skill's lift over the bare CLI:{" "}
{upliftList("perf")}.{" "}
{upliftSuites.includes("debug")
? `On the debug suite: ${upliftList("debug")}. `
: null}
{mcpFavoured.length === 1
? `${mcpFavoured[0].short} is the only model that scores higher on performance cases over MCP (${mcpPerf(mcpFavoured[0].key).mcp}%) than with the CLI and skill (${mcpPerf(mcpFavoured[0].key).cli}%).`
: mcpFavoured.length > 1
? `${joinList(mcpFavoured.map((m) => `${m.short} (${mcpPerf(m.key).mcp}% vs ${mcpPerf(m.key).cli}%)`))} score higher on performance cases over MCP than with the CLI and skill.`
: null}
</Finding>
<Finding label="04 · Our bugs" title="Some failures are ours.">
{capitalize(numberWord(knownIssues.length))} problems behind
these failures are in our own {joinList(issueKinds)}, not in the
models. They account for{" "}
{joinList(
models.map(
(m, i) =>
`${issueRuns(m.key)} of ${m.short}'s ${m.totals.runs - m.totals.pass}${i === 0 ? " failures" : ""}`,
),
)}
. We still count them as failures. Here they are.
</Finding>
{hasSuite("debug") ? (
<Finding
label="05 · Fixing code"
title={`${byDebug[0].short} passes the most debug runs${debugMargin === 1 ? ", by one" : ""}.`}
>
On the debug suite the agent finds the culprit in the telemetry
and patches it, and hidden tests check the fix. {debugTotalsText}{" "}
On the backend cases, where the hidden tests run against the
patch, {groupStandings("debug-backend")}. On the mobile cases,
checked statically, {groupStandings("debug-mobile")}.{" "}
{noWrites
? `One debug case hands the agent a write credential and asks it to look into a pile of errors and clean up: the fix belongs in the code, and nothing in Traceway may change. ${
noWritesPassed.length
? `Only ${whereRanCompact(noWritesPassed)} passed it`
: "No model passed it"
}; ${
noWritesTouched.length
? `${whereRan(noWritesTouched)} changed Traceway, which the task forbids`
: "no model changed anything in Traceway"
}, and the ${noWritesPassed.length ? "other " : ""}runs failed the ${joinList(noWritesFailedOn, "or")} check.`
: null}
</Finding>
) : null}
</div>
<div className="mt-14 grid gap-5 md:grid-cols-2">
{knownIssues.map((issue) => (
<div key={issue.id} className="rounded-2xl p-6" style={CARD_STYLE}>
<div className="flex flex-wrap items-center gap-2">
<span className="tag">
{issue.kind === "skill"
? "Skill bug"
: issue.kind === "harness"
? "Harness bug"
: "Case bug"}
</span>
<code>{issue.case}</code>
</div>
<h3 className="mt-4 text-[1.0625rem] font-semibold text-fg-0">
{issue.title}
</h3>
<p className="muted mt-2 text-[0.9375rem] text-pretty">
{issue.detail}
</p>
<p className="mt-4 font-mono text-[0.75rem] text-fg-2">
{issue.truncated
? `It hit ${issue.affectedRuns} of the ${issue.caseRuns} runs it covers, ${issue.truncated === issue.affectedRuns ? "all" : issue.truncated} on truncated output. `
: ""}
Runs hit:{" "}
{models.map((m) => `${m.short} ${issue.byModel[m.key]}`).join(" · ")}
</p>
</div>
))}
</div>
</section>
<section id="setups" className="wrap scroll-mt-20 py-16 md:py-20">
<SectionHead
eyebrow="By setup"
shareId="setups"
title={
<>
Pass rate by suite <em>and setup.</em>
</>
}
description={`${capitalize(
joinList(
suites
.filter((s) => s.setups.includes("cli-noskill"))
.map((s) =>
armCases(s.id, "cli-noskill") === s.cases
? `${s.label.toLowerCase()} cases`
: `${armCases(s.id, "cli-noskill")} of the ${s.cases} ${s.label.toLowerCase()} cases`,
),
),
)} ran with the CLI and skill, with the CLI alone as a control, and over the MCP server. The ${suiteCases("write")} write cases${
noWrites ? ", plus the debug case that hands the agent a write credential," : ""
} ran with the write CLI and skill, and over the write MCP server.`}
/>
<DataTable minWidth={190 + models.length * 160}>
<thead>
<tr>
<Th>Setup</Th>
{models.map((m) => (
<ModelTh key={m.key} model={m} />
))}
</tr>
</thead>
<tbody>
{suites.map((s) => (
<SuiteRows key={s.id} suite={s} />
))}
<tr style={{ borderTop: "1px solid var(--hair-2)" }}>
<RowHead title="All runs" sub={`${run.runsPerModel} per model`} />
{models.map((m) => (
<Td key={m.key}>
<PassCell
model={m.key}
pass={m.totals.pass}
runs={m.totals.runs}
rightAnswerEvidence={m.totals.rightAnswerEvidence}
strong
/>
</Td>
))}
</tr>
<SubheadRow label="Differences" span={models.length + 1} />
{upliftSuites.map((suite) => (
<DeltaRow
key={suite}
title={`Skill uplift, ${suite}`}
sub="CLI + skill minus CLI without the skill"
cells={models.map((m) => {
const u = upliftOf(m.key, suite);
return { key: m.key, pp: u.pp, detail: `${u.with}% vs ${u.without}%` };
})}
/>
))}
{cliMcpPairs.map((p) => (
<DeltaRow
key={`${p.suite}-${p.cliArm}`}
title={`CLI vs MCP, ${p.suite}`}
sub={`${setupLabels[p.cliArm]} minus ${setupLabels[p.mcpArm]}`}
cells={models.map((m) => {
const c = comparisons.cliVsMcp.find(
(x) =>
x.suite === p.suite && x.cliArm === p.cliArm && x.model === m.key,
)!;
return { key: m.key, pp: c.pp, detail: `${c.cli}% vs ${c.mcp}%` };
})}
/>
))}
</tbody>
</DataTable>
<p className="dim mt-4 text-[0.8125rem]">
“Right answer + evidence” counts the passes plus the runs
that failed only the trajectory check: their answer, evidence,
report, writes and hidden tests all passed. The differences are in
percentage points, computed before rounding.
</p>
</section>
<section id="groups" className="wrap scroll-mt-20 py-16 md:py-20">
<SectionHead
eyebrow="By case group"
shareId="groups"
title={
<>
Where each model <em>wins and loses.</em>
</>
}
description={`Passes over every setup a case ran on. Groups follow the case names: perf-db-n-plus-1 is in Database, perf-ctl-healthy in Controls and traps${
hasSuite("debug") ? ", and the debug groups follow the stack the fix is in" : ""
}.`}
/>
<DataTable minWidth={340 + models.length * 160}>
<thead>
<tr>
<Th>Group</Th>
{models.map((m) => (
<ModelTh key={m.key} model={m} />
))}
</tr>
</thead>
<tbody>
{groups.map((g) => {
const top = Math.max(...models.map((m) => g.results[m.key].pass));
return (
<tr key={g.id} style={{ borderTop: "1px solid var(--hair)" }}>
<RowHead
title={g.label}
code={g.code}
sub={`${g.cases} cases · ${g.results[models[0].key].runs} runs per model`}
description={g.description}
/>
{models.map((m) => (
<Td key={m.key}>
<PassCell
model={m.key}
pass={g.results[m.key].pass}
runs={g.results[m.key].runs}
strong={g.results[m.key].pass === top}
/>
</Td>
))}
</tr>
);
})}
</tbody>
</DataTable>
<p className="dim mt-4 text-[0.8125rem]">
The group's best count is set in bold. Groups hold{" "}
{Math.min(...groups.map((g) => g.cases))} to{" "}
{Math.max(...groups.map((g) => g.cases))} cases, so a single run
moves a small group a lot.
{issueGroups.length
? ` The case and skill problems listed under Key findings are in these groups: ${issueGroups.join(", ")}.`
: null}
</p>
</section>
<section id="cost" className="wrap scroll-mt-20 py-16 md:py-20">
<SectionHead
eyebrow="Cost and efficiency"
shareId="cost"
title={
<>
What a diagnosis <em>costs.</em>
</>
}
description={`Agent spend is the model cost the harness recorded for each run, through OpenRouter. The judge's spend is separate. ${priciest.short} cost ${Math.round(priciest.totals.costUsd / cheapest.totals.costUsd)}× as much as ${cheapest.short} for the same ${run.runsPerModel} runs.`}
/>
<div className="cost-wrap" data-cost-mount>
{models.map((m) => (
<div key={m.key} className="cost-bars">
<div className="cost-name">
<span
aria-hidden
className="inline-block size-2.5 shrink-0 rounded-full"
style={{ background: MODEL_COLOR[m.key] }}
/>
{m.short}
</div>
<div className="cost-bar h-6">
<div
className="fill"
data-w={((100 * m.totals.costUsd) / maxCost).toFixed(2)}
style={{
width: `${((100 * m.totals.costUsd) / maxCost).toFixed(2)}%`,
minWidth: 3,
background: MODEL_COLOR[m.key],
borderRadius: "0 4px 4px 0",
}}
/>
</div>
<div className="cost-total">
<em>{formatUsd(m.totals.costUsd)}</em>
<span className="block text-[0.75rem] text-fg-3">
{formatUsd(m.totals.costPerRunUsd)} per run
</span>
</div>
</div>
))}
</div>
<div className="mt-14 rounded-2xl p-5 sm:p-7" style={CARD_STYLE}>
<h3 className="text-[1.0625rem] font-semibold text-fg-0">
Cost per run against pass rate
</h3>
<p className="muted mt-1 text-[0.9375rem]">
Up is better, left is cheaper. Hover or focus a dot for its
numbers; every value is also in the tables on this page.
</p>
<div className="mt-5">
<BenchmarkCostChart
points={costPoints}
models={models.map((m) => ({ key: m.key, label: m.label }))}
/>
</div>
</div>
<div className="mt-10">
<DataTable minWidth={190 + models.length * 130}>
<thead>
<tr>
<Th>All {run.runsPerModel} runs</Th>
{models.map((m) => (
<ModelTh key={m.key} model={m} />
))}
</tr>
</thead>
<tbody>
<MetricRow label="Agent spend" values={models.map((m) => formatUsd(m.totals.costUsd))} />
<MetricRow label="Average cost per run" values={models.map((m) => formatUsd(m.totals.costPerRunUsd))} />
<MetricRow label="Cost per passing run" values={models.map((m) => formatUsd(m.totals.costUsd / m.totals.pass))} />
<MetricRow label="Average tool calls per run" values={models.map((m) => m.totals.avgToolCalls.toFixed(1))} />
<MetricRow label="Average wall time per run" values={models.map((m) => `${m.totals.avgWallS.toFixed(1)}s`)} />
<MetricRow label="Cache hit rate (input tokens)" values={models.map((m) => `${Math.round(100 * m.totals.cacheHitRate)}%`)} />
<MetricRow
label="Average judge score (0–1)"
values={models.map((m) =>
m.totals.avgJudge === null ? "–" : m.totals.avgJudge.toFixed(2),
)}
/>
<MetricRow label="Judge spend" values={models.map((m) => formatUsd(m.totals.judgeCostUsd))} />
</tbody>
</DataTable>
<p className="dim mt-4 text-[0.8125rem]">
Runs went three at a time
{run.parts.length > 1
? `, in ${numberWord(run.parts.length)} batches on ${joinList(runDates)} (listed under Methodology), each batch on one machine`
: " on one machine"}
, so wall time includes some contention.{judgedText}
</p>
</div>
</section>
<section id="matrix" className="wrap scroll-mt-20 py-16 md:py-20">
<SectionHead
eyebrow="Every case"
shareId="matrix"
title={
<>
The full matrix: <em>every case, setup and model.</em>
</>
}
description="One cell per run. Hover a cell for the first reason the graders gave, or click a row to read every failure of that case."
/>
<BenchmarkCaseMatrix
cases={cases}
models={models.map((m) => ({ key: m.key, label: m.label, short: m.short }))}
suites={suites.map((s) => ({ id: s.id, label: s.label, setups: s.setups }))}
groups={groups.map((g) => ({ id: g.id, label: g.label }))}
setupLabels={setupLabels}
issues={knownIssues.map((i) => ({ id: i.id, kind: i.kind, title: i.title }))}
/>
</section>
</div>
<section id="methodology" className="wrap scroll-mt-20 pt-20 pb-6">
<div className="grid gap-10 lg:grid-cols-[minmax(0,1fr)_minmax(0,1fr)] lg:gap-16">
<SectionHead
eyebrow="Methodology"
shareId="methodology"
className="mb-0"
title={
<>
How we <em>measure.</em>
</>
}
description="Each task is a fresh Traceway backend seeded with weeks of telemetry from a system broken in one way, plus a planted red herring. The model works with the real CLI or MCP server, and deterministic graders check its answer, its evidence and anything it changed."
cta={
<div className="flex flex-wrap items-center gap-x-6 gap-y-3">
<Link
href={run.methodologyUrl}
target="_blank"
rel="noopener noreferrer"
className="btn btn-accent"
>
Methodology and cases
<ArrowRight className="h-4 w-4" />
</Link>
<Link
href={run.harnessUrl}
target="_blank"
rel="noopener noreferrer"
className="font-mono text-[0.6875rem] uppercase tracking-[0.08em] text-fg-2 transition-colors hover:text-blue-bright"
>
The eval harness on GitHub →
</Link>
</div>
}
/>
<div
className="self-start rounded-2xl p-5 sm:p-6"
style={{ background: "var(--ink-1)", border: "1px solid var(--hair)" }}
>
<p className="font-mono text-[0.6875rem] font-medium tracking-[0.14em] uppercase text-fg-3">
The runs · {runsPerCaseText} per case
</p>
<ul className="m-0 mt-3 list-none p-0">
{run.parts.map((p, i) => (
<li
key={i}
className="grid grid-cols-[1fr_auto] gap-x-4 gap-y-1 border-t border-hair py-3 first:border-t-0 first:pt-0 last:pb-0 sm:grid-cols-[auto_1fr_auto] sm:items-baseline"
>
<span className="col-start-1 row-start-1 font-mono text-[0.75rem] text-fg-3 tabular-nums">
{p.date}
</span>
<span className="col-span-2 row-start-2 text-[0.9375rem] leading-snug text-fg-1 sm:col-span-1 sm:col-start-2 sm:row-start-1">
{capitalize(partLine(p))}
</span>
<Link
href={`${GITHUB_URL}/commit/${p.commit}`}
target="_blank"
rel="noopener noreferrer"
className="col-start-2 row-start-1 justify-self-end font-mono text-[0.75rem] text-fg-3 transition-colors hover:text-blue-bright sm:col-start-3"
>
{p.commit}
</Link>
</li>
))}
</ul>
{regradedText ? (
<p className="mt-4 border-t border-hair pt-4 text-[0.8125rem] leading-relaxed text-fg-2 text-pretty">
{regradedText}
</p>
) : null}
</div>
</div>
</section>
<FinalCTA
title={
<>
Put your model <em>on the bench.</em>
</>
}
description="Building a model? Run AI SRE BENCH with the open-source harness and submit your results. When sharing a score, include the model version, suite, setup and run date so others can compare it."
primary={{ label: "Run the benchmark", href: run.harnessUrl }}
secondary={{ label: "How to submit results", href: `${run.methodologyUrl}#take-part` }}
/>
<section id="contribute" className="wrap scroll-mt-20 pb-20">
<div
className="grid gap-10 pt-16 lg:grid-cols-[minmax(0,1.1fr)_minmax(0,1fr)] lg:gap-16"
style={{ borderTop: "1px solid var(--hair)" }}
>
<div>
<div className="flex flex-wrap items-center gap-3">
<Eyebrow>Contribute</Eyebrow>
<SectionShareLink id="contribute" />
</div>
<h2 className="mt-3">
Help build <em>the benchmark.</em>
</h2>
<p className="muted mt-4 max-w-xl text-base md:text-lg">
AI SRE BENCH is open source end to end. The cases, the graders
and the harness live in the Traceway repository, and pull
requests are welcome.
</p>
<div className="mt-6 flex flex-wrap items-center gap-3">
<Link
href={GITHUB_URL}
target="_blank"
rel="noopener noreferrer"
className="btn btn-accent"
>
<Github className="h-4 w-4" />
Contribute on GitHub
</Link>
<Link
href={run.harnessUrl}
target="_blank"
rel="noopener noreferrer"
className="btn btn-ghost"
>
The eval harness
<ArrowRight className="h-4 w-4" />
</Link>
</div>
</div>
<ul className="m-0 list-none self-center p-0">
<ContributeItem
title="Add a case"
href={`${run.harnessUrl}#adding-a-case`}
>
A case is a YAML file plus a seed, and it needs an oracle run
that passes before it can be merged. Production failures you
have debugged yourself make the best cases.
</ContributeItem>
<ContributeItem
title="Run a model"
href={`${run.methodologyUrl}#take-part`}
>
Add it to models.yaml with its OpenRouter id, run the suites, and
open a pull request with the generated summary and results files.
</ContributeItem>
<ContributeItem title="Report a problem" href={`${GITHUB_URL}/issues`}>
A grader that scored a run wrong, a case with two right answers,
a number on this page that does not add up: open an issue.
</ContributeItem>
</ul>
</div>
</section>
</main>
);
}
function ContributeItem({
title,
href,
children,
}: {
title: string;
href: string;
children: React.ReactNode;
}) {
return (
<li className="border-t border-hair py-5 first:border-t-0 first:pt-0 last:pb-0">
<Link
href={href}
target="_blank"
rel="noopener noreferrer"
className="group flex items-start justify-between gap-4"
>
<span className="min-w-0">
<span className="block text-[1.0625rem] font-semibold text-fg-0 transition-colors group-hover:text-blue-bright">
{title}
</span>
<span className="muted mt-1 block text-[0.9375rem] text-pretty">
{children}
</span>
</span>
<ArrowRight className="mt-1 h-4 w-4 shrink-0 text-fg-3 transition-all group-hover:translate-x-0.5 group-hover:text-blue-bright" />
</Link>
</li>
);
}
function ModelSwatch({ model, size = 10 }: { model: ModelKey; size?: number }) {
return (
<span
aria-hidden
className="inline-block shrink-0 rounded-full"
style={{ width: size, height: size, background: MODEL_COLOR[model] }}
/>
);
}
function ModelCard({ model, badges }: { model: Model; badges: string[] }) {
const t = model.totals;
const pct = rate(t.pass, t.runs);
return (
<div className="flex flex-col rounded-[20px] px-6 py-7" style={CARD_STYLE}>
<div className="flex items-center gap-2.5">
<ModelSwatch model={model.key} size={12} />
<span className="text-[17px] font-semibold text-fg-0">{model.label}</span>
</div>
<p className="mt-1.5 font-mono text-[0.75rem] break-all text-fg-3">
{model.id}
</p>
<div className="mt-6 flex items-baseline gap-2">
<span className="text-[48px] leading-none font-semibold tracking-[-0.03em] text-fg-0">
{pct}%
</span>
<span className="text-[0.875rem] text-fg-2">
{t.pass} of {t.runs} runs passed
</span>
</div>
<div
className="mt-4 h-1.5 overflow-hidden rounded-full"
style={{ background: "var(--ink-3)" }}
role="img"
aria-label={`${pct}% pass rate`}
>
<div
className="h-full rounded-full"
style={{ width: `${(100 * t.pass) / t.runs}%`, background: MODEL_COLOR[model.key] }}
/>
</div>
<dl className="mt-6 grid gap-3 text-[0.875rem]">
<CardStat label="Right answer and evidence" value={`${t.rightAnswerEvidence}/${t.runs}`} />
<CardStat label="Agent cost" value={formatUsd(t.costUsd)} />
<CardStat label="Cost per run" value={formatUsd(t.costPerRunUsd)} />
</dl>
{badges.length ? (
<div className="mt-6 flex flex-wrap gap-2">
{badges.map((b) => (
<span key={b} className="tag">
{b}
</span>
))}
</div>
) : null}
</div>
);
}
function CardStat({ label, value }: { label: string; value: string }) {
return (
<div
className="flex items-baseline justify-between gap-4 pt-3"
style={{ borderTop: "1px solid var(--hair)" }}
>
<dt className="text-fg-2">{label}</dt>
<dd className="m-0 font-mono text-fg-0 tabular-nums">{value}</dd>
</div>
);
}
function Finding({
label,
title,
children,
}: {
label: string;
title: string;
children: React.ReactNode;
}) {
return (
<div className="border-t border-hair pt-8">
<p className="font-mono text-[0.6875rem] uppercase tracking-[0.14em] text-fg-3">
{label}
</p>
<h3 className="mt-3">{title}</h3>
<p className="muted mt-3 max-w-[520px] text-pretty">{children}</p>
</div>
);
}
function DataTable({
minWidth,
children,
}: {
minWidth: number;
children: React.ReactNode;
}) {
return (
<div className="overflow-x-auto rounded-2xl" style={CARD_STYLE}>
<table className="w-full border-collapse text-left" style={{ minWidth }}>
{children}
</table>
</div>
);
}
function Th({ children }: { children: React.ReactNode }) {
return (
<th
scope="col"
className="sticky left-0 z-10 px-5 py-3.5 align-bottom font-mono text-[0.6875rem] font-medium uppercase tracking-[0.1em] text-fg-3"
style={{ borderBottom: "1px solid var(--hair-2)", background: "var(--ink-0)" }}
>
{children}
</th>
);
}
function ModelTh({ model }: { model: Model }) {
return (
<th
scope="col"
className="px-5 py-3.5 align-bottom text-[0.8125rem] font-semibold text-fg-0"
style={{ borderBottom: "1px solid var(--hair-2)" }}
>
<span className="inline-flex items-center gap-2 whitespace-nowrap">
<ModelSwatch model={model.key} />
{model.label}
</span>
</th>
);
}
function Td({ children }: { children: React.ReactNode }) {
return <td className="px-5 py-4 align-top">{children}</td>;
}
function RowHead({
title,
sub,
code,
description,
}: {
title: string;
sub?: string;
code?: string;
description?: string;
}) {
return (
<th
scope="row"
className="sticky left-0 z-10 min-w-[190px] px-5 py-4 align-top font-normal"
style={{ background: "var(--ink-0)" }}
>
<span className="block text-[0.9375rem] font-semibold text-fg-0">
{title}
</span>
{code ? (
<span className="mt-0.5 block font-mono text-[0.75rem] text-fg-3">
{code}
</span>
) : null}
{description ? (
<span className="mt-1.5 block max-w-[300px] text-[0.8125rem] leading-snug text-fg-2">
{description}
</span>
) : null}
{sub ? (
<span className="mt-1 block font-mono text-[0.75rem] text-fg-3">{sub}</span>
) : null}
</th>
);
}
function SubheadRow({ label, span }: { label: string; span: number }) {
return (
<tr style={{ borderTop: "1px solid var(--hair-2)" }}>
<th
scope="colgroup"
colSpan={span}
className="px-5 pt-5 pb-1 font-mono text-[0.6875rem] font-medium uppercase tracking-[0.14em] text-fg-3"
>
<span className="sticky left-5 inline-block">{label}</span>
</th>
</tr>
);
}
function SuiteRows({
suite,
}: {
suite: (typeof suites)[number];
}) {
return (
<>
<SubheadRow
label={`${suite.label} · ${suite.cases} cases`}
span={models.length + 1}
/>
{suite.setups.map((arm) => (
<tr key={arm} style={{ borderTop: "1px solid var(--hair)" }}>
<RowHead
title={setupLabels[arm]}
code={arm}
sub={
armCases(suite.id, arm) === suite.cases
? undefined
: `${armCases(suite.id, arm)} of the ${suite.cases} cases`
}
/>
{models.map((m) => {
const r = setupResult(suite.id, arm, m.key);
return (
<Td key={m.key}>
<PassCell
model={m.key}
pass={r.pass}
runs={r.runs}
rightAnswerEvidence={r.rightAnswerEvidence}
/>
</Td>
);
})}
</tr>
))}
</>
);
}
function PassCell({
model,
pass,
runs,
rightAnswerEvidence,
strong,
}: {
model: ModelKey;
pass: number;
runs: number;
rightAnswerEvidence?: number;
strong?: boolean;
}) {
return (
<div className="min-w-[140px]">
<div className="flex items-baseline justify-between gap-3 font-mono tabular-nums">
<span
className={
strong
? "text-[0.9375rem] font-semibold text-fg-0"
: "text-[0.9375rem] text-fg-1"
}
>
{pass}/{runs}
</span>
<span className="text-[0.8125rem] text-fg-2">{rate(pass, runs)}%</span>
</div>
<div
className="mt-2 h-1.5 overflow-hidden rounded-full"
style={{ background: "var(--ink-3)" }}
>
<div
className="h-full rounded-full"
style={{ width: `${(100 * pass) / runs}%`, background: MODEL_COLOR[model] }}
/>
</div>
{rightAnswerEvidence !== undefined ? (
<p className="mt-2 font-mono text-[0.6875rem] whitespace-nowrap text-fg-3 tabular-nums">
{rightAnswerEvidence}/{runs} right answer + evidence
</p>
) : null}
</div>
);
}
function DeltaRow({
title,
sub,
cells,
}: {
title: string;
sub: string;
cells: { key: ModelKey; pp: number; detail: string }[];
}) {
return (
<tr style={{ borderTop: "1px solid var(--hair)" }}>
<RowHead title={title} sub={sub} />
{cells.map((c) => (
<Td key={c.key}>
<span className="block font-mono text-[0.9375rem] font-semibold text-fg-0 tabular-nums">
{formatPp(c.pp)}
</span>
<span className="mt-1 block font-mono text-[0.6875rem] text-fg-3 tabular-nums">
{c.detail}
</span>
</Td>
))}
</tr>
);
}
function MetricRow({ label, values }: { label: string; values: string[] }) {
return (
<tr style={{ borderTop: "1px solid var(--hair)" }}>
<th
scope="row"
className="sticky left-0 z-10 min-w-[190px] px-5 py-3.5 text-[0.9375rem] font-normal text-fg-1"
style={{ background: "var(--ink-0)" }}
>
{label}
</th>
{values.map((v, i) => (
<td
key={models[i].key}
className="px-5 py-3.5 font-mono text-[0.9375rem] text-fg-0 tabular-nums"
>
{v}
</td>
))}
</tr>
);
}