The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
progress
narrow-adventure6779d3e88af9
diff --git a/cli/test/evals/models.yaml b/cli/test/evals/models.yamlindex 4d630118..b0a4c794 100644--- a/cli/test/evals/models.yaml+++ b/cli/test/evals/models.yaml@@ -129,11 +129,13 @@ agents:fallback_price: {prompt: 0.15, completion: 0.60, cache_read: 0.015, cache_write: 0.15}mimo-pro:+ # Xiaomi first party only: with fallbacks on, the smoke run was served by+ # DeepInfra although Xiaomi was up.id: xiaomi/mimo-v2.6-prolabel: MiMo-V2.6 Proprovider:order: [xiaomi]- allow_fallbacks: true+ allow_fallbacks: falsecaching: automaticreasoning_effort: mediummax_tokens_per_turn: 16000diff --git a/website/app/benchmarks/page.tsx b/website/app/benchmarks/page.tsxindex a3b36a00..325fef49 100644--- a/website/app/benchmarks/page.tsx+++ b/website/app/benchmarks/page.tsx@@ -45,9 +45,10 @@ const capitalize = (s: string) => s[0].toUpperCase() + s.slice(1);// batch narrowed to the setups run again after a fix, "the debug CLI + skill// runs ran again on ... at commit ...".const partText = (p: (typeof run.parts)[number], withCommit: boolean) => {- if (!p.arms) return `${joinList(p.suites)} ${withCommit ? "ran " : ""}on ${p.date}${withCommit ? ` at commit ${p.commit}` : ""}`;+ const forModels = p.models ? ` for ${joinList(p.models.map((k) => modelByKey[k].short))}` : "";+ if (!p.arms) return `${joinList(p.suites)}${forModels} ${withCommit ? "ran " : ""}on ${p.date}${withCommit ? ` at commit ${p.commit}` : ""}`;const labels = p.arms.map((a) => setupLabels[a] ?? a);- return `the ${joinList(p.suites)} ${joinList(labels)} runs ${withCommit ? "ran again" : "again"} on ${p.date}${withCommit ? `, with the skill fix of commit ${p.commit}` : ""}`;+ return `the ${joinList(p.suites)} ${joinList(labels)} runs${forModels} ${withCommit ? "ran again" : "again"} on ${p.date}${withCommit ? `, with the skill fix of commit ${p.commit}` : ""}`;};const regradedText = run.regraded? ` Every saved transcript was scored on ${run.regraded} by the same graders: after a grader fix the transcripts are scored again, so no number here mixes two versions of a grader.`@@ -149,6 +150,12 @@ const mcpGapText = (a: ModelKey, b: ModelKey) => {const upliftOrder = [...models].sort((a, b) => upliftOf(b.key).pp - upliftOf(a.key).pp,);+// The model with the lowest perf pass rate on the CLI without the skill.+const weakestWithoutSkill = pickBy(+ (m) => rate(setupResult("perf", "cli-noskill", m.key).pass, setupResult("perf", "cli-noskill", m.key).runs),+ "min",+);+const controlsClean = (m: Model) => controls.results[m.key].pass === controls.results[m.key].runs;const suitePasses = (suiteId: string, key: ModelKey) =>setups@@ -499,8 +506,8 @@ export default function BenchmarksPage() {description={`Every model ran the same ${run.runsPerModel} runs. A run passes when its answer is right, its evidence is grounded, its writes are safe and, where it fixed code, the hidden tests pass. Tool-call counts are reported but never gate.`}/><BenchmarkIndexChart views={indexViews} subtitle={indexSubtitle} />- <div className="mt-8 grid gap-5 md:grid-cols-3">- {models.map((m) => (+ <div className="mt-8 grid gap-5 sm:grid-cols-2 lg:grid-cols-4">+ {byPass.map((m) => (<ModelCard key={m.key} model={m} badges={BADGES[m.key] ?? []} />))}</div>@@ -557,7 +564,14 @@ export default function BenchmarksPage() {}/><div className="grid gap-12 md:grid-cols-2">- <Finding label="01 · Judgment" title={`${byControls[0].short} never false-alarms.`}>+ <Finding+ label="01 · Judgment"+ title={+ controlsClean(byControls[0])+ ? `${joinList(byControls.filter(controlsClean).map((m) => m.short))} ${byControls.filter(controlsClean).length === 1 ? "never false-alarms" : "never false-alarm"}.`+ : `${byControls[0].short} false-alarms least.`+ }+ >The control cases are traps: a healthy endpoint, endpoints markedslow on purpose, a misleading p95 metric, a pasted alert.{" "}{byControls[0].short} passed {controls.results[byControls[0].key].pass}{" "}@@ -586,7 +600,11 @@ export default function BenchmarksPage() {</Finding><Findinglabel="03 · The skill"- title="The skill helps the weakest model most."+ title={+ upliftOrder[0].key === weakestWithoutSkill.key+ ? "The skill helps the weakest model most."+ : `The skill helps ${upliftOrder[0].short} most.`+ }>On the perf suite, CLI + skill against the same CLI without it:{" "}{joinList(@@ -602,7 +620,9 @@ export default function BenchmarksPage() {: null}{mcpFavoured.length === 1? `${mcpFavoured[0].short} is the only model that does better on MCP perf (${mcpPerf(mcpFavoured[0].key).mcp}%) than on CLI + skill (${mcpPerf(mcpFavoured[0].key).cli}%).`- : null}+ : mcpFavoured.length > 1+ ? `${joinList(mcpFavoured.map((m) => `${m.short} (${mcpPerf(m.key).mcp}% vs ${mcpPerf(m.key).cli}%)`))} do better on MCP perf than on CLI + skill.`+ : null}</Finding><Finding label="04 · Our bugs" title="Some failures are ours.">{capitalize(numberWord(knownIssues.length))} problems behind@@ -702,7 +722,7 @@ export default function BenchmarksPage() {noWrites ? ", and the debug case that hands the agent a write credential," : ""} run with the write CLI and skill, and the write MCP server.`}/>- <DataTable minWidth={760}>+ <DataTable minWidth={190 + models.length * 160}><thead><tr><Th>Setup</Th>@@ -777,7 +797,7 @@ export default function BenchmarksPage() {hasSuite("debug") ? ", and the debug groups follow the stack the fix is in" : ""}.`}/>- <DataTable minWidth={820}>+ <DataTable minWidth={340 + models.length * 160}><thead><tr><Th>Group</Th>@@ -882,7 +902,7 @@ export default function BenchmarksPage() {</div><div className="mt-10">- <DataTable minWidth={700}>+ <DataTable minWidth={190 + models.length * 130}><thead><tr><Th>All {run.runsPerModel} runs</Th>diff --git a/website/components/benchmark-comparison-table.tsx b/website/components/benchmark-comparison-table.tsxindex f3f04852..648f5895 100644--- a/website/components/benchmark-comparison-table.tsx+++ b/website/components/benchmark-comparison-table.tsx@@ -35,13 +35,13 @@ export function BenchmarkComparisonTable({return (<div className="overflow-x-auto"><table- className="w-full min-w-[560px] border-separate text-left md:min-w-[680px]"- style={{ borderSpacing: 0 }}+ className="w-full border-separate text-left"+ style={{ borderSpacing: 0, minWidth: 200 + models.length * 120 }}><thead><tr><th- className="sticky left-0 z-10 w-[132px] pb-3 md:static md:w-[26%]"+ className="sticky left-0 z-10 w-[132px] pb-3 md:static md:w-[200px]"style={{ background: "var(--ink-0)" }}/>{models.map((m) => {diff --git a/website/components/benchmark-index-chart.tsx b/website/components/benchmark-index-chart.tsxindex d95b1dfa..7097a521 100644--- a/website/components/benchmark-index-chart.tsx+++ b/website/components/benchmark-index-chart.tsx@@ -55,7 +55,7 @@ export function BenchmarkIndexChart({const view = views.find((v) => v.id === viewId) ?? views[0];const value = (b: IndexBar) => rate(b[metric], b.runs);const bars = [...view.bars].sort((a, b) => value(b) - value(a));- const dense = bars.length > 6;+ const dense = bars.length > 8;const minWidth = dense ? 64 + bars.length * 46 : 0;const metricLabel = METRICS.find((m) => m.id === metric)!.label;