Skip to content

The only tool you need to know what is happening and how to fix it.

cleanup

narrow-adventure4a07674f26bf
diff --git a/.github/workflows/cli-evals.yml b/.github/workflows/cli-evals.yml
index 4ec98a5e..78a04e71 100644
--- a/.github/workflows/cli-evals.yml
+++ b/.github/workflows/cli-evals.yml
@@ -32,9 +32,9 @@ on:
required: false
description: "cli+skill,cli-noskill,mcp,cli-write+skill,mcp-write,cli+setup-skill"
models:
- type: choice
- options: ["opus,astra", "opus", "astra"]
+ type: string
default: "opus,astra"
+ description: "comma-separated agent keys from models.yaml: opus, astra, haiku, deepseek-flash, luna, minimax-flash, mimo-pro, qwen-27b, glm"
runs:
type: number
default: 1
diff --git a/cli/test/evals/README.md b/cli/test/evals/README.md
index 09c65fcb..fdf14326 100644
--- a/cli/test/evals/README.md
+++ b/cli/test/evals/README.md
@@ -18,9 +18,13 @@ tools, and deterministic graders score the result.
| write | The write playbooks (pages, alerts, monitors, archive, post-mortems, slow marks) and the 403 policy | `cli-write+skill`, `mcp-write` |
| setup | Instrumenting an app with the setup skill until its telemetry arrives | `cli+setup-skill` |
-The agents are `anthropic/claude-opus-5.5` and `openai/gpt-6-astra`, and the
-judge is `anthropic/claude-haiku-5.5`. All three run on OpenRouter, in the same
-harness, with the same tools, prompts and budgets (`models.yaml`).
+The agents are the entries of `models.yaml` (`opus`, `astra`, `haiku`,
+`deepseek-flash`, `luna`, `minimax-flash`, `mimo-pro`, `qwen-27b`, `glm`), and
+the judge is `anthropic/claude-haiku-5.5`. All of them run on OpenRouter, in
+the same harness, with the same tools, prompts and budgets, each pinned to
+its vendor's own endpoint. A model that cannot finish the cases inside their
+wall-clock budgets (MiMo-V2.6 Pro, as of 2026-10-22) stays in the catalog
+with a note and off the benchmarks page.
## The manual-only policy
@@ -82,7 +86,7 @@ the `evals` gate. `just eval-dispatch -f preset=smoke -f max_cost_usd=30` from
| `preset` | `smoke`: the cases tagged `smoke`, primary arm. `core`: every case, primary arm. `full`: every arm the case lists |
| `cases` | Comma-separated ids or globs (`perf-db-*`). Overrides the preset's case filter |
| `suites`, `arms` | Narrow the selection |
-| `models` | `opus,astra`, `opus` or `astra` |
+| `models` | Comma-separated agent keys from `models.yaml` (`opus,astra` by default) |
| `runs` | Repeats per case, 1-3 |
| `max_cost_usd` | Cap for the whole dispatch, split per shard (see the policy above). Required |
| `dry_run` | Preflight, harness tests and every oracle only. Costs $0. Use it to validate workflow edits and Dependabot bumps |
diff --git a/cli/test/evals/models.yaml b/cli/test/evals/models.yaml
index b0a4c794..24804c88 100644
--- a/cli/test/evals/models.yaml
+++ b/cli/test/evals/models.yaml
@@ -129,7 +129,10 @@ agents:
fallback_price: {prompt: 0.15, completion: 0.60, cache_read: 0.015, cache_write: 0.15}
mimo-pro:
- # Xiaomi first party only: with fallbacks on, the smoke run was served by
+ # Not benchmarked yet: too slow for the case budgets on 2026-10-22. On
+ # Xiaomi one write case took 4 turns in 8 minutes, and on DeepInfra a perf
+ # case ran out of its 10 minute budget after 14 tool calls. Pinned to
+ # Xiaomi only, since with fallbacks on the smoke run was served by
# DeepInfra although Xiaomi was up.
id: xiaomi/mimo-v2.6-pro
label: MiMo-V2.6 Pro
diff --git a/website/lib/benchmark-format.ts b/website/lib/benchmark-format.ts
index d0f36e3b..8c0f2f59 100644
--- a/website/lib/benchmark-format.ts
+++ b/website/lib/benchmark-format.ts
@@ -8,9 +8,8 @@ export const MODEL_COLOR: Record<ModelKey, string> = {
opus: "var(--series-3)",
luna: "var(--series-4)",
"minimax-flash": "var(--series-5)",
- "mimo-pro": "var(--series-6)",
- "qwen-27b": "var(--series-7)",
- glm: "var(--series-8)",
+ "qwen-27b": "var(--series-6)",
+ glm: "var(--series-7)",
};
export const KIND_LABEL: Record<RunKind, string> = {
diff --git a/website/lib/benchmarks.ts b/website/lib/benchmarks.ts
index 0f1a3e2f..223343ae 100644
--- a/website/lib/benchmarks.ts
+++ b/website/lib/benchmarks.ts
@@ -6,7 +6,6 @@ export type ModelKey =
| "opus"
| "luna"
| "minimax-flash"
- | "mimo-pro"
| "qwen-27b"
| "glm";
diff --git a/website/scripts/build-benchmarks-data.mjs b/website/scripts/build-benchmarks-data.mjs
index 04915c9a..8da5e98a 100644
--- a/website/scripts/build-benchmarks-data.mjs
+++ b/website/scripts/build-benchmarks-data.mjs
@@ -30,14 +30,13 @@ const PAGE_URL = "https://tracewayapp.com/benchmarks";
// Display order. A model's palette slot (lib/benchmark-format.ts) is fixed
// to its key, so adding a model never recolors the others.
-const MODEL_ORDER = ["haiku", "deepseek-flash", "opus", "luna", "minimax-flash", "mimo-pro", "qwen-27b", "glm"];
+const MODEL_ORDER = ["haiku", "deepseek-flash", "opus", "luna", "minimax-flash", "qwen-27b", "glm"];
const MODEL_SHORT = {
haiku: "Haiku",
"deepseek-flash": "DeepSeek",
opus: "Opus",
luna: "Luna",
"minimax-flash": "MiniMax",
- "mimo-pro": "MiMo",
"qwen-27b": "Qwen 27B",
glm: "GLM",
};
@@ -47,7 +46,6 @@ const MODEL_VENDOR = {
opus: "Anthropic",
luna: "OpenAI",
"minimax-flash": "MiniMax",
- "mimo-pro": "Xiaomi",
"qwen-27b": "Alibaba",
glm: "Z.ai",
};
@@ -57,7 +55,6 @@ const MODEL_WEIGHTS = {
opus: "proprietary",
luna: "proprietary",
"minimax-flash": "open",
- "mimo-pro": "open",
"qwen-27b": "open",
glm: "open",
};