The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
cleanup
narrow-adventure4a07674f26bf
diff --git a/.github/workflows/cli-evals.yml b/.github/workflows/cli-evals.ymlindex 4ec98a5e..78a04e71 100644--- a/.github/workflows/cli-evals.yml+++ b/.github/workflows/cli-evals.yml@@ -32,9 +32,9 @@ on:required: falsedescription: "cli+skill,cli-noskill,mcp,cli-write+skill,mcp-write,cli+setup-skill"models:- type: choice- options: ["opus,astra", "opus", "astra"]+ type: stringdefault: "opus,astra"+ description: "comma-separated agent keys from models.yaml: opus, astra, haiku, deepseek-flash, luna, minimax-flash, mimo-pro, qwen-27b, glm"runs:type: numberdefault: 1diff --git a/cli/test/evals/README.md b/cli/test/evals/README.mdindex 09c65fcb..fdf14326 100644--- a/cli/test/evals/README.md+++ b/cli/test/evals/README.md@@ -18,9 +18,13 @@ tools, and deterministic graders score the result.| write | The write playbooks (pages, alerts, monitors, archive, post-mortems, slow marks) and the 403 policy | `cli-write+skill`, `mcp-write` || setup | Instrumenting an app with the setup skill until its telemetry arrives | `cli+setup-skill` |-The agents are `anthropic/claude-opus-5.5` and `openai/gpt-6-astra`, and the-judge is `anthropic/claude-haiku-5.5`. All three run on OpenRouter, in the same-harness, with the same tools, prompts and budgets (`models.yaml`).+The agents are the entries of `models.yaml` (`opus`, `astra`, `haiku`,+`deepseek-flash`, `luna`, `minimax-flash`, `mimo-pro`, `qwen-27b`, `glm`), and+the judge is `anthropic/claude-haiku-5.5`. All of them run on OpenRouter, in+the same harness, with the same tools, prompts and budgets, each pinned to+its vendor's own endpoint. A model that cannot finish the cases inside their+wall-clock budgets (MiMo-V2.6 Pro, as of 2026-10-22) stays in the catalog+with a note and off the benchmarks page.## The manual-only policy@@ -82,7 +86,7 @@ the `evals` gate. `just eval-dispatch -f preset=smoke -f max_cost_usd=30` from| `preset` | `smoke`: the cases tagged `smoke`, primary arm. `core`: every case, primary arm. `full`: every arm the case lists || `cases` | Comma-separated ids or globs (`perf-db-*`). Overrides the preset's case filter || `suites`, `arms` | Narrow the selection |-| `models` | `opus,astra`, `opus` or `astra` |+| `models` | Comma-separated agent keys from `models.yaml` (`opus,astra` by default) || `runs` | Repeats per case, 1-3 || `max_cost_usd` | Cap for the whole dispatch, split per shard (see the policy above). Required || `dry_run` | Preflight, harness tests and every oracle only. Costs $0. Use it to validate workflow edits and Dependabot bumps |diff --git a/cli/test/evals/models.yaml b/cli/test/evals/models.yamlindex b0a4c794..24804c88 100644--- a/cli/test/evals/models.yaml+++ b/cli/test/evals/models.yaml@@ -129,7 +129,10 @@ agents:fallback_price: {prompt: 0.15, completion: 0.60, cache_read: 0.015, cache_write: 0.15}mimo-pro:- # Xiaomi first party only: with fallbacks on, the smoke run was served by+ # Not benchmarked yet: too slow for the case budgets on 2026-10-22. On+ # Xiaomi one write case took 4 turns in 8 minutes, and on DeepInfra a perf+ # case ran out of its 10 minute budget after 14 tool calls. Pinned to+ # Xiaomi only, since with fallbacks on the smoke run was served by# DeepInfra although Xiaomi was up.id: xiaomi/mimo-v2.6-prolabel: MiMo-V2.6 Prodiff --git a/website/lib/benchmark-format.ts b/website/lib/benchmark-format.tsindex d0f36e3b..8c0f2f59 100644--- a/website/lib/benchmark-format.ts+++ b/website/lib/benchmark-format.ts@@ -8,9 +8,8 @@ export const MODEL_COLOR: Record<ModelKey, string> = {opus: "var(--series-3)",luna: "var(--series-4)","minimax-flash": "var(--series-5)",- "mimo-pro": "var(--series-6)",- "qwen-27b": "var(--series-7)",- glm: "var(--series-8)",+ "qwen-27b": "var(--series-6)",+ glm: "var(--series-7)",};export const KIND_LABEL: Record<RunKind, string> = {diff --git a/website/lib/benchmarks.ts b/website/lib/benchmarks.tsindex 0f1a3e2f..223343ae 100644--- a/website/lib/benchmarks.ts+++ b/website/lib/benchmarks.ts@@ -6,7 +6,6 @@ export type ModelKey =| "opus"| "luna"| "minimax-flash"- | "mimo-pro"| "qwen-27b"| "glm";diff --git a/website/scripts/build-benchmarks-data.mjs b/website/scripts/build-benchmarks-data.mjsindex 04915c9a..8da5e98a 100644--- a/website/scripts/build-benchmarks-data.mjs+++ b/website/scripts/build-benchmarks-data.mjs@@ -30,14 +30,13 @@ const PAGE_URL = "https://tracewayapp.com/benchmarks";// Display order. A model's palette slot (lib/benchmark-format.ts) is fixed// to its key, so adding a model never recolors the others.-const MODEL_ORDER = ["haiku", "deepseek-flash", "opus", "luna", "minimax-flash", "mimo-pro", "qwen-27b", "glm"];+const MODEL_ORDER = ["haiku", "deepseek-flash", "opus", "luna", "minimax-flash", "qwen-27b", "glm"];const MODEL_SHORT = {haiku: "Haiku","deepseek-flash": "DeepSeek",opus: "Opus",luna: "Luna","minimax-flash": "MiniMax",- "mimo-pro": "MiMo","qwen-27b": "Qwen 27B",glm: "GLM",};@@ -47,7 +46,6 @@ const MODEL_VENDOR = {opus: "Anthropic",luna: "OpenAI","minimax-flash": "MiniMax",- "mimo-pro": "Xiaomi","qwen-27b": "Alibaba",glm: "Z.ai",};@@ -57,7 +55,6 @@ const MODEL_WEIGHTS = {opus: "proprietary",luna: "proprietary","minimax-flash": "open",- "mimo-pro": "open","qwen-27b": "open",glm: "open",};