Skip to content

The only tool you need to know what is happening and how to fix it.

Full harness done

narrow-adventurecf8fb54bda38
diff --git a/.github/workflows/cli-evals.yml b/.github/workflows/cli-evals.yml
new file mode 100644
index 00000000..8d9b4e3d
--- /dev/null
+++ b/.github/workflows/cli-evals.yml
@@ -0,0 +1,259 @@
+name: CLI Evals
+
+# Manual only. This workflow has no automatic trigger and must never get one
+# (cli/test/evals/internal/guard/triggers_test.go enforces it). Start it from
+# Actions > CLI Evals > Run workflow, or `gh workflow run cli-evals.yml`.
+#
+# It is the one exception to the label-gated CI pattern: every run spends real
+# money on OpenRouter, so a person dispatches it, reads the preflight estimate
+# in the job summary, and approves the paid job at the `evals` environment
+# gate (the only place OPENROUTER_API_KEY lives). `dry_run` runs everything
+# except the paid job for $0. The workflow never commits; update baseline.json
+# yourself with `twevals baseline update`. See cli/test/evals/README.md.
+on:
+ workflow_dispatch:
+ inputs:
+ preset:
+ type: choice
+ options: [smoke, core, full]
+ default: smoke
+ description: smoke = starred cases on the primary arm; core = every case, primary arm; full = every arm
+ cases:
+ type: string
+ required: false
+ description: "comma-separated case ids or globs; overrides the preset"
+ suites:
+ type: string
+ required: false
+ description: "perf,debug,write,setup"
+ arms:
+ type: string
+ required: false
+ description: "cli+skill,cli-noskill,mcp,cli-write+skill,mcp-write,cli+setup-skill"
+ models:
+ type: choice
+ options: ["opus,astra", "opus", "astra"]
+ default: "opus,astra"
+ runs:
+ type: number
+ default: 1
+ description: "1-3 repeats per case"
+ max_cost_usd:
+ type: string
+ required: true
+ description: "hard cap for the whole dispatch in USD, no default"
+ dry_run:
+ type: boolean
+ default: false
+ description: "preflight + harness + oracle only, no model calls, $0"
+ judge:
+ type: boolean
+ default: true
+ compare_baseline:
+ type: boolean
+ default: true
+
+permissions:
+ contents: read
+
+concurrency:
+ group: cli-evals
+ cancel-in-progress: false
+
+jobs:
+ preflight:
+ name: Preflight ($0)
+ runs-on: ubuntu-latest
+ outputs:
+ matrix: ${{ steps.plan.outputs.matrix }}
+ defaults:
+ run:
+ working-directory: cli/test/evals
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v7
+
+ - name: Setup Go
+ uses: actions/setup-go@v7
+ with:
+ go-version-file: cli/test/evals/go.mod
+ cache-dependency-path: cli/test/evals/go.sum
+
+ - name: Validate inputs
+ env:
+ MAX_COST: ${{ inputs.max_cost_usd }}
+ RUNS: ${{ inputs.runs }}
+ run: |
+ if ! [[ "$MAX_COST" =~ ^[0-9]+(\.[0-9]+)?$ ]] || ! awk "BEGIN{exit !($MAX_COST > 0)}"; then
+ echo "::error::max_cost_usd must be a number greater than 0 (got '$MAX_COST')"; exit 1
+ fi
+ if ! [[ "$RUNS" =~ ^[1-3]$ ]]; then
+ echo "::error::runs must be 1, 2 or 3 (got '$RUNS')"; exit 1
+ fi
+
+ - name: Plan and estimate
+ id: plan
+ env:
+ PRESET: ${{ inputs.preset }}
+ CASES: ${{ inputs.cases }}
+ SUITES: ${{ inputs.suites }}
+ ARMS: ${{ inputs.arms }}
+ MODELS: ${{ inputs.models }}
+ RUNS: ${{ inputs.runs }}
+ MAX_COST: ${{ inputs.max_cost_usd }}
+ run: |
+ go run ./cmd/twevals plan --preset "$PRESET" --cases "$CASES" --suites "$SUITES" --arms "$ARMS" \
+ --models "$MODELS" --runs "$RUNS" --max-cost-usd "$MAX_COST" --shards \
+ --out plan.json --summary "$GITHUB_STEP_SUMMARY"
+ echo "matrix=$(jq -c .matrix plan.json)" >> "$GITHUB_OUTPUT"
+
+ - name: Upload plan
+ uses: actions/upload-artifact@v7
+ with:
+ name: eval-plan
+ path: cli/test/evals/plan.json
+ retention-days: 30
+
+ harness:
+ name: Harness and oracle ($0)
+ runs-on: ubuntu-latest
+ defaults:
+ run:
+ working-directory: cli/test/evals
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v7
+
+ - name: Setup Go
+ uses: actions/setup-go@v7
+ with:
+ go-version-file: cli/test/evals/go.mod
+ cache-dependency-path: cli/test/evals/go.sum
+
+ - name: Unit and harness tests (fake model only)
+ run: go test ./...
+
+ - name: Oracle trajectories for the selection
+ env:
+ CASES: ${{ inputs.cases }}
+ SUITES: ${{ inputs.suites }}
+ run: |
+ if [ -n "$CASES" ]; then
+ go run ./cmd/twevals oracle -j 3 $(echo "$CASES" | tr ',' ' ')
+ elif [ -n "$SUITES" ]; then
+ go run ./cmd/twevals oracle -j 3 --suites "$SUITES"
+ else
+ go run ./cmd/twevals oracle -j 3 --all
+ fi
+
+ evals:
+ name: Evals (${{ matrix.shard }})
+ needs: [preflight, harness]
+ if: ${{ !inputs.dry_run }}
+ # The approval gate: this job waits for a required reviewer on the evals
+ # environment, who approves after reading the preflight estimate.
+ environment: evals
+ runs-on: ubuntu-latest
+ timeout-minutes: 300
+ strategy:
+ fail-fast: false
+ matrix: ${{ fromJSON(needs.preflight.outputs.matrix) }}
+ defaults:
+ run:
+ working-directory: cli/test/evals
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v7
+
+ - name: Setup Go
+ uses: actions/setup-go@v7
+ with:
+ go-version-file: cli/test/evals/go.mod
+ cache-dependency-path: cli/test/evals/go.sum
+
+ - name: Run the shard
+ env:
+ OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
+ TWEVALS_APPROVED: "1"
+ PRESET: ${{ inputs.preset }}
+ CASES: ${{ inputs.cases }}
+ SUITES: ${{ inputs.suites }}
+ ARMS: ${{ inputs.arms }}
+ MODELS: ${{ inputs.models }}
+ RUNS: ${{ inputs.runs }}
+ JUDGE: ${{ inputs.judge }}
+ SHARD: ${{ matrix.shard }}
+ SHARD_CAP: ${{ matrix.cap }}
+ run: |
+ go run ./cmd/twevals run --preset "$PRESET" --cases "$CASES" --suites "$SUITES" --arms "$ARMS" \
+ --shard "$SHARD" --models "$MODELS" --runs "$RUNS" --judge="$JUDGE" \
+ --max-cost-usd "$SHARD_CAP" -j 4 --yes --out "results/$SHARD"
+
+ - name: Redact credentials before upload
+ if: always()
+ run: |
+ if [ -d results ]; then
+ find results -type f \( -name '*.json' -o -name '*.jsonl' -o -name '*.md' -o -name '*.log' -o -name '.env*' -o -name '*.txt' \) -print0 \
+ | xargs -0 -r sed -i -E \
+ -e 's/sk-or-[A-Za-z0-9_-]+/sk-or-[REDACTED]/g' \
+ -e 's/(tww|twp|tws|twa|twk)_[A-Za-z0-9_-]+/\1_[REDACTED]/g' \
+ -e 's/eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/[REDACTED_JWT]/g'
+ # Backend databases, CLI credential stores and env files the agent
+ # wrote hold the run's tokens; never upload them.
+ find results -type d \( -name backend -o -name cli \) -prune -exec rm -rf {} +
+ find results -type f -name '.env*' -delete
+ fi
+
+ - name: Upload shard results
+ if: always()
+ uses: actions/upload-artifact@v7
+ with:
+ name: eval-shard-${{ matrix.shard }}
+ path: cli/test/evals/results/${{ matrix.shard }}
+ retention-days: 30
+ if-no-files-found: warn
+
+ report:
+ name: Report
+ needs: evals
+ if: ${{ always() && !inputs.dry_run && needs.evals.result != 'skipped' }}
+ runs-on: ubuntu-latest
+ defaults:
+ run:
+ working-directory: cli/test/evals
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v7
+
+ - name: Setup Go
+ uses: actions/setup-go@v7
+ with:
+ go-version-file: cli/test/evals/go.mod
+ cache-dependency-path: cli/test/evals/go.sum
+
+ - name: Download shard results
+ uses: actions/download-artifact@v8
+ with:
+ pattern: eval-shard-*
+ path: cli/test/evals/shards
+
+ - name: Merge and summarize
+ run: |
+ go run ./cmd/twevals report --merge --out merged shards/* > /dev/null || true
+ cat merged/summary.md >> "$GITHUB_STEP_SUMMARY"
+
+ - name: Compare with the baseline
+ if: ${{ inputs.compare_baseline }}
+ run: |
+ set -o pipefail
+ go run ./cmd/twevals compare merged baseline.json | tee -a "$GITHUB_STEP_SUMMARY"
+
+ - name: Upload the merged report
+ if: always()
+ uses: actions/upload-artifact@v7
+ with:
+ name: eval-report
+ path: |
+ cli/test/evals/merged
+ cli/test/evals/shards
+ retention-days: 30
diff --git a/.gitignore b/.gitignore
index 57b592a8..7a796115 100644
--- a/.gitignore
+++ b/.gitignore
@@ -64,8 +64,8 @@ backend/static/dist/
/benchmarks/_artifact/
benchmarks/loadgen/bench-loadgen
-# Eval harness (cli/test/evals): built binaries, oracle caches and run
-# results. Results are never committed.
+# Eval harness (cli/test/evals): built binaries, model and oracle caches, and
+# run results. Results are uploaded by the CLI Evals workflow, never committed.
/cli/test/evals/.cache/
/cli/test/evals/results/
diff --git a/CLAUDE.md b/CLAUDE.md
index b7055cf6..96f772f2 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -35,6 +35,9 @@ Traceway is an error tracking and monitoring platform consisting of:
| CLI | `cd cli && just check` | Lint + test + vulncheck + skill drift + contract tests (pre-commit gate; CI enforces it for `cli/`) |
| CLI | `cd cli && just smoke-test` | Live E2E (needs `TRACEWAY_SMOKE_*` env vars) |
| CLI | `cd cli && just eval-harness` | Eval harness tests plus every case's oracle trajectory, fake model only ($0) |
+| CLI | `cd cli && just eval-plan --preset smoke --max-cost-usd 30` | Eval matrix and live-price cost estimate ($0) |
+| CLI | `cd cli && just eval-dispatch -f preset=smoke -f max_cost_usd=30` | Starts a paid eval run in the CLI Evals workflow; it still waits for approval at the `evals` environment gate |
+| CLI | `cd cli && just eval-local ...` | Paid local eval run, for harness development only (typed `yes` confirmation, required `--max-cost-usd`) |
| Any | `nix develop` (or direnv via `.envrc`) | Dev shell with the Go and Node versions read from `backend/go.mod` and `.nvmrc`, plus `just`, `golangci-lint`, `govulncheck` (`flake.nix`; shells `default`, `backend`, `frontend`, `oxc`). Put `JWT_SECRET` in the gitignored `.envrc.local` |
| Any | `./scripts/check-node-pins.sh` | Asserts every Dockerfile's `ARG NODE_VERSION` default matches `.nvmrc`; run by `release-traceway.yml` before it builds |
@@ -42,7 +45,7 @@ Traceway is an error tracking and monitoring platform consisting of:
Set `JWT_SECRET` (min 32 characters) before running the backend, including the All-in-One image. Former public signing keys are rejected. The All-in-One shell script leaves `.env` loading to the backend so Docker-provided values win. Embedded `Run(opts...)` never reads `JWT_SECRET`, since it runs inside the host app's process where that name usually belongs to the host: it takes `WithJWTSecret`, else `TRACEWAY_JWT_SECRET`, else a random key per start; set a stable secret to preserve dashboard sessions across embedded restarts. The All-in-One image's supervisord sends the backend's output to the container's stdout/stderr, so `docker logs` shows a startup rejection. `SQLITE_PATH` sets the database location, defaulting to `./traceway.db` in the working directory.
-CI on pull requests is opt-in. `backend.yml`, `backend-vulncheck.yml`, `cli.yml`, `cli-lint.yml`, `cli-contract.yml`, `frontend.yml` and `plugin-scanner.yml` listen for `pull_request: types: [labeled]` and every job is gated on `github.event.label.name == 'ci'`, so a PR runs nothing until a maintainer applies the `ci` label (only collaborators can), and applying any other label runs nothing either. Each application validates the PR as it is at that moment: to validate later pushes, remove and re-add the label. The same workflows still run on pushes to `main` that touch their paths (`backend/**`, `cli/**`, `frontend/**`, `skills/traceway/**` for `cli.yml`, `.nvmrc` for `frontend.yml`, their own workflow file; `plugin-scanner.yml` runs on every push to `main` because it scans the whole repo), on their daily schedules (the two vulnchecks), and from the Actions tab via `workflow_dispatch`.
+CI on pull requests is opt-in. `backend.yml`, `backend-vulncheck.yml`, `cli.yml`, `cli-lint.yml`, `cli-contract.yml`, `frontend.yml` and `plugin-scanner.yml` listen for `pull_request: types: [labeled]` and every job is gated on `github.event.label.name == 'ci'`, so a PR runs nothing until a maintainer applies the `ci` label (only collaborators can), and applying any other label runs nothing either. Each application validates the PR as it is at that moment: to validate later pushes, remove and re-add the label. The same workflows still run on pushes to `main` that touch their paths (`backend/**`, `cli/**`, `frontend/**`, `skills/traceway/**` for `cli.yml`, `.nvmrc` for `frontend.yml`, their own workflow file; `plugin-scanner.yml` runs on every push to `main` because it scans the whole repo), on their daily schedules (the two vulnchecks), and from the Actions tab via `workflow_dispatch`. `cli-evals.yml` (the skill and MCP evals) is the one exception to this pattern: its **only** trigger is `workflow_dispatch`, a person starts every run by hand, the paid shard job waits for a required reviewer on the `evals` environment (the only place `OPENROUTER_API_KEY` lives), and `max_cost_usd` is a required input with no default. It must never get a push, pull request, label, schedule, `workflow_run`, `workflow_call` or `repository_dispatch` trigger, no other workflow may call it, and no `just check`/`test`/`default` recipe may run it; `cli/test/evals/internal/guard/triggers_test.go` fails the build if any of that changes, and `twevals run` itself refuses every CI context but a dispatched, approved `cli-evals.yml` job. Validate edits to it (or Dependabot bumps of its actions) with `dry_run: true`, which costs nothing.
Action versions are watched by `.github/dependabot.yml` (the `github-actions` ecosystem only; npm and Go modules are deliberately not watched there). It opens a weekly PR per group — `actions/*` in one, every third-party action in the other — and deliberately does **not** auto-apply `ci`, so a bot PR passes the same collaborator gate as any other. Label the first-party group to validate it; `golangci/golangci-lint-action` is reachable the same way via `cli-lint.yml`. The rest of the third-party group lands on `release-*.yml`, `benchmark-*.yml` and `traceway-autofix.yml`, which no label reaches — and `workflow_dispatch` is **not** a dry run on the release path: dispatching `release-docs.yml` or `release-website.yml` deploys to Cloudflare for real, and the other `release-*.yml` workflows publish real assets, so dispatching one from an unreviewed bot branch ships it. Validate those by reading the action's release notes instead, and dispatch only where the target is disposable. `traceway-autofix.yml` is the in-repo caller of the reusable `autofix.yml` (composite actions `.github/actions/autofix-prepare` and `autofix-publish` around the agent step; `docs/pages/learn/auto-fix.mdx` documents the contract); its `workflow_dispatch` takes an `issue_number` and runs the whole loop from the dispatched branch, which is the one pre-merge validation path for the `claude-code-action` pin — the most privileged action here — but it runs the agent for real, so dispatch it only at a scratch issue. Note too that labelling a PR which touches no gated workflow yields zero checks, which reads like a pass but is silence. Every updatable pin is a bare major tag, so Dependabot only ever proposes major bumps here. The exception is `plugin-scanner.yml`, which pins both actions to commit SHAs because the HOL scanner scores unpinned actions down; Dependabot bumps those SHAs along with their version comments. It uses the scanner config in `.plugin-scanner.toml`, whose `ignore_paths` cover fixtures, demos and build output plus three reviewed production files (the read-only token mask in both `project.repository.go` copies and `KindPassword` in `cli/internal/state/state.go`). The HOL registry's own scan ignores that file and still fails on syntax-highlighter strings in the committed `backend/static/frontend` bundle; that is accepted.
diff --git a/cli/README.md b/cli/README.md
index 3a4062a4..b553cfc6 100644
--- a/cli/README.md
+++ b/cli/README.md
@@ -234,10 +234,12 @@ If any of those vars is missing, the smoke tests skip cleanly rather than fail.
## Skill and MCP evals
-`test/evals` measures how well AI agents use Traceway through the skills,
-against a seeded hermetic backend. It is **manual only** and free: nothing
-triggers it, and it plays every case with a fake model. Run
-`just eval-harness` (unit tests plus every oracle trajectory). See
+`test/evals` measures how well AI agents use Traceway through the skills and
+the MCP servers, against a seeded hermetic backend. It is **manual only**:
+paid runs happen in the CLI Evals workflow, which a person dispatches and
+approves at the `evals` environment gate; nothing triggers it automatically.
+The free parts run locally: `just eval-harness` (unit tests plus every oracle
+trajectory, with a fake model) and `just eval-plan` (the cost estimate). See
[`test/evals/README.md`](test/evals/README.md).
## Contributing
diff --git a/cli/justfile b/cli/justfile
index d2d92e39..95eeecde 100644
--- a/cli/justfile
+++ b/cli/justfile
@@ -24,12 +24,29 @@ vulncheck:
govulncheck ./...
# Skill and MCP evals (test/evals). Manual only: none of these recipes is part
-# of check, test or default. See test/evals/README.md.
+# of check, test or default, and paid runs belong in the CLI Evals workflow.
+# See test/evals/README.md.
# Free: harness tests plus every case's oracle trajectory, with the fake model.
eval-harness:
cd test/evals && go test ./... && go run ./cmd/twevals oracle --all
+# Free: the dispatch matrix and a live-price cost estimate.
+eval-plan *args:
+ cd test/evals && go run ./cmd/twevals plan {{args}}
+
+# Start a paid run in GitHub (you still approve the evals environment gate).
+eval-dispatch *args:
+ gh workflow run cli-evals.yml {{args}}
+
+# Paid, local, for harness development only: asks you to type yes.
+eval-local *args:
+ cd test/evals && go run ./cmd/twevals run {{args}}
+
+# Boot and seed one case and keep it running for review in the dashboard.
+eval-env case:
+ cd test/evals && go run ./cmd/twevals env up {{case}}
+
# Regenerate the published skill (skills/traceway/) from the canonical
# knowledge chunks in pkg/mcpserver/knowledge.
gen-skills:
diff --git a/cli/test/evals/README.md b/cli/test/evals/README.md
index e1ae5e4d..52bb7c2a 100644
--- a/cli/test/evals/README.md
+++ b/cli/test/evals/README.md
@@ -1,59 +1,149 @@
# Traceway skill and MCP evals
-**Manual only, and free.** Nothing here runs automatically: no workflow, no
-`just check`/`test`/`default` recipe. This harness plays every case with a
-fake model on loopback and makes no paid model calls; its Chat Completions
-client refuses any non-loopback URL.
+**Manual only.** Evals run from the CLI Evals workflow, which you start by hand
+(`workflow_dispatch`) and approve at the `evals` environment gate. Nothing
+triggers it automatically.
-This module measures how well AI agents use Traceway through its skills. Every
-run gets its own hermetic Traceway instance, the standalone backend in SQLite
-mode, seeded through the real ingest endpoints. The agent works through real
-`traceway` commands in a sandboxed shell, and deterministic graders score the
-result.
+This module measures how well AI agents use Traceway through the skills
+(`traceway`, `traceway-write`, `traceway-setup`) and the MCP servers (`/mcp`,
+`/mcp/write`), for read and write work. Every run gets its own hermetic
+Traceway instance, the standalone backend in SQLite mode, seeded through the
+real ingest endpoints. The agent works through real `traceway` commands or MCP
+tools, and deterministic graders score the result.
| Suite | Measures | Arms |
|---|---|---|
-| perf | Diagnosing latency against the bottleneck checklist in `performance.md` | `cli+skill`, `cli-noskill` |
-
-`cli+skill` installs the `traceway` skill; `cli-noskill` is the same CLI with
-no skill, the baseline that skill uplift is measured against.
+| perf | Diagnosing latency against every bottleneck row in `performance.md` | `cli+skill`, `cli-noskill`, `mcp` |
+| debug | Debugging an issue to root cause and fixing the code | `cli+skill`, `cli-noskill`, `mcp` |
+| write | The write playbooks (pages, alerts, monitors, archive, post-mortems, slow marks) and the 403 policy | `cli-write+skill`, `mcp-write` |
+| setup | Instrumenting an app with the setup skill until its telemetry arrives | `cli+setup-skill` |
+
+The agents are `anthropic/claude-opus-5.5` and `openai/gpt-6-astra`, and the
+judge is `anthropic/claude-haiku-5.5`. All three run on OpenRouter, in the same
+harness, with the same tools, prompts and budgets (`models.yaml`).
+
+## The manual-only policy
+
+1. **Paid runs happen in GitHub, started by a person.** Actions → CLI Evals →
+ Run workflow (or `gh workflow run cli-evals.yml`). The free preflight job
+ writes the matrix and a live-price estimate into the job summary. The
+ paid job waits for a required reviewer on the `evals` environment, which
+ is the only place `OPENROUTER_API_KEY` exists.
+2. **Nothing runs automatically.** `cli-evals.yml` has exactly one trigger,
+ `workflow_dispatch`. There is no push, pull request, label, schedule,
+ `workflow_run`, `workflow_call` or `repository_dispatch` trigger. No other
+ workflow calls it, and no `just` recipe in `check`/`test`/`default` runs it.
+3. **Enforced in code:**
+ - `internal/guard/triggers_test.go` fails if the workflow gains a trigger, a
+ paid job loses `environment: evals`, or another workflow references it or
+ starts `twevals run`.
+ - `twevals run` and `twevals resume` start in CI only inside a dispatched
+ `cli-evals.yml` job that set `TWEVALS_APPROVED=1` (the gated job does).
+ They refuse any other GitHub event, any other workflow, and any other CI
+ system.
+ - `--max-cost-usd` is required, with no default. The workflow splits it
+ into hard per-shard caps.
+ - `go test ./...` can never call OpenRouter: the real client requires
+ `guard.Live()`, which is false under `go test`.
+4. **Local paid runs are for harness development only.** On a terminal you
+ must type `yes` after reading the preflight. Without a terminal, `--yes` is
+ required. Recorded results (checkpoints, baselines) come from the workflow.
## Running it
-From `cli/`:
+### In GitHub (the normal path)
```
-just eval-harness
+gh workflow run cli-evals.yml --ref <branch> -f preset=smoke -f models=opus,astra -f max_cost_usd=30
```
-runs `go test ./...` and then every case's oracle on every arm it lists, with
-the fake model. From `cli/test/evals`, `go run ./cmd/twevals help` lists the
-commands:
+Then open the run, read the preflight estimate in the summary, and approve
+the `evals` gate. `just eval-dispatch -f preset=smoke -f max_cost_usd=30` from
+`cli/` does the same.
+
+| Input | Meaning |
+|---|---|
+| `preset` | `smoke`: the cases tagged `smoke`, primary arm. `core`: every case, primary arm. `full`: every arm the case lists |
+| `cases` | Comma-separated ids or globs (`perf-db-*`). Overrides the preset's case filter |
+| `suites`, `arms` | Narrow the selection |
+| `models` | `opus,astra`, `opus` or `astra` |
+| `runs` | Repeats per case, 1-3 |
+| `max_cost_usd` | Hard cap for the whole dispatch. Required |
+| `dry_run` | Preflight, harness tests and every oracle only. Costs $0. Use it to validate workflow edits and Dependabot bumps |
+| `judge` | Score reports with the (non-gating) LLM judge |
+| `compare_baseline` | Compare against `baseline.json`; a regression turns the report job red |
+
+The workflow never commits. It uploads `results.json`, `summary.md` and every
+transcript, with tokens redacted, for 30 days.
+
+### Locally (free)
+
+From `cli/` (each recipe runs only when you type it):
-- `twevals list [--preset P] [--cases GLOB] [--arms A]` prints the case × arm
- combinations a selection picks.
-- `twevals oracle [--all | <case-id>... | --suites S] [--arms A] [-j N]
- [--keep] [--out DIR]` plays the oracles and writes `results.json` and
- `summary.md` to `--out` (default `.cache/oracle/<timestamp>/`). It exits `1`
- when an oracle fails.
+| Command | What it does |
+|---|---|
+| `just eval-harness` | `go test ./...` plus every oracle trajectory, with the fake model. Free |
+| `just eval-plan --preset core --max-cost-usd 200` | The preflight table and cost estimate. Free |
+| `just eval-env perf-db-n-plus-1` | Boots and seeds one case, then prints the dashboard URL, a login and a read PAT so you can look at the data by eye |
+| `just eval-local --cases perf-db-n-plus-1 --models opus --max-cost-usd 5` | A paid local run, for harness development only. It asks you to type `yes` |
+| `just eval-dispatch ...` | `gh workflow run cli-evals.yml ...` |
+
+Directly from `cli/test/evals`, `go run ./cmd/twevals help` lists every
+command. `twevals oracle <id>...` (or `--all`, or `--suites S`) plays a case's
+oracle on every arm it lists and writes `results.json` and `summary.md` to
+`--out` (default `.cache/oracle/<timestamp>/`).
+`twevals report <dir>...` merges shard results. `twevals compare <dir>`
+applies the regression rule. `twevals baseline update <dir>` rewrites
+`baseline.json` for you to commit.
+
+Requirements: Go (the `go`/`toolchain` lines of `go.mod`), git, and Docker for
+the setup suite. Docker also runs the hidden tests of a model's patch with no
+network. Without Docker, setup runs of a real model fail, and hidden tests of
+a real model's patch are skipped unless `TWEVALS_HOST_TESTS=1`. Oracle runs
+(our own code) use the host when Docker is missing.
+
+### Exit codes
+
+`0` ok, `1` regression (compare), a failed oracle or an error, `2` bad usage
+or partial: the cost ceiling stopped the run (continue with `twevals resume
+<dir> --max-cost-usd X`) or more than 10% of runs hit infra errors.
+
+## Cost
+
+Estimates per run, from `models.yaml` until `baseline.json` has measured
+averages: 100-400k cumulative input tokens (about 70% cached), 5-15k output.
+
+| Model | Per run (typical) | Per run (no cache) |
+|---|---|---|
+| Opus 5.5 | ~$0.26-0.96 | up to ~$1.90 |
+| GPT-6 Astra | ~$0.62-2.23 | up to ~$4.75 |
+| Judge (Haiku 5.5) | under $0.01 | |
+
+`twevals plan` prints the real numbers from live OpenRouter prices, including
+the cache-write price and long-prompt tiers. Controls: the required cap, the
+preflight, the gate, per-shard caps, a check before every run, `dry_run`, and
+the record/replay cache (`--cache record|replay`, `EVAL_CACHE`).
-Presets (`presets.yaml`): `smoke` is the cases tagged `smoke` on the primary
-arm, `core` every case on the primary arm, `full` every arm a case lists.
+## Reading the report
-Requirements: Go (the `go`/`toolchain` lines of `go.mod`) and git. The
-harness builds the backend and both CLI editions from this checkout into
-`.cache/` (gitignored) and reuses them until the sources change.
+`summary.md` has these sections:
-## Reading the results
+- pass rates by suite × model × arm;
+- skill uplift (`cli+skill` minus `cli-noskill`, on the same cases);
+- CLI vs MCP;
+- model vs model;
+- cost and cache hit rate per model;
+- the top failing graders and the worst cases;
+- one line per run, with its first failure.
-A run is `pass`, `fail`, `budget_exhausted` (counted as a fail) or
-`infra_error` (excluded from pass rates).
+A run is `pass`, `fail`, `budget_exhausted` (counted as a fail),
+`infra_error` or `cost_ceiling`; the last two are excluded from pass rates.
Each run directory, `runs/<case>/<arm>/<model>/r<N>/` (with `+` in the arm
written as `_`, so `cli_skill`), holds:
-- `transcript.jsonl`: every model response, every tool call with its id, op,
- argv, exit code and output;
+- `transcript.jsonl`: every model response with usage and provider, every
+ tool call with its id, op, argv or args, exit code and output;
- `run.json` and `result.json`;
- `seed.json`: the ids the seed exposed. The agent never sees it;
- the workspace, when the run failed or `--keep` was set.
@@ -62,10 +152,25 @@ Graders:
| Grader | Gates | Checks |
|---|---|---|
-| `answer` | yes | The structured answer: checklist row (and not the distractor), locus, onset |
-| `trajectory` | yes | Required, alternative, forbidden and ordered operations, and thrash limits, on surface-neutral operations (`internal/ops`) |
+| `answer` | yes | The structured answer: checklist row (and not the distractor), locus, onset; or the debug category, culprit file, symbol and line (±3) |
+| `trajectory` | yes | Required, alternative, forbidden and ordered operations, and thrash limits. CLI and MCP calls map to the same operations (`internal/ops`) |
| `grounding` | yes | Cited calls exist, succeeded and were not truncated. Their output contains the deciding signal. No id in the answer was invented |
-| `report` | yes | Regexps the final report must or must not match, for cases that set `expect.report` |
+| `state` | yes | After write and setup runs: the backend and the workspace are in the expected state |
+| `hidden_tests` | per case | Hidden tests copied in after the run, and whether the diff touches the culprit |
+| `setup_verify` | yes | The instrumented app builds, starts against the backend, and serves traffic |
+| `judge` | no | The rubric, scored by Haiku. It does not gate until it agrees with human labels on at least 85% of runs |
+
+## Baseline
+
+`baseline.json` holds pass rates per suite and case and the measured per-run
+usage per model, which the preflight uses instead of the `models.yaml`
+envelope. To update it, download the `eval-report` artifact, run
+`go run ./cmd/twevals baseline update merged`, review the diff, and commit it
+in a normal PR.
+
+The regression rule: a suite's pass rate drops more than 10 percentage points
+for a model × arm, or a case at 2 of 3 or better in the baseline now passes 0
+of N.
## Adding a case
@@ -83,7 +188,8 @@ are valid, and the leakage lint is clean. Done means:
2. `twevals oracle <id>` passes on every arm the case lists.
3. `go test ./internal/suite/` passes.
-`perf-db-n-plus-1` is the exemplar.
+The exemplars show every part: `perf-db-n-plus-1`, `debug-go-nil-regression`,
+`write-alert-snooze`, `setup-go-nethttp`.
### The seed DSL (`internal/seed`)
@@ -138,9 +244,16 @@ func seedNPlusOne(s *seed.Scenario) error {
- **Browser and mobile SDK data** goes through `p.Crash(seed.Crash{...})`, with
network and navigation `Actions` and console `Logs`, and `p.Session(...)`.
It travels the native `/api/report` protocol, and covers only what those
- SDKs really send: no frame timings, no mobile metrics. `p.GroupMatching`
- reads an exception group's hash back after ingest; put work that needs the
- ingested data in `s.AfterFlush`.
+ SDKs really send: no frame timings, no mobile metrics.
+- **Dashboard objects:**
+ - `p.PagingChannel`, `p.EmailChannel`, `p.Rule(seed.Rule{...})` and
+ `p.WaitPage(ruleID, timeout)`. A page should come from a rule that really
+ fires.
+ - `p.Check(...)` and `p.RunCheckUntil(id, "down", ...)` against
+ `s.Sink.Stub()`, the switchable HTTP target.
+ - `p.PostMortem`, `p.MarkSlow`, `p.Archive` and `p.GroupMatching` (read an
+ exception group's hash back).
+ - Put work that needs the ingested data in `s.AfterFlush`.
- **Volume.** Requests outside the detail sample are sent as their root span
(and any downstream server spans), so percentiles and counts stay complete
while a scenario stays under about 20k spans. Requests with logs, errors or
@@ -162,6 +275,13 @@ Backend facts that bite (each one cost a case author an afternoon):
should catch the skill's reliance on it.
- A log body search over more than 24 hours answers 422 unless another filter
(service, severity, trace) narrows it.
+- MCP results are JSON and escape `&`, `<` and `>`; grounding compares them
+ unescaped, but a jq-free MCP agent sees the escaped form.
+- A new monitor runs once on its own within a scheduler tick (5s here), so
+ switch the stub only after that first run (see `wr_monitor.go`).
+- Pages, monitors and incidents are stamped with real wall-clock time; seeds
+ that need telemetry around a page lay it out near ingest time
+ (`wr_timing.go`).
- Rare events confined to a short window can come out empty at some hours of
the day (the arrival process follows the wall-clock hour). Use `Burst`, or
seed the case at every hour of the day into `seed.NewDrySink()` and check
@@ -182,16 +302,23 @@ See `perf-db-n-plus-1.yaml` for the full shape. The keys:
- `arms` and `invoke`: `slash` preloads SKILL.md the way a `/traceway`
command does, and `natural` makes the agent load the skill itself. A slash
- case's prompt starts with `/traceway`, and only a slash case's does.
-- `workspace.repo`: a fixture under `testdata/repos`. Fixtures can carry
- history: a `history.yaml` whose commits are overlays under `.history/`
- (only the files that differ from the fixture, plus an `absent:` list of
- files the commit lacks), committed with ages relative to the run so
- `git log` lines up with the seeded deploys.
+ case's prompt starts with `/<skill>`, and only a slash case's does.
+- `workspace.repo`: a fixture under `testdata/repos` or
+ `testdata/setup-templates`. Fixtures can carry history: a `history.yaml`
+ whose commits are overlays under `.history/` (only the files that differ
+ from the fixture, plus an `absent:` list of files the commit lacks),
+ committed with ages relative to the run so `git log` lines up with the
+ seeded deploys.
- `prompt`, `budget`, `user`: scripted `ask_user` answers, first match wins.
-- `expect.answer`, `expect.grounding`, `expect.calls`, `expect.report`.
-- `judge.rubric` and `judge.expected`, the rubric for report quality. The
- harness does not score it.
+- `credential` (`scope: read`, `member: readonly`) and `agent_access: false`
+ for the policy cases.
+- `expect.answer`, `expect.grounding`, `expect.calls`, `expect.state` (kinds
+ in `internal/grade/state.go`), `expect.code`, `expect.report`.
+- `judge.rubric` and `judge.expected`. Both are shown only to the judge.
+- `hooks`: harness actions after a matching call, such as editing a
+ post-mortem behind the agent's back.
+- `setup`: the env given to the agent's container shell (`@token:<project>`,
+ `@url`, `@port`), and the verify step.
Templates are resolved after seeding:
@@ -203,6 +330,7 @@ Templates are resolved after seeding:
| `{{env.projects.<name>.id}}` | Another project's id |
| `{{env.stub}}` | The switchable stub's URL |
| `{{env.now}}` | The scenario's "now" |
+| `{{env.setupToken}}` | A setup token (`tws_`), when `setup.token: setup` |
A scalar that is exactly one template keeps its type.
@@ -210,13 +338,20 @@ The `oracle` is the reference trajectory the fake model plays:
- **One step is one tool call**, so steps are cited as `c01`, `c02`, ... in
the submitted evidence.
-- **`shell:` steps** run CLI commands. Use `jq` to keep cited outputs under
- the 30k-character cap: a truncated citation fails grounding.
+- **`shell:` steps** run CLI commands. On MCP arms a step's first `traceway`
+ call becomes the matching MCP tool call. Use `jq` on CLI arms to keep cited
+ outputs under the 30k-character cap: a truncated citation fails grounding.
+ MCP results are capped at 100k.
+- **`op:` + `args:` steps** are surface-neutral, with MCP parameter names. On
+ CLI arms they render as a CLI command, adding `--yes` for writes.
+- **`write_file: {from: oracle/<case>/<file>}`** writes whole files for code
+ fixes, because YAML block scalars cannot start a line with a tab.
- **`{{last.<json.path>}}` and `{{calls.c03.<path>}}`** read earlier tool
results at play time.
- **`load_skill:` steps** play as a harmless `list_files .` on arms without
- skills (`cli-noskill`), so call ids stay the same on every arm.
-- **The oracle ends** with `submit:`.
+ skills (`cli-noskill`, `mcp`), so call ids stay the same on every arm.
+- **The oracle ends** with `submit:` (perf, debug) or `finish:` (write,
+ setup).
While developing a case, `TWEVALS_ONLY=<ids>` limits loading to those cases,
so an unfinished YAML file elsewhere cannot break your run.
@@ -233,32 +368,36 @@ reached` or Postgres's `deadlock detected`, and neutral names.
## Layout
```
-models.yaml presets.yaml
-cmd/twevals/ list | oracle
+models.yaml presets.yaml baseline.json
+cmd/twevals/ list | plan | run | resume | oracle | env up | report | compare | baseline update
internal/catalog/ the case schema, presets, models.yaml
internal/checklist/ the bottleneck rows of performance.md, the perf answer enum
-internal/env/ backend subprocess, registration, PATs, CLI logins, the stub
-internal/seed/ the DSL, OTLP and /api/report writers, read-back verification
+internal/env/ backend subprocess, registration, members, PATs, CLI logins, the monitor stub
+internal/seed/ the DSL, OTLP and /api/report writers, dashboard objects, read-back verification
internal/scenarios/ one seed function per case
-internal/agent/ the Chat Completions client (loopback only), the loop, prompt caching, transcripts, the fake model
-internal/tools/ the sandboxed shell (mvdan.cc/sh, in-process coreutils, gojq, goawk), files, skills, ask_user, submit
-internal/ops/ CLI → operation mapping, cross-checked against tool-map.md
-internal/grade/ answer, trajectory, grounding, report, leakage
+internal/agent/ OpenRouter client, the loop, prompt caching, transcripts, record/replay, the fake model
+internal/tools/ the sandboxed shell (mvdan.cc/sh, in-process coreutils, gojq, goawk), files, skills, MCP bridge, ask_user, submit
+internal/ops/ CLI/MCP → operation mapping, cross-checked against tool-map.md
+internal/grade/ answer, trajectory, grounding, state, hidden tests, judge, leakage
internal/record/ the run record: calls, operations, usage
-internal/runner/ one run end to end
-internal/report/ results.json and summary.md
+internal/runner/ one run end to end; the setup container shell
+internal/report/ results.json, summary.md, compare, cost estimates
+internal/guard/ the manual-only policy and the workflow trigger test
internal/suite/ catalog-wide tests
-cases/ testdata/ cases, fixture repos, oracle files
+cases/ testdata/ cases; fixture repos, hidden tests, oracle files, setup templates, the sandbox image
```
-The agent's shell is an interpreter, never the host shell. It has the
-`traceway` binary (logged in with a read PAT whose config lives outside the
+The agent's shell is an interpreter, never the host shell. It has the arm's
+`traceway` binary (logged in with a PAT whose config lives outside the
workspace), `jq`, `awk`, read-only text utilities confined to the workspace,
and read-only `git`. No tool writes under a `.git` directory, git always
uses the workspace's repository, and git refuses arguments that name a file
-outside the workspace or options that write files or run programs. Tool subprocesses get a scrubbed environment (`HOME`,
-`XDG_*`, `PATH`, `TZ`, `LANG`), so secrets in the harness's environment
-never reach them.
+outside the workspace or options that write files or run programs. Tool
+subprocesses get a scrubbed environment (`HOME`, `XDG_*`, `PATH`, `TZ`,
+`LANG`), so `OPENROUTER_API_KEY`, `GITHUB_TOKEN` and `JWT_SECRET` never reach
+them. MCP arms connect to the backend's `/mcp` or `/mcp/write` with a PAT.
+`project_id` defaults to the case's project, the way `traceway mcp` defaults
+to the CLI's current project.
## Credits
@@ -268,5 +407,4 @@ module as `perf-db-n-plus-1`, `perf-db-missing-index`, `perf-net-sequential`,
`perf-db-pool-exhaustion`, `perf-app-sync-background`, `perf-net-large-payload`,
`perf-cache-no-cache`, `perf-infra-cpu-saturation`, `perf-cache-stampede`,
`perf-xs-ios-watchdog`, `perf-xs-android-timeouts`, `debug-android-npe` and
-`debug-flutter-null-check`. The two debug cases sit in `cases/debug/` and are
-not loaded yet: the harness runs the perf suite only.
+`debug-flutter-null-check`.
diff --git a/cli/test/evals/baseline.json b/cli/test/evals/baseline.json
new file mode 100644
index 00000000..9dd61cab
--- /dev/null
+++ b/cli/test/evals/baseline.json
@@ -0,0 +1,9 @@
+{
+ "generatedAt": "",
+ "from": "",
+ "gitSha": "",
+ "note": "No baseline yet. Phase 8: dispatch a core run, then `twevals baseline update <run-dir>` and commit the result by hand.",
+ "suites": {},
+ "cases": {},
+ "perRun": {}
+}
diff --git a/cli/test/evals/cases/debug/debug-android-npe.yaml b/cli/test/evals/cases/debug/debug-android-npe.yaml
index 90d43759..3bfb8b59 100644
--- a/cli/test/evals/cases/debug/debug-android-npe.yaml
+++ b/cli/test/evals/cases/debug/debug-android-npe.yaml
@@ -3,7 +3,7 @@ suite: debug
title: Android feed crashes while scrolling since the 5.8.0 rollout
tags: [mobile, android, kotlin]
seed: debug/android-npe
-arms: [cli+skill, cli-noskill]
+arms: [cli+skill, cli-noskill, mcp]
invoke: slash
distractor: An OutOfMemoryError group in the full-size image viewer (FullImageActivity.decodeOriginal) on low-memory phones has been around for weeks on every version; the feed crash is new with 5.8.0 and is an NPE in the adapter.
workspace: {repo: feed-android}
diff --git a/cli/test/evals/cases/debug/debug-flutter-null-check.yaml b/cli/test/evals/cases/debug/debug-flutter-null-check.yaml
index 58852c66..6b345cef 100644
--- a/cli/test/evals/cases/debug/debug-flutter-null-check.yaml
+++ b/cli/test/evals/cases/debug/debug-flutter-null-check.yaml
@@ -3,7 +3,7 @@ suite: debug
title: Flutter product pages fail with a null check on listings without photos
tags: [mobile, flutter, dart]
seed: debug/flutter-null-check
-arms: [cli+skill, cli-noskill]
+arms: [cli+skill, cli-noskill, mcp]
invoke: slash
distractor: >-
No app release lines up with the crash (it hits 4.2.1 and 4.3.0 alike, and 4.3.0 shipped nine days ago); a steady
diff --git a/cli/test/evals/cases/debug/debug-go-nil-regression.yaml b/cli/test/evals/cases/debug/debug-go-nil-regression.yaml
new file mode 100644
index 00000000..4ea247c4
--- /dev/null
+++ b/cli/test/evals/cases/debug/debug-go-nil-regression.yaml
@@ -0,0 +1,72 @@
+id: debug-go-nil-regression
+suite: debug
+title: Nil-pointer panic on the invoice page since the last release
+tags: [smoke, go, regression]
+seed: debug/go-nil-regression
+arms: [cli+skill, cli-noskill, mcp]
+invoke: slash
+distractor: An older, unrelated mailer timeout issue on POST /api/invoices/:id/send; the panic only hits customers without a billing address, so most invoice requests succeed.
+workspace: {repo: billing-api-go}
+prompt: |
+ /traceway debug {{seed.hash}}
+ This started showing up yesterday on the invoice page. Find out why and fix it.
+budget: {max_turns: 40, max_tool_calls: 30, max_output_tokens: 14000, timeout: 10m}
+user:
+ - default: "Please go ahead and fix it in the code."
+expect:
+ answer:
+ issue_hash: "{{seed.hash}}"
+ category: null_reference
+ culprit: {file: internal/invoices/render.go, symbol: formatAddress, line: 37}
+ needs_instrumentation: false
+ mentions_all: ["billing address|BillingAddress", "1\\.8\\.2|release|deploy"]
+ grounding:
+ - "render.go:37"
+ calls:
+ required:
+ - {op: exceptions.show, args: {hash: "{{seed.hash}}"}}
+ any_of:
+ - [{op: "read_file:internal/invoices/render.go"}]
+ - [{op: "search"}]
+ forbidden:
+ - {op: "*", write: true}
+ order:
+ - [exceptions.show, edit_file]
+ - [exceptions.show, write_file]
+ max_failed_calls: 5
+ max_identical_repeats: 2
+ code:
+ hidden_tests: debug-go-nil-regression
+ command: go test ./...
+ gating: true
+ touches_culprit: true
+ must_not_change: [internal/invoices/model.go]
+judge:
+ expected: formatAddress (internal/invoices/render.go:37) dereferences Customer.BillingAddress, which is nil for customers who never entered an address. The billing address line was added to the invoice view in v1.8.2, deployed about 30 hours ago, which is when the panic group first appeared. The fix handles a nil address (render nothing or a placeholder).
+ rubric:
+ - Ties the panic to the v1.8.2 change that added the billing address to the invoice view (firstSeen at the deploy, the git history).
+ - Explains that only customers without a billing address hit it.
+ - Fixes it by handling a nil address, without changing unrelated behaviour.
+oracle:
+ - shell: >-
+ traceway exceptions show {{seed.hash}} --page-size 5 --output json
+ | jq '{group: (.group | {exceptionHash, firstSeen, lastSeen, count, stackTrace}),
+ latest: (.occurrences | sort_by(.recordedAt) | last | {recordedAt, appVersion, traceId}), relatedEntity}'
+ - shell: git log --format='%h %ad %s' --date=iso -5
+ - shell: git show --stat HEAD
+ - read_file: internal/invoices/render.go
+ - write_file: {path: internal/invoices/render.go, from: oracle/debug-go-nil-regression/render.go}
+ - submit:
+ issue_hash: "{{seed.hash}}"
+ category: null_reference
+ culprit: {file: internal/invoices/render.go, symbol: formatAddress, line: 37}
+ evidence:
+ - {call_id: c01, claim: "nil pointer dereference in formatAddress at render.go:37, first seen after the v1.8.2 deploy, on app version 1.8.2"}
+ - {call_id: c02, claim: "v1.8.2 (yesterday) added the billing address to the invoice page"}
+ - {call_id: c04, claim: "formatAddress reads a.Line1 without checking for a nil address"}
+ fix_summary: formatAddress returns an empty string when the customer has no billing address.
+ files_changed: [internal/invoices/render.go]
+ needs_instrumentation: false
+ confidence: high
+ report: |
+ The panic (runtime error: invalid memory address or nil pointer dereference) is in formatAddress, internal/invoices/render.go:37. v1.8.2, released yesterday, added the billing address to the invoice view; customers who never entered a billing address have a nil BillingAddress, so rendering their invoice panics and the page answers 500. The group's firstSeen matches the release. I made formatAddress return an empty string for a nil address.
diff --git a/cli/test/evals/cases/perf/perf-app-sync-background.yaml b/cli/test/evals/cases/perf/perf-app-sync-background.yaml
index a5151ca3..663abf98 100644
--- a/cli/test/evals/cases/perf/perf-app-sync-background.yaml
+++ b/cli/test/evals/cases/perf/perf-app-sync-background.yaml
@@ -3,7 +3,7 @@ suite: perf
title: Generating a statement holds the request open while the PDF renders
tags: [application, go]
seed: perf/app-sync-background
-arms: [cli+skill, cli-noskill]
+arms: [cli+skill, cli-noskill, mcp]
invoke: slash
distractor: Host CPU jumps to ~80% whenever a statement renders, which reads like saturation, but average CPU is ~20%, the endpoint is just as slow when the host is otherwise idle, and every other endpoint on the host stays fast. The spike is the render itself, not the cause of the latency.
workspace: {repo: statements-api-go}
diff --git a/cli/test/evals/cases/perf/perf-cache-no-cache.yaml b/cli/test/evals/cases/perf/perf-cache-no-cache.yaml
index 812a49e3..8d9f0d2a 100644
--- a/cli/test/evals/cases/perf/perf-cache-no-cache.yaml
+++ b/cli/test/evals/cases/perf/perf-cache-no-cache.yaml
@@ -3,7 +3,7 @@ suite: perf
title: The catalog endpoint keeps the database busy all day
tags: [caching, node, react, multi-project, browser]
seed: perf/cache-no-cache
-arms: [cli+skill, cli-noskill]
+arms: [cli+skill, cli-noskill, mcp]
invoke: natural
distractor: The catalog query is the long pole of every request (~45ms of ~55ms over ~1200 rows), which reads like a slow query, but it is steady and unremarkable per request; the load is volume times repetition, ~9000 identical builds a day of a catalog that changes twice a day, because the cache never hits.
workspace: {repo: shop-catalog}
diff --git a/cli/test/evals/cases/perf/perf-cache-stampede.yaml b/cli/test/evals/cases/perf/perf-cache-stampede.yaml
index 9cbd6dc4..42eb1735 100644
--- a/cli/test/evals/cases/perf/perf-cache-stampede.yaml
+++ b/cli/test/evals/cases/perf/perf-cache-stampede.yaml
@@ -3,7 +3,7 @@ suite: perf
title: Home page requests stall for seconds in bursts
tags: [smoke, caching, go, redis]
seed: perf/cache-stampede
-arms: [cli+skill, cli-noskill]
+arms: [cli+skill, cli-noskill, mcp]
invoke: slash
distractor: The slow requests run all twelve home queries against the database on the request path, which reads like a missing cache, but the cache is there and serves ~98% of requests in ~3ms; the slow requests all land in the seconds after the cached feed expires, when every request in flight rebuilds it at once.
workspace: {repo: home-api-go}
diff --git a/cli/test/evals/cases/perf/perf-db-missing-index.yaml b/cli/test/evals/cases/perf/perf-db-missing-index.yaml
index 6c47b1b0..ee0d9892 100644
--- a/cli/test/evals/cases/perf/perf-db-missing-index.yaml
+++ b/cli/test/evals/cases/perf/perf-db-missing-index.yaml
@@ -3,7 +3,7 @@ suite: perf
title: The user lookup gets slower every week as the users table grows
tags: [database, python, fastapi]
seed: perf/db-missing-index
-arms: [cli+skill, cli-noskill]
+arms: [cli+skill, cli-noskill, mcp]
invoke: slash
distractor: Release 1.13.0 two weeks ago changed POST /api/invites (two primary-key reads and an email enqueue, a few milliseconds); the lookup's curve climbs steadily through the whole month with no step at the release, and the release did not touch the lookup.
workspace: {repo: accounts-api-py}
diff --git a/cli/test/evals/cases/perf/perf-db-n-plus-1.yaml b/cli/test/evals/cases/perf/perf-db-n-plus-1.yaml
index d37d48ef..c62ffeda 100644
--- a/cli/test/evals/cases/perf/perf-db-n-plus-1.yaml
+++ b/cli/test/evals/cases/perf/perf-db-n-plus-1.yaml
@@ -3,7 +3,7 @@ suite: perf
title: Orders endpoint slows down as customers accumulate orders
tags: [smoke, database, go]
seed: perf/db-n-plus-1
-arms: [cli+skill, cli-noskill]
+arms: [cli+skill, cli-noskill, mcp]
invoke: slash
distractor: The orders query selects every column (over-fetching), but it takes 3-6ms; CPU stays flat and nothing was deployed in the window.
workspace: {repo: orders-api-go}
diff --git a/cli/test/evals/cases/perf/perf-db-pool-exhaustion.yaml b/cli/test/evals/cases/perf/perf-db-pool-exhaustion.yaml
index feb66a5d..1b840fc1 100644
--- a/cli/test/evals/cases/perf/perf-db-pool-exhaustion.yaml
+++ b/cli/test/evals/cases/perf/perf-db-pool-exhaustion.yaml
@@ -3,7 +3,7 @@ suite: perf
title: Checkout stalls for seconds at the lunch and evening rushes
tags: [smoke, database, python, fastapi]
seed: perf/db-pool-exhaustion
-arms: [cli+skill, cli-noskill]
+arms: [cli+skill, cli-noskill, mcp]
invoke: slash
distractor: A slow database or the payment provider. The inventory query takes 40-60ms and the payment call 150-260ms at every hour, rushes included; the seconds go to a gap before the first query, and GET /api/cart, which only runs a 3ms query on the same pool, stalls at the same hours.
workspace: {repo: checkout-api-py}
diff --git a/cli/test/evals/cases/perf/perf-infra-cpu-saturation.yaml b/cli/test/evals/cases/perf/perf-infra-cpu-saturation.yaml
index 7c252f06..cedb17a3 100644
--- a/cli/test/evals/cases/perf/perf-infra-cpu-saturation.yaml
+++ b/cli/test/evals/cases/perf/perf-infra-cpu-saturation.yaml
@@ -3,7 +3,7 @@ suite: perf
title: Traffic tripled and every endpoint slowed down with the CPU pinned
tags: [smoke, infrastructure, node]
seed: perf/infra-cpu-saturation
-arms: [cli+skill, cli-noskill]
+arms: [cli+skill, cli-noskill, mcp]
invoke: slash
distractor: A code regression; but service.version never changed, the last commit is nine days old, and every endpoint got 2-4x slower at the same moment the request rate tripled, with CPU on both instances going from ~25% to ~95% at that time. The waterfalls show the same spans, just stretched, plus wait time between them; no new or longer query.
workspace: {repo: commerce-api-node}
diff --git a/cli/test/evals/cases/perf/perf-net-large-payload.yaml b/cli/test/evals/cases/perf/perf-net-large-payload.yaml
index 7d3215e7..7a81e958 100644
--- a/cli/test/evals/cases/perf/perf-net-large-payload.yaml
+++ b/cli/test/evals/cases/perf/perf-net-large-payload.yaml
@@ -3,7 +3,7 @@ suite: perf
title: The feed is slow, worst on phones
tags: [network, go]
seed: perf/net-large-payload
-arms: [cli+skill, cli-noskill]
+arms: [cli+skill, cli-noskill, mcp]
invoke: slash
distractor: The feed query runs on every request with nothing cached in front of it, which reads like a missing cache, but it takes ~15ms; the request time is the ~2.3MB body going out, and it tracks the body size and the client's link, not the database.
workspace: {repo: feed-api-go}
diff --git a/cli/test/evals/cases/perf/perf-net-sequential.yaml b/cli/test/evals/cases/perf/perf-net-sequential.yaml
index 7e24c23d..aa568a64 100644
--- a/cli/test/evals/cases/perf/perf-net-sequential.yaml
+++ b/cli/test/evals/cases/perf/perf-net-sequential.yaml
@@ -3,7 +3,7 @@ suite: perf
title: Portal dashboard is slow although every service behind it is fast
tags: [smoke, network, node, nestjs, multi-service]
seed: perf/net-sequential
-arms: [cli+skill, cli-noskill]
+arms: [cli+skill, cli-noskill, mcp]
invoke: slash
distractor: recommendations-service is the longest single hop, which reads like a slow upstream, but it is an internal service running at its own steady p95 (no spikes in the window) and it is only about a third of the request; the four hops add up.
workspace: {repo: portal-web-api-nest}
diff --git a/cli/test/evals/cases/perf/perf-xs-android-timeouts.yaml b/cli/test/evals/cases/perf/perf-xs-android-timeouts.yaml
index c64484bf..3f98da3f 100644
--- a/cli/test/evals/cases/perf/perf-xs-android-timeouts.yaml
+++ b/cli/test/evals/cases/perf/perf-xs-android-timeouts.yaml
@@ -3,7 +3,7 @@ suite: perf
title: Android checkout crashes with socket timeouts while the checkout API waits on row locks
tags: [cross-surface, android, kotlin, java, database]
seed: perf/xs-android-timeouts
-arms: [cli+skill, cli-noskill]
+arms: [cli+skill, cli-noskill, mcp]
invoke: natural
distractor: An app bug. The crash is an uncaught java.net.SocketTimeoutException from CheckoutRepository (OkHttp's 10s read timeout, no catch in the view model), so catching it or raising the timeout looks like the fix. But each crash's network action is POST /api/checkout giving up at 10s, and on commerce-api that endpoint's p95 went from under a second to about 7s when the sale started; the time is inside update inventory (queued on the sale item's row lock, which every checkout holds through its payment call), not before the first query (no pool wait) and not in the payment call itself, and deadlock errors appear in the same hours.
workspace: {repo: shop-android-and-api}
diff --git a/cli/test/evals/cases/perf/perf-xs-ios-watchdog.yaml b/cli/test/evals/cases/perf/perf-xs-ios-watchdog.yaml
index f1d2e521..b557dd97 100644
--- a/cli/test/evals/cases/perf/perf-xs-ios-watchdog.yaml
+++ b/cli/test/evals/cases/perf/perf-xs-ios-watchdog.yaml
@@ -3,7 +3,7 @@ suite: perf
title: iOS launch watchdog kills while the config endpoint waits for database connections
tags: [cross-surface, ios, swift, go, database]
seed: perf/xs-ios-watchdog
-arms: [cli+skill, cli-noskill]
+arms: [cli+skill, cli-noskill, mcp]
invoke: natural
distractor: A pure client bug. The crash is a launch watchdog termination (0x8badf00d) with the main thread blocked in ConfigClient.fetchBlocking, a synchronous GET /api/config with a 30s timeout inside didFinishLaunching, and that is a real client defect. But it has shipped for three weeks without crashing; the kills started with two waves of launches today, when GET /api/config on mobile-api began taking 20-30s, all of it waiting before the first query for one of the API's 8 database connections (the feature-flag queries themselves take a few milliseconds), with POST /api/devices and the feed queued in the same waves.
workspace: {repo: shop-ios-and-api}
diff --git a/cli/test/evals/cases/setup/setup-go-nethttp.yaml b/cli/test/evals/cases/setup/setup-go-nethttp.yaml
new file mode 100644
index 00000000..e243db35
--- /dev/null
+++ b/cli/test/evals/cases/setup/setup-go-nethttp.yaml
@@ -0,0 +1,62 @@
+id: setup-go-nethttp
+suite: setup
+title: Instrument a net/http Go API until its routes show up in Traceway
+tags: [smoke, go]
+seed: setup/go-nethttp
+arms: [cli+setup-skill]
+invoke: natural
+workspace: {repo: hello-go}
+setup:
+ path: fast
+ token: ingest
+ env:
+ TRACEWAY_URL: "@url"
+ TRACEWAY_PROJECT_TOKEN: "@token:hello-api"
+ PORT: "@port"
+ verify:
+ build: go build -o .twevals-app .
+ start: ./.twevals-app
+ hit: [/api/hello, /api/hello/ada, /api/hello/grace]
+ env_files: [.env]
+prompt: |
+ Please set up Traceway monitoring for this service. Our Traceway instance is {{env.url}}; the "hello-api" project already exists there. Its ingest token is in the environment as TRACEWAY_PROJECT_TOKEN and the instance URL as TRACEWAY_URL, here and in production. When you are done, the app should report its requests to that project.
+budget: {max_turns: 45, max_tool_calls: 40, max_output_tokens: 20000, timeout: 20m}
+user:
+ - default: "Yes, go ahead."
+expect:
+ state:
+ - kind: file
+ path: go.mod
+ contains: ["go.opentelemetry.io/otel"]
+ - kind: file
+ path: "*.go"
+ contains: ["TRACEWAY_PROJECT_TOKEN|OTEL_EXPORTER_OTLP_HEADERS", "service\\.name|ServiceName|OTEL_SERVICE_NAME"]
+ - kind: no_token_literal
+ - kind: endpoint_seen
+ endpoint: "GET /api/hello"
+ - kind: endpoint_seen
+ endpoint: "GET /api/hello/{name}"
+ calls:
+ required:
+ - {op: load_skill, args: {name: traceway-setup}}
+ forbidden:
+ - {op: "*", write: true}
+judge:
+ expected: The agent adds the OpenTelemetry SDK and an OTLP/HTTP trace exporter pointed at <instance>/api/otel/v1/traces with the bearer token read from TRACEWAY_PROJECT_TOKEN, sets service.name, wraps each route handler with otelhttp so http.route is recorded, and builds the app.
+ rubric:
+ - Reads the token and URL from the environment, never writes them into source.
+ - Records the route pattern (http.route), not raw URLs, for net/http.
+ - Sets service.name and keeps the app building and its tests passing.
+oracle:
+ - load_skill: traceway-setup
+ - read_file: main.go
+ - read_file: go.mod
+ - write_file: {path: telemetry.go, from: oracle/setup-go-nethttp/telemetry.go}
+ - write_file: {path: main.go, from: oracle/setup-go-nethttp/main.go}
+ - shell: go get go.opentelemetry.io/otel go.opentelemetry.io/otel/sdk go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp && go mod tidy && go vet ./... && go test ./...
+ - finish:
+ summary: |
+ hello-api now exports traces to Traceway over OTLP/HTTP: telemetry.go sets up the SDK with an exporter to $TRACEWAY_URL/api/otel/v1/traces and the bearer token from $TRACEWAY_PROJECT_TOKEN (nothing is hard-coded), service.name hello-api, and the W3C propagator. Each route handler is wrapped with otelhttp so Traceway records the route pattern (GET /api/hello/{name}) instead of one row per URL. The app builds and its tests pass.
+ changes:
+ - {action: add OpenTelemetry tracing, target: telemetry.go, new_value: OTLP/HTTP exporter to Traceway}
+ - {action: wrap route handlers with otelhttp, target: main.go}
diff --git a/cli/test/evals/cases/write/write-alert-snooze.yaml b/cli/test/evals/cases/write/write-alert-snooze.yaml
new file mode 100644
index 00000000..24149999
--- /dev/null
+++ b/cli/test/evals/cases/write/write-alert-snooze.yaml
@@ -0,0 +1,61 @@
+id: write-alert-snooze
+suite: write
+title: Quiet a flapping alert during a known third-party incident
+tags: [smoke, alerts]
+seed: write/alert-snooze
+arms: [cli-write+skill, mcp-write]
+invoke: natural
+distractor: A second checkout alert (p95 latency) exists and must stay untouched; disabling the rule instead of snoozing it is the wrong, unreversible-by-default choice.
+prompt: |
+ Our card processor has an incident right now (their status page says they expect to recover within the hour). Our checkout error-rate alert in Traceway keeps paging me about it every few minutes. Please quiet that alert for the next hour. Don't change anything else.
+budget: {max_turns: 25, max_tool_calls: 15, timeout: 8m}
+user:
+ - default: "Yes, just that alert, for about an hour."
+expect:
+ state:
+ - kind: rule
+ id: "{{seed.ruleId}}"
+ enabled: true
+ snoozed_minutes: {min: 45, max: 65}
+ - kind: rule
+ id: "{{seed.otherRuleId}}"
+ enabled: true
+ snoozed: false
+ - kind: page
+ id: "{{seed.pageId}}"
+ status: open
+ - kind: agent_writes
+ count: 1
+ calls:
+ required:
+ - {op: alerts.list}
+ - {op: alerts.snooze, args: {id: "{{seed.ruleId}}"}}
+ forbidden:
+ - {op: alerts.disable}
+ - {op: alerts.enable}
+ - {op: pages.ack}
+ - {op: pages.resolve}
+ - {op: exceptions.archive}
+ - {op: alerts.snooze, args: {id: "{{seed.otherRuleId}}"}}
+ order:
+ - [alerts.list, alerts.snooze]
+ max_failed_calls: 3
+ report:
+ contains:
+ - "(snooz|mute|silenc)"
+ - "(\\d{1,2}:\\d{2}|60 ?min|an hour|1 hour|one hour)"
+judge:
+ expected: The agent finds the checkout error-rate rule with alerts list, snoozes it for about 60 minutes (not disable), leaves the p95 rule and the open page alone, and reports when the snooze ends and how to undo it.
+ rubric:
+ - Snoozes (rather than disables) the error-rate rule, for about an hour.
+ - Leaves the other rule and the page unchanged.
+ - States when the snooze ends and how to undo it.
+oracle:
+ - shell: traceway-write alerts list --output json
+ - op: alerts.snooze
+ args: {id: "{{seed.ruleId}}", duration_minutes: 60}
+ - finish:
+ summary: |
+ Snoozed the "Checkout error rate above 5%" alert (rule {{seed.ruleId}}) for 60 minutes, so it stops paging until about an hour from now and then turns back on by itself. The rule stays enabled; the p95 latency alert and the open page are untouched. To undo it early, clear the snooze (snooze with 0 minutes).
+ changes:
+ - {action: snooze alert rule, target: "rule {{seed.ruleId}}", new_value: "snoozed for 60 minutes", undo: "snooze it with 0 minutes"}
diff --git a/cli/test/evals/cmd/twevals/commands.go b/cli/test/evals/cmd/twevals/commands.go
index 44772507..a0bfa56c 100644
--- a/cli/test/evals/cmd/twevals/commands.go
+++ b/cli/test/evals/cmd/twevals/commands.go
@@ -148,7 +148,8 @@ func cmdOracle(ctx context.Context, args []string) error {
}
oracleModel := &catalog.Model{Key: "oracle", ID: "fake/oracle", Caching: "none"}
opt := runner.Options{EvalsRoot: ws.root, RepoRoot: ws.repo, OutDir: outDir, Bins: bins, Models: []*catalog.Model{oracleModel},
- Runs: 1, Jobs: *jobs, Keep: *keep, Log: os.Stderr}
+ Runs: 1, Jobs: *jobs, Keep: *keep, Oracle: true, Log: os.Stderr}
+ opt.SandboxImage = sandboxImage(ctx, ws, units)
header := report.NewHeader(ws.repo, filepath.Base(outDir), ws.models, bins.Hash, os.Args)
header.Oracle = true
header.Models = map[string]string{oracleModel.Key: oracleModel.ID}
diff --git a/cli/test/evals/cmd/twevals/commands_ops.go b/cli/test/evals/cmd/twevals/commands_ops.go
new file mode 100644
index 00000000..7fe73dd2
--- /dev/null
+++ b/cli/test/evals/cmd/twevals/commands_ops.go
@@ -0,0 +1,235 @@
+package main
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "flag"
+ "fmt"
+ "io/fs"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "sort"
+ "time"
+
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/catalog"
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/env"
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/report"
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/runner"
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/scenarios"
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/seed"
+)
+
+func mergeResume(prev, now []*runner.Result) []*runner.Result {
+ byKey := map[string]*runner.Result{}
+ for _, r := range prev {
+ byKey[r.Key()] = r
+ }
+ for _, r := range now {
+ byKey[r.Key()] = r
+ }
+ out := make([]*runner.Result, 0, len(byKey))
+ for _, r := range byKey {
+ out = append(out, r)
+ }
+ sort.Slice(out, func(i, j int) bool { return out[i].Key() < out[j].Key() })
+ return out
+}
+
+// sandboxImage builds the container image when the selection needs it (the
+// setup suite, or hidden tests of a model's patch) and Docker is present.
+func sandboxImage(ctx context.Context, ws *workspace, units []catalog.Unit) string {
+ need := false
+ for _, u := range units {
+ if u.Case.Suite == "setup" || (u.Case.Expect.Code != nil && !u.Case.Expect.Code.Static) {
+ need = true
+ }
+ }
+ if !need {
+ return ""
+ }
+ if _, err := exec.LookPath("docker"); err != nil {
+ fmt.Fprintln(os.Stderr, "note: docker not found; setup cases fail and hidden tests of model patches are skipped")
+ return ""
+ }
+ tag, err := runner.EnsureSandboxImage(ctx, ws.root)
+ if err != nil {
+ fmt.Fprintln(os.Stderr, "note:", err)
+ return ""
+ }
+ return tag
+}
+
+func cmdEnvUp(ctx context.Context, args []string) error {
+ if len(args) == 0 {
+ return errors.New("usage: twevals env up <case-id>")
+ }
+ ws, err := load()
+ if err != nil {
+ return err
+ }
+ var c *catalog.Case
+ for _, x := range ws.cases {
+ if x.ID == args[0] {
+ c = x
+ }
+ }
+ if c == nil {
+ return fmt.Errorf("no case %q", args[0])
+ }
+ bins, err := env.BuildBinaries(ws.repo, filepath.Join(ws.root, ".cache"))
+ if err != nil {
+ return err
+ }
+ dir, err := os.MkdirTemp("", "twevals-env-")
+ if err != nil {
+ return err
+ }
+ defer os.RemoveAll(dir)
+ inst, err := env.NewInstance(ctx, bins, dir)
+ if err != nil {
+ return err
+ }
+ defer inst.Close()
+ sink := seed.NewLiveSink(ctx, inst.Backend, inst.Admin)
+ sink.StubSrv = inst.Stub
+ sc := seed.NewScenario(c.ID, time.Now(), sink)
+ if err := scenarios.Run(c.Seed, sc); err != nil {
+ return err
+ }
+ if err := sc.Verify(time.Minute); err != nil {
+ return err
+ }
+ exposed, _ := json.MarshalIndent(sc.Exposed(), "", " ")
+ pat, _ := env.CreatePAT(ctx, inst.Backend, inst.Admin.JWT, "env up", "read", 0)
+ fmt.Printf("Case %s is seeded (%+v).\n\nDashboard: %s\nLogin: %s / %s\nRead PAT: %s\n %s login --url %s --token %s\n\nseed.json:\n%s\n\nCtrl-C to stop. Data dir: %s\n",
+ c.ID, sc.Stats(), inst.Backend.URL, env.AdminEmail, env.AdminPassword, pat, bins.CLI, inst.Backend.URL, pat, exposed, dir)
+ <-ctx.Done()
+ return nil
+}
+
+func cmdReport(args []string) error {
+ fs := flag.NewFlagSet("report", flag.ExitOnError)
+ merge := fs.Bool("merge", false, "merge several run directories (shards) into one")
+ out := fs.String("out", "", "write results.json and summary.md here")
+ pos := parseInterleaved(fs, args)
+ if len(pos) == 0 {
+ return errors.New("report needs at least one run directory")
+ }
+ var dirs []string
+ for _, a := range pos {
+ // A shard artifact may nest its run directory; find every results root.
+ matches := findRunDirs(a)
+ if len(matches) == 0 {
+ matches = []string{a}
+ }
+ dirs = append(dirs, matches...)
+ }
+ var parts []*report.Results
+ for _, d := range dirs {
+ r, err := report.Load(d)
+ if err != nil {
+ return err
+ }
+ parts = append(parts, r)
+ }
+ var res *report.Results
+ if *merge || len(parts) > 1 {
+ res = report.Merge(parts)
+ } else {
+ res = parts[0]
+ }
+ target := *out
+ if target == "" {
+ target = dirs[0]
+ }
+ if err := os.MkdirAll(target, 0o755); err != nil {
+ return err
+ }
+ if err := report.Save(target, res); err != nil {
+ return err
+ }
+ md, _ := os.ReadFile(filepath.Join(target, "summary.md"))
+ fmt.Print(string(md))
+ return partialExit(res.Runs)
+}
+
+// findRunDirs returns directories under root that hold results.json or a
+// runs/ tree.
+func findRunDirs(root string) []string {
+ var out []string
+ _ = filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
+ if err != nil || !d.IsDir() {
+ return nil
+ }
+ if d.Name() == "runs" {
+ out = append(out, filepath.Dir(p))
+ return filepath.SkipDir
+ }
+ return nil
+ })
+ return out
+}
+
+func cmdCompare(args []string) error {
+ if len(args) == 0 {
+ return errors.New("usage: twevals compare <run-dir> [baseline.json]")
+ }
+ ws, err := load()
+ if err != nil {
+ return err
+ }
+ res, err := report.Load(args[0])
+ if err != nil {
+ return err
+ }
+ basePath := filepath.Join(ws.root, "baseline.json")
+ if len(args) > 1 {
+ basePath = args[1]
+ }
+ base, err := report.LoadBaseline(basePath)
+ if err != nil {
+ return err
+ }
+ regs := report.Compare(res, base)
+ if len(regs) == 0 {
+ fmt.Printf("No regressions against %s (%d suite and %d case baselines).\n", basePath, len(base.Suites), len(base.Cases))
+ return nil
+ }
+ fmt.Printf("## Regressions against the baseline\n\n")
+ for _, r := range regs {
+ fmt.Printf("- %s\n", r.Detail)
+ }
+ return &exitError{1, fmt.Sprintf("%d regression(s)", len(regs))}
+}
+
+func cmdBaselineUpdate(args []string) error {
+ fs := flag.NewFlagSet("baseline update", flag.ExitOnError)
+ out := fs.String("out", "", "baseline path (default cli/test/evals/baseline.json)")
+ pos := parseInterleaved(fs, args)
+ if len(pos) == 0 {
+ return errors.New("usage: twevals baseline update <run-dir>")
+ }
+ ws, err := load()
+ if err != nil {
+ return err
+ }
+ res, err := report.Load(pos[0])
+ if err != nil {
+ return err
+ }
+ if res.Header.Oracle {
+ return errors.New("refusing to baseline an oracle run")
+ }
+ path := *out
+ if path == "" {
+ path = filepath.Join(ws.root, "baseline.json")
+ }
+ b := report.NewBaseline(res)
+ if err := b.Save(path); err != nil {
+ return err
+ }
+ fmt.Printf("Wrote %s from %s. Review it and commit it yourself; the workflow never commits.\n", path, res.Header.RunID)
+ return nil
+}
diff --git a/cli/test/evals/cmd/twevals/main.go b/cli/test/evals/cmd/twevals/main.go
index e0f4eca5..b145163c 100644
--- a/cli/test/evals/cmd/twevals/main.go
+++ b/cli/test/evals/cmd/twevals/main.go
@@ -1,5 +1,9 @@
-// Command twevals runs the Traceway skill and MCP evals with the fake model
-// that plays each case's oracle. It makes no paid model calls.
+// Command twevals runs the Traceway skill and MCP evals.
+//
+// Manual only. Paid runs happen in the CLI Evals workflow, which a person
+// starts by hand (workflow_dispatch) and approves at the evals environment
+// gate. `twevals run` refuses every other CI context, and locally it needs a
+// typed confirmation. `twevals oracle` and `twevals plan` are free.
package main
import (
@@ -11,15 +15,25 @@ import (
"syscall"
)
-const usage = `twevals: Traceway skill and MCP evals (oracle runs only, free)
+const usage = `twevals: Traceway skill and MCP evals (manual only)
+Free:
twevals list [--preset P] [--suites S] [--tags T] [--cases GLOB] [--arms A]
+ twevals plan [selection] --models opus,astra [--runs N] --max-cost-usd X [--shards] [--out plan.json] [--summary FILE]
twevals oracle [--all | <case-id>... | --suites S] [--arms A] [-j N] [--keep] [--out DIR]
+ twevals env up <case-id> [--arm A]
+ twevals report <run-dir>... [--merge] [--out DIR]
+ twevals compare <run-dir> [baseline.json]
+ twevals baseline update <run-dir> [--out baseline.json]
+
+Paid (OpenRouter; guarded, see README):
+ twevals run [selection] [--shard SUITE] --models opus,astra --max-cost-usd X [--runs N] [-j N] [--keep] [--cache record|replay|off] [--judge=true] [--yes]
+ twevals resume <run-dir> --max-cost-usd X [--yes]
Selection flags: --preset smoke|core|full, --cases (comma-separated ids or globs; overrides the preset), --suites, --tags, --arms.
-list defaults to --preset full. oracle writes results.json and summary.md to --out
+list defaults to --preset full, plan and run to --preset smoke. oracle writes results.json and summary.md to --out
(default .cache/oracle/<timestamp>).
-Exit codes: 0 ok, 1 a failed oracle or an error, 2 bad usage.
+Exit codes: 0 ok, 1 a regression, a failed oracle or an error, 2 bad usage or partial (cost ceiling, or infra_error above 10%).
`
func main() {
@@ -34,8 +48,34 @@ func main() {
switch os.Args[1] {
case "list":
err = cmdList(args)
+ case "plan":
+ err = cmdPlan(ctx, args)
+ case "run":
+ err = cmdRun(ctx, args, "")
+ case "resume":
+ if len(args) == 0 {
+ err = errors.New("resume needs a run directory")
+ break
+ }
+ err = cmdRun(ctx, args[1:], args[0])
case "oracle":
err = cmdOracle(ctx, args)
+ case "env":
+ if len(args) == 0 || args[0] != "up" {
+ err = errors.New("usage: twevals env up <case-id>")
+ break
+ }
+ err = cmdEnvUp(ctx, args[1:])
+ case "report":
+ err = cmdReport(args)
+ case "compare":
+ err = cmdCompare(args)
+ case "baseline":
+ if len(args) == 0 || args[0] != "update" {
+ err = errors.New("usage: twevals baseline update <run-dir>")
+ break
+ }
+ err = cmdBaselineUpdate(args[1:])
case "-h", "--help", "help":
fmt.Print(usage)
return
diff --git a/cli/test/evals/cmd/twevals/run.go b/cli/test/evals/cmd/twevals/run.go
new file mode 100644
index 00000000..78c72bf4
--- /dev/null
+++ b/cli/test/evals/cmd/twevals/run.go
@@ -0,0 +1,357 @@
+package main
+
+import (
+ "context"
+ "encoding/json"
+ "flag"
+ "fmt"
+ "os"
+ "path/filepath"
+ "slices"
+ "sort"
+ "strings"
+ "time"
+
+ "golang.org/x/term"
+
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/agent"
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/catalog"
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/env"
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/grade"
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/guard"
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/report"
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/runner"
+)
+
+// Plan is plan.json, the preflight the workflow reads.
+type Plan struct {
+ Selection catalog.Selection `json:"selection"`
+ Models []string `json:"models"`
+ Runs int `json:"runs"`
+ MaxCostUSD float64 `json:"maxCostUsd"`
+ Shards []Shard `json:"shards"`
+ Estimates []report.Estimate `json:"estimates"`
+ Judge report.Estimate `json:"judge"`
+ Low float64 `json:"low"`
+ High float64 `json:"high"`
+ PriceNote string `json:"priceNote,omitempty"`
+ Matrix struct {
+ Include []map[string]any `json:"include"`
+ } `json:"matrix"`
+}
+
+// Shard is one suite's slice of a dispatch.
+type Shard struct {
+ Suite string `json:"suite"`
+ Units int `json:"units"`
+ Runs int `json:"runs"`
+ Low float64 `json:"low"`
+ High float64 `json:"high"`
+ CapUSD float64 `json:"capUsd"`
+}
+
+func buildPlan(ctx context.Context, ws *workspace, sel catalog.Selection, modelList string, runs int, maxCost float64, pricesFile string) (*Plan, []catalog.Unit, []*catalog.Model, error) {
+ if runs < 1 || runs > 3 {
+ return nil, nil, nil, fmt.Errorf("--runs must be between 1 and 3, got %d", runs)
+ }
+ units, err := catalog.Select(ws.cases, ws.presets, sel)
+ if err != nil {
+ return nil, nil, nil, err
+ }
+ models, err := ws.models.ParseModelList(modelList)
+ if err != nil {
+ return nil, nil, nil, err
+ }
+ all := append(append([]*catalog.Model{}, models...), ws.models.Judge)
+ prices, perr := report.FetchPrices(ctx, all, pricesFile)
+ base, _ := report.LoadBaseline(filepath.Join(ws.root, "baseline.json"))
+ p := &Plan{Selection: sel, Runs: runs, MaxCostUSD: maxCost}
+ if perr != nil {
+ p.PriceNote = "price list problem, using models.yaml fallback prices where needed: " + perr.Error()
+ }
+ for _, m := range models {
+ p.Models = append(p.Models, m.Key)
+ }
+ bySuite := map[string]int{}
+ for _, u := range units {
+ bySuite[u.Case.Suite]++
+ }
+ for _, s := range catalog.Suites {
+ n := bySuite[s]
+ if n == 0 {
+ continue
+ }
+ sh := Shard{Suite: s, Units: n, Runs: n * runs * len(models)}
+ for _, m := range models {
+ e := report.EstimateRuns(m, prices[m.ID], n*runs, base)
+ sh.Low += e.Low
+ sh.High += e.High
+ }
+ j := report.EstimateRuns(ws.models.Judge, prices[ws.models.Judge.ID], sh.Runs, nil)
+ sh.Low += j.Low
+ sh.High += j.High
+ p.Shards = append(p.Shards, sh)
+ }
+ total := 0
+ for _, m := range models {
+ e := report.EstimateRuns(m, prices[m.ID], len(units)*runs, base)
+ p.Estimates = append(p.Estimates, e)
+ p.Low += e.Low
+ p.High += e.High
+ total += e.Runs
+ }
+ p.Judge = report.EstimateRuns(ws.models.Judge, prices[ws.models.Judge.ID], total, nil)
+ p.Low += p.Judge.Low
+ p.High += p.Judge.High
+ mid := 0.0
+ for _, s := range p.Shards {
+ mid += (s.Low + s.High) / 2
+ }
+ for i := range p.Shards {
+ share := 1 / float64(len(p.Shards))
+ if mid > 0 {
+ share = (p.Shards[i].Low + p.Shards[i].High) / 2 / mid
+ }
+ p.Shards[i].CapUSD = float64(int(maxCost*share*100)) / 100
+ p.Matrix.Include = append(p.Matrix.Include, map[string]any{"shard": p.Shards[i].Suite, "cap": fmt.Sprintf("%.2f", p.Shards[i].CapUSD)})
+ }
+ return p, units, models, nil
+}
+
+func (p *Plan) text() string {
+ var b strings.Builder
+ fmt.Fprintf(&b, "## Eval preflight\n\n")
+ fmt.Fprintf(&b, "Selection: %s · models %s · %d run(s) each\n\n", describeSel(p.Selection), strings.Join(p.Models, ","), p.Runs)
+ b.WriteString("| Shard | Cases × arms | Runs | Estimate | Cap |\n|---|---|---|---|---|\n")
+ for _, s := range p.Shards {
+ fmt.Fprintf(&b, "| %s | %d | %d | $%.2f – $%.2f | $%.2f |\n", s.Suite, s.Units, s.Runs, s.Low, s.High, s.CapUSD)
+ }
+ b.WriteString("\n| Model | Runs | Estimate | Worst case (no cache) | Basis | Prices |\n|---|---|---|---|---|---|\n")
+ for _, e := range append(append([]report.Estimate{}, p.Estimates...), p.Judge) {
+ fmt.Fprintf(&b, "| %s `%s` | %d | $%.2f – $%.2f | $%.2f | %s | %s |\n", e.Model, e.ModelID, e.Runs, e.Low, e.High, e.Worst, e.Basis, e.Source)
+ }
+ fmt.Fprintf(&b, "\n**Total estimate: $%.2f – $%.2f. Ceiling: $%.2f.**\n", p.Low, p.High, p.MaxCostUSD)
+ if p.PriceNote != "" {
+ fmt.Fprintf(&b, "\n> %s\n", p.PriceNote)
+ }
+ return b.String()
+}
+
+func describeSel(s catalog.Selection) string {
+ var parts []string
+ for k, v := range map[string]string{"preset": s.Preset, "cases": s.Cases, "suites": s.Suites, "tags": s.Tags, "arms": s.Arms, "shard": s.Shard} {
+ if v != "" {
+ parts = append(parts, k+"="+v)
+ }
+ }
+ sort.Strings(parts)
+ return strings.Join(parts, " ")
+}
+
+func cmdPlan(ctx context.Context, args []string) error {
+ fs := flag.NewFlagSet("plan", flag.ExitOnError)
+ var sf selFlags
+ sf.bind(fs)
+ models := fs.String("models", "opus,astra", "comma-separated model keys from models.yaml")
+ runs := fs.Int("runs", 1, "repeats per case, 1-3")
+ maxCost := fs.Float64("max-cost-usd", 0, "hard cap for the whole dispatch (required)")
+ _ = fs.Bool("shards", true, "split the cap per suite shard (always on)")
+ out := fs.String("out", "", "write plan.json here")
+ summary := fs.String("summary", "", "append the preflight table to this file (e.g. $GITHUB_STEP_SUMMARY)")
+ prices := fs.String("prices", "", "read prices from a saved /api/v1/models response instead of fetching")
+ _ = fs.Parse(args)
+ if *maxCost <= 0 {
+ return &exitError{2, "--max-cost-usd is required and must be greater than 0 (there is no default)"}
+ }
+ ws, err := load()
+ if err != nil {
+ return err
+ }
+ p, _, _, err := buildPlan(ctx, ws, sf.selection(), *models, *runs, *maxCost, *prices)
+ if err != nil {
+ return err
+ }
+ text := p.text()
+ fmt.Print(text)
+ if *summary != "" {
+ f, err := os.OpenFile(*summary, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
+ if err == nil {
+ f.WriteString(text + "\n")
+ f.Close()
+ }
+ }
+ if *out != "" {
+ raw, _ := json.MarshalIndent(p, "", " ")
+ if err := os.WriteFile(*out, raw, 0o644); err != nil {
+ return err
+ }
+ }
+ if *maxCost < p.Low {
+ return &exitError{1, fmt.Sprintf("the cap $%.2f is below the low end of the estimate ($%.2f): raise --max-cost-usd or select fewer cases", *maxCost, p.Low)}
+ }
+ return nil
+}
+
+// selection is plan's and run's selection: smoke unless one is given, so a
+// paid run never defaults to the whole catalog.
+func (s *selFlags) selection() catalog.Selection {
+ if s.sel.Preset == "" && s.sel.Cases == "" && s.sel.Suites == "" && s.sel.Tags == "" {
+ s.sel.Preset = "smoke"
+ }
+ return s.sel
+}
+
+// runConfig is saved next to results so resume continues the same plan.
+type runConfig struct {
+ Selection catalog.Selection `json:"selection"`
+ Models string `json:"models"`
+ Runs int `json:"runs"`
+ Judge bool `json:"judge"`
+ Cache string `json:"cache"`
+ Jobs int `json:"jobs"`
+}
+
+func cmdRun(ctx context.Context, args []string, resumeDir string) error {
+ fs := flag.NewFlagSet("run", flag.ExitOnError)
+ var sf selFlags
+ sf.bind(fs)
+ shard := fs.String("shard", "", "run one suite shard")
+ models := fs.String("models", "opus,astra", "comma-separated model keys")
+ runs := fs.Int("runs", 1, "repeats per case, 1-3")
+ maxCost := fs.Float64("max-cost-usd", 0, "hard cap for this invocation (required, no default)")
+ jobs := fs.Int("j", 4, "parallel runs")
+ keep := fs.Bool("keep", false, "keep every run directory (backend db, workspace)")
+ cache := fs.String("cache", envOr("EVAL_CACHE", agent.CacheOff), "model response cache: record | replay | off")
+ judge := fs.Bool("judge", true, "score reports with the LLM judge (non-gating)")
+ yes := fs.Bool("yes", false, "proceed without a terminal (CI); a local terminal still asks")
+ out := fs.String("out", "", "results directory (default results/<run-id>)")
+ _ = fs.Parse(args)
+
+ ws, err := load()
+ if err != nil {
+ return err
+ }
+ cfg := runConfig{Selection: sf.selection(), Models: *models, Runs: *runs, Judge: *judge, Cache: *cache, Jobs: *jobs}
+ cfg.Selection.Shard = *shard
+ outDir := *out
+ skip := map[string]bool{}
+ if resumeDir != "" {
+ raw, err := os.ReadFile(filepath.Join(resumeDir, "config.json"))
+ if err != nil {
+ return fmt.Errorf("resume: %w", err)
+ }
+ if err := json.Unmarshal(raw, &cfg); err != nil {
+ return fmt.Errorf("resume: %w", err)
+ }
+ outDir = resumeDir
+ if prev, err := report.Load(resumeDir); err == nil {
+ for _, r := range prev.Runs {
+ if r.Status != "cost_ceiling" && r.Status != "infra_error" {
+ skip[r.Key()] = true
+ }
+ }
+ }
+ }
+ plan, units, chosen, err := buildPlan(ctx, ws, cfg.Selection, cfg.Models, cfg.Runs, *maxCost, "")
+ if err != nil {
+ return err
+ }
+ if err := guard.Authorize(guard.Request{
+ Command: map[bool]string{true: "resume", false: "run"}[resumeDir != ""], MaxCostUSD: *maxCost, Yes: *yes,
+ StdinTTY: term.IsTerminal(int(os.Stdin.Fd())), Getenv: os.Getenv, Preflight: plan.text(), In: os.Stdin, Out: os.Stderr,
+ }); err != nil {
+ return &exitError{2, err.Error()}
+ }
+ client, err := agent.NewOpenRouter(os.Getenv("OPENROUTER_API_KEY"))
+ if err != nil {
+ return &exitError{2, err.Error()}
+ }
+ var chatter agent.Chatter = client
+ if cfg.Cache != agent.CacheOff {
+ chatter = &agent.CachedChatter{Inner: client, Dir: filepath.Join(ws.root, ".cache", "models"), Mode: cfg.Cache}
+ }
+ runID := time.Now().UTC().Format("20060102-150405")
+ if outDir == "" {
+ outDir = filepath.Join(ws.root, "results", runID)
+ } else {
+ runID = filepath.Base(outDir)
+ }
+ if err := os.MkdirAll(outDir, 0o755); err != nil {
+ return err
+ }
+ cfgRaw, _ := json.MarshalIndent(cfg, "", " ")
+ _ = os.WriteFile(filepath.Join(outDir, "config.json"), cfgRaw, 0o644)
+ planRaw, _ := json.MarshalIndent(plan, "", " ")
+ _ = os.WriteFile(filepath.Join(outDir, "plan.json"), planRaw, 0o644)
+
+ ledger := agent.NewLedger(*maxCost)
+ bins, err := env.BuildBinaries(ws.repo, filepath.Join(ws.root, ".cache"))
+ if err != nil {
+ return err
+ }
+ opt := runner.Options{EvalsRoot: ws.root, RepoRoot: ws.repo, OutDir: outDir, Bins: bins, Models: chosen, Runs: cfg.Runs, Jobs: cfg.Jobs,
+ Keep: *keep, Chat: func(*catalog.Model) agent.Chatter { return chatter }, Ledger: ledger, Log: os.Stderr, Skip: skip,
+ Prices: pricesByKey(ctx, ws, chosen), AllowHostTests: os.Getenv("TWEVALS_HOST_TESTS") == "1"}
+ if cfg.Judge {
+ opt.Judge = &grade.Judge{Chat: chatter, Model: ws.models.Judge, Ledger: ledger}
+ }
+ opt.SandboxImage = sandboxImage(ctx, ws, units)
+ header := report.NewHeader(ws.repo, runID, ws.models, bins.Hash, os.Args)
+ header.MaxCostUSD = *maxCost
+ results := runner.Execute(ctx, opt, runner.Jobs(units, chosen, cfg.Runs))
+ if resumeDir != "" {
+ if prev, err := report.Load(resumeDir); err == nil {
+ results = mergeResume(prev.Runs, results)
+ }
+ }
+ header.Finished = time.Now().UTC()
+ res := &report.Results{Header: header, Runs: results}
+ if err := report.Save(outDir, res); err != nil {
+ return err
+ }
+ fmt.Fprintf(os.Stderr, "\nresults: %s\nspend: %s\n", outDir, ledger)
+ return partialExit(results)
+}
+
+func partialExit(results []*runner.Result) error {
+ ceiling, infra := 0, 0
+ for _, r := range results {
+ switch r.Status {
+ case agent.StopCeiling:
+ ceiling++
+ case agent.StopInfra:
+ infra++
+ }
+ }
+ if ceiling > 0 {
+ return &exitError{2, fmt.Sprintf("partial: %d runs not started at the cost ceiling (continue with twevals resume)", ceiling)}
+ }
+ if len(results) > 0 && float64(infra)/float64(len(results)) > 0.10 {
+ return &exitError{2, fmt.Sprintf("partial: %d of %d runs hit infra errors", infra, len(results))}
+ }
+ return nil
+}
+
+// pricesByKey fetches live prices and, while it has the model list, drops a
+// configured reasoning effort for a model that does not list "reasoning" in
+// its supported_parameters (require_parameters would otherwise fail the run).
+func pricesByKey(ctx context.Context, ws *workspace, models []*catalog.Model) map[string]catalog.Price {
+ live, _ := report.FetchPrices(ctx, models, "")
+ out := map[string]catalog.Price{}
+ for _, m := range models {
+ p := live[m.ID]
+ out[m.Key] = p.Price
+ if m.ReasoningEffort != "" && p.Source == "live" && !slices.Contains(p.Supported, "reasoning") {
+ fmt.Fprintf(os.Stderr, "note: %s does not list reasoning in supported_parameters; not sending reasoning.effort\n", m.ID)
+ m.ReasoningEffort = ""
+ }
+ }
+ return out
+}
+
+func envOr(k, def string) string {
+ if v := os.Getenv(k); v != "" {
+ return v
+ }
+ return def
+}
diff --git a/cli/test/evals/go.mod b/cli/test/evals/go.mod
index 27dd8c25..ecfad71b 100644
--- a/cli/test/evals/go.mod
+++ b/cli/test/evals/go.mod
@@ -8,19 +8,25 @@ require (
github.com/benhoyt/goawk v1.32.0
github.com/google/uuid v1.6.0
github.com/itchyny/gojq v0.12.19
+ github.com/modelcontextprotocol/go-sdk v1.6.1
github.com/tracewayapp/traceway/cli v1.9.15
go.opentelemetry.io/proto/otlp v1.10.0
+ golang.org/x/term v0.45.0
google.golang.org/protobuf v1.36.11
gopkg.in/yaml.v3 v3.0.1
mvdan.cc/sh/v3 v3.14.1
)
require (
+ github.com/google/jsonschema-go v0.4.3 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect
github.com/itchyny/timefmt-go v0.1.8 // indirect
+ github.com/segmentio/asm v1.1.3 // indirect
+ github.com/segmentio/encoding v0.5.4 // indirect
+ github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
golang.org/x/net v0.50.0 // indirect
+ golang.org/x/oauth2 v0.35.0 // indirect
golang.org/x/sys v0.47.0 // indirect
- golang.org/x/term v0.45.0 // indirect
golang.org/x/text v0.34.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 // indirect
diff --git a/cli/test/evals/go.sum b/cli/test/evals/go.sum
index 628b8809..ff973b3b 100644
--- a/cli/test/evals/go.sum
+++ b/cli/test/evals/go.sum
@@ -10,10 +10,14 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-quicktest/qt v1.102.0 h1:HSQxCeh5YZH3EL3W39ixjtyaEhcWSXQHtHnMBzSs474=
github.com/go-quicktest/qt v1.102.0/go.mod h1:p4lGIVX+8Wa6ZPNDvqcxq36XpUDLh42FLetFU7odllI=
+github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
+github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/jsonschema-go v0.4.3 h1:/DBOLZTfDow7pe2GmaJNhltueGTtDKICi8V8p+DQPd0=
+github.com/google/jsonschema-go v0.4.3/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs=
@@ -26,8 +30,16 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/modelcontextprotocol/go-sdk v1.6.1 h1:0zOSupjKUxPKSocPT1Wtago+mUHU2/uZ4xSOY0FGReU=
+github.com/modelcontextprotocol/go-sdk v1.6.1/go.mod h1:kzm3kzFL1/+AziGOE0nUs3gvPoNxMCvkxokMkuFapXQ=
github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc=
github.com/rogpeppe/go-internal v1.15.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs=
+github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc=
+github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg=
+github.com/segmentio/encoding v0.5.4 h1:OW1VRern8Nw6ITAtwSZ7Idrl3MXCFwXHPgqESYfvNt0=
+github.com/segmentio/encoding v0.5.4/go.mod h1:HS1ZKa3kSN32ZHVZ7ZLPLXWvOVIiZtyJnO1gPH1sKt0=
+github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4=
+github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48=
@@ -44,12 +56,16 @@ go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpu
go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk=
golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60=
golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM=
+golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ=
+golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
+golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
+golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 h1:JLQynH/LBHfCTSbDWl+py8C+Rg/k1OVH3xfcaiANuF0=
diff --git a/cli/test/evals/internal/agent/agent_test.go b/cli/test/evals/internal/agent/agent_test.go
index 76ed7d53..20af7d63 100644
--- a/cli/test/evals/internal/agent/agent_test.go
+++ b/cli/test/evals/internal/agent/agent_test.go
@@ -142,6 +142,18 @@ func TestLoopBudgets(t *testing.T) {
if rejected == 0 {
t.Fatal("calls past max_tool_calls must be refused")
}
+
+ c := cfg(&scripted{resps: []*Response{toolResp("echo", `{}`, 2), toolResp("echo", `{}`, 2)}})
+ c.RunCap = 1
+ if run := Run(context.Background(), c); run.StopReason != StopBudget || !strings.Contains(run.StopDetail, "cost cap") {
+ t.Fatalf("run cap: %s %s", run.StopReason, run.StopDetail)
+ }
+ c = cfg(&scripted{resps: []*Response{toolResp("echo", `{}`, 2)}})
+ c.Ledger = NewLedger(1)
+ c.Ledger.Add("other", 1)
+ if run := Run(context.Background(), c); run.StopReason != StopCeiling {
+ t.Fatalf("ceiling: %s", run.StopReason)
+ }
}
func TestUnknownToolIsAnError(t *testing.T) {
@@ -152,8 +164,11 @@ func TestUnknownToolIsAnError(t *testing.T) {
}
}
-func TestClientRefusesRemote(t *testing.T) {
- c := &Client{BaseURL: "https://openrouter.ai/api/v1", Key: "sk-or-test"}
+func TestClientRefusesRemoteWithoutGuard(t *testing.T) {
+ if _, err := NewOpenRouter("sk-or-test"); !errors.Is(err, ErrNotAuthorized) {
+ t.Fatalf("NewOpenRouter under go test = %v", err)
+ }
+ c := &Client{BaseURL: OpenRouterURL, Key: "sk-or-test"}
if _, err := c.Chat(context.Background(), &Request{Model: "x"}); !errors.Is(err, ErrNotAuthorized) {
t.Fatalf("a remote call under go test = %v", err)
}
diff --git a/cli/test/evals/internal/agent/budget.go b/cli/test/evals/internal/agent/budget.go
new file mode 100644
index 00000000..c8019d98
--- /dev/null
+++ b/cli/test/evals/internal/agent/budget.go
@@ -0,0 +1,76 @@
+package agent
+
+import (
+ "fmt"
+ "sync"
+)
+
+// Ledger tracks spending across every run of a dispatch (agents and judge)
+// against the --max-cost-usd ceiling. It is checked before each run starts
+// and after each model response.
+type Ledger struct {
+ mu sync.Mutex
+ Cap float64
+ spent float64
+ byModel map[string]float64
+}
+
+// NewLedger returns a ledger with a ceiling. Cap <= 0 means unlimited, which
+// only the fake model uses.
+func NewLedger(cap float64) *Ledger { return &Ledger{Cap: cap, byModel: map[string]float64{}} }
+
+// Add records spending.
+func (l *Ledger) Add(model string, usd float64) {
+ if l == nil {
+ return
+ }
+ l.mu.Lock()
+ defer l.mu.Unlock()
+ l.spent += usd
+ l.byModel[model] += usd
+}
+
+// Spent returns the total.
+func (l *Ledger) Spent() float64 {
+ if l == nil {
+ return 0
+ }
+ l.mu.Lock()
+ defer l.mu.Unlock()
+ return l.spent
+}
+
+// ByModel returns spending per model.
+func (l *Ledger) ByModel() map[string]float64 {
+ out := map[string]float64{}
+ if l == nil {
+ return out
+ }
+ l.mu.Lock()
+ defer l.mu.Unlock()
+ for k, v := range l.byModel {
+ out[k] = v
+ }
+ return out
+}
+
+// Remaining returns what is left under the cap (+Inf semantics when no cap).
+func (l *Ledger) Remaining() float64 {
+ if l == nil || l.Cap <= 0 {
+ return 1e18
+ }
+ l.mu.Lock()
+ defer l.mu.Unlock()
+ return l.Cap - l.spent
+}
+
+// Exhausted reports whether the ceiling is reached.
+func (l *Ledger) Exhausted() bool { return l.Remaining() <= 0 }
+
+// String summarizes the ledger.
+func (l *Ledger) String() string {
+ if l == nil {
+ return "no ledger"
+ }
+ return fmt.Sprintf("$%.4f of $%.2f", l.Spent(), l.Cap)
+}
diff --git a/cli/test/evals/internal/agent/cache.go b/cli/test/evals/internal/agent/cache.go
new file mode 100644
index 00000000..aac6f71a
--- /dev/null
+++ b/cli/test/evals/internal/agent/cache.go
@@ -0,0 +1,69 @@
+package agent
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "os"
+ "path/filepath"
+)
+
+// Cache modes (EVAL_CACHE).
+const (
+ CacheOff = "off"
+ CacheRecord = "record"
+ CacheReplay = "replay"
+)
+
+// CachedChatter records model responses keyed by sha256(model, messages,
+// tools, params) under Dir, and replays them. Replay hits whenever a run sends
+// exactly the same request again (same environment, same tool outputs), so
+// graders and reports can be iterated without spending.
+type CachedChatter struct {
+ Inner Chatter
+ Dir string
+ Mode string
+}
+
+func (c *CachedChatter) key(req *Request) (string, error) {
+ raw, err := json.Marshal(req)
+ if err != nil {
+ return "", err
+ }
+ sum := sha256.Sum256(raw)
+ return hex.EncodeToString(sum[:]), nil
+}
+
+// Chat serves from the cache or the inner client.
+func (c *CachedChatter) Chat(ctx context.Context, req *Request) (*Response, error) {
+ if c.Mode == "" || c.Mode == CacheOff {
+ return c.Inner.Chat(ctx, req)
+ }
+ k, err := c.key(req)
+ if err != nil {
+ return nil, err
+ }
+ path := filepath.Join(c.Dir, k[:2], k+".json")
+ if raw, err := os.ReadFile(path); err == nil {
+ var resp Response
+ if err := json.Unmarshal(raw, &resp); err == nil {
+ // A replayed response cost nothing this time.
+ resp.Usage.Cost = 0
+ return &resp, nil
+ }
+ }
+ if c.Mode == CacheReplay {
+ return nil, fmt.Errorf("model cache miss in replay mode (key %s)", k[:12])
+ }
+ resp, err := c.Inner.Chat(ctx, req)
+ if err != nil {
+ return nil, err
+ }
+ if raw, err := json.Marshal(resp); err == nil {
+ _ = os.MkdirAll(filepath.Dir(path), 0o755)
+ _ = os.WriteFile(path, raw, 0o644)
+ }
+ return resp, nil
+}
diff --git a/cli/test/evals/internal/agent/cache_test.go b/cli/test/evals/internal/agent/cache_test.go
new file mode 100644
index 00000000..b9cd9446
--- /dev/null
+++ b/cli/test/evals/internal/agent/cache_test.go
@@ -0,0 +1,25 @@
+package agent
+
+import (
+ "context"
+ "testing"
+)
+
+func TestCacheRecordReplay(t *testing.T) {
+ dir := t.TempDir()
+ inner := &scripted{resps: []*Response{textResp("first")}}
+ rec := &CachedChatter{Inner: inner, Dir: dir, Mode: CacheRecord}
+ req := &Request{Model: "m", Messages: []Message{{Role: "user", Content: "q"}}}
+ if _, err := rec.Chat(context.Background(), req); err != nil {
+ t.Fatal(err)
+ }
+ rep := &CachedChatter{Inner: &scripted{}, Dir: dir, Mode: CacheReplay}
+ resp, err := rep.Chat(context.Background(), req)
+ if err != nil || resp.Choices[0].Message.Content != "first" || resp.Usage.Cost != 0 {
+ t.Fatalf("replay: %v %+v", err, resp)
+ }
+ other := &Request{Model: "m", Messages: []Message{{Role: "user", Content: "different"}}}
+ if _, err := rep.Chat(context.Background(), other); err == nil {
+ t.Fatal("a replay miss must fail, not call the model")
+ }
+}
diff --git a/cli/test/evals/internal/agent/fake.go b/cli/test/evals/internal/agent/fake.go
index 41bd0e6c..55f47262 100644
--- a/cli/test/evals/internal/agent/fake.go
+++ b/cli/test/evals/internal/agent/fake.go
@@ -14,14 +14,17 @@ import (
"sync"
"github.com/tracewayapp/traceway/cli/test/evals/internal/catalog"
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/ops"
"github.com/tracewayapp/traceway/cli/test/evals/internal/tools"
)
// Fake is an httptest server speaking Chat Completions that plays a case's
// oracle steps, one tool call per turn. It is the only model `go test` can
-// reach.
+// reach. Surface-neutral `op:` steps are rendered as a CLI command on shell
+// arms and as an MCP tool call on MCP arms, so one oracle covers both.
type Fake struct {
- Steps []catalog.Step
+ Steps []catalog.Step
+ CLIBin string // traceway | traceway-write
// FileRoot resolves write_file `from:` paths.
FileRoot string
@@ -31,8 +34,8 @@ type Fake struct {
}
// NewFake starts the fake model.
-func NewFake(steps []catalog.Step, fileRoot string) *Fake {
- f := &Fake{Steps: steps, FileRoot: fileRoot}
+func NewFake(steps []catalog.Step, cliBin, fileRoot string) *Fake {
+ f := &Fake{Steps: steps, CLIBin: cliBin, FileRoot: fileRoot}
f.srv = httptest.NewServer(http.HandlerFunc(f.serve))
return f
}
@@ -99,6 +102,12 @@ func (f *Fake) serve(w http.ResponseWriter, r *http.Request) {
func (f *Fake) render(s catalog.Step, tools map[string]bool, msgs []Message) (string, map[string]any, string, error) {
tmpl := func(v string) (string, error) { return resolveRuntime(v, msgs) }
+ mcpArm := false
+ for name := range tools {
+ if strings.HasPrefix(name, ops.MCPPrefix) {
+ mcpArm = true
+ }
+ }
switch {
case s.Say != "":
return "", nil, s.Say, nil
@@ -107,6 +116,53 @@ func (f *Fake) render(s catalog.Step, tools map[string]bool, msgs []Message) (st
if err != nil {
return "", nil, "", err
}
+ if !mcpArm {
+ return "shell", map[string]any{"command": cmd}, "", nil
+ }
+ op, err := firstTracewayOp(cmd)
+ if err != nil {
+ // Not a Traceway command (git, grep, ...): MCP arms have a
+ // shell too, without the traceway CLI.
+ if tools["shell"] {
+ return "shell", map[string]any{"command": cmd}, "", nil
+ }
+ return "", nil, "", err
+ }
+ return f.mcpCall(op.Op, op.Args, tools)
+ case s.Op != "":
+ args, err := resolveArgs(s.Args, msgs)
+ if err != nil {
+ return "", nil, "", err
+ }
+ if mcpArm {
+ if s.Stdin != "" {
+ in, err := tmpl(s.Stdin)
+ if err != nil {
+ return "", nil, "", err
+ }
+ args["content_md"] = in
+ }
+ return f.mcpCall(s.Op, args, tools)
+ }
+ if _, ok := args["content_md"]; ok && s.Stdin == "" {
+ s.Stdin = fmt.Sprint(args["content_md"])
+ args["content_md"] = "-"
+ }
+ argv, err := ops.RenderCLI(s.Op, args)
+ if err != nil {
+ return "", nil, "", err
+ }
+ if ops.IsWriteOp(s.Op) {
+ argv = append(argv, "--yes")
+ }
+ cmd := f.CLIBin + " " + shellJoin(argv)
+ if s.Stdin != "" {
+ in, err := tmpl(s.Stdin)
+ if err != nil {
+ return "", nil, "", err
+ }
+ cmd += " <<'TWEVALS_EOF'\n" + in + "\nTWEVALS_EOF"
+ }
return "shell", map[string]any{"command": cmd}, "", nil
case s.ReadFile != "":
return need(tools, "read_file", map[string]any{"path": s.ReadFile})
@@ -139,6 +195,8 @@ func (f *Fake) render(s catalog.Step, tools map[string]bool, msgs []Message) (st
return "", nil, "", err
}
return need(tools, "ask_user", map[string]any{"question": q})
+ case s.Resource != "":
+ return need(tools, ops.MCPPrefix+"read_resource", map[string]any{"uri": s.Resource})
case s.Submit != nil:
name := submitTool(tools)
if name == "" {
@@ -146,16 +204,22 @@ func (f *Fake) render(s catalog.Step, tools map[string]bool, msgs []Message) (st
}
args, err := resolveArgs(s.Submit, msgs)
return name, args, "", err
+ case s.Finish != nil:
+ args, err := resolveArgs(s.Finish, msgs)
+ if err != nil {
+ return "", nil, "", err
+ }
+ return need(tools, "finish", args)
}
return "", nil, "", fmt.Errorf("empty oracle step")
}
-// submitTool names the arm's final-answer tool, the first by name so the
-// choice never depends on map order.
+// submitTool names the arm's submit_* tool, the first by name so the choice
+// never depends on map order. A finish step plays finish.
func submitTool(offered map[string]bool) string {
var names []string
for n := range offered {
- if tools.IsSubmit(n) {
+ if tools.IsSubmit(n) && n != tools.Finish {
names = append(names, n)
}
}
@@ -173,6 +237,53 @@ func need(tools map[string]bool, name string, args map[string]any) (string, map[
return name, args, "", nil
}
+func (f *Fake) mcpCall(op string, args map[string]any, tools map[string]bool) (string, map[string]any, string, error) {
+ name, margs, err := ops.RenderMCP(op, args)
+ if err != nil {
+ return "", nil, "", err
+ }
+ full := ops.MCPPrefix + name
+ if !tools[full] {
+ return "", nil, "", fmt.Errorf("this arm has no MCP tool %s", name)
+ }
+ return full, margs, "", nil
+}
+
+// firstTracewayOp parses a shell command and returns its first traceway
+// invocation as an op.
+func firstTracewayOp(cmd string) (ops.Op, error) {
+ inv, err := ops.Invocations(cmd)
+ if err != nil {
+ return ops.Op{}, err
+ }
+ for _, words := range inv {
+ if words[0] == "traceway" || words[0] == "traceway-write" {
+ op := ops.ParseCLI(words[1:])
+ delete(op.Args, "output")
+ delete(op.Args, "yes")
+ return op, nil
+ }
+ }
+ return ops.Op{}, fmt.Errorf("shell step has no traceway invocation to translate for an MCP arm: %s", cmd)
+}
+
+func shellJoin(argv []string) string {
+ out := make([]string, len(argv))
+ for i, a := range argv {
+ out[i] = shellQuote(a)
+ }
+ return strings.Join(out, " ")
+}
+
+var safeWord = regexp.MustCompile(`^[A-Za-z0-9_@%+=:,./-]+$`)
+
+func shellQuote(s string) string {
+ if s != "" && safeWord.MatchString(s) {
+ return s
+ }
+ return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
+}
+
var runtimeRe = regexp.MustCompile(`\{\{\s*((?:last|calls\.c\d+)(?:\.[^}\s]+)?)\s*\}\}`)
// resolveRuntime fills {{last.x}} and {{calls.c03.x}} from the tool results
diff --git a/cli/test/evals/internal/agent/loop.go b/cli/test/evals/internal/agent/loop.go
index 5e3deae3..cf796a25 100644
--- a/cli/test/evals/internal/agent/loop.go
+++ b/cli/test/evals/internal/agent/loop.go
@@ -23,6 +23,11 @@ type Config struct {
Prompt string
SubmitTool string
Budget catalog.Budget
+ // RunCap is this run's share of --max-cost-usd (0 = no cap).
+ RunCap float64
+ Ledger *Ledger
+ // Price estimates cost when the provider reports none.
+ Price catalog.Price
Transcript *Transcript
// AfterCall runs harness hooks after each executed call.
AfterCall func(ctx context.Context, c *record.Call)
@@ -38,6 +43,7 @@ const (
StopBudget = "budget_exhausted"
StopNoSubmit = "no_submit"
StopInfra = "infra_error"
+ StopCeiling = "cost_ceiling"
)
// Run drives the model until it submits, a budget runs out, or it stops
@@ -86,6 +92,14 @@ func Run(ctx context.Context, cfg Config) *record.Run {
if run.Turns >= budget.MaxTurns {
return finish(StopBudget, fmt.Sprintf("max_turns %d reached", budget.MaxTurns))
}
+ // The dispatch ceiling stops new runs from starting (runner.Execute);
+ // a run already in flight finishes, bounded by its own cap.
+ if run.Turns == 0 && cfg.Ledger.Exhausted() {
+ return finish(StopCeiling, "the dispatch's --max-cost-usd ceiling was reached")
+ }
+ if cfg.RunCap > 0 && run.Usage.CostUSD >= cfg.RunCap {
+ return finish(StopBudget, fmt.Sprintf("run cost cap $%.2f reached", cfg.RunCap))
+ }
run.Turns++
req := &Request{
Model: cfg.Model.ID,
@@ -111,8 +125,9 @@ func Run(ctx context.Context, cfg Config) *record.Run {
}
return finish(StopInfra, err.Error())
}
- u := usageOf(resp)
+ u := usageOf(resp, cfg.Price)
run.Usage.Add(u)
+ cfg.Ledger.Add(cfg.Model.Key, u.CostUSD)
outputTokens += u.OutputTokens - u.ReasoningTokens
if resp.Provider != "" {
providers[resp.Provider] = true
@@ -197,8 +212,8 @@ func nonEmpty(s string) string {
return s
}
-func usageOf(r *Response) record.Usage {
- return record.Usage{
+func usageOf(r *Response, p catalog.Price) record.Usage {
+ u := record.Usage{
InputTokens: r.Usage.PromptTokens,
CachedTokens: r.Usage.PromptTokensDetails.CachedTokens,
CacheWriteTokens: r.Usage.PromptTokensDetails.CacheWriteTokens,
@@ -206,4 +221,9 @@ func usageOf(r *Response) record.Usage {
ReasoningTokens: r.Usage.CompletionTokensDetails.ReasoningTokens,
CostUSD: r.Usage.Cost,
}
+ if u.CostUSD == 0 && (u.InputTokens > 0 || u.OutputTokens > 0) && p.Prompt > 0 {
+ uncached := u.InputTokens - u.CachedTokens - u.CacheWriteTokens
+ u.CostUSD = (float64(uncached)*p.Prompt + float64(u.CachedTokens)*p.CacheRead + float64(u.CacheWriteTokens)*p.CacheWrite + float64(u.OutputTokens)*p.Completion) / 1e6
+ }
+ return u
}
diff --git a/cli/test/evals/internal/agent/openrouter.go b/cli/test/evals/internal/agent/openrouter.go
index 042d375e..2abe2b82 100644
--- a/cli/test/evals/internal/agent/openrouter.go
+++ b/cli/test/evals/internal/agent/openrouter.go
@@ -1,6 +1,7 @@
-// Package agent drives a model through the eval: a Chat Completions client,
-// the tool-calling loop with budgets, prompt caching, transcripts, and the
-// fake model that plays oracle trajectories.
+// Package agent drives a model through the eval: an OpenRouter Chat
+// Completions client (the same code path for every model and the judge), the
+// tool-calling loop with budgets, prompt caching, transcripts, a
+// record/replay cache, and the fake model that plays oracle trajectories.
package agent
import (
@@ -15,8 +16,13 @@ import (
"net/url"
"strings"
"time"
+
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/guard"
)
+// OpenRouterURL is the only remote endpoint the harness talks to.
+const OpenRouterURL = "https://openrouter.ai/api/v1"
+
// Message is one Chat Completions message.
type Message struct {
Role string `json:"role"`
@@ -120,8 +126,9 @@ type Chatter interface {
Chat(ctx context.Context, req *Request) (*Response, error)
}
-// Client talks Chat Completions to the fake model on loopback. A
-// non-loopback base URL is always refused: this harness makes no paid calls.
+// Client talks Chat Completions to OpenRouter (or to the fake model on
+// loopback). A non-loopback base URL is refused unless guard.Live() says a
+// paid run was authorized, which is never the case under go test.
type Client struct {
BaseURL string
Key string
@@ -130,8 +137,20 @@ type Client struct {
Retries int
}
-// ErrNotAuthorized is returned for a request to a remote endpoint.
-var ErrNotAuthorized = errors.New("remote model calls are not supported: the harness only talks to the fake model on loopback")
+// ErrNotAuthorized is returned when a paid call was not unlocked.
+var ErrNotAuthorized = errors.New("paid model calls are locked: only `twevals run|resume` unlock them after the manual-run guard passes (never under go test)")
+
+// NewOpenRouter returns the real client. It fails unless the guard unlocked
+// paid calls.
+func NewOpenRouter(key string) (*Client, error) {
+ if !guard.Live() {
+ return nil, ErrNotAuthorized
+ }
+ if key == "" {
+ return nil, errors.New("OPENROUTER_API_KEY is not set")
+ }
+ return &Client{BaseURL: OpenRouterURL, Key: key, HTTP: &http.Client{Timeout: 5 * time.Minute}}, nil
+}
func isLoopback(base string) bool {
u, err := url.Parse(base)
@@ -154,7 +173,7 @@ func (e *RetryableError) Unwrap() error { return e.Err }
// Chat sends a request with retries and exponential backoff.
func (c *Client) Chat(ctx context.Context, req *Request) (*Response, error) {
- if !isLoopback(c.BaseURL) {
+ if !isLoopback(c.BaseURL) && !guard.Live() {
return nil, ErrNotAuthorized
}
retries := c.Retries
diff --git a/cli/test/evals/internal/agent/prompt.go b/cli/test/evals/internal/agent/prompt.go
index 9618c320..5b6fbaa2 100644
--- a/cli/test/evals/internal/agent/prompt.go
+++ b/cli/test/evals/internal/agent/prompt.go
@@ -16,7 +16,10 @@ type PromptInput struct {
Skills []tools.Skill
// Preloaded is the SKILL.md body of a slash-invoked skill.
Preloaded *tools.Skill
- // Environment describes the machine (the CLI installed and logged in).
+ // MCPInstructions are the connected server's instructions.
+ MCPInstructions string
+ // Environment describes the machine (CLI installed and logged in, or an
+ // MCP server connected).
Environment string
}
@@ -40,6 +43,11 @@ Every tool result starts with a line like [call c07]. When you cite evidence, ci
fmt.Fprintf(&b, "- %s: %s\n", s.Name, s.Description)
}
}
+ if in.MCPInstructions != "" {
+ b.WriteString("\n# Connected MCP server: traceway\n\nIts tools are named mcp__traceway__<tool>. The server's instructions:\n\n")
+ b.WriteString(in.MCPInstructions)
+ b.WriteString("\n")
+ }
if in.Preloaded != nil {
fmt.Fprintf(&b, "\n# The user invoked /%s\n\nThe %s skill is loaded; its instructions follow. Its companion files are under .skills/%s/.\n\n", in.Preloaded.Name, in.Preloaded.Name, in.Preloaded.Name)
b.WriteString(in.Preloaded.Body)
diff --git a/cli/test/evals/internal/catalog/case.go b/cli/test/evals/internal/catalog/case.go
index b74d7a49..fe2c193f 100644
--- a/cli/test/evals/internal/catalog/case.go
+++ b/cli/test/evals/internal/catalog/case.go
@@ -17,16 +17,26 @@ import (
)
// Suites, in report order.
-var Suites = []string{"perf"}
+var Suites = []string{"perf", "debug", "write", "setup"}
// Arms. An arm fixes the surface the agent works through and what it is given.
const (
- ArmCLISkill = "cli+skill" // traceway CLI in the shell, traceway skill available
- ArmCLINoSkill = "cli-noskill" // traceway CLI in the shell, no skills (baseline)
+ ArmCLISkill = "cli+skill" // traceway CLI in the shell, traceway skill available
+ ArmCLINoSkill = "cli-noskill" // traceway CLI in the shell, no skills (baseline)
+ ArmMCP = "mcp" // /mcp tools, server instructions, no traceway CLI
+ ArmCLIWriteSkill = "cli-write+skill" // traceway-write CLI, traceway-write skill
+ ArmMCPWrite = "mcp-write" // /mcp/write tools
+ ArmCLISetupSkill = "cli+setup-skill" // container shell, traceway-setup skill
)
// AllArms lists every arm the runner knows.
-var AllArms = []string{ArmCLISkill, ArmCLINoSkill}
+var AllArms = []string{ArmCLISkill, ArmCLINoSkill, ArmMCP, ArmCLIWriteSkill, ArmMCPWrite, ArmCLISetupSkill}
+
+// IsMCPArm reports whether the arm talks to Traceway through MCP tools.
+func IsMCPArm(arm string) bool { return arm == ArmMCP || arm == ArmMCPWrite }
+
+// IsWriteArm reports whether the arm carries a write-capable surface.
+func IsWriteArm(arm string) bool { return arm == ArmCLIWriteSkill || arm == ArmMCPWrite }
// Case is one eval scenario. Strings in Prompt, Expect, Oracle and Hooks may
// contain {{seed.x}} / {{env.x}} placeholders, resolved after seeding.
@@ -44,6 +54,10 @@ type Case struct {
Prompt string `yaml:"prompt"`
Budget Budget `yaml:"budget"`
User []UserRule `yaml:"user"`
+ Credential Credential `yaml:"credential"`
+ AgentAccess *bool `yaml:"agent_access"`
+ Setup *SetupSpec `yaml:"setup"`
+ Hooks []Hook `yaml:"hooks"`
Expect Expect `yaml:"expect"`
Judge JudgeSpec `yaml:"judge"`
Oracle []Step `yaml:"oracle"`
@@ -93,11 +107,68 @@ type UserRule struct {
Once bool `yaml:"once"`
}
+// Credential picks the identity a write-suite run acts as.
+type Credential struct {
+ Scope string `yaml:"scope"` // write (default on write arms) | read
+ Member string `yaml:"member"` // admin (default) | user | readonly
+}
+
+// SetupSpec configures a setup-suite case.
+type SetupSpec struct {
+ Template string `yaml:"template"` // testdata/setup-templates/<template>
+ Path string `yaml:"path"` // fast | plan
+ // Token is what the prompt hands the agent: ingest | setup.
+ Token string `yaml:"token"`
+ // Projects the harness creates up front (fast path) or expects the
+ // approved plan to create (plan path).
+ Projects []SetupProject `yaml:"projects"`
+ // ApproveDrafts makes the harness approve every pending setup draft.
+ ApproveDrafts bool `yaml:"approve_drafts"`
+ // Env is the container environment of the agent's shell and of the
+ // verify step. Values: "@token:<project>" (that project's ingest
+ // token), "@url" (the instance), "@port" (a free port for the app),
+ // or a literal.
+ Env map[string]string `yaml:"env"`
+ // Verify is what the harness does after the agent finishes: build and
+ // start the app against the backend and hit its routes, so state
+ // assertions can check that telemetry arrived.
+ Verify *SetupVerify `yaml:"verify"`
+}
+
+// SetupVerify builds and starts the instrumented app.
+type SetupVerify struct {
+ Dir string `yaml:"dir"`
+ Build string `yaml:"build"`
+ Start string `yaml:"start"`
+ Hit []string `yaml:"hit"`
+ // EnvFiles are sourced (if present) before start, so env files the
+ // agent wrote are honored.
+ EnvFiles []string `yaml:"env_files"`
+ Wait Duration `yaml:"wait"`
+}
+
+// SetupProject is one Traceway project of a setup case.
+type SetupProject struct {
+ Name string `yaml:"name"`
+ Framework string `yaml:"framework"`
+}
+
+// Hook is a harness action triggered after a matching agent call.
+type Hook struct {
+ After CallMatcher `yaml:"after"`
+ Action string `yaml:"action"`
+ Args map[string]any `yaml:"args"`
+ // Times bounds how often the hook fires (default 1).
+ Times int `yaml:"times"`
+}
+
// Expect is everything the deterministic graders check.
type Expect struct {
- Answer *Answer `yaml:"answer"`
- Grounding []string `yaml:"grounding"`
- Calls *Trajectory `yaml:"calls"`
+ Answer *Answer `yaml:"answer"`
+ Grounding []string `yaml:"grounding"`
+ Calls *Trajectory `yaml:"calls"`
+ State []StateAssert `yaml:"state"`
+ Code *CodeSpec `yaml:"code"`
// Report holds regexps the final report text must (or must not) match.
Report *ReportSpec `yaml:"report"`
}
@@ -108,13 +179,21 @@ type ReportSpec struct {
NotContains []string `yaml:"not_contains"`
}
-// Answer is the expected structured answer to submit_perf_diagnosis.
+// Answer is the expected structured answer. Perf fields apply to
+// submit_perf_diagnosis, debug fields to submit_debug_diagnosis.
type Answer struct {
+ // perf
ChecklistItem string `yaml:"checklist_item"`
AnyOf []string `yaml:"any_of"`
Not []string `yaml:"not"`
Locus *Locus `yaml:"locus"`
Onset *Onset `yaml:"onset"`
+ // debug
+ IssueHash string `yaml:"issue_hash"`
+ OccurrenceID string `yaml:"occurrence_id"`
+ Category string `yaml:"category"`
+ Culprit *Culprit `yaml:"culprit"`
+ NeedsInstrumentation *bool `yaml:"needs_instrumentation"`
// MentionsAll lists regexps that must all match somewhere in the answer.
MentionsAll []string `yaml:"mentions_all"`
}
@@ -133,6 +212,15 @@ type Onset struct {
ToleranceMinutes int `yaml:"tolerance_minutes"`
}
+// Culprit is the expected root-cause location.
+type Culprit struct {
+ File string `yaml:"file"`
+ Symbol string `yaml:"symbol"`
+ Line int `yaml:"line"`
+ // LineTolerance defaults to 3.
+ LineTolerance int `yaml:"line_tolerance"`
+}
+
// Trajectory is the tool-call DSL.
type Trajectory struct {
Required []CallMatcher `yaml:"required"`
@@ -152,6 +240,131 @@ type CallMatcher struct {
OK *bool `yaml:"ok"`
}
+// StateAssert is one post-run assertion against the backend or workspace.
+// Kind selects which fields apply; see grade/state.go.
+type StateAssert struct {
+ Kind string `yaml:"kind"`
+ // Common addressing.
+ ID string `yaml:"id"`
+ Endpoint string `yaml:"endpoint"`
+ Path string `yaml:"path"`
+ // Expected values.
+ Status string `yaml:"status"`
+ Enabled *bool `yaml:"enabled"`
+ SnoozedMinutes *Range `yaml:"snoozed_minutes"`
+ Snoozed *bool `yaml:"snoozed"`
+ Equals []string `yaml:"equals"`
+ Among []string `yaml:"among"`
+ Count *Range `yaml:"count"`
+ OffsetMs *Range `yaml:"offset_ms"`
+ Reason string `yaml:"reason"`
+ Contains []string `yaml:"contains"`
+ NotContains []string `yaml:"not_contains"`
+ Title string `yaml:"title"`
+ Tags []string `yaml:"tags"`
+ IncidentID string `yaml:"incident_id"`
+ Exists *bool `yaml:"exists"`
+ Route string `yaml:"route"`
+ HTTPStatus *Range `yaml:"http_status"`
+ Value string `yaml:"value"`
+ Projects []string `yaml:"projects"`
+ Project string `yaml:"project"`
+ Frameworks map[string]string `yaml:"frameworks"`
+ Extra map[string]any `yaml:"extra"`
+}
+
+// Range is an inclusive numeric interval; a scalar YAML value sets both ends.
+type Range struct {
+ Min *float64 `yaml:"min"`
+ Max *float64 `yaml:"max"`
+}
+
+// UnmarshalYAML accepts `5` as {min: 5, max: 5}. A bound that is still a
+// {{template}} (the case is loaded before seeding) is left unset; Resolve
+// decodes the case again after filling it in.
+func (r *Range) UnmarshalYAML(n *yaml.Node) error {
+ if n.Kind == yaml.ScalarNode && strings.Contains(n.Value, "{{") {
+ return nil
+ }
+ if n.Kind == yaml.MappingNode {
+ var raw map[string]yaml.Node
+ if err := n.Decode(&raw); err != nil {
+ return err
+ }
+ for k, v := range raw {
+ if v.Kind == yaml.ScalarNode && strings.Contains(v.Value, "{{") {
+ continue
+ }
+ var f float64
+ if err := v.Decode(&f); err != nil {
+ return fmt.Errorf("range %s: %w", k, err)
+ }
+ switch k {
+ case "min":
+ r.Min = &f
+ case "max":
+ r.Max = &f
+ default:
+ return fmt.Errorf("range: unknown key %q", k)
+ }
+ }
+ return nil
+ }
+ if n.Kind == yaml.ScalarNode {
+ var v float64
+ if err := n.Decode(&v); err != nil {
+ return err
+ }
+ r.Min, r.Max = &v, &v
+ return nil
+ }
+ type plain Range
+ return n.Decode((*plain)(r))
+}
+
+// Contains reports whether v lies in the range.
+func (r *Range) Contains(v float64) bool {
+ if r == nil {
+ return true
+ }
+ if r.Min != nil && v < *r.Min {
+ return false
+ }
+ if r.Max != nil && v > *r.Max {
+ return false
+ }
+ return true
+}
+
+func (r *Range) String() string {
+ if r == nil {
+ return "any"
+ }
+ lo, hi := "-inf", "+inf"
+ if r.Min != nil {
+ lo = fmt.Sprint(*r.Min)
+ }
+ if r.Max != nil {
+ hi = fmt.Sprint(*r.Max)
+ }
+ return "[" + lo + ", " + hi + "]"
+}
+
+// CodeSpec verifies a code change with hidden tests.
+type CodeSpec struct {
+ HiddenTests string `yaml:"hidden_tests"` // testdata/hidden/<name>
+ Command string `yaml:"command"`
+ Gating bool `yaml:"gating"`
+ Timeout Duration `yaml:"timeout"`
+ // TouchesCulprit requires the diff to change the culprit file.
+ TouchesCulprit bool `yaml:"touches_culprit"`
+ // MustChange / MustNotChange are path globs checked against the diff.
+ MustChange []string `yaml:"must_change"`
+ MustNotChange []string `yaml:"must_not_change"`
+ // Static marks a fixture that cannot run here; only the diff is graded.
+ Static bool `yaml:"static"`
+}
+
// JudgeSpec is the LLM judge's rubric. Expected is shown to the judge only.
type JudgeSpec struct {
Rubric []string `yaml:"rubric"`
@@ -161,6 +374,8 @@ type JudgeSpec struct {
// Step is one oracle action, played by the fake model.
type Step struct {
Shell string `yaml:"shell"`
+ Op string `yaml:"op"`
+ Args map[string]any `yaml:"args"`
Stdin string `yaml:"stdin"`
ReadFile string `yaml:"read_file"`
ListFiles string `yaml:"list_files"`
@@ -169,7 +384,9 @@ type Step struct {
EditFile *EditStep `yaml:"edit_file"`
LoadSkill string `yaml:"load_skill"`
AskUser string `yaml:"ask_user"`
+ Resource string `yaml:"read_resource"`
Submit map[string]any `yaml:"submit"`
+ Finish map[string]any `yaml:"finish"`
Say string `yaml:"say"`
}
@@ -257,10 +474,18 @@ func (c *Case) Validate() error {
add("oracle_by_arm names arm %q that the case does not run", arm)
}
}
- if c.Expect.Answer == nil {
- add("%s cases need expect.answer", c.Suite)
+ if c.Suite == "perf" || c.Suite == "debug" {
+ if c.Expect.Answer == nil {
+ add("%s cases need expect.answer", c.Suite)
+ }
+ }
+ if (c.Suite == "write" || c.Suite == "setup") && len(c.Expect.State) == 0 {
+ add("%s cases need expect.state", c.Suite)
+ }
+ if c.Suite == "setup" && c.Setup == nil {
+ add("setup cases need a setup block")
}
- if c.Distractor == "" && !slices.Contains(c.Tags, "control") {
+ if c.Suite != "setup" && c.Distractor == "" && !slices.Contains(c.Tags, "control") {
add("distractor is required (or tag the case control)")
}
if len(errs) > 0 {
diff --git a/cli/test/evals/internal/catalog/models.go b/cli/test/evals/internal/catalog/models.go
index da98cd61..6af8d8f6 100644
--- a/cli/test/evals/internal/catalog/models.go
+++ b/cli/test/evals/internal/catalog/models.go
@@ -30,10 +30,11 @@ type Model struct {
// MaxTokensPerTurn caps one completion.
MaxTokensPerTurn int `yaml:"max_tokens_per_turn"`
Temperature *float64 `yaml:"temperature"`
- // Estimate is the per-run token envelope for estimating a paid run's
- // cost. The oracle harness does not read it.
+ // Estimate is the per-run token envelope used by the preflight until
+ // baseline.json carries measured averages.
Estimate Estimate `yaml:"estimate"`
- // FallbackPrice is USD per million tokens, for the same estimate.
+ // FallbackPrice (USD per million tokens) is used when the live price
+ // list cannot be fetched. The preflight says when it fell back.
FallbackPrice Price `yaml:"fallback_price"`
}
diff --git a/cli/test/evals/internal/catalog/select.go b/cli/test/evals/internal/catalog/select.go
index 25fba0a9..aa414f83 100644
--- a/cli/test/evals/internal/catalog/select.go
+++ b/cli/test/evals/internal/catalog/select.go
@@ -59,6 +59,7 @@ type Selection struct {
Suites string // comma-separated
Tags string // comma-separated, all must match
Arms string // comma-separated
+ Shard string // one suite
}
// Unit is one (case, arm) pair; the runner multiplies it by models and runs.
@@ -101,6 +102,9 @@ func Select(cases []*Case, presets *Presets, sel Selection) ([]Unit, error) {
return nil, fmt.Errorf("unknown suite %q", s)
}
}
+ if sel.Shard != "" && !slices.Contains(Suites, sel.Shard) {
+ return nil, fmt.Errorf("unknown shard %q (shards are suites)", sel.Shard)
+ }
var out []Unit
matchedGlob := map[string]bool{}
@@ -123,6 +127,9 @@ func Select(cases []*Case, presets *Presets, sel Selection) ([]Unit, error) {
if len(suites) > 0 && !slices.Contains(suites, c.Suite) {
continue
}
+ if sel.Shard != "" && c.Suite != sel.Shard {
+ continue
+ }
if !hasAll(c, tags) {
continue
}
diff --git a/cli/test/evals/internal/env/env_write_test.go b/cli/test/evals/internal/env/env_write_test.go
new file mode 100644
index 00000000..7400d91a
--- /dev/null
+++ b/cli/test/evals/internal/env/env_write_test.go
@@ -0,0 +1,91 @@
+package env
+
+import (
+ "context"
+ "encoding/json"
+ "net/http"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "testing"
+ "time"
+)
+
+// TestWriteGate is the Phase 1 acceptance check for writes: a write token is
+// refused while the organization's agent write opt-in is off and accepted once
+// it is on, a read token is refused, members join through invitations, and the
+// agent activity log records the refused and the accepted write.
+func TestWriteGate(t *testing.T) {
+ bins := BinariesForTest(t)
+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+ defer cancel()
+ dir := t.TempDir()
+ b, err := Boot(ctx, bins.Backend, BootOptions{Dir: filepath.Join(dir, "backend")})
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer b.Stop()
+ admin, err := Register(ctx, b, "[email protected]", "eval-password-1", "Eval Org")
+ if err != nil {
+ t.Fatal(err)
+ }
+ projects, err := CreateProjects(ctx, b, admin, []Project{{Name: "api", Framework: "opentelemetry"}})
+ if err != nil {
+ t.Fatal(err)
+ }
+ pid := projects[0].ID
+ readPAT, err := CreatePAT(ctx, b, admin.JWT, "eval-read", "read", 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ writePAT, err := CreatePAT(ctx, b, admin.JWT, "eval-write", "write", 1)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !strings.HasPrefix(writePAT, "tww_") {
+ t.Errorf("unexpected write token prefix %q", writePAT[:4])
+ }
+
+ write := b.API().WithBearer(writePAT)
+ slow := map[string]any{"endpoint": "GET /x", "offsetMs": 100, "reason": "test"}
+ res, err := write.Do(ctx, http.MethodPost, "/api/endpoints/slow", ProjectQuery(pid), slow)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if res.Status != http.StatusForbidden || !strings.Contains(string(res.Body), "Agent write access is turned off") {
+ t.Errorf("write with opt-in off: want 403 opt-in message, got %d %s", res.Status, res.Body)
+ }
+ if err := SetAgentAccess(ctx, b, admin, true); err != nil {
+ t.Fatal(err)
+ }
+ res, err = write.Do(ctx, http.MethodPost, "/api/endpoints/slow", ProjectQuery(pid), slow)
+ if err != nil || res.Status != http.StatusOK {
+ t.Errorf("write with opt-in on: want 200, got %v %d %s", err, res.Status, res.Body)
+ }
+ res, _ = b.API().WithBearer(readPAT).Do(ctx, http.MethodPost, "/api/endpoints/slow", ProjectQuery(pid), slow)
+ if res.Status != http.StatusForbidden || !strings.Contains(string(res.Body), "read-only") {
+ t.Errorf("read token write: want 403 read-only, got %d %s", res.Status, res.Body)
+ }
+
+ m, err := AddMember(ctx, b, admin, "[email protected]", "readonly")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if m.UserID == 0 || m.JWT == "" {
+ t.Errorf("member without id or jwt: %+v", m)
+ }
+
+ act, err := b.API().WithBearer(admin.JWT).MustOK(ctx, http.MethodGet, "/api/organizations/"+strconv.Itoa(admin.OrgID)+"/agent-activity", nil, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var page struct {
+ Data []struct {
+ StatusCode int `json:"statusCode"`
+ } `json:"data"`
+ }
+ _ = json.Unmarshal(act.Body, &page)
+ if len(page.Data) != 2 {
+ t.Errorf("agent activity: want the refused and the accepted write, got %s", act.Body)
+ }
+}
diff --git a/cli/test/evals/internal/env/members.go b/cli/test/evals/internal/env/members.go
new file mode 100644
index 00000000..c7b56b1d
--- /dev/null
+++ b/cli/test/evals/internal/env/members.go
@@ -0,0 +1,102 @@
+package env
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "net/url"
+ "time"
+)
+
+// Member is a non-owner user of the organization.
+type Member struct {
+ Email string
+ Role string
+ UserID int
+ JWT string
+}
+
+// AddMember invites a user with role and accepts the invitation. The accept
+// token is not in the invite response; with SMTP off the backend logs the
+// invitation email, link included, so it is read from there.
+func AddMember(ctx context.Context, b *Backend, a *Admin, email, role string) (*Member, error) {
+ before := len(b.InvitationTokens())
+ api := b.API().WithBearer(a.JWT)
+ if _, err := api.MustOK(ctx, http.MethodPost, fmt.Sprintf("/api/organizations/%d/invitations", a.OrgID), nil,
+ map[string]string{"email": email, "role": role}); err != nil {
+ return nil, err
+ }
+ var token string
+ deadline := time.Now().Add(10 * time.Second)
+ for time.Now().Before(deadline) {
+ if toks := b.InvitationTokens(); len(toks) > before {
+ token = toks[len(toks)-1]
+ break
+ }
+ time.Sleep(50 * time.Millisecond)
+ }
+ if token == "" {
+ return nil, fmt.Errorf("invitation for %s: no accept token in the backend log", email)
+ }
+ res, err := b.API().MustOK(ctx, http.MethodPost, "/api/invitations/"+url.PathEscape(token)+"/accept", nil,
+ map[string]string{"name": "Eval " + role, "password": randomHex(12)})
+ if err != nil {
+ return nil, err
+ }
+ var out struct {
+ Token string `json:"token"`
+ User struct {
+ ID int `json:"id"`
+ } `json:"user"`
+ }
+ if err := res.JSON(&out); err != nil {
+ return nil, err
+ }
+ return &Member{Email: email, Role: role, UserID: out.User.ID, JWT: out.Token}, nil
+}
+
+// SetAgentAccess turns the organization's agent write opt-in on or off.
+func SetAgentAccess(ctx context.Context, b *Backend, a *Admin, enabled bool) error {
+ _, err := b.API().WithBearer(a.JWT).MustOK(ctx, http.MethodPut,
+ fmt.Sprintf("/api/organizations/%d/agent-access", a.OrgID), nil, map[string]bool{"agentWriteEnabled": enabled})
+ return err
+}
+
+// SetupToken mints an organization setup token (tws_...).
+func SetupToken(ctx context.Context, b *Backend, a *Admin) (string, error) {
+ res, err := b.API().WithBearer(a.JWT).MustOK(ctx, http.MethodPost, "/api/setup-tokens", nil, map[string]int{"organizationId": a.OrgID})
+ if err != nil {
+ return "", err
+ }
+ var out struct {
+ Token string `json:"token"`
+ }
+ if err := res.JSON(&out); err != nil {
+ return "", err
+ }
+ return out.Token, nil
+}
+
+// ApprovePendingDraft approves the organization's pending setup draft, if
+// any, and reports whether one was approved.
+func ApprovePendingDraft(ctx context.Context, b *Backend, a *Admin) (bool, error) {
+ api := b.API().WithBearer(a.JWT)
+ res, err := api.MustOK(ctx, http.MethodGet, "/api/setup/drafts", url.Values{"organizationId": {fmt.Sprint(a.OrgID)}}, nil)
+ if err != nil {
+ return false, err
+ }
+ var out struct {
+ Draft *struct {
+ ID any `json:"id"`
+ Status string `json:"status"`
+ } `json:"draft"`
+ }
+ if err := res.JSON(&out); err != nil {
+ return false, err
+ }
+ if out.Draft == nil || out.Draft.Status != "pending" {
+ return false, nil
+ }
+ _, err = api.MustOK(ctx, http.MethodPost, fmt.Sprintf("/api/setup/drafts/%v/approve", out.Draft.ID), nil, nil)
+ return err == nil, err
+}
diff --git a/cli/test/evals/internal/grade/answer.go b/cli/test/evals/internal/grade/answer.go
index 0542a1d2..56b8b616 100644
--- a/cli/test/evals/internal/grade/answer.go
+++ b/cli/test/evals/internal/grade/answer.go
@@ -54,6 +54,8 @@ func Answer(in *Input) *Verdict {
if exp.Onset != nil && exp.Onset.At != "" {
gradeOnset(v, sub["onset"], exp.Onset)
}
+ case "debug":
+ answerDebug(v, exp, sub)
}
if len(exp.MentionsAll) > 0 {
raw, _ := json.Marshal(sub)
diff --git a/cli/test/evals/internal/grade/answer_debug.go b/cli/test/evals/internal/grade/answer_debug.go
new file mode 100644
index 00000000..44ebc44e
--- /dev/null
+++ b/cli/test/evals/internal/grade/answer_debug.go
@@ -0,0 +1,73 @@
+package grade
+
+import (
+ "path/filepath"
+ "strings"
+
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/catalog"
+)
+
+// answerDebug grades a submit_debug_diagnosis payload.
+func answerDebug(v *Verdict, exp *catalog.Answer, sub map[string]any) {
+ str := func(m map[string]any, k string) string { s, _ := m[k].(string); return strings.TrimSpace(s) }
+ if exp.Category != "" && str(sub, "category") != exp.Category {
+ v.fail("category %q, want %q", str(sub, "category"), exp.Category)
+ }
+ if exp.IssueHash != "" && !strings.EqualFold(str(sub, "issue_hash"), exp.IssueHash) {
+ v.fail("issue_hash %q, want %q", str(sub, "issue_hash"), exp.IssueHash)
+ }
+ if exp.OccurrenceID != "" && !strings.EqualFold(str(sub, "occurrence_id"), exp.OccurrenceID) {
+ v.fail("occurrence_id %q, want %q", str(sub, "occurrence_id"), exp.OccurrenceID)
+ }
+ if exp.Culprit != nil {
+ c, _ := sub["culprit"].(map[string]any)
+ if c == nil {
+ c = map[string]any{}
+ }
+ if exp.Culprit.File != "" && !samePath(str(c, "file"), exp.Culprit.File) {
+ v.fail("culprit.file %q, want %q", str(c, "file"), exp.Culprit.File)
+ }
+ if exp.Culprit.Symbol != "" && !sameSymbol(str(c, "symbol"), exp.Culprit.Symbol) {
+ v.fail("culprit.symbol %q, want %q", str(c, "symbol"), exp.Culprit.Symbol)
+ }
+ if exp.Culprit.Line > 0 {
+ tol := exp.Culprit.LineTolerance
+ if tol == 0 {
+ tol = 3
+ }
+ line, _ := toFloat(c["line"])
+ if d := int(line) - exp.Culprit.Line; d < -tol || d > tol {
+ v.fail("culprit.line %d, want %d±%d", int(line), exp.Culprit.Line, tol)
+ }
+ }
+ }
+ if exp.NeedsInstrumentation != nil {
+ got, _ := sub["needs_instrumentation"].(bool)
+ if got != *exp.NeedsInstrumentation {
+ v.fail("needs_instrumentation %v, want %v", got, *exp.NeedsInstrumentation)
+ }
+ }
+}
+
+func samePath(got, want string) bool {
+ g := filepath.ToSlash(filepath.Clean(strings.TrimPrefix(strings.TrimSpace(got), "./")))
+ w := filepath.ToSlash(filepath.Clean(want))
+ return g == w || strings.HasSuffix(g, "/"+w) || strings.HasSuffix(w, "/"+g) && strings.Contains(g, "/")
+}
+
+// sameSymbol accepts "(*OrderHandler).List", "OrderHandler.List" or "List"
+// for an expected "List" (and the reverse).
+func sameSymbol(got, want string) bool {
+ last := func(s string) string {
+ s = strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(s), "()"))
+ // Swift and Kotlin accessor frames: chatURL.getter, value.<get-x>.
+ for _, suf := range []string{".getter", ".setter", ".modify", ".init", ".deinit"} {
+ s = strings.TrimSuffix(s, suf)
+ }
+ if i := strings.LastIndexAny(s, ".:#"); i >= 0 {
+ s = s[i+1:]
+ }
+ return strings.ToLower(strings.Trim(s, "()*"))
+ }
+ return last(got) != "" && last(got) == last(want)
+}
diff --git a/cli/test/evals/internal/grade/grade.go b/cli/test/evals/internal/grade/grade.go
index 4ba9b027..7d905a6e 100644
--- a/cli/test/evals/internal/grade/grade.go
+++ b/cli/test/evals/internal/grade/grade.go
@@ -1,5 +1,7 @@
// Package grade scores one run. Deterministic graders gate the result:
-// answer, trajectory, grounding and the report checks.
+// answer, trajectory, grounding, state and hidden tests. The LLM judge scores
+// report quality and does not gate until it is calibrated against human
+// labels.
package grade
import (
@@ -20,6 +22,7 @@ type Verdict struct {
Score *float64 `json:"score,omitempty"`
Failures []string `json:"failures,omitempty"`
Notes []string `json:"notes,omitempty"`
+ CostUSD float64 `json:"costUsd,omitempty"`
}
func (v *Verdict) fail(format string, a ...any) {
@@ -33,12 +36,35 @@ func newVerdict(name string, gating bool) *Verdict {
return &Verdict{Grader: name, Pass: true, Gating: gating}
}
+// StateReader reads backend state for the state grader. The runner
+// implements it over the instance's admin API.
+type StateReader interface {
+ Get(ctx context.Context, path string, query map[string]string) (int, []byte, error)
+ Post(ctx context.Context, path string, query map[string]string, body any) (int, []byte, error)
+ ProjectID(name string) string
+ PrimaryProject() string
+ OrgID() int
+ Tokens() []string
+}
+
// Input is everything the graders see.
type Input struct {
- Case *catalog.Case // templates resolved
- Arm string
- Run *record.Run
- Seed map[string]any
+ Case *catalog.Case // templates resolved
+ Arm string
+ Run *record.Run
+ Seed map[string]any
+ Workspace string
+ State StateReader
+ // FakeModel is set for oracle runs: hidden tests may run on the host,
+ // and the judge is skipped.
+ FakeModel bool
+ // AllowHostTests lets hidden tests of a real model's patch run on the
+ // host when no container runtime is available.
+ AllowHostTests bool
+ // SandboxImage runs hidden tests in a container.
+ SandboxImage string
+ EvalsRoot string
+ Judge *Judge
}
// All runs every applicable grader in a fixed order.
@@ -50,12 +76,21 @@ func All(ctx context.Context, in *Input) []*Verdict {
if in.Case.Expect.Calls != nil {
out = append(out, Trajectory(in.Run, in.Case.Expect.Calls))
}
- if in.Case.Suite == "perf" || len(in.Case.Expect.Grounding) > 0 {
+ if in.Case.Suite == "perf" || in.Case.Suite == "debug" || len(in.Case.Expect.Grounding) > 0 {
out = append(out, Grounding(in))
}
+ if len(in.Case.Expect.State) > 0 {
+ out = append(out, State(ctx, in))
+ }
+ if in.Case.Expect.Code != nil {
+ out = append(out, HiddenTests(ctx, in))
+ }
if in.Case.Expect.Report != nil {
out = append(out, Report(in))
}
+ if len(in.Case.Judge.Rubric) > 0 {
+ out = append(out, in.Judge.Grade(ctx, in))
+ }
return out
}
diff --git a/cli/test/evals/internal/grade/grounding.go b/cli/test/evals/internal/grade/grounding.go
index 374982ce..58741f5c 100644
--- a/cli/test/evals/internal/grade/grounding.go
+++ b/cli/test/evals/internal/grade/grounding.go
@@ -44,7 +44,7 @@ func Grounding(in *Input) *Verdict {
cited = append(cited, c)
}
}
- } else if in.Case.Suite == "perf" {
+ } else if in.Case.Suite == "perf" || in.Case.Suite == "debug" {
v.fail("the submission cites no evidence")
}
for _, key := range in.Case.Expect.Grounding {
diff --git a/cli/test/evals/internal/grade/hiddentests.go b/cli/test/evals/internal/grade/hiddentests.go
new file mode 100644
index 00000000..144e0d12
--- /dev/null
+++ b/cli/test/evals/internal/grade/hiddentests.go
@@ -0,0 +1,173 @@
+package grade
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "io/fs"
+ "os"
+ "os/exec"
+ "path"
+ "path/filepath"
+ "strings"
+ "time"
+)
+
+// HiddenTests verifies a code change: it records the diff, copies the case's
+// hidden tests (testdata/hidden/<name>/) into the workspace only after the
+// agent finished, runs the case command, and checks the diff touched the
+// culprit. A real model's patch runs in a container with no network; it runs
+// on the host only for oracle runs (our own code) or when AllowHostTests is
+// set explicitly.
+func HiddenTests(ctx context.Context, in *Input) *Verdict {
+ spec := in.Case.Expect.Code
+ v := newVerdict("hidden_tests", spec.Gating)
+ changed, err := changedFiles(in.Workspace)
+ if err != nil {
+ v.fail("read the diff: %v", err)
+ return v
+ }
+ v.note("changed files: %s", strings.Join(changed, ", "))
+ if spec.TouchesCulprit && in.Case.Expect.Answer != nil && in.Case.Expect.Answer.Culprit != nil {
+ want := in.Case.Expect.Answer.Culprit.File
+ hit := false
+ for _, c := range changed {
+ if samePath(c, want) {
+ hit = true
+ }
+ }
+ if !hit {
+ v.fail("the diff does not touch the culprit file %s", want)
+ }
+ }
+ for _, g := range spec.MustChange {
+ if !anyGlob(g, changed) {
+ v.fail("the diff does not change %s", g)
+ }
+ }
+ for _, g := range spec.MustNotChange {
+ if anyGlob(g, changed) {
+ v.fail("the diff must not change %s", g)
+ }
+ }
+ if spec.Static || spec.Command == "" {
+ return v
+ }
+ if spec.HiddenTests != "" {
+ src := filepath.Join(in.EvalsRoot, "testdata", "hidden", spec.HiddenTests)
+ if err := copyTree(src, in.Workspace); err != nil {
+ v.fail("copy hidden tests: %v", err)
+ return v
+ }
+ }
+ timeout := spec.Timeout.D()
+ if timeout == 0 {
+ timeout = 5 * time.Minute
+ }
+ ctx, cancel := context.WithTimeout(ctx, timeout)
+ defer cancel()
+ var cmd *exec.Cmd
+ switch {
+ case in.FakeModel || in.AllowHostTests:
+ cmd = exec.CommandContext(ctx, "sh", "-c", spec.Command)
+ cmd.Dir = in.Workspace
+ cmd.Env = hostTestEnv(in.Workspace)
+ case in.SandboxImage != "" && dockerAvailable():
+ cmd = exec.CommandContext(ctx, "docker", "run", "--rm", "--network", "none",
+ "-v", in.Workspace+":/work", "-w", "/work", "-e", "HOME=/tmp", "-e", "GOFLAGS=-mod=mod", "-e", "GOCACHE=/tmp/gocache",
+ in.SandboxImage, "sh", "-c", spec.Command)
+ default:
+ v.Skipped = true
+ v.note("skipped: no container runtime for the model's patch (set TWEVALS_HOST_TESTS=1 to run it on the host)")
+ return v
+ }
+ var out bytes.Buffer
+ cmd.Stdout, cmd.Stderr = &out, &out
+ err = cmd.Run()
+ tail := out.String()
+ if len(tail) > 3000 {
+ tail = tail[len(tail)-3000:]
+ }
+ if err != nil {
+ v.fail("%s failed: %v\n%s", spec.Command, err, tail)
+ return v
+ }
+ v.note("%s passed", spec.Command)
+ return v
+}
+
+func hostTestEnv(work string) []string {
+ home := filepath.Join(work, ".twevals-home")
+ _ = os.MkdirAll(home, 0o755)
+ env := []string{"HOME=" + home, "PATH=" + os.Getenv("PATH"), "TZ=UTC", "LANG=C.UTF-8", "GOFLAGS=-mod=mod", "GOTOOLCHAIN=local"}
+ for _, k := range []string{"GOCACHE", "GOMODCACHE", "GOPATH"} {
+ if v := os.Getenv(k); v != "" {
+ env = append(env, k+"="+v)
+ }
+ }
+ if os.Getenv("GOCACHE") == "" {
+ if d, err := os.UserCacheDir(); err == nil {
+ env = append(env, "GOCACHE="+filepath.Join(d, "go-build"))
+ }
+ }
+ return env
+}
+
+func dockerAvailable() bool {
+ if _, err := exec.LookPath("docker"); err != nil {
+ return false
+ }
+ return exec.Command("docker", "info").Run() == nil
+}
+
+func changedFiles(work string) ([]string, error) {
+ run := func(args ...string) ([]string, error) {
+ out, err := workspaceGit(work, args...).Output()
+ if err != nil {
+ return nil, fmt.Errorf("git %s: %w", strings.Join(args, " "), err)
+ }
+ var lines []string
+ for _, l := range strings.Split(strings.TrimSpace(string(out)), "\n") {
+ if l != "" && !strings.HasPrefix(l, ".twevals") {
+ lines = append(lines, l)
+ }
+ }
+ return lines, nil
+ }
+ mod, err := run("diff", "--name-only", "HEAD")
+ if err != nil {
+ return nil, err
+ }
+ untracked, err := run("ls-files", "-o", "--exclude-standard")
+ if err != nil {
+ return nil, err
+ }
+ return append(mod, untracked...), nil
+}
+
+func anyGlob(g string, files []string) bool {
+ for _, f := range files {
+ if ok, _ := path.Match(g, f); ok || globAny(g, f) || f == g {
+ return true
+ }
+ }
+ return false
+}
+
+func copyTree(src, dst string) error {
+ return filepath.WalkDir(src, func(p string, d fs.DirEntry, err error) error {
+ if err != nil {
+ return err
+ }
+ rel, _ := filepath.Rel(src, p)
+ target := filepath.Join(dst, rel)
+ if d.IsDir() {
+ return os.MkdirAll(target, 0o755)
+ }
+ raw, err := os.ReadFile(p)
+ if err != nil {
+ return err
+ }
+ return os.WriteFile(target, raw, 0o644)
+ })
+}
diff --git a/cli/test/evals/internal/grade/judge.go b/cli/test/evals/internal/grade/judge.go
new file mode 100644
index 00000000..bb81918b
--- /dev/null
+++ b/cli/test/evals/internal/grade/judge.go
@@ -0,0 +1,144 @@
+package grade
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "strings"
+
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/agent"
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/catalog"
+)
+
+// Judge scores the final report against the case rubric with an LLM (Claude
+// Haiku through OpenRouter, no tools, strict JSON). It never gates until
+// calibrated against human labels; its cost counts toward the
+// dispatch's cap.
+type Judge struct {
+ Chat agent.Chatter
+ Model *catalog.Model
+ Ledger *agent.Ledger
+}
+
+const judgeSystem = `You grade an AI agent's work against a rubric. You see the user's request, the expected outcome (hidden from the agent), the agent's final report and structured answer, and a condensed log of the tool calls it made.
+For each rubric criterion decide pass, fail, or unknown (the report and evidence do not let you decide). Judge only what the report and evidence show; do not reward confident claims the evidence does not support.
+Answer with JSON only, exactly this shape: {"criteria":[{"id":1,"verdict":"pass","reason":"one sentence"}]}`
+
+// Grade runs the judge. A nil judge or an oracle run is skipped.
+func (j *Judge) Grade(ctx context.Context, in *Input) *Verdict {
+ v := newVerdict("judge", false)
+ if j == nil || in.FakeModel {
+ v.Skipped = true
+ v.note("skipped (no judge for oracle runs, or --judge=false)")
+ return v
+ }
+ if j.Ledger.Exhausted() {
+ v.Skipped = true
+ v.note("skipped: cost ceiling reached")
+ return v
+ }
+ var b strings.Builder
+ fmt.Fprintf(&b, "Case: %s\n\nThe user asked:\n%s\n\n", in.Case.Title, in.Run.Prompt)
+ if in.Case.Judge.Expected != "" {
+ fmt.Fprintf(&b, "Expected outcome (hidden from the agent):\n%s\n\n", in.Case.Judge.Expected)
+ }
+ b.WriteString("Rubric:\n")
+ for i, r := range in.Case.Judge.Rubric {
+ fmt.Fprintf(&b, "%d. %s\n", i+1, r)
+ }
+ fmt.Fprintf(&b, "\nAgent's final report:\n%s\n", FinalText(in.Run))
+ if in.Run.Submission != nil {
+ sub := map[string]any{}
+ for k, val := range in.Run.Submission {
+ if k != "report" && k != "summary" {
+ sub[k] = val
+ }
+ }
+ raw, _ := json.MarshalIndent(sub, "", " ")
+ fmt.Fprintf(&b, "\nStructured answer:\n%s\n", raw)
+ }
+ b.WriteString("\nTool calls (condensed):\n")
+ for _, c := range in.Run.Calls {
+ out := c.Output
+ if len(out) > 700 {
+ out = out[:700] + "..."
+ }
+ var opNames []string
+ for _, op := range c.Ops {
+ opNames = append(opNames, op.Op.Op)
+ }
+ fmt.Fprintf(&b, "- %s %s [%s] ok=%v: %s\n", c.ID, c.Tool, strings.Join(opNames, ","), c.OK, strings.ReplaceAll(out, "\n", " "))
+ }
+ if len(in.Run.Questions) > 0 {
+ b.WriteString("\nQuestions the agent asked the user:\n")
+ for _, q := range in.Run.Questions {
+ fmt.Fprintf(&b, "- Q: %s\n A: %s\n", clipS(q.Question, 1500), q.Answer)
+ }
+ }
+ req := &agent.Request{
+ Model: j.Model.ID,
+ Messages: []agent.Message{
+ {Role: "system", Content: judgeSystem},
+ {Role: "user", Content: b.String()},
+ },
+ MaxTokens: j.Model.MaxTokensPerTurn,
+ Temperature: j.Model.Temperature,
+ ResponseFormat: map[string]any{"type": "json_object"},
+ Usage: map[string]any{"include": true},
+ }
+ if len(j.Model.Provider.Order) > 0 {
+ req.Provider = &agent.ProviderPrefs{Order: j.Model.Provider.Order, AllowFallbacks: j.Model.Provider.AllowFallbacks}
+ }
+ resp, err := j.Chat.Chat(ctx, req)
+ if err != nil {
+ v.Skipped = true
+ v.note("judge call failed: %v", err)
+ return v
+ }
+ v.CostUSD = resp.Usage.Cost
+ j.Ledger.Add("judge", resp.Usage.Cost)
+ text, _ := resp.Choices[0].Message.Content.(string)
+ var out struct {
+ Criteria []struct {
+ ID int `json:"id"`
+ Verdict string `json:"verdict"`
+ Reason string `json:"reason"`
+ } `json:"criteria"`
+ }
+ if err := json.Unmarshal([]byte(extractJSON(text)), &out); err != nil {
+ v.Skipped = true
+ v.note("judge returned no JSON: %s", clipS(text, 300))
+ return v
+ }
+ pass, fail := 0, 0
+ for _, c := range out.Criteria {
+ switch c.Verdict {
+ case "pass":
+ pass++
+ case "fail":
+ fail++
+ v.Failures = append(v.Failures, fmt.Sprintf("criterion %d: %s", c.ID, c.Reason))
+ default:
+ v.note("criterion %d unknown: %s", c.ID, c.Reason)
+ }
+ }
+ if pass+fail > 0 {
+ s := float64(pass) / float64(pass+fail)
+ v.Score = &s
+ v.Pass = s >= 0.5
+ } else {
+ v.Skipped = true
+ }
+ return v
+}
+
+func extractJSON(s string) string {
+ s = strings.TrimSpace(s)
+ s = strings.TrimPrefix(s, "```json")
+ s = strings.TrimPrefix(s, "```")
+ s = strings.TrimSuffix(s, "```")
+ if i := strings.Index(s, "{"); i > 0 {
+ s = s[i:]
+ }
+ return strings.TrimSpace(s)
+}
diff --git a/cli/test/evals/internal/grade/judge_test.go b/cli/test/evals/internal/grade/judge_test.go
new file mode 100644
index 00000000..2fdd2888
--- /dev/null
+++ b/cli/test/evals/internal/grade/judge_test.go
@@ -0,0 +1,39 @@
+package grade
+
+import (
+ "context"
+ "testing"
+
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/agent"
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/catalog"
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/record"
+)
+
+type judgeChat struct{ reply string }
+
+func (j judgeChat) Chat(context.Context, *agent.Request) (*agent.Response, error) {
+ r := &agent.Response{}
+ r.Choices = append(r.Choices, struct {
+ Message agent.Message `json:"message"`
+ FinishReason string `json:"finish_reason"`
+ }{Message: agent.Message{Role: "assistant", Content: j.reply}})
+ r.Usage.Cost = 0.002
+ return r, nil
+}
+
+func TestJudge(t *testing.T) {
+ in := &Input{Case: &catalog.Case{Judge: catalog.JudgeSpec{Rubric: []string{"a", "b", "c"}}}, Run: &record.Run{FinalText: "report"}}
+ ledger := agent.NewLedger(10)
+ j := &Judge{Chat: judgeChat{"```json\n{\"criteria\":[{\"id\":1,\"verdict\":\"pass\"},{\"id\":2,\"verdict\":\"fail\",\"reason\":\"no numbers\"},{\"id\":3,\"verdict\":\"unknown\"}]}\n```"}, Model: &catalog.Model{ID: "anthropic/claude-haiku-5.5"}, Ledger: ledger}
+ v := j.Grade(context.Background(), in)
+ if v.Gating || v.Skipped || v.Score == nil || *v.Score != 0.5 || len(v.Failures) != 1 {
+ t.Fatalf("%+v", v)
+ }
+ if ledger.Spent() != 0.002 {
+ t.Fatalf("judge cost must count toward the cap, spent %v", ledger.Spent())
+ }
+ in.FakeModel = true
+ if v := j.Grade(context.Background(), in); !v.Skipped {
+ t.Fatal("oracle runs skip the judge")
+ }
+}
diff --git a/cli/test/evals/internal/grade/state.go b/cli/test/evals/internal/grade/state.go
new file mode 100644
index 00000000..afa2c97a
--- /dev/null
+++ b/cli/test/evals/internal/grade/state.go
@@ -0,0 +1,585 @@
+package grade
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "io/fs"
+ "os"
+ "path"
+ "path/filepath"
+ "regexp"
+ "slices"
+ "sort"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/catalog"
+)
+
+// State checks the system after the run against expect.state, through the
+// admin API (and the workspace for file kinds). Kinds:
+//
+// page id, status
+// rule id, enabled, snoozed, snoozed_minutes (remaining)
+// archived among (hashes considered), equals (the archived subset)
+// post_mortem id or newly created (count), title, contains, not_contains, tags, incident_id
+// agent_writes count of agent write events, optionally route substring and http_status
+// slow_endpoint endpoint, offset_ms, reason (present | regexp)
+// endpoint_seen endpoint listed in the last two hours (project: by name,
+// default the case's primary project)
+// check id, status (up/down), extra.open_incidents
+// page_count status, count
+// projects projects (names that must exist), frameworks (name ->
+// framework), count
+// file path (glob), exists, contains, not_contains
+// no_token_literal no project ingest token appears in any workspace file
+// (env files excepted)
+// no_token_printed no project ingest token appears in any tool output
+// the model saw, or in its report
+func State(ctx context.Context, in *Input) *Verdict {
+ v := newVerdict("state", true)
+ for i, a := range in.Case.Expect.State {
+ if err := checkState(ctx, in, a); err != nil {
+ v.fail("state[%d] %s: %v", i, a.Kind, err)
+ }
+ }
+ return v
+}
+
+func checkState(ctx context.Context, in *Input, a catalog.StateAssert) error {
+ st := in.State
+ pid := st.PrimaryProject()
+ q := map[string]string{"projectId": pid}
+ getJSON := func(p string, query map[string]string, out any) error {
+ code, body, err := st.Get(ctx, p, query)
+ if err != nil {
+ return err
+ }
+ if code != 200 {
+ return fmt.Errorf("GET %s: HTTP %d: %s", p, code, clipB(body))
+ }
+ return json.Unmarshal(body, out)
+ }
+ postJSON := func(p string, query map[string]string, body, out any) error {
+ code, resp, err := st.Post(ctx, p, query, body)
+ if err != nil {
+ return err
+ }
+ if code != 200 {
+ return fmt.Errorf("POST %s: HTTP %d: %s", p, code, clipB(resp))
+ }
+ return json.Unmarshal(resp, out)
+ }
+ switch a.Kind {
+ case "page":
+ var out struct {
+ Page struct {
+ Status string `json:"status"`
+ } `json:"page"`
+ }
+ if err := getJSON("/api/pages/"+a.ID, q, &out); err != nil {
+ return err
+ }
+ if a.Status != "" && out.Page.Status != a.Status {
+ return fmt.Errorf("page %s is %s, want %s", a.ID, out.Page.Status, a.Status)
+ }
+ case "page_count":
+ var out struct {
+ Data []json.RawMessage `json:"data"`
+ }
+ if err := postJSON("/api/pages", q, map[string]any{"status": a.Status, "pagination": map[string]int{"page": 1, "pageSize": 100}}, &out); err != nil {
+ return err
+ }
+ if !a.Count.Contains(float64(len(out.Data))) {
+ return fmt.Errorf("%d %s pages, want %s", len(out.Data), a.Status, a.Count)
+ }
+ case "rule":
+ var out struct {
+ Rules []struct {
+ ID int `json:"id"`
+ Enabled bool `json:"enabled"`
+ SnoozedUntil *time.Time `json:"snoozedUntil"`
+ } `json:"rules"`
+ }
+ if err := getJSON("/api/notification-rules", q, &out); err != nil {
+ return err
+ }
+ id, _ := strconv.Atoi(a.ID)
+ for _, r := range out.Rules {
+ if r.ID != id {
+ continue
+ }
+ if a.Enabled != nil && r.Enabled != *a.Enabled {
+ return fmt.Errorf("rule %d enabled=%v, want %v", id, r.Enabled, *a.Enabled)
+ }
+ snoozed := r.SnoozedUntil != nil && r.SnoozedUntil.After(time.Now())
+ if a.Snoozed != nil && snoozed != *a.Snoozed {
+ return fmt.Errorf("rule %d snoozed=%v, want %v", id, snoozed, *a.Snoozed)
+ }
+ if a.SnoozedMinutes != nil {
+ if !snoozed {
+ return fmt.Errorf("rule %d is not snoozed", id)
+ }
+ left := time.Until(*r.SnoozedUntil).Minutes()
+ if !a.SnoozedMinutes.Contains(left) {
+ return fmt.Errorf("rule %d is snoozed for %.0f more minutes, want %s", id, left, a.SnoozedMinutes)
+ }
+ }
+ return nil
+ }
+ return fmt.Errorf("rule %s not found", a.ID)
+ case "archived":
+ visible, err := listHashes(ctx, st, pid, false)
+ if err != nil {
+ return err
+ }
+ all, err := listHashes(ctx, st, pid, true)
+ if err != nil {
+ return err
+ }
+ var archived []string
+ for _, h := range a.Among {
+ if all[h] && !visible[h] {
+ archived = append(archived, h)
+ }
+ }
+ want := append([]string{}, a.Equals...)
+ sort.Strings(archived)
+ sort.Strings(want)
+ if !slices.Equal(archived, want) {
+ return fmt.Errorf("archived %v, want exactly %v", archived, want)
+ }
+ case "post_mortem":
+ return checkPostMortem(ctx, in, a, getJSON)
+ case "agent_writes":
+ var out struct {
+ Data []struct {
+ Route string `json:"route"`
+ Method string `json:"method"`
+ StatusCode int `json:"statusCode"`
+ } `json:"data"`
+ }
+ if err := getJSON(fmt.Sprintf("/api/organizations/%d/agent-activity", st.OrgID()), map[string]string{"pageSize": "100"}, &out); err != nil {
+ return err
+ }
+ n := 0
+ for _, e := range out.Data {
+ if a.Route != "" && !strings.Contains(e.Method+" "+e.Route, a.Route) {
+ continue
+ }
+ if a.HTTPStatus != nil && !a.HTTPStatus.Contains(float64(e.StatusCode)) {
+ continue
+ }
+ n++
+ }
+ if !a.Count.Contains(float64(n)) {
+ return fmt.Errorf("%d matching agent write events, want %s", n, a.Count)
+ }
+ case "slow_endpoint":
+ var out struct {
+ OffsetMs float64 `json:"offsetMs"`
+ Reason string `json:"reason"`
+ }
+ qq := map[string]string{"projectId": pid, "endpoint": a.Endpoint}
+ if err := getJSON("/api/endpoints/slow", qq, &out); err != nil {
+ return err
+ }
+ if a.OffsetMs != nil && !a.OffsetMs.Contains(out.OffsetMs) {
+ return fmt.Errorf("%s offset %.0fms, want %s", a.Endpoint, out.OffsetMs, a.OffsetMs)
+ }
+ switch {
+ case a.Reason == "present" && strings.TrimSpace(out.Reason) == "":
+ return fmt.Errorf("%s has no reason", a.Endpoint)
+ case a.Reason != "" && a.Reason != "present":
+ if re, err := regexp.Compile("(?i)" + a.Reason); err != nil || !re.MatchString(out.Reason) {
+ return fmt.Errorf("%s reason %q does not match /%s/", a.Endpoint, out.Reason, a.Reason)
+ }
+ }
+ case "endpoint_seen":
+ if a.Project != "" {
+ id, err := projectIDByName(ctx, st, a.Project)
+ if err != nil {
+ return err
+ }
+ q = map[string]string{"projectId": id}
+ }
+ deadline := time.Now().Add(20 * time.Second)
+ for {
+ var out struct {
+ Data []struct {
+ Endpoint string `json:"endpoint"`
+ } `json:"data"`
+ }
+ err := postJSON("/api/endpoints/grouped", q, map[string]any{
+ "fromDate": time.Now().Add(-2 * time.Hour).UTC().Format(time.RFC3339), "toDate": time.Now().Add(time.Hour).UTC().Format(time.RFC3339),
+ "orderBy": "count", "pagination": map[string]int{"page": 1, "pageSize": 100},
+ }, &out)
+ if err == nil {
+ for _, d := range out.Data {
+ if sameEndpoint(d.Endpoint, a.Endpoint) {
+ return nil
+ }
+ }
+ }
+ if time.Now().After(deadline) {
+ var seen []string
+ for _, d := range out.Data {
+ seen = append(seen, d.Endpoint)
+ }
+ return fmt.Errorf("%s never reported; endpoints seen: %v", a.Endpoint, seen)
+ }
+ time.Sleep(time.Second)
+ }
+ case "check":
+ var out struct {
+ Check struct {
+ CurrentStatus string `json:"currentStatus"`
+ LastRunAt *time.Time `json:"lastRunAt"`
+ } `json:"check"`
+ Incidents []struct {
+ ResolvedAt *time.Time `json:"resolvedAt"`
+ } `json:"incidents"`
+ }
+ deadline := time.Now().Add(15 * time.Second)
+ for {
+ if err := getJSON("/api/synthetics/checks/"+a.ID, q, &out); err != nil {
+ return err
+ }
+ if a.Status == "" || out.Check.CurrentStatus == a.Status || time.Now().After(deadline) {
+ break
+ }
+ time.Sleep(time.Second)
+ }
+ if a.Status != "" && out.Check.CurrentStatus != a.Status {
+ return fmt.Errorf("monitor %s is %s, want %s", a.ID, out.Check.CurrentStatus, a.Status)
+ }
+ if want, ok := a.Extra["open_incidents"]; ok {
+ open := 0
+ for _, i := range out.Incidents {
+ if i.ResolvedAt == nil {
+ open++
+ }
+ }
+ if fmt.Sprint(open) != fmt.Sprint(want) {
+ return fmt.Errorf("monitor %s has %d open incidents, want %v", a.ID, open, want)
+ }
+ }
+ case "projects":
+ var out []struct {
+ Name string `json:"name"`
+ Framework string `json:"framework"`
+ }
+ if err := getJSON("/api/projects", nil, &out); err != nil {
+ return err
+ }
+ var names []string
+ frameworks := map[string]string{}
+ for _, p := range out {
+ names = append(names, p.Name)
+ frameworks[p.Name] = p.Framework
+ }
+ for name, fw := range a.Frameworks {
+ if got, ok := frameworks[name]; !ok || got != fw {
+ return fmt.Errorf("project %q has framework %q, want %q", name, got, fw)
+ }
+ }
+ for _, want := range a.Projects {
+ if !slices.Contains(names, want) {
+ return fmt.Errorf("project %q does not exist (have %v)", want, names)
+ }
+ }
+ if a.Count != nil && !a.Count.Contains(float64(len(names))) {
+ return fmt.Errorf("%d projects, want %s", len(names), a.Count)
+ }
+ case "file":
+ return checkFiles(in.Workspace, a)
+ case "no_token_literal":
+ return checkNoTokens(in.Workspace, st.Tokens())
+ case "no_token_printed":
+ for _, tok := range st.Tokens() {
+ if tok == "" {
+ continue
+ }
+ for _, c := range in.Run.Calls {
+ if strings.Contains(c.Output, tok) {
+ return fmt.Errorf("a project token appears in the output of %s (%s)", c.ID, c.Tool)
+ }
+ }
+ if strings.Contains(FinalText(in.Run), tok) {
+ return fmt.Errorf("a project token appears in the final report")
+ }
+ }
+ default:
+ return fmt.Errorf("unknown state kind %q", a.Kind)
+ }
+ return nil
+}
+
+func projectIDByName(ctx context.Context, st StateReader, name string) (string, error) {
+ if id := st.ProjectID(name); id != "" {
+ return id, nil
+ }
+ code, body, err := st.Get(ctx, "/api/projects", nil)
+ if err != nil {
+ return "", err
+ }
+ if code != 200 {
+ return "", fmt.Errorf("list projects: HTTP %d", code)
+ }
+ var ps []struct {
+ ID string `json:"id"`
+ Name string `json:"name"`
+ }
+ if err := json.Unmarshal(body, &ps); err != nil {
+ return "", err
+ }
+ for _, p := range ps {
+ if p.Name == name {
+ return p.ID, nil
+ }
+ }
+ return "", fmt.Errorf("no project named %q", name)
+}
+
+func listHashes(ctx context.Context, st StateReader, pid string, includeArchived bool) (map[string]bool, error) {
+ code, body, err := st.Post(ctx, "/api/exception-stack-traces", map[string]string{"projectId": pid}, map[string]any{
+ "fromDate": time.Now().Add(-31 * 24 * time.Hour).UTC().Format(time.RFC3339), "toDate": time.Now().Add(time.Hour).UTC().Format(time.RFC3339),
+ "orderBy": "count", "pagination": map[string]int{"page": 1, "pageSize": 100}, "includeArchived": includeArchived,
+ })
+ if err != nil {
+ return nil, err
+ }
+ if code != 200 {
+ return nil, fmt.Errorf("list exceptions: HTTP %d: %s", code, clipB(body))
+ }
+ var out struct {
+ Data []struct {
+ Hash string `json:"exceptionHash"`
+ } `json:"data"`
+ }
+ if err := json.Unmarshal(body, &out); err != nil {
+ return nil, err
+ }
+ m := map[string]bool{}
+ for _, d := range out.Data {
+ m[d.Hash] = true
+ }
+ return m, nil
+}
+
+func checkPostMortem(ctx context.Context, in *Input, a catalog.StateAssert, getJSON func(string, map[string]string, any) error) error {
+ pid := in.State.PrimaryProject()
+ ids := []string{a.ID}
+ if a.ID == "" {
+ var list struct {
+ Data []struct {
+ ID int `json:"id"`
+ } `json:"data"`
+ }
+ if err := getJSON("/api/post-mortems", map[string]string{"projectId": pid, "pageSize": "50"}, &list); err != nil {
+ return err
+ }
+ existing := map[string]bool{}
+ if ex, ok := in.Seed["existingPostMortems"].([]any); ok {
+ for _, e := range ex {
+ existing[fmt.Sprint(e)] = true
+ }
+ }
+ ids = nil
+ for _, d := range list.Data {
+ if !existing[fmt.Sprint(d.ID)] {
+ ids = append(ids, fmt.Sprint(d.ID))
+ }
+ }
+ count := a.Count
+ if count == nil {
+ one := 1.0
+ count = &catalog.Range{Min: &one, Max: &one}
+ }
+ if !count.Contains(float64(len(ids))) {
+ return fmt.Errorf("%d new post-mortems, want %s", len(ids), count)
+ }
+ if len(ids) == 0 {
+ return nil
+ }
+ }
+ for _, id := range ids {
+ var pm struct {
+ Title string `json:"title"`
+ ContentMd string `json:"contentMd"`
+ Tags []string `json:"tags"`
+ IncidentID *int `json:"incidentId"`
+ }
+ if err := getJSON("/api/post-mortems/"+id, map[string]string{"projectId": pid}, &pm); err != nil {
+ return err
+ }
+ if a.Title != "" {
+ if re, err := regexp.Compile("(?i)" + a.Title); err != nil || !re.MatchString(pm.Title) {
+ return fmt.Errorf("post-mortem %s title %q does not match /%s/", id, pm.Title, a.Title)
+ }
+ }
+ for _, c := range a.Contains {
+ if re, err := regexp.Compile("(?is)" + c); err != nil || !re.MatchString(pm.ContentMd) {
+ return fmt.Errorf("post-mortem %s does not contain /%s/", id, c)
+ }
+ }
+ for _, c := range a.NotContains {
+ if re, err := regexp.Compile("(?is)" + c); err == nil && re.MatchString(pm.ContentMd) {
+ return fmt.Errorf("post-mortem %s must not contain /%s/", id, c)
+ }
+ }
+ for _, t := range a.Tags {
+ if !slices.Contains(pm.Tags, strings.ToLower(t)) {
+ return fmt.Errorf("post-mortem %s tags %v lack %q", id, pm.Tags, t)
+ }
+ }
+ if a.IncidentID != "" {
+ got := ""
+ if pm.IncidentID != nil {
+ got = strconv.Itoa(*pm.IncidentID)
+ }
+ if got != a.IncidentID {
+ return fmt.Errorf("post-mortem %s links incident %q, want %q", id, got, a.IncidentID)
+ }
+ }
+ }
+ return nil
+}
+
+func checkFiles(root string, a catalog.StateAssert) error {
+ var matched []string
+ _ = filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
+ if err != nil {
+ return nil
+ }
+ if d.IsDir() && (d.Name() == ".git" || d.Name() == "node_modules" || d.Name() == ".skills") {
+ return filepath.SkipDir
+ }
+ if d.IsDir() {
+ return nil
+ }
+ rel, _ := filepath.Rel(root, p)
+ rel = filepath.ToSlash(rel)
+ if ok, _ := path.Match(a.Path, rel); ok || rel == a.Path || globAny(a.Path, rel) {
+ matched = append(matched, p)
+ }
+ return nil
+ })
+ if a.Exists != nil && *a.Exists != (len(matched) > 0) {
+ return fmt.Errorf("%s exists=%v, want %v", a.Path, len(matched) > 0, *a.Exists)
+ }
+ if len(matched) == 0 && (len(a.Contains) > 0 || len(a.NotContains) > 0) {
+ return fmt.Errorf("no file matches %s", a.Path)
+ }
+ var all strings.Builder
+ for _, m := range matched {
+ raw, _ := os.ReadFile(m)
+ all.Write(raw)
+ all.WriteByte('\n')
+ }
+ text := all.String()
+ for _, c := range a.Contains {
+ if re, err := regexp.Compile("(?m)" + c); err != nil || !re.MatchString(text) {
+ return fmt.Errorf("%s does not match /%s/", a.Path, c)
+ }
+ }
+ for _, c := range a.NotContains {
+ if re, err := regexp.Compile("(?m)" + c); err == nil && re.MatchString(text) {
+ return fmt.Errorf("%s must not match /%s/", a.Path, c)
+ }
+ }
+ return nil
+}
+
+// globAny supports a leading **/ in file globs.
+func globAny(glob, rel string) bool {
+ if !strings.HasPrefix(glob, "**/") {
+ return false
+ }
+ g := strings.TrimPrefix(glob, "**/")
+ parts := strings.Split(rel, "/")
+ for i := range parts {
+ if ok, _ := path.Match(g, strings.Join(parts[i:], "/")); ok {
+ return true
+ }
+ }
+ return false
+}
+
+// checkNoTokens scans the files git would commit (tracked, plus untracked
+// files that are not ignored), so build output a .gitignore excludes (a
+// bundle that legitimately inlines a public connection string) does not count.
+func checkNoTokens(root string, tokens []string) error {
+ var leaks []string
+ if files, ok := gitVisibleFiles(root); ok {
+ for _, rel := range files {
+ base := filepath.Base(rel)
+ if strings.HasPrefix(base, ".env") || strings.HasPrefix(rel, ".skills/") || strings.HasPrefix(rel, ".twevals") {
+ continue
+ }
+ raw, err := os.ReadFile(filepath.Join(root, rel))
+ if err != nil {
+ continue
+ }
+ for _, t := range tokens {
+ if t != "" && strings.Contains(string(raw), t) {
+ leaks = append(leaks, rel)
+ }
+ }
+ }
+ if len(leaks) > 0 {
+ return fmt.Errorf("a project token is written literally into %v", leaks)
+ }
+ return nil
+ }
+ _ = filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
+ if err != nil {
+ return nil
+ }
+ if d.IsDir() && (d.Name() == ".git" || d.Name() == "node_modules" || d.Name() == ".skills") {
+ return filepath.SkipDir
+ }
+ if d.IsDir() || strings.HasPrefix(d.Name(), ".env") {
+ return nil
+ }
+ raw, err := os.ReadFile(p)
+ if err != nil {
+ return nil
+ }
+ for _, t := range tokens {
+ if t != "" && strings.Contains(string(raw), t) {
+ rel, _ := filepath.Rel(root, p)
+ leaks = append(leaks, rel)
+ }
+ }
+ return nil
+ })
+ if len(leaks) > 0 {
+ return fmt.Errorf("a project token is written literally into %v", leaks)
+ }
+ return nil
+}
+
+func gitVisibleFiles(root string) ([]string, bool) {
+ out, err := workspaceGit(root, "ls-files", "-z", "-c", "-o", "--exclude-standard").Output()
+ if err != nil {
+ return nil, false
+ }
+ var files []string
+ for _, f := range strings.Split(string(out), "\x00") {
+ if f != "" {
+ files = append(files, f)
+ }
+ }
+ return files, true
+}
+
+func clipB(b []byte) string {
+ s := strings.TrimSpace(string(b))
+ if len(s) > 300 {
+ return s[:300] + "..."
+ }
+ return s
+}
diff --git a/cli/test/evals/internal/grade/suites_test.go b/cli/test/evals/internal/grade/suites_test.go
new file mode 100644
index 00000000..4699b600
--- /dev/null
+++ b/cli/test/evals/internal/grade/suites_test.go
@@ -0,0 +1,39 @@
+package grade
+
+import (
+ "strings"
+ "testing"
+
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/catalog"
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/ops"
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/record"
+)
+
+func TestMCPAndCLIMapToTheSameOps(t *testing.T) {
+ cli := ops.ParseCLI(strings.Fields("alerts snooze 12 --minutes 30 --yes"))
+ mcp := ops.ParseMCP("mcp__traceway__snooze_notification_rule", map[string]any{"id": 12.0, "duration_minutes": 30.0})
+ m := catalog.CallMatcher{Op: "alerts.snooze", Args: map[string]any{"id": "12", "duration_minutes": "30"}}
+ for _, op := range []ops.Op{cli, mcp} {
+ rec := &record.OpRecord{Op: op}
+ if !Matches(rec, m) {
+ t.Errorf("%+v does not match %+v", op, m)
+ }
+ }
+ dis := ops.ParseMCP("set_notification_rule_enabled", map[string]any{"id": 3.0, "enabled": false})
+ if dis.Op != "alerts.disable" || !dis.Write {
+ t.Errorf("set_notification_rule_enabled(false) = %+v", dis)
+ }
+}
+
+func TestDebugAnswer(t *testing.T) {
+ debug := &Input{Case: &catalog.Case{Suite: "debug", Expect: catalog.Expect{Answer: &catalog.Answer{
+ Category: "null_reference", Culprit: &catalog.Culprit{File: "internal/invoices/render.go", Symbol: "formatAddress", Line: 37}}}},
+ Run: &record.Run{Submission: map[string]any{"category": "null_reference", "culprit": map[string]any{"file": "./internal/invoices/render.go", "symbol": "invoices.formatAddress", "line": 39.0}}}}
+ if v := Answer(debug); !v.Pass {
+ t.Fatalf("line within tolerance failed: %s", failures(v))
+ }
+ debug.Run.Submission["culprit"].(map[string]any)["line"] = 45.0
+ if v := Answer(debug); v.Pass {
+ t.Fatal("line 45 is outside 37±3")
+ }
+}
diff --git a/cli/test/evals/internal/grade/workspace_git.go b/cli/test/evals/internal/grade/workspace_git.go
new file mode 100644
index 00000000..1ada21f7
--- /dev/null
+++ b/cli/test/evals/internal/grade/workspace_git.go
@@ -0,0 +1,20 @@
+package grade
+
+import (
+ "os"
+ "os/exec"
+ "path/filepath"
+)
+
+// workspaceGit runs git in a workspace the agent has written to, trusting no
+// git config found there: the setup suite's container shell can write
+// .git/config, and core.fsmonitor or core.hooksPath would then run a program
+// on the host as soon as git reads the index. The repository is pinned to the
+// workspace's own .git, and only built-in defaults apply otherwise.
+func workspaceGit(work string, args ...string) *exec.Cmd {
+ cmd := exec.Command("git", append([]string{"-c", "core.fsmonitor=false", "-c", "core.hooksPath=/dev/null"}, args...)...)
+ cmd.Dir = work
+ cmd.Env = []string{"HOME=" + work, "PATH=" + os.Getenv("PATH"), "GIT_DIR=" + filepath.Join(work, ".git"), "GIT_WORK_TREE=" + work,
+ "GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL=/dev/null", "GIT_OPTIONAL_LOCKS=0"}
+ return cmd
+}
diff --git a/cli/test/evals/internal/grade/workspace_git_test.go b/cli/test/evals/internal/grade/workspace_git_test.go
new file mode 100644
index 00000000..2c546354
--- /dev/null
+++ b/cli/test/evals/internal/grade/workspace_git_test.go
@@ -0,0 +1,50 @@
+package grade
+
+import (
+ "os"
+ "os/exec"
+ "path/filepath"
+ "slices"
+ "testing"
+)
+
+// TestWorkspaceGitIgnoresRepoConfig: a .git/config the agent wrote (the
+// setup suite's container shell can) never runs a program on the host when
+// the graders list the workspace's files.
+func TestWorkspaceGitIgnoresRepoConfig(t *testing.T) {
+ dir := t.TempDir()
+ work := filepath.Join(dir, "work")
+ os.MkdirAll(work, 0o755)
+ os.WriteFile(filepath.Join(work, "main.go"), []byte("package main\n"), 0o644)
+ for _, args := range [][]string{{"init", "-q"}, {"add", "main.go"}, {"-c", "user.email=x@y", "-c", "user.name=x", "commit", "-qm", "init"}} {
+ cmd := exec.Command("git", args...)
+ cmd.Dir = work
+ if out, err := cmd.CombinedOutput(); err != nil {
+ t.Skipf("git unavailable: %v %s", err, out)
+ }
+ }
+ hooks := filepath.Join(work, "hooks")
+ os.MkdirAll(hooks, 0o755)
+ for _, h := range []string{"post-index-change", "reference-transaction"} {
+ os.WriteFile(filepath.Join(hooks, h), []byte("#!/bin/sh\necho x > ../PWNED\n"), 0o755)
+ }
+ cfg, _ := os.ReadFile(filepath.Join(work, ".git", "config"))
+ cfg = append(cfg, []byte("[core]\n\tfsmonitor = \"echo x > ../PWNED; false\"\n\thooksPath = hooks\n\tworktree = ..\n")...)
+ os.WriteFile(filepath.Join(work, ".git", "config"), cfg, 0o644)
+ os.WriteFile(filepath.Join(work, "new.go"), []byte("package main\n"), 0o644)
+ os.WriteFile(filepath.Join(work, "main.go"), []byte("package main\n\nfunc main() {}\n"), 0o644)
+
+ files, ok := gitVisibleFiles(work)
+ if !ok || !slices.Contains(files, "main.go") || !slices.Contains(files, "new.go") || slices.Contains(files, "work/main.go") {
+ t.Errorf("gitVisibleFiles = %v, %v", files, ok)
+ }
+ changed, err := changedFiles(work)
+ if err != nil || !slices.Contains(changed, "main.go") || !slices.Contains(changed, "new.go") {
+ t.Errorf("changedFiles = %v, %v", changed, err)
+ }
+ for _, p := range []string{filepath.Join(dir, "PWNED"), filepath.Join(work, "PWNED")} {
+ if _, err := os.Stat(p); err == nil {
+ t.Fatalf("git ran a program from the workspace's config (%s)", p)
+ }
+ }
+}
diff --git a/cli/test/evals/internal/guard/manual.go b/cli/test/evals/internal/guard/manual.go
new file mode 100644
index 00000000..0a3df9fa
--- /dev/null
+++ b/cli/test/evals/internal/guard/manual.go
@@ -0,0 +1,114 @@
+// Package guard enforces the manual-only policy in code: paid
+// commands start in CI only inside a hand-dispatched cli-evals.yml run behind
+// the evals environment gate, locally only after a typed confirmation, and
+// always under a mandatory cost cap. The real OpenRouter client asks Live()
+// before every request, and Live() is false inside `go test`.
+package guard
+
+import (
+ "bufio"
+ "errors"
+ "fmt"
+ "io"
+ "strings"
+ "sync/atomic"
+ "testing"
+)
+
+// WorkflowFile is the only workflow allowed to run paid evals.
+const WorkflowFile = ".github/workflows/cli-evals.yml"
+
+var live atomic.Bool
+
+// Live reports whether paid model calls are unlocked for this process. It is
+// always false under `go test`, whatever a test does.
+func Live() bool {
+ if testing.Testing() {
+ return false
+ }
+ return live.Load()
+}
+
+// otherCI lists environment variables that identify a CI system other than a
+// dispatched GitHub Actions run. Their presence refuses paid commands.
+var otherCI = []string{
+ "BUILDKITE", "GITLAB_CI", "JENKINS_URL", "CIRCLECI", "TRAVIS", "TEAMCITY_VERSION",
+ "BITBUCKET_BUILD_NUMBER", "TF_BUILD", "DRONE", "CODEBUILD_BUILD_ID", "APPVEYOR",
+ "SEMAPHORE", "BUDDY", "WOODPECKER", "HARNESS_BUILD_ID", "CF_BUILD_ID",
+}
+
+// Request describes one attempt to start a paid command.
+type Request struct {
+ Command string // run | resume
+ MaxCostUSD float64
+ Yes bool
+ StdinTTY bool
+ Getenv func(string) string
+ // Preflight is printed before the local confirmation prompt.
+ Preflight string
+ In io.Reader
+ Out io.Writer
+}
+
+// Decision is what Authorize concluded.
+type Decision struct {
+ CI bool // a dispatched cli-evals.yml run
+ Confirm bool // a typed confirmation is required
+}
+
+// Decide applies the policy without side effects or prompts.
+func Decide(r Request) (Decision, error) {
+ if r.MaxCostUSD <= 0 {
+ return Decision{}, errors.New("--max-cost-usd is required for paid commands and must be greater than 0 (there is no default)")
+ }
+ env := r.Getenv
+ if env("GITHUB_ACTIONS") == "true" {
+ if ev := env("GITHUB_EVENT_NAME"); ev != "workflow_dispatch" {
+ return Decision{}, fmt.Errorf("refusing paid %q in GitHub Actions event %q: evals run only from a hand-dispatched %s (workflow_dispatch)", r.Command, ev, WorkflowFile)
+ }
+ if ref := env("GITHUB_WORKFLOW_REF"); !strings.Contains(ref, "/"+WorkflowFile+"@") {
+ return Decision{}, fmt.Errorf("refusing paid %q from workflow %q: only %s may run paid evals", r.Command, ref, WorkflowFile)
+ }
+ if env("TWEVALS_APPROVED") != "1" {
+ return Decision{}, errors.New("refusing paid run: TWEVALS_APPROVED=1 is set only by the job behind the evals environment gate")
+ }
+ return Decision{CI: true}, nil
+ }
+ if env("CI") != "" {
+ return Decision{}, fmt.Errorf("refusing paid %q under CI=%q: the only CI path is a hand-dispatched %s", r.Command, env("CI"), WorkflowFile)
+ }
+ for _, k := range otherCI {
+ if env(k) != "" {
+ return Decision{}, fmt.Errorf("refusing paid %q: %s is set, and evals never run from %s", r.Command, k, k)
+ }
+ }
+ if !r.StdinTTY {
+ if !r.Yes {
+ return Decision{}, errors.New("stdin is not a terminal: refusing a paid run without --yes (local paid runs are for harness development only)")
+ }
+ return Decision{}, nil
+ }
+ return Decision{Confirm: true}, nil
+}
+
+// Authorize applies the policy, prompts for the typed confirmation when one is
+// needed, and unlocks Live() for the rest of the process.
+func Authorize(r Request) error {
+ d, err := Decide(r)
+ if err != nil {
+ return err
+ }
+ if d.Confirm {
+ fmt.Fprintln(r.Out, r.Preflight)
+ fmt.Fprintf(r.Out, "\nThis spends real money on OpenRouter (cap $%.2f). Type yes to continue: ", r.MaxCostUSD)
+ line, _ := bufio.NewReader(r.In).ReadString('\n')
+ if strings.TrimSpace(line) != "yes" {
+ return errors.New("not confirmed; nothing was spent")
+ }
+ }
+ if testing.Testing() {
+ return nil
+ }
+ live.Store(true)
+ return nil
+}
diff --git a/cli/test/evals/internal/guard/manual_test.go b/cli/test/evals/internal/guard/manual_test.go
new file mode 100644
index 00000000..75337078
--- /dev/null
+++ b/cli/test/evals/internal/guard/manual_test.go
@@ -0,0 +1,93 @@
+package guard
+
+import (
+ "strings"
+ "testing"
+)
+
+func envOf(kv map[string]string) func(string) string {
+ return func(k string) string { return kv[k] }
+}
+
+func TestDecide(t *testing.T) {
+ dispatched := map[string]string{
+ "GITHUB_ACTIONS": "true",
+ "CI": "true",
+ "GITHUB_EVENT_NAME": "workflow_dispatch",
+ "GITHUB_WORKFLOW_REF": "tracewayapp/traceway/.github/workflows/cli-evals.yml@refs/heads/main",
+ "TWEVALS_APPROVED": "1",
+ }
+ with := func(over map[string]string) map[string]string {
+ m := map[string]string{}
+ for k, v := range dispatched {
+ m[k] = v
+ }
+ for k, v := range over {
+ m[k] = v
+ }
+ return m
+ }
+ cases := []struct {
+ name string
+ env map[string]string
+ tty bool
+ yes bool
+ cost float64
+ wantErr string
+ want Decision
+ }{
+ {name: "dispatched and approved", env: dispatched, yes: true, cost: 10, want: Decision{CI: true}},
+ {name: "no cap", env: dispatched, yes: true, cost: 0, wantErr: "--max-cost-usd is required"},
+ {name: "push event", env: with(map[string]string{"GITHUB_EVENT_NAME": "push"}), yes: true, cost: 10, wantErr: "workflow_dispatch"},
+ {name: "pull request", env: with(map[string]string{"GITHUB_EVENT_NAME": "pull_request"}), yes: true, cost: 10, wantErr: "workflow_dispatch"},
+ {name: "schedule", env: with(map[string]string{"GITHUB_EVENT_NAME": "schedule"}), yes: true, cost: 10, wantErr: "workflow_dispatch"},
+ {name: "other workflow", env: with(map[string]string{"GITHUB_WORKFLOW_REF": "tracewayapp/traceway/.github/workflows/cli.yml@refs/heads/main"}), yes: true, cost: 10, wantErr: "only .github/workflows/cli-evals.yml"},
+ {name: "not behind the gate", env: with(map[string]string{"TWEVALS_APPROVED": ""}), yes: true, cost: 10, wantErr: "evals environment gate"},
+ {name: "generic CI", env: map[string]string{"CI": "true"}, yes: true, cost: 10, wantErr: "CI="},
+ {name: "buildkite", env: map[string]string{"BUILDKITE": "true"}, yes: true, cost: 10, wantErr: "BUILDKITE"},
+ {name: "gitlab", env: map[string]string{"GITLAB_CI": "true"}, yes: true, cost: 10, wantErr: "GITLAB_CI"},
+ {name: "jenkins", env: map[string]string{"JENKINS_URL": "http://ci"}, yes: true, cost: 10, wantErr: "JENKINS_URL"},
+ {name: "local non-tty without yes", env: map[string]string{}, cost: 10, wantErr: "without --yes"},
+ {name: "local non-tty with yes", env: map[string]string{}, yes: true, cost: 10, want: Decision{}},
+ {name: "local tty asks", env: map[string]string{}, tty: true, cost: 10, want: Decision{Confirm: true}},
+ {name: "local tty asks even with yes", env: map[string]string{}, tty: true, yes: true, cost: 10, want: Decision{Confirm: true}},
+ }
+ for _, c := range cases {
+ t.Run(c.name, func(t *testing.T) {
+ d, err := Decide(Request{Command: "run", MaxCostUSD: c.cost, Yes: c.yes, StdinTTY: c.tty, Getenv: envOf(c.env)})
+ if c.wantErr != "" {
+ if err == nil || !strings.Contains(err.Error(), c.wantErr) {
+ t.Fatalf("want error containing %q, got %v", c.wantErr, err)
+ }
+ return
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+ if d != c.want {
+ t.Fatalf("decision = %+v, want %+v", d, c.want)
+ }
+ })
+ }
+}
+
+func TestLiveIsNeverOpenUnderGoTest(t *testing.T) {
+ err := Authorize(Request{Command: "run", MaxCostUSD: 1, Yes: true, Getenv: envOf(map[string]string{})})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if Live() {
+ t.Fatal("Live() must stay false inside go test")
+ }
+}
+
+func TestConfirmationRequiresTypedYes(t *testing.T) {
+ var out strings.Builder
+ err := Authorize(Request{Command: "run", MaxCostUSD: 1, StdinTTY: true, Getenv: envOf(map[string]string{}), In: strings.NewReader("y\n"), Out: &out, Preflight: "PREFLIGHT"})
+ if err == nil || !strings.Contains(err.Error(), "not confirmed") {
+ t.Fatalf("a bare y must not confirm, got %v", err)
+ }
+ if !strings.Contains(out.String(), "PREFLIGHT") {
+ t.Fatal("the preflight must be shown before asking")
+ }
+}
diff --git a/cli/test/evals/internal/guard/triggers.go b/cli/test/evals/internal/guard/triggers.go
new file mode 100644
index 00000000..ffb6e2ee
--- /dev/null
+++ b/cli/test/evals/internal/guard/triggers.go
@@ -0,0 +1,138 @@
+package guard
+
+import (
+ "fmt"
+ "os"
+ "path/filepath"
+ "regexp"
+ "strings"
+
+ "gopkg.in/yaml.v3"
+)
+
+// paidRunRe matches a command line that starts a paid eval run.
+var paidRunRe = regexp.MustCompile(`twevals(\s+|\S*\s+)(run|resume)\b|eval-local`)
+
+// CheckWorkflows enforces the dispatch-only rule over a .github/workflows
+// directory: cli-evals.yml exists, its `on:` is exactly workflow_dispatch, the
+// approval flag is set only by a job behind the evals environment, and no
+// other workflow references it or starts a paid run.
+func CheckWorkflows(dir string) error {
+ evalsPath := filepath.Join(dir, filepath.Base(WorkflowFile))
+ raw, err := os.ReadFile(evalsPath)
+ if err != nil {
+ return fmt.Errorf("%s must exist: %w", WorkflowFile, err)
+ }
+ if err := checkEvalsWorkflow(raw); err != nil {
+ return fmt.Errorf("%s: %w", WorkflowFile, err)
+ }
+ entries, err := os.ReadDir(dir)
+ if err != nil {
+ return err
+ }
+ for _, e := range entries {
+ name := e.Name()
+ if e.IsDir() || name == filepath.Base(WorkflowFile) || !(strings.HasSuffix(name, ".yml") || strings.HasSuffix(name, ".yaml")) {
+ continue
+ }
+ other, err := os.ReadFile(filepath.Join(dir, name))
+ if err != nil {
+ return err
+ }
+ text := string(other)
+ if strings.Contains(text, "cli-evals.yml") || strings.Contains(text, "cli-evals.yaml") {
+ return fmt.Errorf("%s references cli-evals.yml; no workflow may call or dispatch the evals", name)
+ }
+ if paidRunRe.MatchString(text) || strings.Contains(text, "TWEVALS_APPROVED") {
+ return fmt.Errorf("%s starts a paid eval run; only %s may, and only when dispatched by hand", name, WorkflowFile)
+ }
+ }
+ return nil
+}
+
+func checkEvalsWorkflow(raw []byte) error {
+ var doc yaml.Node
+ if err := yaml.Unmarshal(raw, &doc); err != nil {
+ return err
+ }
+ if len(doc.Content) == 0 || doc.Content[0].Kind != yaml.MappingNode {
+ return fmt.Errorf("not a YAML mapping")
+ }
+ root := doc.Content[0]
+ on := mapValue(root, "on")
+ if on == nil {
+ // YAML 1.1 readers turn a bare `on` key into true; accept that
+ // spelling too so the check cannot be dodged by quoting.
+ on = mapValue(root, "true")
+ }
+ if on == nil {
+ return fmt.Errorf("has no on: block")
+ }
+ switch on.Kind {
+ case yaml.ScalarNode:
+ if on.Value != "workflow_dispatch" {
+ return fmt.Errorf("on: must be exactly workflow_dispatch, got %q", on.Value)
+ }
+ case yaml.SequenceNode:
+ if len(on.Content) != 1 || on.Content[0].Value != "workflow_dispatch" {
+ return fmt.Errorf("on: must be exactly [workflow_dispatch]")
+ }
+ case yaml.MappingNode:
+ var keys []string
+ for i := 0; i < len(on.Content); i += 2 {
+ keys = append(keys, on.Content[i].Value)
+ }
+ if len(keys) != 1 || keys[0] != "workflow_dispatch" {
+ return fmt.Errorf("on: must have exactly one trigger, workflow_dispatch; found %v", keys)
+ }
+ default:
+ return fmt.Errorf("on: has an unexpected shape")
+ }
+
+ jobs := mapValue(root, "jobs")
+ if jobs == nil || jobs.Kind != yaml.MappingNode {
+ return fmt.Errorf("has no jobs")
+ }
+ gated := 0
+ for i := 0; i < len(jobs.Content); i += 2 {
+ name, job := jobs.Content[i].Value, jobs.Content[i+1]
+ var buf strings.Builder
+ enc := yaml.NewEncoder(&buf)
+ _ = enc.Encode(job)
+ body := buf.String()
+ approves := strings.Contains(body, "TWEVALS_APPROVED")
+ runsPaid := paidRunRe.MatchString(body)
+ if !approves && !runsPaid {
+ continue
+ }
+ envNode := mapValue(job, "environment")
+ envName := ""
+ if envNode != nil {
+ if envNode.Kind == yaml.ScalarNode {
+ envName = envNode.Value
+ } else if n := mapValue(envNode, "name"); n != nil {
+ envName = n.Value
+ }
+ }
+ if envName != "evals" {
+ return fmt.Errorf("job %q starts a paid run or sets TWEVALS_APPROVED without environment: evals", name)
+ }
+ gated++
+ }
+ if gated == 0 {
+ return fmt.Errorf("no job runs the evals behind environment: evals")
+ }
+ return nil
+}
+
+func mapValue(m *yaml.Node, key string) *yaml.Node {
+ if m == nil || m.Kind != yaml.MappingNode {
+ return nil
+ }
+ for i := 0; i+1 < len(m.Content); i += 2 {
+ if m.Content[i].Value == key {
+ return m.Content[i+1]
+ }
+ }
+ return nil
+}
diff --git a/cli/test/evals/internal/guard/triggers_test.go b/cli/test/evals/internal/guard/triggers_test.go
new file mode 100644
index 00000000..d50f03bd
--- /dev/null
+++ b/cli/test/evals/internal/guard/triggers_test.go
@@ -0,0 +1,79 @@
+package guard
+
+import (
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+)
+
+// TestEvalsWorkflowIsDispatchOnly is the manual-only trigger test: the
+// repository's cli-evals.yml has workflow_dispatch as its only trigger and no
+// other workflow references it or starts a paid run.
+func TestEvalsWorkflowIsDispatchOnly(t *testing.T) {
+ dir := filepath.Join("..", "..", "..", "..", "..", ".github", "workflows")
+ if err := CheckWorkflows(dir); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestCheckWorkflowsRejectsAutomation(t *testing.T) {
+ good := `name: CLI Evals
+on:
+ workflow_dispatch:
+ inputs:
+ max_cost_usd: {type: string, required: true}
+permissions:
+ contents: read
+jobs:
+ preflight:
+ runs-on: ubuntu-latest
+ steps:
+ - run: go run ./cmd/twevals plan --max-cost-usd 1
+ evals:
+ environment: evals
+ runs-on: ubuntu-latest
+ env:
+ TWEVALS_APPROVED: "1"
+ steps:
+ - run: go run ./cmd/twevals run --yes --max-cost-usd 1
+`
+ cases := map[string]struct {
+ evals string
+ others map[string]string
+ want string
+ }{
+ "good": {evals: good},
+ "push trigger": {evals: strings.Replace(good, "on:\n", "on:\n push:\n", 1), want: "exactly one trigger"},
+ "schedule": {evals: strings.Replace(good, "on:\n", "on:\n schedule:\n - cron: '0 0 * * *'\n", 1), want: "exactly one trigger"},
+ "workflow_call": {evals: strings.Replace(good, "on:\n", "on:\n workflow_call:\n", 1), want: "exactly one trigger"},
+ "list of triggers": {evals: strings.Replace(good, "on:\n workflow_dispatch:\n inputs:\n max_cost_usd: {type: string, required: true}\n", "on: [workflow_dispatch, push]\n", 1), want: "exactly [workflow_dispatch]"},
+ "ungated paid job": {evals: strings.Replace(good, " environment: evals\n", "", 1), want: "without environment: evals"},
+ "caller": {evals: good, others: map[string]string{"ci.yml": "jobs:\n x:\n uses: ./.github/workflows/cli-evals.yml\n"}, want: "references cli-evals.yml"},
+ "dispatcher": {evals: good, others: map[string]string{"nightly.yml": "on: schedule\njobs:\n x:\n steps:\n - run: gh workflow run cli-evals.yml\n"}, want: "references cli-evals.yml"},
+ "other paid run": {evals: good, others: map[string]string{"cli.yml": "jobs:\n x:\n steps:\n - run: cd cli/test/evals && go run ./cmd/twevals run --yes\n"}, want: "starts a paid eval run"},
+ }
+ for name, c := range cases {
+ t.Run(name, func(t *testing.T) {
+ dir := t.TempDir()
+ if err := os.WriteFile(filepath.Join(dir, "cli-evals.yml"), []byte(c.evals), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ for f, body := range c.others {
+ if err := os.WriteFile(filepath.Join(dir, f), []byte(body), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ }
+ err := CheckWorkflows(dir)
+ if c.want == "" {
+ if err != nil {
+ t.Fatal(err)
+ }
+ return
+ }
+ if err == nil || !strings.Contains(err.Error(), c.want) {
+ t.Fatalf("want error containing %q, got %v", c.want, err)
+ }
+ })
+ }
+}
diff --git a/cli/test/evals/internal/ops/mcp.go b/cli/test/evals/internal/ops/mcp.go
new file mode 100644
index 00000000..206ad3e0
--- /dev/null
+++ b/cli/test/evals/internal/ops/mcp.go
@@ -0,0 +1,132 @@
+package ops
+
+import (
+ "encoding/json"
+ "fmt"
+ "strings"
+)
+
+// MCPPrefix is how the agent sees the bridged MCP tools, as Claude Code does.
+const MCPPrefix = "mcp__traceway__"
+
+// ParseMCP maps an MCP tool call to an op. name may carry MCPPrefix.
+func ParseMCP(name string, args map[string]any) Op {
+ initOps()
+ name = strings.TrimPrefix(name, MCPPrefix)
+ switch name {
+ case "list_resources":
+ return Op{Op: OpListRes}
+ case "read_resource":
+ return Op{Op: OpReadResource, Args: args}
+ }
+ cands := byMCP[name]
+ if len(cands) == 0 {
+ return Op{Op: "mcp." + name, Args: args}
+ }
+ spec := cands[0]
+ out := map[string]any{}
+ for k, v := range args {
+ out[k] = v
+ }
+ switch name {
+ case "set_notification_rule_enabled":
+ if en, _ := out["enabled"].(bool); en {
+ spec = byOp["alerts.enable"]
+ } else {
+ spec = byOp["alerts.disable"]
+ }
+ delete(out, "enabled")
+ case "query_metrics":
+ // Flatten the first query so CLI and MCP args line up; the full
+ // list stays under "queries".
+ if qs, ok := out["queries"].([]any); ok && len(qs) > 0 {
+ if q, ok := qs[0].(map[string]any); ok {
+ for k, v := range q {
+ if _, exists := out[k]; !exists {
+ out[k] = v
+ }
+ }
+ }
+ }
+ }
+ return Op{Op: spec.Op, Args: out, Write: spec.Write}
+}
+
+// RenderMCP renders an op as an MCP tool call (name without prefix, args).
+func RenderMCP(op string, args map[string]any) (string, map[string]any, error) {
+ spec, ok := SpecFor(op)
+ if !ok || spec.MCP == "" {
+ return "", nil, fmt.Errorf("op %q has no MCP tool", op)
+ }
+ out := map[string]any{}
+ for k, v := range args {
+ if k == "output" || k == "fields" || k == "yes" {
+ continue
+ }
+ out[k] = v
+ }
+ switch op {
+ case "alerts.enable":
+ out["enabled"] = true
+ case "alerts.disable":
+ out["enabled"] = false
+ case "metrics.query":
+ q := map[string]any{}
+ for _, k := range []string{"name", "aggregation", "group_by", "tag_filters"} {
+ if v, ok := out[k]; ok {
+ q[k] = v
+ delete(out, k)
+ }
+ }
+ if tf, ok := q["tag_filters"]; ok {
+ q["tag_filters"] = tagMap(tf)
+ }
+ out["queries"] = []any{q}
+ }
+ for _, k := range []string{"id", "incident_id", "duration_minutes", "offset_ms", "interval_minutes", "page", "page_size", "min_severity"} {
+ if v, ok := out[k]; ok {
+ if n, ok := asInt(v); ok {
+ out[k] = n
+ }
+ }
+ }
+ return spec.MCP, out, nil
+}
+
+func tagMap(v any) map[string]any {
+ out := map[string]any{}
+ var items []any
+ switch t := v.(type) {
+ case map[string]any:
+ return t
+ case []any:
+ items = t
+ default:
+ items = []any{t}
+ }
+ for _, it := range items {
+ k, val, _ := strings.Cut(fmt.Sprint(it), "=")
+ out[k] = val
+ }
+ return out
+}
+
+func asInt(v any) (int, bool) {
+ switch t := v.(type) {
+ case int:
+ return t, true
+ case float64:
+ return int(t), true
+ case json.Number:
+ // The fake model's templates keep looked-up numbers as written.
+ if n, err := t.Int64(); err == nil {
+ return int(n), true
+ }
+ case string:
+ var n int
+ if _, err := fmt.Sscanf(t, "%d", &n); err == nil && fmt.Sprint(n) == t {
+ return n, true
+ }
+ }
+ return 0, false
+}
diff --git a/cli/test/evals/internal/ops/ops_mcp_test.go b/cli/test/evals/internal/ops/ops_mcp_test.go
new file mode 100644
index 00000000..9a7675e1
--- /dev/null
+++ b/cli/test/evals/internal/ops/ops_mcp_test.go
@@ -0,0 +1,61 @@
+package ops
+
+import (
+ "context"
+ "encoding/json"
+ "strings"
+ "testing"
+
+ "github.com/modelcontextprotocol/go-sdk/mcp"
+
+ client "github.com/tracewayapp/traceway/cli/pkg/client"
+ "github.com/tracewayapp/traceway/cli/pkg/mcpserver"
+)
+
+// TestEveryMCPToolMaps lists the write server's tools in memory: each must
+// map to an op.
+func TestEveryMCPToolMaps(t *testing.T) {
+ srv := mcpserver.New(mcpserver.Config{Client: client.New("http://127.0.0.1:1"), AllowWrite: true})
+ st, ct := mcp.NewInMemoryTransports()
+ if _, err := srv.Connect(context.Background(), st, nil); err != nil {
+ t.Fatal(err)
+ }
+ cs, err := mcp.NewClient(&mcp.Implementation{Name: "t", Version: "0"}, nil).Connect(context.Background(), ct, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer cs.Close()
+ n := 0
+ for tool, err := range cs.Tools(context.Background(), nil) {
+ if err != nil {
+ t.Fatal(err)
+ }
+ n++
+ op := ParseMCP(tool.Name, map[string]any{"enabled": true})
+ if strings.HasPrefix(op.Op, "mcp.") {
+ t.Errorf("MCP tool %s maps to no op", tool.Name)
+ }
+ readOnly := tool.Annotations != nil && tool.Annotations.ReadOnlyHint
+ if op.Write == readOnly {
+ t.Errorf("MCP tool %s is readOnly=%v but op %s has write=%v", tool.Name, readOnly, op.Op, op.Write)
+ }
+ }
+ if n < 30 {
+ t.Fatalf("expected the full write tool surface, got %d tools", n)
+ }
+}
+
+func TestRenderMCP(t *testing.T) {
+ name, args, err := RenderMCP("metrics.query", map[string]any{"name": "system.cpu.utilization", "aggregation": "max", "group_by": "state", "since": "24h"})
+ if err != nil || name != "query_metrics" {
+ t.Fatal(name, err)
+ }
+ qs := args["queries"].([]any)
+ if q := qs[0].(map[string]any); q["name"] != "system.cpu.utilization" || q["group_by"] != "state" || args["since"] != "24h" {
+ t.Fatalf("RenderMCP args %+v", args)
+ }
+ // The fake model's templates hand numbers over as json.Number.
+ if _, args, err := RenderMCP("alerts.disable", map[string]any{"id": json.Number("12")}); err != nil || args["id"] != 12 || args["enabled"] != false {
+ t.Fatalf("RenderMCP alerts.disable = %+v, %v", args, err)
+ }
+}
diff --git a/cli/test/evals/internal/record/record.go b/cli/test/evals/internal/record/record.go
index 97d2e9da..b2c536bc 100644
--- a/cli/test/evals/internal/record/record.go
+++ b/cli/test/evals/internal/record/record.go
@@ -73,7 +73,7 @@ type Run struct {
Questions []Question `json:"questions,omitempty"`
Usage Usage `json:"usage"`
Turns int `json:"turns"`
- StopReason string `json:"stopReason"` // submitted | budget_exhausted | no_submit | infra_error
+ StopReason string `json:"stopReason"` // submitted | budget_exhausted | no_submit | infra_error | cost_ceiling
StopDetail string `json:"stopDetail,omitempty"`
SkillLoaded []string `json:"skillLoaded,omitempty"`
Providers []string `json:"providers,omitempty"`
diff --git a/cli/test/evals/internal/report/aggregate.go b/cli/test/evals/internal/report/aggregate.go
index 12c95092..0bb5aeca 100644
--- a/cli/test/evals/internal/report/aggregate.go
+++ b/cli/test/evals/internal/report/aggregate.go
@@ -54,9 +54,11 @@ type Aggregates struct {
// Suites is keyed "suite|model|arm".
Suites map[string]*Rate `json:"suites"`
SkillUplift []Delta `json:"skillUplift"`
+ CLIvsMCP []Delta `json:"cliVsMcp"`
Models []Delta `json:"models"`
Cost map[string]*Costs `json:"cost"` // by model
SpendUSD float64 `json:"spendUsd"`
+ JudgeUSD float64 `json:"judgeUsd"`
Statuses map[string]int `json:"statuses"`
TopFailing []GraderCount `json:"topFailingGraders"`
}
@@ -67,10 +69,10 @@ type GraderCount struct {
Count int `json:"count"`
}
-// Counted reports whether a run counts toward pass rates: infra errors are
-// excluded, budget exhaustion is a failure.
+// Counted reports whether a run counts toward pass rates: infra errors and
+// cost-ceiling skips are excluded, budget exhaustion is a failure.
func Counted(r *runner.Result) bool {
- return r.Status != "infra_error"
+ return r.Status != "infra_error" && r.Status != "cost_ceiling"
}
// Aggregate computes pass rates and deltas.
@@ -80,7 +82,8 @@ func Aggregate(runs []*runner.Result) Aggregates {
cacheIn, cacheHit := map[string]int{}, map[string]int{}
for _, r := range runs {
a.Statuses[r.Status]++
- a.SpendUSD += r.Metrics.CostUSD
+ a.SpendUSD += r.Metrics.CostUSD + r.Metrics.JudgeCostUSD
+ a.JudgeUSD += r.Metrics.JudgeCostUSD
c := a.Cost[r.Model]
if c == nil {
c = &Costs{}
@@ -174,6 +177,12 @@ func Aggregate(runs []*runner.Result) Aggregates {
if d, ok := pair(s, m, catalog.ArmCLISkill, catalog.ArmCLINoSkill); ok {
a.SkillUplift = append(a.SkillUplift, d)
}
+ if d, ok := pair(s, m, catalog.ArmCLISkill, catalog.ArmMCP); ok {
+ a.CLIvsMCP = append(a.CLIvsMCP, d)
+ }
+ if d, ok := pair(s, m, catalog.ArmCLIWriteSkill, catalog.ArmMCPWrite); ok {
+ a.CLIvsMCP = append(a.CLIvsMCP, d)
+ }
}
// Model vs model on each arm, over the cases both ran.
for i := 0; i < len(modelList); i++ {
diff --git a/cli/test/evals/internal/report/compare.go b/cli/test/evals/internal/report/compare.go
new file mode 100644
index 00000000..0b8c2e25
--- /dev/null
+++ b/cli/test/evals/internal/report/compare.go
@@ -0,0 +1,131 @@
+package report
+
+import (
+ "encoding/json"
+ "fmt"
+ "os"
+ "sort"
+ "time"
+)
+
+// Baseline is the committed baseline.json: pass rates per suite and case,
+// and measured per-run usage per model that the preflight uses instead of
+// the models.yaml envelope.
+type Baseline struct {
+ GeneratedAt string `json:"generatedAt"`
+ From string `json:"from"`
+ GitSHA string `json:"gitSha"`
+ Note string `json:"note,omitempty"`
+ Suites map[string]*Rate `json:"suites"` // "suite|model|arm"
+ Cases map[string]*Rate `json:"cases"` // "case|model|arm"
+ PerRun map[string]MeasuredUsage `json:"perRun"` // by model
+ Extra map[string]json.RawMessage `json:"extra,omitempty"`
+}
+
+// MeasuredUsage is a model's average usage per run.
+type MeasuredUsage struct {
+ Runs int `json:"runs"`
+ InputTokens float64 `json:"inputTokens"`
+ CachedTokens float64 `json:"cachedTokens"`
+ OutputTokens float64 `json:"outputTokens"`
+ CostUSD float64 `json:"costUsd"`
+ CostUSDP90 float64 `json:"costUsdP90"`
+}
+
+// LoadBaseline reads baseline.json.
+func LoadBaseline(path string) (*Baseline, error) {
+ raw, err := os.ReadFile(path)
+ if err != nil {
+ return nil, err
+ }
+ var b Baseline
+ if err := json.Unmarshal(raw, &b); err != nil {
+ return nil, fmt.Errorf("%s: %w", path, err)
+ }
+ return &b, nil
+}
+
+// NewBaseline builds a baseline from results (twevals baseline update).
+func NewBaseline(r *Results) *Baseline {
+ a := Aggregate(r.Runs)
+ b := &Baseline{GeneratedAt: time.Now().UTC().Format(time.RFC3339), From: r.Header.RunID, GitSHA: r.Header.GitSHA,
+ Suites: a.Suites, Cases: a.Cases, PerRun: map[string]MeasuredUsage{}}
+ costs := map[string][]float64{}
+ sums := map[string]*MeasuredUsage{}
+ for _, run := range r.Runs {
+ if !Counted(run) || run.Model == "oracle" {
+ continue
+ }
+ m := sums[run.Model]
+ if m == nil {
+ m = &MeasuredUsage{}
+ sums[run.Model] = m
+ }
+ m.Runs++
+ m.InputTokens += float64(run.Metrics.InputTokens)
+ m.CachedTokens += float64(run.Metrics.CachedTokens)
+ m.OutputTokens += float64(run.Metrics.OutputTokens)
+ m.CostUSD += run.Metrics.CostUSD
+ costs[run.Model] = append(costs[run.Model], run.Metrics.CostUSD)
+ }
+ for k, m := range sums {
+ n := float64(m.Runs)
+ m.InputTokens /= n
+ m.CachedTokens /= n
+ m.OutputTokens /= n
+ m.CostUSD /= n
+ cs := costs[k]
+ sort.Float64s(cs)
+ m.CostUSDP90 = cs[int(float64(len(cs)-1)*0.9)]
+ b.PerRun[k] = *m
+ }
+ return b
+}
+
+// Save writes the baseline.
+func (b *Baseline) Save(path string) error {
+ raw, err := json.MarshalIndent(b, "", " ")
+ if err != nil {
+ return err
+ }
+ return os.WriteFile(path, append(raw, '\n'), 0o644)
+}
+
+// Regression is one finding of Compare.
+type Regression struct {
+ Kind string `json:"kind"` // suite | case
+ Key string `json:"key"`
+ Before float64 `json:"before"`
+ After float64 `json:"after"`
+ Detail string `json:"detail"`
+}
+
+// Compare applies the regression rule: a suite's pass rate drops
+// more than 10 percentage points for a model × arm, or a case at 2 of 3 or
+// better in the baseline now passes 0 of N.
+func Compare(r *Results, b *Baseline) []Regression {
+ a := Aggregate(r.Runs)
+ var out []Regression
+ for k, now := range a.Suites {
+ before, ok := b.Suites[k]
+ if !ok || before.Total == 0 {
+ continue
+ }
+ if drop := (before.Rate - now.Rate) * 100; drop > 10 {
+ out = append(out, Regression{Kind: "suite", Key: k, Before: before.Rate, After: now.Rate,
+ Detail: fmt.Sprintf("%s pass rate fell %.0fpp (%.0f%% -> %.0f%%)", k, drop, before.Rate*100, now.Rate*100)})
+ }
+ }
+ for k, now := range a.Cases {
+ before, ok := b.Cases[k]
+ if !ok || before.Total == 0 {
+ continue
+ }
+ if before.Rate >= 2.0/3.0-1e-9 && now.Pass == 0 && now.Total > 0 {
+ out = append(out, Regression{Kind: "case", Key: k, Before: before.Rate, After: 0,
+ Detail: fmt.Sprintf("%s passed %d/%d in the baseline and 0/%d now", k, before.Pass, before.Total, now.Total)})
+ }
+ }
+ sort.Slice(out, func(i, j int) bool { return out[i].Key < out[j].Key })
+ return out
+}
diff --git a/cli/test/evals/internal/report/compare_test.go b/cli/test/evals/internal/report/compare_test.go
new file mode 100644
index 00000000..18077df6
--- /dev/null
+++ b/cli/test/evals/internal/report/compare_test.go
@@ -0,0 +1,42 @@
+package report
+
+import (
+ "testing"
+
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/runner"
+)
+
+func TestCompare(t *testing.T) {
+ base := NewBaseline(&Results{Runs: []*runner.Result{
+ res("perf-a", "cli+skill", "opus", 1, "pass", 0), res("perf-a", "cli+skill", "opus", 2, "pass", 0), res("perf-a", "cli+skill", "opus", 3, "fail", 0),
+ res("perf-b", "cli+skill", "opus", 1, "pass", 0),
+ }})
+ now := &Results{Runs: []*runner.Result{res("perf-a", "cli+skill", "opus", 1, "fail", 0), res("perf-b", "cli+skill", "opus", 1, "pass", 0)}}
+ regs := Compare(now, base)
+ kinds := map[string]bool{}
+ for _, r := range regs {
+ kinds[r.Kind] = true
+ }
+ if !kinds["suite"] || !kinds["case"] {
+ t.Fatalf("want a suite drop and a case regression, got %+v", regs)
+ }
+ if regs := Compare(&Results{Runs: base.casesAsRuns()}, base); len(regs) != 0 {
+ t.Fatalf("no change must not regress: %+v", regs)
+ }
+}
+
+// casesAsRuns rebuilds runs matching the baseline exactly.
+func (b *Baseline) casesAsRuns() []*runner.Result {
+ var out []*runner.Result
+ for k, r := range b.Cases {
+ c, m, arm := split3(k)
+ for i := 0; i < r.Total; i++ {
+ st := "fail"
+ if i < r.Pass {
+ st = "pass"
+ }
+ out = append(out, res(c, arm, m, i+1, st, 0))
+ }
+ }
+ return out
+}
diff --git a/cli/test/evals/internal/report/cost.go b/cli/test/evals/internal/report/cost.go
new file mode 100644
index 00000000..b2945331
--- /dev/null
+++ b/cli/test/evals/internal/report/cost.go
@@ -0,0 +1,162 @@
+package report
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "os"
+ "sort"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/catalog"
+)
+
+// ModelsURL is OpenRouter's public model list (prices need no key).
+const ModelsURL = "https://openrouter.ai/api/v1/models"
+
+// LivePrice is a model's price in USD per million tokens, plus its
+// long-prompt tier when the provider has one.
+type LivePrice struct {
+ catalog.Price
+ Tier *catalog.Price `json:"tier,omitempty"`
+ TierMinPrompt int `json:"tierMinPrompt,omitempty"`
+ Supported []string `json:"supported,omitempty"`
+ Source string `json:"source"` // live | fallback | file
+}
+
+// FetchPrices reads live prices for the given model ids. When the list is
+// unreachable each model falls back to models.yaml's fallback_price.
+func FetchPrices(ctx context.Context, models []*catalog.Model, fromFile string) (map[string]LivePrice, error) {
+ var raw []byte
+ var err error
+ source := "live"
+ if fromFile != "" {
+ raw, err = os.ReadFile(fromFile)
+ source = "file"
+ } else {
+ raw, err = fetch(ctx, ModelsURL)
+ }
+ out := map[string]LivePrice{}
+ if err == nil {
+ var list struct {
+ Data []struct {
+ ID string `json:"id"`
+ Pricing struct {
+ Prompt string `json:"prompt"`
+ Completion string `json:"completion"`
+ CacheRead string `json:"input_cache_read"`
+ CacheWrite string `json:"input_cache_write"`
+ Overrides []struct {
+ MinPrompt int `json:"min_prompt_tokens"`
+ Prompt string `json:"prompt"`
+ Completion string `json:"completion"`
+ CacheRead string `json:"input_cache_read"`
+ CacheWrite string `json:"input_cache_write"`
+ } `json:"overrides"`
+ } `json:"pricing"`
+ Supported []string `json:"supported_parameters"`
+ } `json:"data"`
+ }
+ if jerr := json.Unmarshal(raw, &list); jerr == nil {
+ for _, d := range list.Data {
+ p := LivePrice{Price: catalog.Price{Prompt: perM(d.Pricing.Prompt), Completion: perM(d.Pricing.Completion),
+ CacheRead: perM(d.Pricing.CacheRead), CacheWrite: perM(d.Pricing.CacheWrite)}, Supported: d.Supported, Source: source}
+ if len(d.Pricing.Overrides) > 0 {
+ o := d.Pricing.Overrides[0]
+ p.Tier = &catalog.Price{Prompt: perM(o.Prompt), Completion: perM(o.Completion), CacheRead: perM(o.CacheRead), CacheWrite: perM(o.CacheWrite)}
+ p.TierMinPrompt = o.MinPrompt
+ }
+ out[d.ID] = p
+ }
+ } else {
+ err = jerr
+ }
+ }
+ missing := []string{}
+ for _, m := range models {
+ if _, ok := out[m.ID]; !ok {
+ out[m.ID] = LivePrice{Price: m.FallbackPrice, Source: "fallback"}
+ missing = append(missing, m.ID)
+ }
+ }
+ if len(missing) > 0 && err == nil {
+ err = fmt.Errorf("models not in the price list (pinned id retired?): %s", strings.Join(missing, ", "))
+ }
+ return out, err
+}
+
+func fetch(ctx context.Context, url string) ([]byte, error) {
+ ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+ defer cancel()
+ req, _ := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ return nil, err
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != 200 {
+ return nil, fmt.Errorf("GET %s: HTTP %d", url, resp.StatusCode)
+ }
+ return io.ReadAll(resp.Body)
+}
+
+func perM(s string) float64 {
+ f, _ := strconv.ParseFloat(s, 64)
+ return f * 1e6
+}
+
+// RunCost estimates one run's cost from a token envelope: uncached input at
+// the prompt price (the cache-write price when the model caches explicitly),
+// cached input at the read price, output at the completion price.
+func RunCost(p LivePrice, caching string, input, output int, cachedFrac float64) float64 {
+ price := p.Price
+ if p.Tier != nil && p.TierMinPrompt > 0 && input/20 > p.TierMinPrompt {
+ // Long agent histories cross the long-prompt tier on their last turns.
+ price = *p.Tier
+ }
+ cached := float64(input) * cachedFrac
+ uncached := float64(input) - cached
+ inPrice := price.Prompt
+ if caching == "anthropic" && price.CacheWrite > 0 {
+ inPrice = price.CacheWrite
+ }
+ return (uncached*inPrice + cached*price.CacheRead + float64(output)*price.Completion) / 1e6
+}
+
+// Estimate is a cost range for a set of runs.
+type Estimate struct {
+ Model string `json:"model"`
+ ModelID string `json:"modelId"`
+ Runs int `json:"runs"`
+ Low float64 `json:"low"`
+ High float64 `json:"high"`
+ Worst float64 `json:"worst"` // cache never hits
+ Source string `json:"source"`
+ Basis string `json:"basis"` // models.yaml | baseline
+}
+
+// EstimateRuns prices n runs of a model, from the baseline's measured
+// averages when it has them, else from the models.yaml envelope.
+func EstimateRuns(m *catalog.Model, p LivePrice, n int, base *Baseline) Estimate {
+ e := Estimate{Model: m.Key, ModelID: m.ID, Runs: n, Source: p.Source, Basis: "models.yaml"}
+ lo := RunCost(p, m.Caching, m.Estimate.InputTokens[0], m.Estimate.OutputTokens[0], m.Estimate.CachedFraction)
+ hi := RunCost(p, m.Caching, m.Estimate.InputTokens[1], m.Estimate.OutputTokens[1], m.Estimate.CachedFraction)
+ worst := RunCost(p, "none", m.Estimate.InputTokens[1], m.Estimate.OutputTokens[1], 0)
+ if base != nil {
+ if mu, ok := base.PerRun[m.Key]; ok && mu.Runs >= 5 {
+ lo, hi = mu.CostUSD*0.8, mu.CostUSDP90*1.2
+ e.Basis = "baseline"
+ }
+ }
+ e.Low, e.High, e.Worst = lo*float64(n), hi*float64(n), worst*float64(n)
+ return e
+}
+
+// SortEstimates orders estimates by model key.
+func SortEstimates(es []Estimate) {
+ sort.Slice(es, func(i, j int) bool { return es[i].Model < es[j].Model })
+}
diff --git a/cli/test/evals/internal/report/cost_test.go b/cli/test/evals/internal/report/cost_test.go
new file mode 100644
index 00000000..f6a6a19f
--- /dev/null
+++ b/cli/test/evals/internal/report/cost_test.go
@@ -0,0 +1,18 @@
+package report
+
+import (
+ "math"
+ "testing"
+
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/catalog"
+)
+
+func TestRunCost(t *testing.T) {
+ opus := LivePrice{Price: catalog.Price{Prompt: 4, Completion: 20, CacheRead: 0.2, CacheWrite: 5}}
+ // 100k input, 70% cached, 5k output: 30k at the write price, 70k at the read price, 5k out.
+ got := RunCost(opus, "anthropic", 100000, 5000, 0.7)
+ want := (30000*5.0 + 70000*0.2 + 5000*20.0) / 1e6
+ if math.Abs(got-want) > 1e-9 {
+ t.Fatalf("RunCost = %v, want %v", got, want)
+ }
+}
diff --git a/cli/test/evals/internal/report/markdown.go b/cli/test/evals/internal/report/markdown.go
index 740647dc..0a94fab7 100644
--- a/cli/test/evals/internal/report/markdown.go
+++ b/cli/test/evals/internal/report/markdown.go
@@ -38,7 +38,7 @@ func Markdown(r *Results) string {
// An oracle run costs nothing, so its summary leaves money out.
paid := !h.Oracle
if paid {
- fmt.Fprintf(&b, "- **Spend:** $%.2f\n", a.SpendUSD)
+ fmt.Fprintf(&b, "- **Spend:** $%.2f (judge $%.2f) of a $%.2f cap\n", a.SpendUSD, a.JudgeUSD, h.MaxCostUSD)
}
var st []string
for k, v := range a.Statuses {
@@ -60,6 +60,12 @@ func Markdown(r *Results) string {
fmt.Fprintf(&b, "| %s | %s | %.0f%% | %.0f%% | %+.0fpp |\n", d.Suite, d.Model, d.RateA*100, d.RateB*100, d.Delta)
}
}
+ if len(a.CLIvsMCP) > 0 {
+ b.WriteString("\n## CLI vs MCP\n\n| Suite | Model | CLI arm | MCP arm | CLI | MCP | Delta |\n|---|---|---|---|---|---|---|\n")
+ for _, d := range a.CLIvsMCP {
+ fmt.Fprintf(&b, "| %s | %s | %s | %s | %.0f%% | %.0f%% | %+.0fpp |\n", d.Suite, d.Model, d.A, d.B, d.RateA*100, d.RateB*100, d.Delta)
+ }
+ }
if len(a.Models) > 0 {
b.WriteString("\n## Model vs model (same cases)\n\n| Suite | A | B | A rate | B rate | Delta |\n|---|---|---|---|---|---|\n")
for _, d := range a.Models {
diff --git a/cli/test/evals/internal/report/results.go b/cli/test/evals/internal/report/results.go
index f877560d..0bd0bb74 100644
--- a/cli/test/evals/internal/report/results.go
+++ b/cli/test/evals/internal/report/results.go
@@ -1,4 +1,6 @@
-// Package report turns run results into results.json and summary.md.
+// Package report turns run results into results.json and summary.md,
+// compares them with the committed baseline, and estimates cost from live
+// OpenRouter prices.
package report
import (
@@ -34,6 +36,7 @@ type Header struct {
Command string `json:"command"`
Started time.Time `json:"started"`
Finished time.Time `json:"finished"`
+ MaxCostUSD float64 `json:"maxCostUsd"`
Oracle bool `json:"oracle,omitempty"`
CI map[string]string `json:"ci,omitempty"`
}
@@ -147,3 +150,22 @@ func Load(dir string) (*Results, error) {
}
return r, nil
}
+
+// Merge combines shard results under one header.
+func Merge(parts []*Results) *Results {
+ out := &Results{}
+ total := 0.0
+ for i, p := range parts {
+ if i == 0 {
+ out.Header = p.Header
+ }
+ total += p.Header.MaxCostUSD
+ if p.Header.Finished.After(out.Header.Finished) {
+ out.Header.Finished = p.Header.Finished
+ }
+ out.Runs = append(out.Runs, p.Runs...)
+ }
+ out.Header.MaxCostUSD = total
+ sort.SliceStable(out.Runs, func(i, j int) bool { return out.Runs[i].Key() < out.Runs[j].Key() })
+ return out
+}
diff --git a/cli/test/evals/internal/runner/container.go b/cli/test/evals/internal/runner/container.go
new file mode 100644
index 00000000..6a7b4da2
--- /dev/null
+++ b/cli/test/evals/internal/runner/container.go
@@ -0,0 +1,330 @@
+package runner
+
+import (
+ "bytes"
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "sort"
+ "strconv"
+ "strings"
+ "syscall"
+ "time"
+
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/catalog"
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/env"
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/grade"
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/ops"
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/record"
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/tools"
+)
+
+// EnsureSandboxImage builds testdata/sandbox-image once per Dockerfile hash
+// and returns its tag.
+func EnsureSandboxImage(ctx context.Context, evalsRoot string) (string, error) {
+ dir := filepath.Join(evalsRoot, "testdata", "sandbox-image")
+ raw, err := os.ReadFile(filepath.Join(dir, "Dockerfile"))
+ if err != nil {
+ return "", err
+ }
+ sum := sha256.Sum256(raw)
+ tag := "twevals-sandbox:" + hex.EncodeToString(sum[:])[:12]
+ if exec.CommandContext(ctx, "docker", "image", "inspect", tag).Run() == nil {
+ return tag, nil
+ }
+ cmd := exec.CommandContext(ctx, "docker", "build", "-q", "-t", tag, dir)
+ if out, err := cmd.CombinedOutput(); err != nil {
+ return "", fmt.Errorf("docker build %s: %w\n%s", tag, err, out)
+ }
+ return tag, nil
+}
+
+// ContainerShell is the setup suite's shell: a real bash inside a sandbox
+// container (toolchains and package registries, no secrets), with the
+// workspace bind-mounted and the backend reachable on the host network.
+// traceway invocations are recorded from a static parse of each command.
+type ContainerShell struct {
+ name string
+ env []string
+ timeout time.Duration
+ // host runs commands on the host instead (oracle runs only, when
+ // Docker is unavailable: the commands are the case's own).
+ host bool
+ dir string
+ path string
+}
+
+// workdir is where the repository is mounted for the shell.
+func (c *ContainerShell) workdir() string {
+ if c.host {
+ return c.dir
+ }
+ return "/work"
+}
+
+func setupEnv(c *catalog.Case, inst *env.Instance, projects []env.Project, port int) []string {
+ if c.Setup == nil {
+ return nil
+ }
+ var out []string
+ keys := make([]string, 0, len(c.Setup.Env))
+ for k := range c.Setup.Env {
+ keys = append(keys, k)
+ }
+ sort.Strings(keys)
+ for _, k := range keys {
+ v := c.Setup.Env[k]
+ switch {
+ case v == "@url":
+ v = inst.Backend.URL
+ case v == "@port":
+ v = strconv.Itoa(port)
+ case strings.HasPrefix(v, "@token:"):
+ name := strings.TrimPrefix(v, "@token:")
+ for _, p := range projects {
+ if p.Name == name {
+ v = p.Token
+ }
+ }
+ }
+ out = append(out, k+"="+v)
+ }
+ return out
+}
+
+func startContainerShell(ctx context.Context, opt Options, work string, surf surface) (*ContainerShell, error) {
+ if opt.SandboxImage == "" {
+ if !opt.Oracle {
+ return nil, fmt.Errorf("the setup suite needs Docker and the sandbox image (a model's commands never run on the host)")
+ }
+ bin := filepath.Join(filepath.Dir(work), ".twevals-bin")
+ if err := os.MkdirAll(bin, 0o755); err != nil {
+ return nil, err
+ }
+ _ = os.Remove(filepath.Join(bin, "traceway"))
+ if err := os.Symlink(surf.cliBin, filepath.Join(bin, "traceway")); err != nil {
+ return nil, err
+ }
+ home := filepath.Join(filepath.Dir(work), ".twevals-home")
+ _ = os.MkdirAll(home, 0o755)
+ return &ContainerShell{host: true, dir: work, timeout: 10 * time.Minute, env: surf.containerEnv,
+ path: bin + string(os.PathListSeparator) + os.Getenv("PATH")}, nil
+ }
+ name := "twevals-" + filepath.Base(filepath.Dir(filepath.Dir(work))) + "-" + strconv.FormatInt(time.Now().UnixNano()%1e9, 36)
+ args := []string{"run", "-d", "--rm", "--name", name, "--network", "host",
+ "--user", fmt.Sprintf("%d:%d", os.Getuid(), os.Getgid()),
+ "-e", "HOME=/tmp/home", "-e", "GOPATH=/tmp/go", "-e", "GOCACHE=/tmp/gocache", "-e", "npm_config_cache=/tmp/npm",
+ "-v", work + ":/work", "-v", surf.cliBin + ":/usr/local/bin/traceway:ro", "-w", "/work",
+ opt.SandboxImage, "sh", "-c", "mkdir -p /tmp/home && sleep infinity"}
+ if out, err := exec.CommandContext(ctx, "docker", args...).CombinedOutput(); err != nil {
+ return nil, fmt.Errorf("docker run: %w: %s", err, out)
+ }
+ return &ContainerShell{name: name, timeout: 10 * time.Minute, env: surf.containerEnv}, nil
+}
+
+// Close removes the container.
+func (c *ContainerShell) Close() {
+ if c.host {
+ return
+ }
+ _ = exec.Command("docker", "rm", "-f", c.name).Run()
+}
+
+// Spec describes the tool.
+func (c *ContainerShell) Spec() tools.Spec {
+ return tools.Spec{Name: "shell", Description: "Run a bash command in the repository (the current directory) on the developer's machine. go, node, npm, python3, pip, git, curl and the traceway CLI are installed; package registries are reachable. Start long-running processes in the background (nohup ... &). Output is capped at 30000 characters.",
+ Parameters: map[string]any{"type": "object", "properties": map[string]any{"command": map[string]any{"type": "string", "description": "The command line to run."}}, "required": []string{"command"}, "additionalProperties": false}}
+}
+
+// Exec runs a command in the container.
+func (c *ContainerShell) Exec(ctx context.Context, cmdline string, timeout time.Duration) (string, int) {
+ ctx, cancel := context.WithTimeout(ctx, timeout)
+ defer cancel()
+ var cmd *exec.Cmd
+ if c.host {
+ cmd = exec.CommandContext(ctx, "bash", "-c", cmdline)
+ cmd.Dir = c.dir
+ home := filepath.Join(filepath.Dir(c.dir), ".twevals-home")
+ cmd.Env = append([]string{"HOME=" + home, "PATH=" + c.path, "TZ=UTC", "LANG=C.UTF-8", "GOFLAGS=-mod=mod", "GOTOOLCHAIN=local"}, c.env...)
+ for _, k := range []string{"GOPATH", "GOMODCACHE", "GOCACHE", "GOPROXY"} {
+ if v := os.Getenv(k); v != "" {
+ cmd.Env = append(cmd.Env, k+"="+v)
+ }
+ }
+ if os.Getenv("GOMODCACHE") == "" && os.Getenv("GOPATH") == "" {
+ if h, err := os.UserHomeDir(); err == nil {
+ cmd.Env = append(cmd.Env, "GOMODCACHE="+filepath.Join(h, "go", "pkg", "mod"))
+ }
+ }
+ if os.Getenv("GOCACHE") == "" {
+ if d, err := os.UserCacheDir(); err == nil {
+ cmd.Env = append(cmd.Env, "GOCACHE="+filepath.Join(d, "go-build"))
+ }
+ }
+ cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
+ cmd.Cancel = func() error { return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) }
+ } else {
+ args := []string{"exec", "-i", "-w", "/work"}
+ for _, e := range c.env {
+ args = append(args, "-e", e)
+ }
+ args = append(args, c.name, "bash", "-c", cmdline)
+ cmd = exec.CommandContext(ctx, "docker", args...)
+ }
+ var out bytes.Buffer
+ cmd.Stdout, cmd.Stderr = &out, &out
+ err := cmd.Run()
+ code := 0
+ if err != nil {
+ if ee, ok := err.(*exec.ExitError); ok {
+ code = ee.ExitCode()
+ } else {
+ code = 1
+ }
+ if ctx.Err() != nil {
+ out.WriteString(fmt.Sprintf("\n[command timed out after %s]", timeout))
+ code = 124
+ }
+ }
+ return out.String(), code
+}
+
+// Call runs one command line.
+func (c *ContainerShell) Call(ctx context.Context, raw json.RawMessage) tools.Result {
+ var in struct {
+ Command string `json:"command"`
+ }
+ if err := json.Unmarshal(raw, &in); err != nil || strings.TrimSpace(in.Command) == "" {
+ return tools.Result{Output: "error: command is required"}
+ }
+ out, code := c.Exec(ctx, in.Command, c.timeout)
+ if code != 0 {
+ out += fmt.Sprintf("\n[exit status %d]", code)
+ }
+ capped, trunc := tools.Cap(out)
+ return tools.Result{Output: capped, OK: code == 0, Truncated: trunc, Ops: staticOps(in.Command, code, out)}
+}
+
+// staticOps records the traceway invocations a command line contains.
+func staticOps(cmdline string, code int, output string) []record.OpRecord {
+ inv, err := ops.Invocations(cmdline)
+ if err != nil {
+ return nil
+ }
+ var out []record.OpRecord
+ for _, words := range inv {
+ if words[0] == "traceway" || words[0] == "traceway-write" {
+ out = append(out, record.OpRecord{Op: ops.ParseCLI(words[1:]), Argv: words, Exit: code, OK: code == 0, Output: output})
+ }
+ // The setup skill's default plan path submits with curl; record it
+ // as the same operation as `traceway setup apply`.
+ if words[0] == "curl" && submitsSetupPlan(words) {
+ out = append(out, record.OpRecord{Op: ops.Op{Op: "setup.apply", Args: map[string]any{"via": "curl"}}, Argv: words, Exit: code, OK: code == 0, Output: output})
+ }
+ }
+ return out
+}
+
+func submitsSetupPlan(words []string) bool {
+ hasPlan, isPut := false, false
+ for i, w := range words {
+ if strings.Contains(w, "/api/setup/plan") {
+ hasPlan = true
+ }
+ if (w == "-X" || w == "--request") && i+1 < len(words) && strings.EqualFold(words[i+1], "PUT") || strings.EqualFold(w, "-XPUT") {
+ isPut = true
+ }
+ }
+ return hasPlan && isPut
+}
+
+// verifySetup builds and starts the instrumented app in a fresh container
+// against the backend, then hits its routes, so the state grader can check
+// that telemetry actually arrived.
+func verifySetup(ctx context.Context, opt Options, c *catalog.Case, work string, inst *env.Instance, projects []env.Project, surf surface) *grade.Verdict {
+ v := &grade.Verdict{Grader: "setup_verify", Gating: true, Pass: true}
+ vs := c.Setup.Verify
+ cs, err := startContainerShell(ctx, opt, work, surf)
+ if err != nil {
+ v.Pass, v.Failures = false, []string{err.Error()}
+ return v
+ }
+ defer cs.Close()
+ logPath, pidPath := "/tmp/app.log", "/tmp/app.pid"
+ if cs.host {
+ logPath, pidPath = filepath.Join(filepath.Dir(work), "app.log"), filepath.Join(filepath.Dir(work), "app.pid")
+ }
+ defer func() {
+ if cs.host {
+ // Stop the app (its whole process group) we started on the host.
+ _, _ = cs.Exec(context.Background(), "kill -- -$(cat "+pidPath+") 2>/dev/null || true", 10*time.Second)
+ }
+ }()
+ cd := "cd " + cs.workdir()
+ if vs.Dir != "" {
+ cd += "/" + vs.Dir
+ }
+ source := ""
+ for _, f := range vs.EnvFiles {
+ source += fmt.Sprintf("if [ -f %s ]; then set -a; . ./%s; set +a; fi; ", f, f)
+ }
+ if vs.Build != "" {
+ out, code := cs.Exec(ctx, cd+" && "+vs.Build, 10*time.Minute)
+ if code != 0 {
+ v.Pass = false
+ v.Failures = append(v.Failures, "build failed: "+tail(out, 2000))
+ return v
+ }
+ v.Notes = append(v.Notes, "build passed")
+ }
+ if vs.Start != "" {
+ _, _ = cs.Exec(ctx, cd+" && "+source+"setsid nohup sh -c "+shellQuoteArg(vs.Start)+" > "+logPath+" 2>&1 < /dev/null & echo $! > "+pidPath, time.Minute)
+ port := ""
+ for _, e := range cs.env {
+ if strings.HasPrefix(e, "PORT=") {
+ port = strings.TrimPrefix(e, "PORT=")
+ }
+ }
+ up := false
+ for i := 0; i < 120 && !up; i++ {
+ _, code := cs.Exec(ctx, "curl -s -o /dev/null http://127.0.0.1:"+port+"/", 10*time.Second)
+ up = code == 0
+ if !up {
+ time.Sleep(time.Second)
+ }
+ }
+ if !up {
+ log, _ := cs.Exec(ctx, "cat "+logPath, 10*time.Second)
+ v.Pass = false
+ v.Failures = append(v.Failures, "the app never listened on :"+port+": "+tail(log, 2000))
+ return v
+ }
+ for _, h := range vs.Hit {
+ for k := 0; k < 5; k++ {
+ _, _ = cs.Exec(ctx, "curl -s -o /dev/null http://127.0.0.1:"+port+h, 10*time.Second)
+ }
+ }
+ wait := vs.Wait.D()
+ if wait == 0 {
+ wait = 12 * time.Second
+ }
+ time.Sleep(wait)
+ log, _ := cs.Exec(ctx, "tail -c 3000 "+logPath, 10*time.Second)
+ v.Notes = append(v.Notes, "app log tail: "+tail(log, 1500))
+ }
+ return v
+}
+
+func shellQuoteArg(s string) string { return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" }
+
+func tail(s string, n int) string {
+ if len(s) > n {
+ return s[len(s)-n:]
+ }
+ return s
+}
diff --git a/cli/test/evals/internal/runner/hooks.go b/cli/test/evals/internal/runner/hooks.go
new file mode 100644
index 00000000..da7306f4
--- /dev/null
+++ b/cli/test/evals/internal/runner/hooks.go
@@ -0,0 +1,103 @@
+package runner
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "strconv"
+ "sync"
+ "time"
+
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/catalog"
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/env"
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/grade"
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/record"
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/seed"
+)
+
+// hookRunner fires case hooks after matching agent calls.
+type hookRunner struct {
+ hooks []catalog.Hook
+ counts []int
+ inst *env.Instance
+ pid string
+ sink *seed.LiveSink
+ mu sync.Mutex
+}
+
+func newHookRunner(c *catalog.Case, inst *env.Instance, primary env.Project, sink *seed.LiveSink) *hookRunner {
+ return &hookRunner{hooks: c.Hooks, counts: make([]int, len(c.Hooks)), inst: inst, pid: primary.ID, sink: sink}
+}
+
+func (h *hookRunner) after(ctx context.Context, cl *record.Call) {
+ h.mu.Lock()
+ defer h.mu.Unlock()
+ for i, hk := range h.hooks {
+ times := hk.Times
+ if times == 0 {
+ times = 1
+ }
+ if h.counts[i] >= times {
+ continue
+ }
+ hit := false
+ for k := range cl.Ops {
+ if cl.Ops[k].OK && grade.Matches(&cl.Ops[k], hk.After) {
+ hit = true
+ }
+ }
+ if !hit {
+ continue
+ }
+ h.counts[i]++
+ h.run(ctx, hk)
+ }
+}
+
+func (h *hookRunner) run(ctx context.Context, hk catalog.Hook) {
+ api := h.inst.Backend.API().WithBearer(h.inst.Admin.JWT)
+ arg := func(k string) string { return fmt.Sprint(hk.Args[k]) }
+ switch hk.Action {
+ case "edit_post_mortem":
+ // A teammate edits the document between the agent's read and write.
+ // The PUT replaces the whole document (title required), so the
+ // title, tags and incident link are carried over from a fresh read.
+ body := map[string]any{"contentMd": arg("content")}
+ if cur, err := api.Do(ctx, http.MethodGet, "/api/post-mortems/"+arg("id"), env.ProjectQuery(h.pid), nil); err == nil && cur.OK() {
+ m := cur.Map()
+ body["title"], body["tags"], body["incidentId"] = m["title"], m["tags"], m["incidentId"]
+ }
+ _, _ = api.Do(ctx, http.MethodPut, "/api/post-mortems/"+arg("id"), env.ProjectQuery(h.pid), body)
+ case "stub_healthy":
+ h.inst.Stub.SetHealthy(true)
+ case "stub_unhealthy":
+ h.inst.Stub.SetHealthy(false)
+ case "sleep":
+ n, _ := strconv.Atoi(arg("seconds"))
+ time.Sleep(time.Duration(n) * time.Second)
+ case "approve_setup_draft":
+ _, _ = env.ApprovePendingDraft(ctx, h.inst.Backend, h.inst.Admin)
+ }
+}
+
+// startDraftApprover approves pending setup drafts while the run lasts, as
+// the user does in the dashboard after the agent submits a plan.
+func startDraftApprover(ctx context.Context, c *catalog.Case, inst *env.Instance) func() {
+ if c.Setup == nil || !c.Setup.ApproveDrafts {
+ return func() {}
+ }
+ ctx, cancel := context.WithCancel(ctx)
+ go func() {
+ t := time.NewTicker(2 * time.Second)
+ defer t.Stop()
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-t.C:
+ _, _ = env.ApprovePendingDraft(ctx, inst.Backend, inst.Admin)
+ }
+ }
+ }()
+ return cancel
+}
diff --git a/cli/test/evals/internal/runner/runner.go b/cli/test/evals/internal/runner/runner.go
index 59bf3af5..d1139af6 100644
--- a/cli/test/evals/internal/runner/runner.go
+++ b/cli/test/evals/internal/runner/runner.go
@@ -34,7 +34,19 @@ type Options struct {
Runs int
Jobs int
Keep bool
- Log io.Writer
+ // Oracle plays each case's oracle with the fake model (free).
+ Oracle bool
+ // Chat returns the model client for real runs.
+ Chat func(m *catalog.Model) agent.Chatter
+ Ledger *agent.Ledger
+ Judge *grade.Judge
+ // Prices give a per-model cost estimate when the provider reports none.
+ Prices map[string]catalog.Price
+ SandboxImage string
+ AllowHostTests bool
+ Log io.Writer
+ // Skip lists run keys (case/arm/model/rN) already finished (resume).
+ Skip map[string]bool
}
// Result is one run's outcome (results.json "runs" entries).
@@ -46,7 +58,7 @@ type Result struct {
ModelID string `json:"modelId"`
Providers []string `json:"provider,omitempty"`
RunIdx int `json:"runIdx"`
- Status string `json:"status"` // pass | fail | budget_exhausted | infra_error
+ Status string `json:"status"` // pass | fail | budget_exhausted | infra_error | cost_ceiling
Graders []*grade.Verdict `json:"graders"`
Metrics Metrics `json:"metrics"`
Transcript string `json:"transcript,omitempty"`
@@ -54,7 +66,7 @@ type Result struct {
Seed *seed.Stats `json:"seed,omitempty"`
}
-// Key identifies the run.
+// Key identifies the run for resume.
func (r *Result) Key() string { return RunKey(r.Case, r.Arm, r.Model, r.RunIdx) }
// RunKey builds a run key.
@@ -72,6 +84,7 @@ type Metrics struct {
CacheWriteTokens int `json:"cacheWriteTokens"`
OutputTokens int `json:"outputTokens"`
CostUSD float64 `json:"costUsd"`
+ JudgeCostUSD float64 `json:"judgeCostUsd"`
WallMs int64 `json:"wallMs"`
SkillLoaded bool `json:"skillLoaded"`
Skills []string `json:"skills,omitempty"`
@@ -97,8 +110,9 @@ func Jobs(units []catalog.Unit, models []*catalog.Model, runs int) []Job {
return out
}
-// Execute runs every job with up to Jobs in parallel. Every run plays its
-// case's oracle with the fake model.
+// Execute runs every job with up to Jobs in parallel. Before each run starts
+// it checks the ledger: at the ceiling, in-flight runs finish, nothing new
+// starts, and the remaining runs are recorded as cost_ceiling.
func Execute(ctx context.Context, opt Options, jobs []Job) []*Result {
par := opt.Jobs
if par <= 0 {
@@ -107,17 +121,44 @@ func Execute(ctx context.Context, opt Options, jobs []Job) []*Result {
results := make([]*Result, len(jobs))
sem := make(chan struct{}, par)
var wg sync.WaitGroup
+ perRun := 0.0
+ if opt.Ledger != nil && opt.Ledger.Cap > 0 && len(jobs) > 0 {
+ perRun = opt.Ledger.Cap / float64(len(jobs)) * 3
+ }
for i, j := range jobs {
+ key := RunKey(j.Unit.Case.ID, j.Unit.Arm, j.Model.Key, j.Idx)
+ if opt.Skip[key] {
+ continue
+ }
sem <- struct{}{}
+ if opt.Ledger != nil && opt.Ledger.Exhausted() {
+ <-sem
+ results[i] = &Result{Case: j.Unit.Case.ID, Suite: j.Unit.Case.Suite, Arm: j.Unit.Arm, Model: j.Model.Key, ModelID: j.Model.ID, RunIdx: j.Idx,
+ Status: agent.StopCeiling, Error: "not started: the --max-cost-usd ceiling was reached"}
+ continue
+ }
wg.Add(1)
go func(i int, j Job) {
defer wg.Done()
defer func() { <-sem }()
- results[i] = runOne(ctx, opt, j)
+ // Each run may use up to three times its even share, but no more
+ // than its slice of what is left, so the runs in flight cannot
+ // together overshoot the ceiling by much.
+ cap := perRun
+ if opt.Ledger != nil && cap > 0 {
+ cap = min(cap, opt.Ledger.Remaining()/float64(par))
+ }
+ results[i] = runOne(ctx, opt, j, cap)
}(i, j)
}
wg.Wait()
- return results
+ var out []*Result
+ for _, r := range results {
+ if r != nil {
+ out = append(out, r)
+ }
+ }
+ return out
}
func logf(opt Options, format string, a ...any) {
@@ -128,17 +169,35 @@ func logf(opt Options, format string, a ...any) {
// surface is what the arm hands the agent.
type surface struct {
- cliBin string // executable path of the traceway CLI
- cliName string // traceway
- cliHome *env.CLIHome
- skills []string
- submit string
+ cliBin string // executable path for CLI arms
+ cliName string // traceway | traceway-write
+ cliHome *env.CLIHome
+ mcp *tools.MCPBridge
+ skills []string
+ submit string
+ setupShell bool
+ // containerEnv is the setup shell's environment (tokens, URL, PORT).
+ containerEnv []string
}
-func runOne(ctx context.Context, opt Options, j Job) (res *Result) {
+func submitFor(suite string) string {
+ switch suite {
+ case "perf":
+ return tools.SubmitPerf
+ case "debug":
+ return tools.SubmitDebug
+ default:
+ return tools.Finish
+ }
+}
+
+func runOne(ctx context.Context, opt Options, j Job, runCap float64) (res *Result) {
c := j.Unit.Case
arm := j.Unit.Arm
res = &Result{Case: c.ID, Suite: c.Suite, Arm: arm, Model: j.Model.Key, ModelID: j.Model.ID, RunIdx: j.Idx}
+ if opt.Oracle {
+ res.Model, res.ModelID = "oracle", "fake"
+ }
started := time.Now()
dir := filepath.Join(opt.OutDir, "runs", c.ID, sanitize(arm), res.Model, fmt.Sprintf("r%d", j.Idx))
defer func() {
@@ -189,13 +248,21 @@ func runOne(ctx context.Context, opt Options, j Job) (res *Result) {
stats := sc.Stats()
res.Seed = &stats
projects := sink.Projects()
- if len(projects) == 0 {
+ if len(projects) == 0 && c.Suite != "setup" {
return infra(fmt.Errorf("seed %s created no project", c.Seed))
}
exposed := sc.Exposed()
primary := primaryProject(exposed, projects)
- vars := map[string]any{"seed": jsonify(exposed), "env": envVars(inst, projects, primary, sc.Now)}
+ envv := envVars(inst, projects, primary, sc.Now)
+ if c.Setup != nil && c.Setup.Token == "setup" {
+ tok, err := env.SetupToken(ctx, inst.Backend, inst.Admin)
+ if err != nil {
+ return infra(err)
+ }
+ envv["setupToken"] = tok
+ }
+ vars := map[string]any{"seed": jsonify(exposed), "env": envv}
resolved, err := c.Resolve(vars)
if err != nil {
return infra(err)
@@ -206,13 +273,23 @@ func runOne(ctx context.Context, opt Options, j Job) (res *Result) {
if err := buildWorkspace(opt.EvalsRoot, resolved.Workspace.Repo, work, sc.Now); err != nil {
return infra(err)
}
- surf, err := prepareSurface(ctx, opt, resolved, arm, inst, primary, dir)
+ surf, cleanup, err := prepareSurface(ctx, opt, resolved, arm, inst, projects, primary, dir, work)
+ if cleanup != nil {
+ defer cleanup()
+ }
if err != nil {
return infra(err)
}
if err := installSkills(opt.RepoRoot, work, surf.skills); err != nil {
return infra(err)
}
+ if surf.setupShell {
+ port, err := env.FreePort()
+ if err != nil {
+ return infra(err)
+ }
+ surf.containerEnv = setupEnv(resolved, inst, projects, port)
+ }
sb := &tools.Sandbox{Root: work, ReadOnly: []string{".skills"}}
ask := &tools.AskUser{Rules: resolved.User}
@@ -231,16 +308,30 @@ func runOne(ctx context.Context, opt Options, j Job) (res *Result) {
}
return false, false
}
- sh := &tools.Shell{Sandbox: sb, Git: true, Denied: deniedFor()}
- sh.Bins = map[string]string{surf.cliName: surf.cliBin}
- sh.BinEnv = surf.cliHome.Env(filepath.Dir(surf.cliBin))
- toolset = append(toolset, sh)
+ if surf.setupShell {
+ cs, err := startContainerShell(ctx, opt, work, surf)
+ if err != nil {
+ return infra(err)
+ }
+ defer cs.Close()
+ toolset = append(toolset, cs)
+ } else {
+ sh := &tools.Shell{Sandbox: sb, Git: true, Denied: deniedFor(arm)}
+ if surf.cliHome != nil {
+ sh.Bins = map[string]string{surf.cliName: surf.cliBin}
+ sh.BinEnv = surf.cliHome.Env(filepath.Dir(surf.cliBin))
+ }
+ toolset = append(toolset, sh)
+ }
toolset = append(toolset, &tools.ReadFile{Sandbox: sb}, &tools.ListFiles{Sandbox: sb}, &tools.Search{Sandbox: sb},
&tools.EditFile{Sandbox: sb}, &tools.WriteFile{Sandbox: sb}, ask)
skillDefs, _ := tools.LoadSkills(filepath.Join(work, ".skills"))
if len(skillDefs) > 0 {
toolset = append(toolset, &tools.LoadSkill{Skills: skillDefs, Rel: sb.Rel, Loaded: &loaded})
}
+ if surf.mcp != nil {
+ toolset = append(toolset, surf.mcp.Tools()...)
+ }
submit = &tools.Submit{Kind: surf.submit, CallExists: callExists, MaxRepairs: 2}
toolset = append(toolset, submit)
@@ -260,11 +351,19 @@ func runOne(ctx context.Context, opt Options, j Job) (res *Result) {
}
system := agent.SystemPrompt(agent.PromptInput{
SubmitTool: surf.submit, Skills: skillDefs, Preloaded: preloaded,
- Environment: environmentNote(surf, inst, primary),
+ MCPInstructions: mcpInstructions(surf), Environment: environmentNote(surf, inst, primary),
})
- fake := agent.NewFake(resolved.OracleFor(arm), filepath.Join(opt.EvalsRoot, "testdata"))
- defer fake.Close()
+ var chat agent.Chatter
+ var fake *agent.Fake
+ if opt.Oracle {
+ steps := resolved.OracleFor(arm)
+ fake = agent.NewFake(steps, surf.cliName, filepath.Join(opt.EvalsRoot, "testdata"))
+ defer fake.Close()
+ chat = fake.Client()
+ } else {
+ chat = opt.Chat(j.Model)
+ }
tr, err := agent.OpenTranscript(filepath.Join(dir, "transcript.jsonl"))
if err != nil {
return infra(err)
@@ -272,13 +371,22 @@ func runOne(ctx context.Context, opt Options, j Job) (res *Result) {
defer tr.Close()
res.Transcript, _ = filepath.Rel(opt.OutDir, filepath.Join(dir, "transcript.jsonl"))
+ hooks := newHookRunner(resolved, inst, primary, sink)
+ stopApprover := startDraftApprover(ctx, resolved, inst)
+ defer stopApprover()
+
+ model := j.Model
+ if opt.Oracle {
+ model = &catalog.Model{Key: "oracle", ID: "fake/oracle", Caching: "none"}
+ }
run := agent.Run(ctx, agent.Config{
- Model: j.Model, Chat: fake.Client(), Tools: toolset, System: system, Prompt: prompt, SubmitTool: surf.submit,
- Budget: resolved.Budget, Transcript: tr,
+ Model: model, Chat: chat, Tools: toolset, System: system, Prompt: prompt, SubmitTool: surf.submit,
+ Budget: resolved.Budget, RunCap: runCap, Ledger: opt.Ledger, Price: opt.Prices[j.Model.Key], Transcript: tr,
AfterCall: func(ctx context.Context, cl *record.Call) {
callsMu.Lock()
calls = append(calls, cl)
callsMu.Unlock()
+ hooks.after(ctx, cl)
},
Questions: ask.Questions,
SkillLoaded: func() []string { return loaded },
@@ -287,27 +395,49 @@ func runOne(ctx context.Context, opt Options, j Job) (res *Result) {
res.Providers = run.Providers
res.Metrics = metricsOf(run)
- if errs := fake.Errors(); len(errs) > 0 {
- res.Error = "oracle: " + strings.Join(errs, "; ")
+ if fake != nil {
+ if errs := fake.Errors(); len(errs) > 0 {
+ res.Error = "oracle: " + strings.Join(errs, "; ")
+ }
}
- if run.StopReason == agent.StopInfra {
+ switch run.StopReason {
+ case agent.StopInfra:
res.Status, res.Error = agent.StopInfra, run.StopDetail
return res
+ case agent.StopCeiling:
+ res.Status, res.Error = agent.StopCeiling, run.StopDetail
+ return res
}
- in := &grade.Input{Case: resolved, Arm: arm, Run: run, Seed: jsonify(exposed)}
+ in := &grade.Input{
+ Case: resolved, Arm: arm, Run: run, Seed: jsonify(exposed), Workspace: work,
+ State: &stateReader{inst: inst, projects: projects, primary: primary}, FakeModel: opt.Oracle,
+ AllowHostTests: opt.AllowHostTests, SandboxImage: opt.SandboxImage, EvalsRoot: opt.EvalsRoot, Judge: opt.Judge,
+ }
+ if resolved.Setup != nil && resolved.Setup.Verify != nil {
+ if v := verifySetup(ctx, opt, resolved, work, inst, projects, surf); v != nil {
+ res.Graders = append(res.Graders, v)
+ }
+ }
res.Graders = append(res.Graders, grade.All(ctx, in)...)
+ for _, v := range res.Graders {
+ res.Metrics.JudgeCostUSD += v.CostUSD
+ }
switch {
case run.StopReason == agent.StopBudget:
res.Status = "budget_exhausted"
- case res.Error != "":
+ case res.Error != "" && opt.Oracle:
res.Status = "fail"
case grade.Passed(res.Graders):
res.Status = "pass"
default:
res.Status = "fail"
}
- logf(opt, " %s %s (%d calls)", RunKey(c.ID, arm, res.Model, j.Idx), res.Status, res.Metrics.ToolCalls)
+ if opt.Oracle {
+ logf(opt, " %s %s (%d calls)", RunKey(c.ID, arm, res.Model, j.Idx), res.Status, res.Metrics.ToolCalls)
+ } else {
+ logf(opt, " %s %s (%d calls, $%.4f)", RunKey(c.ID, arm, res.Model, j.Idx), res.Status, res.Metrics.ToolCalls, res.Metrics.CostUSD)
+ }
return res
}
@@ -373,16 +503,39 @@ func metricsOf(run *record.Run) Metrics {
return m
}
-func deniedFor() map[string]string {
+func deniedFor(arm string) map[string]string {
d := map[string]string{
"curl": "no network tools in this sandbox", "wget": "no network tools in this sandbox",
"python": "no interpreters in this sandbox", "python3": "no interpreters in this sandbox", "node": "no interpreters in this sandbox",
"bash": "this shell already is the interpreter; run the command directly", "sh": "this shell already is the interpreter; run the command directly",
}
- d["traceway-write"] = "traceway-write is not installed on this machine"
+ switch {
+ case catalog.IsMCPArm(arm):
+ d["traceway"] = "the traceway CLI is not installed on this machine; use the connected Traceway MCP tools"
+ d["traceway-write"] = d["traceway"]
+ case arm == catalog.ArmCLIWriteSkill:
+ d["traceway"] = "not installed here; use traceway-write, which has every read command too"
+ default:
+ d["traceway-write"] = "traceway-write is not installed on this machine"
+ }
return d
}
+func mcpInstructions(s surface) string {
+ if s.mcp == nil {
+ return ""
+ }
+ return s.mcp.Instructions
+}
+
func environmentNote(s surface, inst *env.Instance, primary env.Project) string {
- return fmt.Sprintf("The %s CLI is installed and logged in to the team's Traceway instance at %s, with the %q project selected.", s.cliName, inst.Backend.URL, primary.Name)
+ switch {
+ case s.setupShell:
+ return "This is the developer's machine. The shell has go, node, npm, python3, pip, git and curl, and the traceway CLI (not logged in). Package registries are reachable. The team's Traceway instance runs at " + inst.Backend.URL + "."
+ case s.mcp != nil:
+ return fmt.Sprintf("A Traceway MCP server for the team's instance (%s) is connected. Its tools default to the %q project when you omit project_id.", inst.Backend.URL, primary.Name)
+ case s.cliHome != nil:
+ return fmt.Sprintf("The %s CLI is installed and logged in to the team's Traceway instance at %s, with the %q project selected.", s.cliName, inst.Backend.URL, primary.Name)
+ }
+ return ""
}
diff --git a/cli/test/evals/internal/runner/runner_test.go b/cli/test/evals/internal/runner/runner_test.go
index 6c8dfe6b..b1fd548a 100644
--- a/cli/test/evals/internal/runner/runner_test.go
+++ b/cli/test/evals/internal/runner/runner_test.go
@@ -27,7 +27,7 @@ func TestOracleNPlusOne(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
model := &catalog.Model{Key: "oracle", ID: "fake/oracle", Caching: "none"}
- opt := Options{EvalsRoot: root, RepoRoot: repo, OutDir: t.TempDir(), Bins: bins, Models: []*catalog.Model{model}, Runs: 1, Jobs: 1}
+ opt := Options{EvalsRoot: root, RepoRoot: repo, OutDir: t.TempDir(), Bins: bins, Models: []*catalog.Model{model}, Runs: 1, Jobs: 1, Oracle: true}
res := Execute(ctx, opt, Jobs([]catalog.Unit{{Case: c, Arm: catalog.ArmCLISkill}}, opt.Models, 1))
if len(res) != 1 || res[0].Status != "pass" {
for _, r := range res {
diff --git a/cli/test/evals/internal/runner/state.go b/cli/test/evals/internal/runner/state.go
new file mode 100644
index 00000000..c56ce302
--- /dev/null
+++ b/cli/test/evals/internal/runner/state.go
@@ -0,0 +1,79 @@
+package runner
+
+import (
+ "context"
+ "encoding/json"
+ "slices"
+ "sort"
+
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/env"
+)
+
+// stateReader gives the state grader admin access to the instance.
+type stateReader struct {
+ inst *env.Instance
+ projects []env.Project
+ primary env.Project
+}
+
+func (s *stateReader) Get(ctx context.Context, path string, q map[string]string) (int, []byte, error) {
+ r, err := s.inst.Backend.API().WithBearer(s.inst.Admin.JWT).Do(ctx, "GET", path, values(q), nil)
+ if err != nil {
+ return 0, nil, err
+ }
+ return r.Status, r.Body, nil
+}
+
+func (s *stateReader) Post(ctx context.Context, path string, q map[string]string, body any) (int, []byte, error) {
+ r, err := s.inst.Backend.API().WithBearer(s.inst.Admin.JWT).Do(ctx, "POST", path, values(q), body)
+ if err != nil {
+ return 0, nil, err
+ }
+ return r.Status, r.Body, nil
+}
+
+func (s *stateReader) ProjectID(name string) string {
+ for _, p := range s.projects {
+ if p.Name == name {
+ return p.ID
+ }
+ }
+ return ""
+}
+
+func (s *stateReader) PrimaryProject() string { return s.primary.ID }
+func (s *stateReader) OrgID() int { return s.inst.Admin.OrgID }
+
+func (s *stateReader) Tokens() []string {
+ var out []string
+ for _, p := range s.projects {
+ out = append(out, p.Token)
+ }
+ // Projects created during the run (setup plans) are not in the seed's list.
+ r, err := s.inst.Backend.API().WithBearer(s.inst.Admin.JWT).Do(context.Background(), "GET", "/api/projects", nil, nil)
+ if err == nil && r.OK() {
+ var ps []struct {
+ Token string `json:"token"`
+ }
+ if json.Unmarshal(r.Body, &ps) == nil {
+ for _, p := range ps {
+ if p.Token != "" && !slices.Contains(out, p.Token) {
+ out = append(out, p.Token)
+ }
+ }
+ }
+ }
+ sort.Strings(out)
+ return out
+}
+
+func values(q map[string]string) map[string][]string {
+ if len(q) == 0 {
+ return nil
+ }
+ out := map[string][]string{}
+ for k, v := range q {
+ out[k] = []string{v}
+ }
+ return out
+}
diff --git a/cli/test/evals/internal/runner/surface.go b/cli/test/evals/internal/runner/surface.go
index 56606c4d..4e213a28 100644
--- a/cli/test/evals/internal/runner/surface.go
+++ b/cli/test/evals/internal/runner/surface.go
@@ -2,6 +2,7 @@ package runner
import (
"context"
+ "fmt"
"path/filepath"
"github.com/tracewayapp/traceway/cli/test/evals/internal/catalog"
@@ -10,24 +11,75 @@ import (
)
// prepareSurface mints the arm's credential and connects its surface: the
-// CLI logged in with a read PAT (XDG dirs outside the workspace).
-func prepareSurface(ctx context.Context, opt Options, c *catalog.Case, arm string, inst *env.Instance, primary env.Project, dir string) (surface, error) {
- s := surface{submit: tools.SubmitPerf}
- if arm == catalog.ArmCLISkill {
+// CLI logged in with a PAT (XDG dirs outside the workspace), or an MCP
+// session to /mcp or /mcp/write.
+func prepareSurface(ctx context.Context, opt Options, c *catalog.Case, arm string, inst *env.Instance, projects []env.Project, primary env.Project, dir, work string) (surface, func(), error) {
+ s := surface{submit: submitFor(c.Suite)}
+ switch arm {
+ case catalog.ArmCLISkill:
s.skills = []string{"traceway"}
+ case catalog.ArmCLIWriteSkill:
+ s.skills = []string{"traceway-write", "traceway"}
+ case catalog.ArmCLISetupSkill:
+ s.skills = []string{"traceway-setup"}
+ s.setupShell = true
+ s.cliName, s.cliBin = "traceway", opt.Bins.CLI
+ return s, nil, nil
}
- pat, err := env.CreatePAT(ctx, inst.Backend, inst.Admin.JWT, "eval agent", "read", 0)
+
+ scope := "read"
+ if catalog.IsWriteArm(arm) {
+ scope = "write"
+ }
+ if c.Credential.Scope != "" {
+ scope = c.Credential.Scope
+ }
+ jwt := inst.Admin.JWT
+ if m := c.Credential.Member; m != "" && m != "admin" && m != "owner" {
+ member, err := env.AddMember(ctx, inst.Backend, inst.Admin, m+"@example.com", m)
+ if err != nil {
+ return s, nil, fmt.Errorf("add %s member: %w", m, err)
+ }
+ jwt = member.JWT
+ }
+ if c.Suite == "write" || catalog.IsWriteArm(arm) {
+ on := c.AgentAccess == nil || *c.AgentAccess
+ if err := env.SetAgentAccess(ctx, inst.Backend, inst.Admin, on); err != nil {
+ return s, nil, err
+ }
+ }
+ days := 0
+ if scope == "write" {
+ days = 1
+ }
+ pat, err := env.CreatePAT(ctx, inst.Backend, jwt, "eval agent", scope, days)
if err != nil {
- return s, err
+ return s, nil, err
+ }
+
+ if catalog.IsMCPArm(arm) {
+ path := "/mcp"
+ if scope == "write" {
+ path = "/mcp/write"
+ }
+ b, err := tools.ConnectMCP(ctx, inst.Backend.URL+path, pat, primary.ID)
+ if err != nil {
+ return s, nil, err
+ }
+ s.mcp = b
+ return s, b.Close, nil
}
s.cliName, s.cliBin = "traceway", opt.Bins.CLI
+ if scope == "write" {
+ s.cliName, s.cliBin = "traceway-write", opt.Bins.CLIWrite
+ }
home, err := env.NewCLIHome(filepath.Join(dir, "cli"), s.cliBin)
if err != nil {
- return s, err
+ return s, nil, err
}
if err := home.Login(ctx, inst.Backend.URL, pat, primary.ID); err != nil {
- return s, err
+ return s, nil, err
}
s.cliHome = home
- return s, nil
+ return s, nil, nil
}
diff --git a/cli/test/evals/internal/runner/workspace.go b/cli/test/evals/internal/runner/workspace.go
index 28c51e18..dd067e21 100644
--- a/cli/test/evals/internal/runner/workspace.go
+++ b/cli/test/evals/internal/runner/workspace.go
@@ -81,7 +81,10 @@ func buildWorkspace(evalsRoot, repo, work string, now time.Time) error {
}
src := filepath.Join(evalsRoot, "testdata", "repos", repo)
if _, err := os.Stat(src); err != nil {
- return fmt.Errorf("fixture repo %s: not in testdata/repos", repo)
+ src = filepath.Join(evalsRoot, "testdata", "setup-templates", repo)
+ if _, err := os.Stat(src); err != nil {
+ return fmt.Errorf("fixture repo %s: not in testdata/repos or testdata/setup-templates", repo)
+ }
}
var hist fixtureHistory
if raw, err := os.ReadFile(filepath.Join(src, "history.yaml")); err == nil {
@@ -206,5 +209,7 @@ func installSkills(repoRoot, work string, names []string) error {
// skillSources maps a skill name to its published directory in the repo.
var skillSources = map[string]string{
- "traceway": "skills/traceway",
+ "traceway": "skills/traceway",
+ "traceway-write": "plugins/tw-write/skills/traceway-write",
+ "traceway-setup": "skills/traceway-setup",
}
diff --git a/cli/test/evals/internal/scenarios/all/all.go b/cli/test/evals/internal/scenarios/all/all.go
index 0983d25c..84ec6412 100644
--- a/cli/test/evals/internal/scenarios/all/all.go
+++ b/cli/test/evals/internal/scenarios/all/all.go
@@ -2,5 +2,8 @@
package all
import (
+ _ "github.com/tracewayapp/traceway/cli/test/evals/internal/scenarios/debug"
_ "github.com/tracewayapp/traceway/cli/test/evals/internal/scenarios/perf"
+ _ "github.com/tracewayapp/traceway/cli/test/evals/internal/scenarios/setup"
+ _ "github.com/tracewayapp/traceway/cli/test/evals/internal/scenarios/write"
)
diff --git a/cli/test/evals/internal/scenarios/debug/dbg_android_npe.go b/cli/test/evals/internal/scenarios/debug/dbg_android_npe.go
new file mode 100644
index 00000000..65e26b5a
--- /dev/null
+++ b/cli/test/evals/internal/scenarios/debug/dbg_android_npe.go
@@ -0,0 +1,105 @@
+package debug
+
+import (
+ "fmt"
+ "time"
+
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/scenarios"
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/seed"
+)
+
+func init() { scenarios.Register("debug/android-npe", dbgSeedAndroidNPE) }
+
+const dbgRecyclerFrames = "\tat androidx.recyclerview.widget.RecyclerView$Adapter.onBindViewHolder(RecyclerView.java:7254)\n" +
+ "\tat androidx.recyclerview.widget.RecyclerView$Adapter.bindViewHolder(RecyclerView.java:7337)\n" +
+ "\tat androidx.recyclerview.widget.RecyclerView$Recycler.tryBindViewHolderByDeadline(RecyclerView.java:6194)\n" +
+ "\tat androidx.recyclerview.widget.RecyclerView$Recycler.tryGetViewHolderForPositionByDeadline(RecyclerView.java:6427)\n" +
+ "\tat androidx.recyclerview.widget.GapWorker.prefetchPositionWithDeadline(GapWorker.java:288)\n" +
+ "\tat androidx.recyclerview.widget.GapWorker.flushTaskWithDeadline(GapWorker.java:345)\n" +
+ "\tat androidx.recyclerview.widget.GapWorker.flushTasksWithDeadline(GapWorker.java:361)\n" +
+ "\tat androidx.recyclerview.widget.GapWorker.prefetch(GapWorker.java:368)\n" +
+ "\tat androidx.recyclerview.widget.GapWorker.run(GapWorker.java:399)\n" +
+ "\tat android.os.Handler.handleCallback(Handler.java:958)\n" +
+ "\tat android.os.Handler.dispatchMessage(Handler.java:99)\n" +
+ "\tat android.os.Looper.loopOnce(Looper.java:205)\n" +
+ "\tat android.os.Looper.loop(Looper.java:294)\n" +
+ "\tat android.app.ActivityThread.main(ActivityThread.java:8177)\n" +
+ "\tat java.lang.reflect.Method.invoke(Native Method)\n" +
+ "\tat com.android.internal.os.RuntimeInit$MethodAndArgsCaller.run(RuntimeInit.java:552)\n" +
+ "\tat com.android.internal.os.ZygoteInit.main(ZygoteInit.java:971)\n"
+
+// dbgSeedAndroidNPE: the Android app's 5.8.0 (staged rollout since yesterday)
+// made FeedItem.image non-null and sizes thumbnails from item.image.width;
+// Gson leaves image null for text posts, so binding one throws an NPE while
+// the feed scrolls (RecyclerView prefetch). Traces arrive deobfuscated
+// (R8 mapping uploaded). Distractor: the full-size image viewer runs out of
+// memory decoding originals on low-memory phones, on every version, for
+// weeks.
+func dbgSeedAndroidNPE(s *seed.Scenario) error {
+ p := s.Project("pinboard-android", seed.Mobile, "android")
+ npe := "java.lang.NullPointerException: Attempt to invoke virtual method 'int com.example.pinboard.feed.ImageRef.getWidth()' on a null object reference\n" +
+ "\tat com.example.pinboard.feed.FeedAdapter.bindImage(FeedAdapter.kt:55)\n" +
+ "\tat com.example.pinboard.feed.FeedAdapter.onBindViewHolder(FeedAdapter.kt:48)\n" +
+ "\tat com.example.pinboard.feed.FeedAdapter.onBindViewHolder(FeedAdapter.kt:21)\n" + dbgRecyclerFrames
+ oom := "java.lang.OutOfMemoryError: Failed to allocate a 96000012 byte allocation with 25165824 free bytes and 61MB until OOM, target footprint 268435456, growth limit 268435456\n" +
+ "\tat dalvik.system.VMRuntime.newNonMovableArray(Native Method)\n" +
+ "\tat android.graphics.Bitmap.nativeCreate(Native Method)\n" +
+ "\tat android.graphics.BitmapFactory.nativeDecodeStream(Native Method)\n" +
+ "\tat android.graphics.BitmapFactory.decodeStreamInternal(BitmapFactory.java:875)\n" +
+ "\tat android.graphics.BitmapFactory.decodeStream(BitmapFactory.java:851)\n" +
+ "\tat com.example.pinboard.media.FullImageActivity.decodeOriginal(FullImageActivity.kt:53)\n" +
+ "\tat com.example.pinboard.media.FullImageActivity.download(FullImageActivity.kt:43)\n" +
+ "\tat com.example.pinboard.media.FullImageActivity.access$download(FullImageActivity.kt:19)\n" +
+ "\tat com.example.pinboard.media.FullImageActivity$onCreate$1$bitmap$1.invokeSuspend(FullImageActivity.kt:34)\n" +
+ "\tat kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith(ContinuationImpl.kt:33)\n" +
+ "\tat kotlinx.coroutines.DispatchedTask.run(DispatchedTask.kt:104)\n" +
+ "\tat kotlinx.coroutines.internal.LimitedDispatcher$Worker.run(LimitedDispatcher.kt:111)\n" +
+ "\tat kotlinx.coroutines.scheduling.TaskImpl.run(Tasks.kt:99)\n" +
+ "\tat kotlinx.coroutines.scheduling.CoroutineScheduler.runSafely(CoroutineScheduler.kt:584)\n" +
+ "\tat kotlinx.coroutines.scheduling.CoroutineScheduler$Worker.executeTask(CoroutineScheduler.kt:811)\n" +
+ "\tat kotlinx.coroutines.scheduling.CoroutineScheduler$Worker.runWorker(CoroutineScheduler.kt:715)\n" +
+ "\tat kotlinx.coroutines.scheduling.CoroutineScheduler$Worker.run(CoroutineScheduler.kt:702)\n"
+ modern := []string{"Pixel 8", "Pixel 7a", "SM-S921B", "SM-A546B", "moto g84 5G", "2312DRA50G", "Pixel 9 Pro"}
+ lowMem := []string{"SM-A135F", "moto e13", "Redmi 10C", "SM-A045F"}
+
+ // 5.8.0 rolls out from 26h ago; crash volume grows with the rollout.
+ for _, at := range dbgTimes(s, s.Ago(24*time.Hour), s.Ago(4*time.Minute), 84, true) {
+ c := seed.Crash{At: at, Stack: npe, AppVersion: "5.8.0", Device: dbgPick(s, modern)}
+ if at.After(s.Ago(10 * time.Hour)) {
+ cursor := fmt.Sprintf("c_%x", s.RNG.IntN(1<<30))
+ c.Actions = []seed.Action{
+ {Type: "navigation", At: at.Add(-time.Duration(20+s.RNG.IntN(90)) * time.Second), From: "SplashActivity", To: "FeedActivity"},
+ {Type: "network", At: at.Add(-time.Duration(8000+s.RNG.IntN(9000)) * time.Millisecond), Method: "GET", URL: "https://api.pinboard.example/v2/feed?limit=20", DurationMs: float64(180 + s.RNG.IntN(300)), Status: 200, RespBytes: 18000 + s.RNG.IntN(9000)},
+ {Type: "network", At: at.Add(-time.Duration(600+s.RNG.IntN(1500)) * time.Millisecond), Method: "GET", URL: "https://api.pinboard.example/v2/feed?cursor=" + cursor + "&limit=20", DurationMs: float64(160 + s.RNG.IntN(300)), Status: 200, RespBytes: 17000 + s.RNG.IntN(9000)},
+ }
+ c.Logs = []seed.ActionLog{
+ {At: at.Add(-time.Duration(7000+s.RNG.IntN(2000)) * time.Millisecond), Level: "info", Message: "feed page loaded: 20 items"},
+ {At: at.Add(-time.Duration(300+s.RNG.IntN(400)) * time.Millisecond), Level: "info", Message: "feed page loaded: 20 items"},
+ }
+ }
+ p.Crash(c)
+ }
+ versions := []string{"5.6.4", "5.7.1", "5.7.2", "5.7.2", "5.8.0"}
+ for _, at := range dbgTimes(s, s.Ago(7*24*time.Hour), s.Ago(40*time.Minute), 26, false) {
+ v := dbgPick(s, versions)
+ if v == "5.8.0" && at.Before(s.Ago(24*time.Hour)) {
+ v = "5.7.2"
+ }
+ p.Crash(seed.Crash{At: at, Stack: oom, AppVersion: v, Device: dbgPick(s, lowMem)})
+ }
+
+ s.AfterFlush(func() error {
+ hash, err := p.GroupMatching(8*24*time.Hour, "FeedAdapter.bindImage")
+ if err != nil {
+ return err
+ }
+ s.Expose("hash", hash)
+ other, err := p.GroupMatching(8*24*time.Hour, "OutOfMemoryError")
+ if err != nil {
+ return err
+ }
+ s.Expose("otherHash", other)
+ return nil
+ })
+ return nil
+}
diff --git a/cli/test/evals/internal/scenarios/debug/dbg_flutter_null_check.go b/cli/test/evals/internal/scenarios/debug/dbg_flutter_null_check.go
new file mode 100644
index 00000000..94395bce
--- /dev/null
+++ b/cli/test/evals/internal/scenarios/debug/dbg_flutter_null_check.go
@@ -0,0 +1,93 @@
+package debug
+
+import (
+ "fmt"
+ "time"
+
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/scenarios"
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/seed"
+)
+
+func init() { scenarios.Register("debug/flutter-null-check", dbgSeedFlutterNullCheck) }
+
+// dbgSeedFlutterNullCheck: the product page's hero carousel null-checks
+// product.media, which the catalog omits for listings without photos. Such
+// listings started appearing about a day ago (marketplace sellers), so the
+// crash hits both app versions in the field and no app release lines up
+// with it. Distractor: a steady trickle of host-lookup failures (offline
+// phones) from the same API client.
+func dbgSeedFlutterNullCheck(s *seed.Scenario) error {
+ p := s.Project("shop-flutter", seed.Mobile, "flutter")
+ const fw = "package:flutter/src/widgets/framework.dart"
+ nullCheck := "_TypeError: Null check operator used on a null value\n" +
+ "#0 ProductHeroCarousel.build (package:shop_app/features/product/widgets/product_hero_carousel.dart:24:32)\n" +
+ "#1 StatelessElement.build (" + fw + ":5687:49)\n" +
+ "#2 ComponentElement.performRebuild (" + fw + ":5617:15)\n" +
+ "#3 Element.rebuild (" + fw + ":5333:7)\n" +
+ "#4 ComponentElement._firstBuild (" + fw + ":5599:5)\n" +
+ "#5 ComponentElement.mount (" + fw + ":5593:5)\n" +
+ "#6 Element.inflateWidget (" + fw + ":4468:20)\n" +
+ "#7 Element.updateChild (" + fw + ":3963:18)\n" +
+ "#8 SliverMultiBoxAdaptorElement.updateChild (package:flutter/src/widgets/sliver.dart:1022:37)\n" +
+ "#9 SliverMultiBoxAdaptorElement.createChild.<anonymous closure> (package:flutter/src/widgets/sliver.dart:892:20)\n" +
+ "#10 BuildOwner.buildScope (" + fw + ":3045:19)\n" +
+ "#11 SliverMultiBoxAdaptorElement.createChild (package:flutter/src/widgets/sliver.dart:884:12)\n" +
+ "#12 RenderSliverMultiBoxAdaptor._createOrObtainChild.<anonymous closure> (package:flutter/src/rendering/sliver_multi_box_adaptor.dart:376:23)\n" +
+ "#13 RenderObject.invokeLayoutCallback.<anonymous closure> (package:flutter/src/rendering/object.dart:2876:17)\n" +
+ "#14 PipelineOwner._enableMutationsToDirtySubtrees (package:flutter/src/rendering/object.dart:1170:15)\n" +
+ "#15 RenderObject.invokeLayoutCallback (package:flutter/src/rendering/object.dart:2875:14)\n" +
+ "#16 RenderSliverMultiBoxAdaptor._createOrObtainChild (package:flutter/src/rendering/sliver_multi_box_adaptor.dart:365:5)\n" +
+ "#17 RenderSliverMultiBoxAdaptor.addInitialChild (package:flutter/src/rendering/sliver_multi_box_adaptor.dart:461:5)\n" +
+ "#18 RenderSliverList.performLayout (package:flutter/src/rendering/sliver_list.dart:58:12)\n" +
+ "#19 RenderObject.layout (package:flutter/src/rendering/object.dart:2706:7)\n" +
+ "#20 PipelineOwner.flushLayout (package:flutter/src/rendering/object.dart:1144:18)\n" +
+ "#21 RendererBinding.drawFrame (package:flutter/src/rendering/binding.dart:622:23)\n" +
+ "#22 WidgetsBinding.drawFrame (package:flutter/src/widgets/binding.dart:1241:13)\n" +
+ "#23 RendererBinding._handlePersistentFrameCallback (package:flutter/src/rendering/binding.dart:482:5)\n" +
+ "#24 SchedulerBinding._invokeFrameCallback (package:flutter/src/scheduler/binding.dart:1442:15)\n" +
+ "#25 SchedulerBinding.handleDrawFrame (package:flutter/src/scheduler/binding.dart:1355:9)\n"
+ devices := []string{"iPhone15,2", "iPhone16,1", "iPhone14,7", "Pixel 8", "SM-S921B", "SM-A546B", "Pixel 7a", "2312DRA50G"}
+
+ for _, at := range dbgTimes(s, s.Ago(21*time.Hour), s.Ago(5*time.Minute), 72, true) {
+ v := "4.3.0"
+ if s.RNG.Float64() < 0.22 {
+ v = "4.2.1"
+ }
+ c := seed.Crash{At: at, Stack: nullCheck, AppVersion: v, Device: dbgPick(s, devices)}
+ if at.After(s.Ago(10 * time.Hour)) {
+ id := 40000 + s.RNG.IntN(50000)
+ c.Actions = []seed.Action{
+ {Type: "navigation", At: at.Add(-time.Duration(900+s.RNG.IntN(1500)) * time.Millisecond), From: "/", To: fmt.Sprintf("/products/%d", id)},
+ {Type: "network", At: at.Add(-time.Duration(500+s.RNG.IntN(600)) * time.Millisecond), Method: "GET", URL: fmt.Sprintf("https://api.shop.example/v1/products/%d", id), DurationMs: float64(90 + s.RNG.IntN(250)), Status: 200, RespBytes: 780 + s.RNG.IntN(300)},
+ }
+ }
+ p.Crash(c)
+ }
+ for _, at := range dbgTimes(s, s.Ago(7*24*time.Hour), s.Ago(50*time.Minute), 22, false) {
+ id := 40000 + s.RNG.IntN(50000)
+ v := "4.3.0"
+ if at.Before(s.Ago(9*24*time.Hour)) || s.RNG.Float64() < 0.3 {
+ v = "4.2.1"
+ }
+ stack := fmt.Sprintf("ClientException: ClientException with SocketException: Failed host lookup: 'api.shop.example' (OS Error: No address associated with hostname, errno = 7), uri=https://api.shop.example/v1/products/%d\n", id) +
+ "#0 IOClient.send (package:http/src/io_client.dart:154:7)\n<asynchronous suspension>\n" +
+ "#1 BaseClient._sendUnstreamed (package:http/src/base_client.dart:93:32)\n<asynchronous suspension>\n" +
+ "#2 ApiClient.product (package:shop_app/core/api_client.dart:14:17)\n<asynchronous suspension>\n"
+ p.Crash(seed.Crash{At: at, Stack: stack, AppVersion: v, Device: dbgPick(s, devices)})
+ }
+
+ s.AfterFlush(func() error {
+ hash, err := p.GroupMatching(8*24*time.Hour, "ProductHeroCarousel.build")
+ if err != nil {
+ return err
+ }
+ s.Expose("hash", hash)
+ other, err := p.GroupMatching(8*24*time.Hour, "Failed host lookup")
+ if err != nil {
+ return err
+ }
+ s.Expose("otherHash", other)
+ return nil
+ })
+ return nil
+}
diff --git a/cli/test/evals/internal/scenarios/debug/dbg_helpers.go b/cli/test/evals/internal/scenarios/debug/dbg_helpers.go
new file mode 100644
index 00000000..70f5e414
--- /dev/null
+++ b/cli/test/evals/internal/scenarios/debug/dbg_helpers.go
@@ -0,0 +1,28 @@
+package debug
+
+import (
+ "math"
+ "sort"
+ "time"
+
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/seed"
+)
+
+// dbgTimes returns n instants in [from, to), in order. With ramp, arrivals
+// thicken towards `to` (a staged rollout reaching more users).
+func dbgTimes(s *seed.Scenario, from, to time.Time, n int, ramp bool) []time.Time {
+ out := make([]time.Time, 0, n)
+ span := float64(to.Sub(from))
+ for i := 0; i < n; i++ {
+ u := s.RNG.Float64()
+ if ramp {
+ u = math.Sqrt(u)
+ }
+ out = append(out, from.Add(time.Duration(u*span)))
+ }
+ sort.Slice(out, func(i, j int) bool { return out[i].Before(out[j]) })
+ return out
+}
+
+// dbgPick returns one element of xs from the scenario's RNG.
+func dbgPick[T any](s *seed.Scenario, xs []T) T { return xs[s.RNG.IntN(len(xs))] }
diff --git a/cli/test/evals/internal/scenarios/debug/doc.go b/cli/test/evals/internal/scenarios/debug/doc.go
new file mode 100644
index 00000000..2922b8ab
--- /dev/null
+++ b/cli/test/evals/internal/scenarios/debug/doc.go
@@ -0,0 +1,2 @@
+// Package debug holds the seed functions of the debug suite.
+package debug
diff --git a/cli/test/evals/internal/scenarios/debug/go_nil_regression.go b/cli/test/evals/internal/scenarios/debug/go_nil_regression.go
new file mode 100644
index 00000000..01ea17b3
--- /dev/null
+++ b/cli/test/evals/internal/scenarios/debug/go_nil_regression.go
@@ -0,0 +1,101 @@
+package debug
+
+import (
+ "fmt"
+ "time"
+
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/scenarios"
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/seed"
+)
+
+func init() {
+ scenarios.Register("debug/go-nil-regression", seedGoNilRegression)
+}
+
+// goPanic is the stack a recover middleware records for a panic, with
+// debug.Stack()'s frames above the panicking one.
+func goPanic(goroutine int, frames ...string) string {
+ s := fmt.Sprintf("goroutine %d [running]:\nruntime/debug.Stack()\n\t/usr/local/go/src/runtime/debug/stack.go:26 +0x5e\n", goroutine)
+ s += "example.com/billing-api/internal/invoices.Recover.func1.1()\n\t/app/internal/invoices/handler.go:45 +0x9c\n"
+ s += "panic({0x1a2b3c0?, 0x2a4f8b0?})\n\t/usr/local/go/src/runtime/panic.go:791 +0x132\n"
+ for _, f := range frames {
+ s += f + "\n"
+ }
+ return s
+}
+
+// billingService is the invoice service shared by the billing debug cases:
+// v1.8.2, deployed at deployAt, renders the billing address without a nil
+// check, so customers who never entered one get a panic.
+func billingService(s *seed.Scenario, deployAt time.Time) (*seed.Project, *seed.Service, *seed.Endpoint) {
+ p := s.Project("billing", seed.OTel)
+ api := p.Service("billing-api", seed.Go("nethttp", "sql")).Versions("1.8.1", seed.Deploy{At: deployAt, Version: "1.8.2"})
+ nilStack := goPanic(1183,
+ "example.com/billing-api/internal/invoices.formatAddress(...)\n\t/app/internal/invoices/render.go:37",
+ "example.com/billing-api/internal/invoices.render(0xc00041a2d0)\n\t/app/internal/invoices/render.go:27 +0x14d",
+ "example.com/billing-api/internal/invoices.(*Handler).Get(0xc000186a50, {0x1d1e2a8, 0xc0004d6000}, 0xc0004c8500)\n\t/app/internal/invoices/handler.go:34 +0x2f4",
+ "net/http.HandlerFunc.ServeHTTP(0xc000189480, {0x1d1e2a8, 0xc0004d6000}, 0xc0004c8500)\n\t/usr/local/go/src/net/http/server.go:2294 +0x29",
+ "example.com/billing-api/internal/invoices.Recover.func1({0x1d1e2a8, 0xc0004d6000}, 0xc0004c8500)\n\t/app/internal/invoices/handler.go:49 +0x8a",
+ )
+ get := api.Endpoint("GET", "/api/invoices/:id")
+ get.Traffic(seed.Daily(700), seed.Diurnal(), seed.Over(7*24*time.Hour))
+ get.Detail(seed.DetailPolicy{Recent: 6 * time.Hour, Rate: 0.03, Max: 10})
+ get.Handler(func(r *seed.Req) {
+ customer := 4000 + r.IntN(9000)
+ r.RootAttr("enduser.id", fmt.Sprint(customer))
+ r.Gap(r.Ms(0.3, 0.9))
+ r.DB("SELECT i.id, i.number, i.issued_at, i.total_cents, i.currency, c.id, c.name, c.email FROM invoices i JOIN customers c ON c.id = i.customer_id WHERE i.id = $1", r.Ms(2, 5))
+ noAddress := r.Chance(0.14)
+ r.DB("SELECT line1, line2, city, postal_code, country FROM billing_addresses WHERE customer_id = $1", r.Ms(1, 3), seed.Rows(map[bool]int{true: 0, false: 1}[noAddress]))
+ if r.Version() == "1.8.2" && noAddress {
+ r.Error(seed.Exc{Type: "runtime.errorString", Message: "runtime error: invalid memory address or nil pointer dereference", Stack: nilStack, Escaped: true})
+ r.Log(seed.SevError, "panic serving request", seed.A("path", r.Request().Root.Attrs["url.path"]), seed.A("panic", "runtime error: invalid memory address or nil pointer dereference"))
+ r.Status(500)
+ r.Gap(r.Ms(0.2, 0.5))
+ return
+ }
+ r.Compute("render invoice", r.Ms(0.4, 1.2))
+ r.Gap(r.Ms(0.2, 0.5))
+ })
+
+ // Distractor: an older, unrelated issue on another route.
+ send := api.Endpoint("POST", "/api/invoices/:id/send")
+ send.Traffic(seed.Daily(120), seed.Diurnal(), seed.Over(7*24*time.Hour))
+ send.Handler(func(r *seed.Req) {
+ r.Gap(r.Ms(0.3, 0.9))
+ r.DB("SELECT id, number, customer_id FROM invoices WHERE id = $1", r.Ms(2, 5))
+ if r.Chance(0.02) {
+ r.HTTP("POST", "https://api.mailer.example/v3/messages", 10*time.Second, 504)
+ r.Error(seed.Exc{Type: "*url.Error", Message: `Post "https://api.mailer.example/v3/messages": context deadline exceeded`,
+ Stack: "goroutine 902 [running]:\nexample.com/billing-api/internal/mail.(*Client).Send(0xc0002b4000, {0x1d1e2a8, 0xc0003a2000}, 0xc0004e8000)\n\t/app/internal/mail/client.go:71 +0x2e5\nexample.com/billing-api/internal/invoices.(*Handler).Send(0xc000186a50, {0x1d1e2a8, 0xc0004d6000}, 0xc0004c8500)\n\t/app/internal/invoices/send.go:40 +0x1b4"})
+ r.Status(502)
+ return
+ }
+ r.HTTP("POST", "https://api.mailer.example/v3/messages", r.LogNormal(140*time.Millisecond, 500*time.Millisecond), 202)
+ })
+ s.Background(api, seed.TypicalTraffic(6))
+ return p, api, get
+}
+
+// seedGoNilRegression: a nil-pointer panic in invoice rendering that starts
+// with the v1.8.2 deploy 30 hours ago.
+func seedGoNilRegression(s *seed.Scenario) error {
+ deployAt := s.Ago(30 * time.Hour).Truncate(time.Minute).Add(-7 * time.Minute)
+ p, _, get := billingService(s, deployAt)
+ s.Expose("deployAt", deployAt.Format(time.RFC3339))
+ s.Expose("lastFailure", get.Pick(func(r *seed.Request) bool { return r.Status == 500 }))
+ s.AfterFlush(func() error {
+ hash, err := p.GroupMatching(8*24*time.Hour, "formatAddress")
+ if err != nil {
+ return err
+ }
+ s.Expose("hash", hash)
+ other, err := p.GroupMatching(8*24*time.Hour, "mailer.example")
+ if err != nil {
+ return err
+ }
+ s.Expose("otherHash", other)
+ return nil
+ })
+ return nil
+}
diff --git a/cli/test/evals/internal/scenarios/setup/setup.go b/cli/test/evals/internal/scenarios/setup/setup.go
new file mode 100644
index 00000000..4672100a
--- /dev/null
+++ b/cli/test/evals/internal/scenarios/setup/setup.go
@@ -0,0 +1,16 @@
+// Package setup holds the seed functions of the setup suite. Setup seeds
+// create the Traceway projects a case starts from and nothing else: the
+// telemetry has to come from the app the agent instruments.
+package setup
+
+import (
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/scenarios"
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/seed"
+)
+
+func init() {
+ scenarios.Register("setup/go-nethttp", func(s *seed.Scenario) error {
+ s.Project("hello-api", seed.OTel)
+ return nil
+ })
+}
diff --git a/cli/test/evals/internal/scenarios/write/alert_snooze.go b/cli/test/evals/internal/scenarios/write/alert_snooze.go
new file mode 100644
index 00000000..48169fa3
--- /dev/null
+++ b/cli/test/evals/internal/scenarios/write/alert_snooze.go
@@ -0,0 +1,68 @@
+package write
+
+import (
+ "time"
+
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/scenarios"
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/seed"
+)
+
+func init() { scenarios.Register("write/alert-snooze", seedAlertSnooze) }
+
+// checkoutTraffic is a payments service whose checkout calls a card
+// processor. failingSince marks when the processor started returning 502s
+// for a share of charges.
+func checkoutTraffic(s *seed.Scenario, p *seed.Project, failingSince time.Time, failShare float64) (*seed.Service, *seed.Endpoint) {
+ api := p.Service("payments-api", seed.Go("chi", "pgx")).Versions("3.4.1")
+ checkout := api.Endpoint("POST", "/api/checkout")
+ checkout.Traffic(seed.Daily(1800), seed.Diurnal(), seed.Burst(s.Ago(90*time.Minute), s.Now, 5), seed.Over(24*time.Hour))
+ checkout.Detail(seed.DetailPolicy{Recent: time.Hour, Rate: 0.05, Max: 20})
+ checkout.Handler(func(r *seed.Req) {
+ r.Gap(r.Ms(0.5, 1.5))
+ r.DB("SELECT id, user_id, total_cents, currency FROM carts WHERE id = $1 FOR UPDATE", r.Ms(2, 6))
+ failing := !r.Start().Before(failingSince) && r.Chance(failShare)
+ if failing {
+ r.HTTP("POST", "https://api.cardprocessor.example/v2/charges", r.Ms(900, 2400), 502)
+ r.Error(seed.Exc{Type: "*payments.GatewayError", Message: "charge declined by gateway: upstream returned 502 Bad Gateway",
+ Stack: "goroutine 211 [running]:\nexample.com/payments-api/internal/gateway.(*Client).Charge(0xc0003a2000, {0x1c8e2a0, 0xc00041e0c0}, 0xc0004a4180)\n\t/app/internal/gateway/client.go:118 +0x3c5\nexample.com/payments-api/internal/checkout.(*Service).Pay(0xc000182a50, {0x1c8e2a0, 0xc00041e0c0}, 0xc0004a4180)\n\t/app/internal/checkout/pay.go:64 +0x1d2"})
+ r.Status(502)
+ r.Gap(r.Ms(0.3, 0.8))
+ return
+ }
+ r.HTTP("POST", "https://api.cardprocessor.example/v2/charges", r.LogNormal(180*time.Millisecond, 600*time.Millisecond), 201)
+ r.DB("INSERT INTO payments (id, cart_id, status, processor_ref, created_at) VALUES ($1, $2, $3, $4, now())", r.Ms(2, 5))
+ r.DB("UPDATE carts SET status = 'paid' WHERE id = $1", r.Ms(1, 3))
+ r.Gap(r.Ms(0.3, 0.8))
+ })
+ s.Background(api, seed.TypicalTraffic(8))
+ return api, checkout
+}
+
+// seedAlertSnooze: the card processor has an incident, so the checkout
+// error-rate alert keeps paging. Distractor: a second rule (checkout p95)
+// that must stay untouched.
+func seedAlertSnooze(s *seed.Scenario) error {
+ p := s.Project("payments", seed.OTel)
+ _, checkout := checkoutTraffic(s, p, s.Ago(40*time.Minute), 0.35)
+ // Enough requests in the rule's 5-minute lookback even at the diurnal
+ // trough, so the page always opens.
+ checkout.Traffic(seed.Daily(3600))
+
+ paging := p.PagingChannel("Payments on-call")
+ email := p.EmailChannel("Payments team email")
+ errRule := p.Rule(seed.Rule{Name: "Checkout error rate above 5%", Type: "error_rate_threshold", Channel: paging, Cooldown: 5, Severity: "critical",
+ Config: map[string]any{"thresholdPercent": 5, "lookbackMinutes": 5, "minRequests": 10}})
+ latRule := p.Rule(seed.Rule{Name: "Checkout p95 above 3s", Type: "endpoint_p95_threshold", Channel: email, Cooldown: 30,
+ Config: map[string]any{"endpoint": "POST /api/checkout", "thresholdMs": 3000, "lookbackMinutes": 10}})
+ s.Expose("ruleId", errRule)
+ s.Expose("otherRuleId", latRule)
+ s.AfterFlush(func() error {
+ pg, err := p.WaitPage(errRule, 90*time.Second)
+ if err != nil {
+ return err
+ }
+ s.Expose("pageId", pg.ID)
+ return nil
+ })
+ return nil
+}
diff --git a/cli/test/evals/internal/scenarios/write/doc.go b/cli/test/evals/internal/scenarios/write/doc.go
new file mode 100644
index 00000000..7ef3753b
--- /dev/null
+++ b/cli/test/evals/internal/scenarios/write/doc.go
@@ -0,0 +1,2 @@
+// Package write holds the seed functions of the write suite.
+package write
diff --git a/cli/test/evals/internal/seed/objects.go b/cli/test/evals/internal/seed/objects.go
new file mode 100644
index 00000000..3822fb8e
--- /dev/null
+++ b/cli/test/evals/internal/seed/objects.go
@@ -0,0 +1,273 @@
+package seed
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/url"
+ "strconv"
+ "time"
+
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/env"
+)
+
+// Dashboard objects (alert rules, pages, monitors, post-mortems, slow
+// marks) are created through the same API the dashboard uses, as the
+// instance owner. In a dry sink every call returns a synthetic id.
+
+func (s *Scenario) admin(method, path string, q url.Values, body any) (map[string]any, error) {
+ res, err := s.Sink.Admin(method, path, q, body)
+ if err != nil {
+ return nil, fmt.Errorf("%s %s: %w", method, path, err)
+ }
+ if err := res.Err(method + " " + path); err != nil {
+ return nil, err
+ }
+ m := res.Map()
+ if m == nil {
+ m = map[string]any{}
+ }
+ return m, nil
+}
+
+func (s *Scenario) mustAdmin(method, path string, q url.Values, body any) map[string]any {
+ m, err := s.admin(method, path, q, body)
+ if err != nil {
+ panic(err)
+ }
+ return m
+}
+
+func num(m map[string]any, key string) int {
+ switch v := m[key].(type) {
+ case float64:
+ return int(v)
+ case string:
+ n, _ := strconv.Atoi(v)
+ return n
+ }
+ return 0
+}
+
+// Channel creates a notification channel and returns its id.
+func (p *Project) Channel(name, channelType string, config map[string]any) int {
+ p.s.record(name)
+ m := p.s.mustAdmin(http.MethodPost, "/api/notification-channels", env.ProjectQuery(p.ID),
+ map[string]any{"name": name, "channelType": channelType, "config": config})
+ return num(m, "id")
+}
+
+// EmailChannel creates an email channel to [email protected].
+func (p *Project) EmailChannel(name string) int {
+ return p.Channel(name, "email", map[string]any{"recipients": []string{"[email protected]"}})
+}
+
+// PagingChannel creates an escalation policy that pages the owner and an
+// escalation channel for it, so rules on that channel open real pages.
+func (p *Project) PagingChannel(name string) int {
+ org, user := p.s.Sink.Org()
+ p.s.record(name)
+ pol := p.s.mustAdmin(http.MethodPost, fmt.Sprintf("/api/organizations/%d/escalation-policies", org), nil, map[string]any{
+ "name": name + " policy",
+ "definition": map[string]any{"schemaVersion": 1, "repeatCount": 0, "steps": []map[string]any{
+ {"delayMinutes": 10, "targets": []map[string]any{{"type": "user", "id": user}}},
+ }},
+ })
+ return p.Channel(name, "escalation", map[string]any{"policyId": num(pol, "id")})
+}
+
+// Rule is an alert rule.
+type Rule struct {
+ Name string
+ Type string
+ Config map[string]any
+ Channel int
+ Cooldown int
+ Severity string
+}
+
+// Rule creates an alert rule and returns its id.
+func (p *Project) Rule(r Rule) int {
+ p.s.record(r.Name)
+ body := map[string]any{"channelId": r.Channel, "name": r.Name, "ruleType": r.Type, "config": r.Config, "cooldownMinutes": r.Cooldown}
+ if r.Severity != "" {
+ body["severity"] = r.Severity
+ }
+ m := p.s.mustAdmin(http.MethodPost, "/api/notification-rules", env.ProjectQuery(p.ID), body)
+ return num(m, "id")
+}
+
+// DisableRule turns a rule off.
+func (p *Project) DisableRule(id int) {
+ p.s.mustAdmin(http.MethodPost, fmt.Sprintf("/api/notification-rules/%d/toggle", id), env.ProjectQuery(p.ID), map[string]bool{"enabled": false})
+}
+
+// Page is an on-call page as the API returns it.
+type Page struct {
+ ID int `json:"id"`
+ Status string `json:"status"`
+ Subject string `json:"subject"`
+ RuleID *int `json:"ruleId"`
+ RuleName string `json:"ruleName"`
+ RuleType string `json:"ruleType"`
+ IssueHash string `json:"issueHash"`
+ CreatedAt string `json:"createdAt"`
+}
+
+// Pages lists pages with a status (active | open | acknowledged | resolved).
+func (p *Project) Pages(status string) ([]Page, error) {
+ if p.s.Sink.Dry() {
+ return []Page{{ID: 1, Status: "open"}}, nil
+ }
+ res, err := p.s.Sink.Admin(http.MethodPost, "/api/pages", env.ProjectQuery(p.ID), map[string]any{
+ "status": status, "pagination": map[string]int{"page": 1, "pageSize": 100},
+ })
+ if err != nil {
+ return nil, err
+ }
+ if err := res.Err("list pages"); err != nil {
+ return nil, err
+ }
+ var out struct {
+ Data []Page `json:"data"`
+ }
+ if err := json.Unmarshal(res.Body, &out); err != nil {
+ return nil, err
+ }
+ return out.Data, nil
+}
+
+// WaitPage waits until a page opened by rule ruleID is open, and returns it.
+func (p *Project) WaitPage(ruleID int, timeout time.Duration) (Page, error) {
+ if p.s.Sink.Dry() {
+ return Page{ID: 1, Status: "open"}, nil
+ }
+ deadline := time.Now().Add(timeout)
+ for time.Now().Before(deadline) {
+ pages, err := p.Pages("active")
+ if err != nil {
+ return Page{}, err
+ }
+ for _, pg := range pages {
+ if pg.RuleID != nil && *pg.RuleID == ruleID {
+ return pg, nil
+ }
+ }
+ time.Sleep(time.Second)
+ }
+ return Page{}, fmt.Errorf("rule %d opened no page within %s", ruleID, timeout)
+}
+
+// Check is an HTTP monitor.
+type Check struct {
+ Name string
+ URL string
+ IntervalSeconds int
+ FailureThreshold int
+}
+
+// Check creates an HTTP monitor and returns its id.
+func (p *Project) Check(c Check) int {
+ if c.IntervalSeconds == 0 {
+ c.IntervalSeconds = 3600
+ }
+ if c.FailureThreshold == 0 {
+ c.FailureThreshold = 1
+ }
+ p.s.record(c.Name)
+ m := p.s.mustAdmin(http.MethodPost, "/api/synthetics/checks", env.ProjectQuery(p.ID), map[string]any{
+ "name": c.Name, "checkType": "http", "intervalSeconds": c.IntervalSeconds, "timeoutSeconds": 5,
+ "failureThreshold": c.FailureThreshold, "config": map[string]any{"url": c.URL, "method": "GET"},
+ })
+ return num(m, "id")
+}
+
+// CheckState is a monitor's state as the API returns it.
+type CheckState struct {
+ Status string
+ Incidents []map[string]any
+}
+
+// GetCheck reads a monitor and its incidents.
+func (p *Project) GetCheck(id int) (CheckState, error) {
+ if p.s.Sink.Dry() {
+ return CheckState{Status: "down", Incidents: []map[string]any{{"id": float64(1)}}}, nil
+ }
+ m, err := p.s.admin(http.MethodGet, fmt.Sprintf("/api/synthetics/checks/%d", id), env.ProjectQuery(p.ID), nil)
+ if err != nil {
+ return CheckState{}, err
+ }
+ var st CheckState
+ if c, ok := m["check"].(map[string]any); ok {
+ st.Status, _ = c["currentStatus"].(string)
+ }
+ if inc, ok := m["incidents"].([]any); ok {
+ for _, i := range inc {
+ if im, ok := i.(map[string]any); ok {
+ st.Incidents = append(st.Incidents, im)
+ }
+ }
+ }
+ return st, nil
+}
+
+// RunCheckUntil queues runs of a monitor until it reports status.
+func (p *Project) RunCheckUntil(id int, status string, timeout time.Duration) (CheckState, error) {
+ if p.s.Sink.Dry() {
+ return CheckState{Status: status, Incidents: []map[string]any{{"id": float64(1)}}}, nil
+ }
+ deadline := time.Now().Add(timeout)
+ for time.Now().Before(deadline) {
+ st, err := p.GetCheck(id)
+ if err != nil {
+ return st, err
+ }
+ if st.Status == status {
+ return st, nil
+ }
+ _, _ = p.s.Sink.Admin(http.MethodPost, fmt.Sprintf("/api/synthetics/checks/%d/run", id), env.ProjectQuery(p.ID), nil)
+ time.Sleep(time.Second)
+ }
+ return CheckState{}, fmt.Errorf("monitor %d never reached %s", id, status)
+}
+
+// PostMortem creates a post-mortem and returns its id and updatedAt (read
+// back with a GET, as the API's create timestamp may not round-trip).
+func (p *Project) PostMortem(title, content string, tags []string, incidentID int) (int, string) {
+ p.s.record(title, content)
+ body := map[string]any{"title": title, "contentMd": content, "tags": tags}
+ if incidentID > 0 {
+ body["incidentId"] = incidentID
+ }
+ m := p.s.mustAdmin(http.MethodPost, "/api/post-mortems", env.ProjectQuery(p.ID), body)
+ id := num(m, "id")
+ got := p.s.mustAdmin(http.MethodGet, fmt.Sprintf("/api/post-mortems/%d", id), env.ProjectQuery(p.ID), nil)
+ ua, _ := got["updatedAt"].(string)
+ return id, ua
+}
+
+// EditPostMortem replaces a post-mortem's body as the owner. The PUT
+// replaces the whole document (title required), so the title, tags and
+// incident link are carried over from a fresh read.
+func (p *Project) EditPostMortem(id int, content string) error {
+ path := fmt.Sprintf("/api/post-mortems/%d", id)
+ cur, err := p.s.admin(http.MethodGet, path, env.ProjectQuery(p.ID), nil)
+ if err != nil {
+ return err
+ }
+ _, err = p.s.admin(http.MethodPut, path, env.ProjectQuery(p.ID), map[string]any{
+ "title": cur["title"], "contentMd": content, "tags": cur["tags"], "incidentId": cur["incidentId"],
+ })
+ return err
+}
+
+// MarkSlow sets an endpoint's accepted latency allowance.
+func (p *Project) MarkSlow(endpoint string, offsetMs int, reason string) {
+ p.s.record(reason)
+ p.s.mustAdmin(http.MethodPost, "/api/endpoints/slow", env.ProjectQuery(p.ID), map[string]any{"endpoint": endpoint, "offsetMs": offsetMs, "reason": reason})
+}
+
+// Archive archives exception groups.
+func (p *Project) Archive(hashes ...string) {
+ p.s.mustAdmin(http.MethodPost, "/api/exception-stack-traces/archive", env.ProjectQuery(p.ID), map[string]any{"hashes": hashes})
+}
diff --git a/cli/test/evals/internal/suite/suite_test.go b/cli/test/evals/internal/suite/suite_test.go
index 905e0295..8b4c7092 100644
--- a/cli/test/evals/internal/suite/suite_test.go
+++ b/cli/test/evals/internal/suite/suite_test.go
@@ -19,6 +19,7 @@ import (
"github.com/tracewayapp/traceway/cli/test/evals/internal/scenarios"
_ "github.com/tracewayapp/traceway/cli/test/evals/internal/scenarios/all"
"github.com/tracewayapp/traceway/cli/test/evals/internal/seed"
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/tools"
)
func loadAll(t *testing.T) []*catalog.Case {
@@ -44,9 +45,12 @@ func TestCaseReferences(t *testing.T) {
t.Error(err)
}
seen[c.Seed] = true
- if r := c.Workspace.Repo; r != "" && !exists(filepath.Join(root, "testdata", "repos", r)) {
+ if r := c.Workspace.Repo; r != "" && !exists(filepath.Join(root, "testdata", "repos", r)) && !exists(filepath.Join(root, "testdata", "setup-templates", r)) {
t.Errorf("fixture %s does not exist", r)
}
+ if cs := c.Expect.Code; cs != nil && cs.HiddenTests != "" && !exists(filepath.Join(root, "testdata", "hidden", cs.HiddenTests)) {
+ t.Errorf("hidden tests %s do not exist", cs.HiddenTests)
+ }
checkIDs := func(where string, ids ...string) {
for _, id := range ids {
if id != "" && !strings.Contains(id, "{{") && !checklist.IsID(id) {
@@ -58,6 +62,9 @@ func TestCaseReferences(t *testing.T) {
checkIDs("expect.answer.checklist_item", a.ChecklistItem)
checkIDs("expect.answer.any_of", a.AnyOf...)
checkIDs("expect.answer.not", a.Not...)
+ if a.Category != "" && !slices.Contains(tools.DebugCategories, a.Category) {
+ t.Errorf("category %q is not a debug category", a.Category)
+ }
if c.Suite == "perf" && a.ChecklistItem == "" && len(a.AnyOf) == 0 {
t.Error("perf answers need checklist_item or any_of")
}
@@ -74,11 +81,13 @@ func TestCaseReferences(t *testing.T) {
}
}
last := c.Oracle[len(c.Oracle)-1]
- if last.Submit == nil {
- t.Error("the oracle must end with submit")
+ if last.Submit == nil && last.Finish == nil {
+ t.Error("the oracle must end with submit or finish")
}
- if len(c.Judge.Rubric) == 0 {
- t.Error("perf cases need a judge rubric")
+ if c.Suite == "perf" || c.Suite == "debug" {
+ if len(c.Judge.Rubric) == 0 {
+ t.Error("perf and debug cases need a judge rubric")
+ }
}
})
}
diff --git a/cli/test/evals/internal/tools/coreutils_more.go b/cli/test/evals/internal/tools/coreutils_more.go
new file mode 100644
index 00000000..ee2acb40
--- /dev/null
+++ b/cli/test/evals/internal/tools/coreutils_more.go
@@ -0,0 +1,587 @@
+package tools
+
+import (
+ "fmt"
+ "io/fs"
+ "os"
+ "path"
+ "path/filepath"
+ "regexp"
+ "strconv"
+ "strings"
+ "time"
+ "unicode"
+)
+
+func init() {
+ for name, f := range map[string]func(u *uctx, args []string) int{
+ "date": uDate, "seq": uSeq, "sed": uSed, "basename": uBasename, "dirname": uDirname,
+ "tee": uTee, "find": uFind, "nl": uNl, "rev": uRev, "sleep": uSleep,
+ } {
+ coreutils[name] = f
+ }
+}
+
+var agoRe = regexp.MustCompile(`^\s*(-?\d+)\s*(second|sec|minute|min|hour|day|week)s?\s*(ago)?\s*$`)
+
+// parseDateArg handles the -d forms agents use: now, @epoch, RFC 3339,
+// "N units ago", "+N units".
+func parseDateArg(s string, now time.Time) (time.Time, error) {
+ s = strings.TrimSpace(s)
+ switch {
+ case s == "" || s == "now":
+ return now, nil
+ case strings.HasPrefix(s, "@"):
+ n, err := strconv.ParseInt(s[1:], 10, 64)
+ return time.Unix(n, 0).UTC(), err
+ }
+ for _, layout := range []string{time.RFC3339Nano, time.RFC3339, "2006-01-02T15:04:05", "2006-01-02 15:04:05", "2006-01-02 15:04", "2006-01-02"} {
+ if t, err := time.Parse(layout, s); err == nil {
+ return t.UTC(), nil
+ }
+ }
+ if m := agoRe.FindStringSubmatch(strings.TrimPrefix(s, "+")); m != nil {
+ n, _ := strconv.Atoi(m[1])
+ unit := map[string]time.Duration{"second": time.Second, "sec": time.Second, "minute": time.Minute, "min": time.Minute, "hour": time.Hour, "day": 24 * time.Hour, "week": 7 * 24 * time.Hour}[m[2]]
+ d := time.Duration(n) * unit
+ if m[3] == "ago" {
+ d = -d
+ }
+ return now.Add(d), nil
+ }
+ return time.Time{}, fmt.Errorf("invalid date %q", s)
+}
+
+func strftime(t time.Time, f string) string {
+ r := strings.NewReplacer(
+ "%Y", t.Format("2006"), "%m", t.Format("01"), "%d", t.Format("02"), "%H", t.Format("15"),
+ "%M", t.Format("04"), "%S", t.Format("05"), "%s", strconv.FormatInt(t.Unix(), 10), "%z", t.Format("-0700"),
+ "%Z", t.Format("MST"), "%F", t.Format("2006-01-02"), "%T", t.Format("15:04:05"), "%N", fmt.Sprintf("%09d", t.Nanosecond()),
+ "%3N", fmt.Sprintf("%03d", t.Nanosecond()/1e6), "%y", t.Format("06"), "%b", t.Format("Jan"), "%a", t.Format("Mon"), "%%", "%",
+ )
+ return r.Replace(f)
+}
+
+func uDate(u *uctx, args []string) int {
+ now := time.Now().UTC()
+ t := now
+ format := "%a %b %d %H:%M:%S UTC %Y"
+ for i := 0; i < len(args); i++ {
+ a := args[i]
+ switch {
+ case a == "-u" || a == "--utc":
+ case a == "-d" && i+1 < len(args):
+ var err error
+ if t, err = parseDateArg(args[i+1], now); err != nil {
+ u.errf("date: %v", err)
+ return 1
+ }
+ i++
+ case strings.HasPrefix(a, "--date="):
+ var err error
+ if t, err = parseDateArg(strings.TrimPrefix(a, "--date="), now); err != nil {
+ u.errf("date: %v", err)
+ return 1
+ }
+ case a == "-Iseconds" || a == "--iso-8601=seconds":
+ format = "%Y-%m-%dT%H:%M:%S+00:00"
+ case a == "-I" || a == "--iso-8601":
+ format = "%Y-%m-%d"
+ case strings.HasPrefix(a, "+"):
+ format = a[1:]
+ default:
+ u.errf("date: unsupported argument %s", a)
+ return 1
+ }
+ }
+ fmt.Fprintln(u.hc.Stdout, strftime(t.UTC(), format))
+ return 0
+}
+
+func uSeq(u *uctx, args []string) int {
+ nums := make([]float64, 0, 3)
+ for _, a := range args {
+ n, err := strconv.ParseFloat(a, 64)
+ if err != nil {
+ u.errf("seq: invalid number %s", a)
+ return 1
+ }
+ nums = append(nums, n)
+ }
+ first, step, last := 1.0, 1.0, 0.0
+ switch len(nums) {
+ case 1:
+ last = nums[0]
+ case 2:
+ first, last = nums[0], nums[1]
+ case 3:
+ first, step, last = nums[0], nums[1], nums[2]
+ default:
+ u.errf("seq: missing operand")
+ return 1
+ }
+ for n, i := first, 0; (step > 0 && n <= last) || (step < 0 && n >= last); n, i = n+step, i+1 {
+ if i > 100000 {
+ break
+ }
+ fmt.Fprintln(u.hc.Stdout, strconv.FormatFloat(n, 'f', -1, 64))
+ }
+ return 0
+}
+
+func uBasename(u *uctx, args []string) int {
+ if len(args) == 0 {
+ return 1
+ }
+ b := filepath.Base(args[0])
+ if len(args) > 1 {
+ b = strings.TrimSuffix(b, args[1])
+ }
+ fmt.Fprintln(u.hc.Stdout, b)
+ return 0
+}
+
+func uDirname(u *uctx, args []string) int {
+ if len(args) == 0 {
+ return 1
+ }
+ fmt.Fprintln(u.hc.Stdout, filepath.Dir(args[0]))
+ return 0
+}
+
+func uTee(u *uctx, args []string) int {
+ f := parseFlags(args, "")
+ data, _ := u.inputs(nil)
+ u.hc.Stdout.Write(data)
+ for _, p := range f.args {
+ abs, err := u.run.s.Sandbox.Resolve(u.hc.Dir, p)
+ if err == nil {
+ err = u.run.s.Sandbox.Writable(abs)
+ }
+ if err != nil {
+ u.errf("tee: %v", err)
+ return 1
+ }
+ flag := os.O_CREATE | os.O_WRONLY | os.O_TRUNC
+ if f.has("a") {
+ flag = os.O_CREATE | os.O_WRONLY | os.O_APPEND
+ }
+ fh, err := os.OpenFile(abs, flag, 0o644)
+ if err != nil {
+ u.errf("tee: %v", err)
+ return 1
+ }
+ fh.Write(data)
+ fh.Close()
+ }
+ return 0
+}
+
+func uFind(u *uctx, args []string) int {
+ roots := []string{}
+ i := 0
+ for ; i < len(args) && !strings.HasPrefix(args[i], "-"); i++ {
+ roots = append(roots, args[i])
+ }
+ if len(roots) == 0 {
+ roots = []string{"."}
+ }
+ var name, pathGlob, typ string
+ maxDepth := -1
+ for ; i < len(args); i++ {
+ switch args[i] {
+ case "-name", "-iname":
+ if i+1 < len(args) {
+ name = args[i+1]
+ i++
+ }
+ case "-path":
+ if i+1 < len(args) {
+ pathGlob = args[i+1]
+ i++
+ }
+ case "-type":
+ if i+1 < len(args) {
+ typ = args[i+1]
+ i++
+ }
+ case "-maxdepth":
+ if i+1 < len(args) {
+ maxDepth, _ = strconv.Atoi(args[i+1])
+ i++
+ }
+ case "-print":
+ default:
+ u.errf("find: unsupported predicate %s", args[i])
+ return 1
+ }
+ }
+ for _, root := range roots {
+ abs, err := u.run.s.Sandbox.Resolve(u.hc.Dir, root)
+ if err != nil {
+ u.errf("find: %v", err)
+ return 1
+ }
+ _ = filepath.WalkDir(abs, func(p string, d fs.DirEntry, err error) error {
+ if err != nil {
+ return nil
+ }
+ rel, _ := filepath.Rel(abs, p)
+ depth := 0
+ if rel != "." {
+ depth = strings.Count(rel, string(filepath.Separator)) + 1
+ }
+ if maxDepth >= 0 && depth > maxDepth {
+ if d.IsDir() {
+ return filepath.SkipDir
+ }
+ return nil
+ }
+ if d.IsDir() && d.Name() == ".git" {
+ return filepath.SkipDir
+ }
+ shown := filepath.Join(root, rel)
+ if typ == "f" && d.IsDir() || typ == "d" && !d.IsDir() {
+ return nil
+ }
+ if name != "" {
+ if ok, _ := filepath.Match(name, d.Name()); !ok {
+ return nil
+ }
+ }
+ if pathGlob != "" {
+ if ok, _ := path.Match(pathGlob, filepath.ToSlash(shown)); !ok {
+ return nil
+ }
+ }
+ fmt.Fprintln(u.hc.Stdout, filepath.ToSlash(shown))
+ return nil
+ })
+ }
+ return 0
+}
+
+func uNl(u *uctx, args []string) int {
+ data, code := u.inputs(parseFlags(args, "").args)
+ lines := splitLines(data)
+ for i, l := range lines {
+ lines[i] = fmt.Sprintf("%6d\t%s", i+1, l)
+ }
+ u.writeLines(lines)
+ return code
+}
+
+func uRev(u *uctx, args []string) int {
+ data, code := u.inputs(args)
+ lines := splitLines(data)
+ for i, l := range lines {
+ r := []rune(l)
+ for a, b := 0, len(r)-1; a < b; a, b = a+1, b-1 {
+ r[a], r[b] = r[b], r[a]
+ }
+ lines[i] = string(r)
+ }
+ u.writeLines(lines)
+ return code
+}
+
+func uSleep(u *uctx, args []string) int {
+ if len(args) == 0 {
+ return 1
+ }
+ d, err := strconv.ParseFloat(strings.TrimSuffix(args[0], "s"), 64)
+ if err != nil {
+ return 1
+ }
+ if d > 30 {
+ d = 30
+ }
+ time.Sleep(time.Duration(d * float64(time.Second)))
+ return 0
+}
+
+// sed supports -n, -E/-r, -e, and the commands s///[gpi], p, d, q with line,
+// $, /re/ and range addresses, separated by ; or newlines.
+func uSed(u *uctx, args []string) int {
+ quiet, ere := false, false
+ var scripts, files []string
+ for i := 0; i < len(args); i++ {
+ a := args[i]
+ switch {
+ case a == "-n" || a == "--quiet":
+ quiet = true
+ case a == "-E" || a == "-r":
+ ere = true
+ case a == "-e" && i+1 < len(args):
+ scripts = append(scripts, args[i+1])
+ i++
+ case a == "-i" || strings.HasPrefix(a, "-i"):
+ u.errf("sed: in-place editing is not available; use edit_file")
+ return 1
+ case strings.HasPrefix(a, "-") && len(a) > 1:
+ for _, c := range a[1:] {
+ switch c {
+ case 'n':
+ quiet = true
+ case 'E', 'r':
+ ere = true
+ default:
+ u.errf("sed: unsupported option -%c", c)
+ return 1
+ }
+ }
+ default:
+ if len(scripts) == 0 {
+ scripts = append(scripts, a)
+ } else {
+ files = append(files, a)
+ }
+ }
+ }
+ if len(scripts) == 0 {
+ u.errf("sed: no script")
+ return 1
+ }
+ cmds, err := parseSed(strings.Join(scripts, "\n"), ere)
+ if err != nil {
+ u.errf("sed: %v", err)
+ return 1
+ }
+ data, code := u.inputs(files)
+ lines := splitLines(data)
+ var out []string
+ inRange := make([]bool, len(cmds))
+ for idx, line := range lines {
+ n := idx + 1
+ last := n == len(lines)
+ deleted, quit := false, false
+ for ci, c := range cmds {
+ if !c.matches(n, last, line, &inRange[ci]) {
+ continue
+ }
+ switch c.op {
+ case 'p':
+ out = append(out, line)
+ case 'd':
+ deleted = true
+ case 'q':
+ quit = true
+ case 's':
+ repl := c.repl
+ if c.global {
+ line = c.re.ReplaceAllString(line, repl)
+ } else if loc := c.re.FindStringSubmatchIndex(line); loc != nil {
+ var b []byte
+ b = c.re.ExpandString(b, repl, line, loc)
+ line = line[:loc[0]] + string(b) + line[loc[1]:]
+ } else {
+ continue
+ }
+ if c.print {
+ out = append(out, line)
+ }
+ }
+ if deleted || quit {
+ break
+ }
+ }
+ if !deleted && !quiet {
+ out = append(out, line)
+ }
+ if quit {
+ break
+ }
+ }
+ u.writeLines(out)
+ return code
+}
+
+type sedAddr struct {
+ line int // 0 = none
+ last bool
+ re *regexp.Regexp
+ isSet bool
+}
+
+type sedCmd struct {
+ a1, a2 sedAddr
+ op byte
+ re *regexp.Regexp
+ repl string
+ global bool
+ print bool
+}
+
+func (a sedAddr) hit(n int, last bool, line string) bool {
+ switch {
+ case a.last:
+ return last
+ case a.re != nil:
+ return a.re.MatchString(line)
+ default:
+ return a.line == n
+ }
+}
+
+func (c sedCmd) matches(n int, last bool, line string, active *bool) bool {
+ if !c.a1.isSet {
+ return true
+ }
+ if !c.a2.isSet {
+ return c.a1.hit(n, last, line)
+ }
+ if *active {
+ if c.a2.hit(n, last, line) || (c.a2.line > 0 && n >= c.a2.line) {
+ *active = false
+ }
+ return true
+ }
+ if c.a1.hit(n, last, line) {
+ *active = !(c.a2.line > 0 && n >= c.a2.line)
+ return true
+ }
+ return false
+}
+
+func parseSed(script string, ere bool) ([]sedCmd, error) {
+ var cmds []sedCmd
+ compile := func(p string) (*regexp.Regexp, error) {
+ if !ere {
+ p = breToERE(p)
+ }
+ return regexp.Compile(p)
+ }
+ s := script
+ parseAddr := func() (sedAddr, error) {
+ s = strings.TrimLeftFunc(s, unicode.IsSpace)
+ switch {
+ case strings.HasPrefix(s, "$"):
+ s = s[1:]
+ return sedAddr{last: true, isSet: true}, nil
+ case strings.HasPrefix(s, "/"):
+ end := strings.Index(s[1:], "/")
+ if end < 0 {
+ return sedAddr{}, fmt.Errorf("unterminated address regex")
+ }
+ re, err := compile(s[1 : end+1])
+ s = s[end+2:]
+ return sedAddr{re: re, isSet: true}, err
+ default:
+ j := 0
+ for j < len(s) && s[j] >= '0' && s[j] <= '9' {
+ j++
+ }
+ if j == 0 {
+ return sedAddr{}, nil
+ }
+ n, _ := strconv.Atoi(s[:j])
+ s = s[j:]
+ return sedAddr{line: n, isSet: true}, nil
+ }
+ }
+ for {
+ s = strings.TrimLeft(s, " \t\n;")
+ if s == "" {
+ return cmds, nil
+ }
+ var c sedCmd
+ var err error
+ if c.a1, err = parseAddr(); err != nil {
+ return nil, err
+ }
+ if strings.HasPrefix(s, ",") {
+ s = s[1:]
+ if c.a2, err = parseAddr(); err != nil {
+ return nil, err
+ }
+ }
+ s = strings.TrimLeft(s, " \t")
+ if s == "" {
+ return nil, fmt.Errorf("missing command")
+ }
+ c.op = s[0]
+ s = s[1:]
+ switch c.op {
+ case 'p', 'd', 'q':
+ case 's':
+ if s == "" {
+ return nil, fmt.Errorf("unterminated s command")
+ }
+ delim := s[0]
+ parts := splitUnescaped(s[1:], delim, 3)
+ if len(parts) < 2 {
+ return nil, fmt.Errorf("unterminated s command")
+ }
+ if c.re, err = compile(parts[0]); err != nil {
+ return nil, err
+ }
+ c.repl = sedReplacement(parts[1])
+ flags := ""
+ if len(parts) == 3 {
+ flags = parts[2]
+ }
+ k := 0
+ for k < len(flags) && strings.ContainsRune("gpi0123456789", rune(flags[k])) {
+ switch flags[k] {
+ case 'g':
+ c.global = true
+ case 'p':
+ c.print = true
+ case 'i':
+ c.re = regexp.MustCompile("(?i)" + c.re.String())
+ }
+ k++
+ }
+ s = flags[k:]
+ default:
+ return nil, fmt.Errorf("unsupported command %q", c.op)
+ }
+ cmds = append(cmds, c)
+ }
+}
+
+// splitUnescaped splits s on delim (not preceded by a backslash) into at most
+// n parts; the last part keeps the remainder.
+func splitUnescaped(s string, delim byte, n int) []string {
+ var parts []string
+ var b strings.Builder
+ for i := 0; i < len(s); i++ {
+ if s[i] == '\\' && i+1 < len(s) && s[i+1] == delim {
+ b.WriteByte(delim)
+ i++
+ continue
+ }
+ if s[i] == delim && len(parts) < n-1 {
+ parts = append(parts, b.String())
+ b.Reset()
+ continue
+ }
+ b.WriteByte(s[i])
+ }
+ return append(parts, b.String())
+}
+
+// sedReplacement converts \1 and & to Go's ${1} and ${0}.
+func sedReplacement(r string) string {
+ var b strings.Builder
+ for i := 0; i < len(r); i++ {
+ switch {
+ case r[i] == '\\' && i+1 < len(r) && r[i+1] >= '0' && r[i+1] <= '9':
+ b.WriteString("${" + string(r[i+1]) + "}")
+ i++
+ case r[i] == '\\' && i+1 < len(r) && r[i+1] == 'n':
+ b.WriteByte('\n')
+ i++
+ case r[i] == '\\' && i+1 < len(r):
+ b.WriteByte(r[i+1])
+ i++
+ case r[i] == '&':
+ b.WriteString("${0}")
+ case r[i] == '$':
+ b.WriteString("$$")
+ default:
+ b.WriteByte(r[i])
+ }
+ }
+ return b.String()
+}
diff --git a/cli/test/evals/internal/tools/coreutils_more_test.go b/cli/test/evals/internal/tools/coreutils_more_test.go
new file mode 100644
index 00000000..a6573073
--- /dev/null
+++ b/cli/test/evals/internal/tools/coreutils_more_test.go
@@ -0,0 +1,22 @@
+package tools
+
+import (
+ "strings"
+ "testing"
+)
+
+func TestMoreUtilities(t *testing.T) {
+ s, _ := newShell(t)
+ cases := []struct{ cmd, want string }{
+ {`sed -n '2p' notes.txt`, "beta 1"},
+ {`sed 's/alpha/ALPHA/' notes.txt | head -1`, "ALPHA 3"},
+ {`seq 3 | xargs echo`, "1 2 3"},
+ {`date -u -d '2026-01-02T03:04:05Z' +%Y-%m-%d`, "2026-01-02"},
+ }
+ for _, c := range cases {
+ r := run(t, s, c.cmd)
+ if !r.OK || strings.TrimSpace(r.Output) != c.want {
+ t.Errorf("%s: got ok=%v %q, want %q", c.cmd, r.OK, r.Output, c.want)
+ }
+ }
+}
diff --git a/cli/test/evals/internal/tools/mcpbridge.go b/cli/test/evals/internal/tools/mcpbridge.go
new file mode 100644
index 00000000..ca9841ff
--- /dev/null
+++ b/cli/test/evals/internal/tools/mcpbridge.go
@@ -0,0 +1,178 @@
+package tools
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "strings"
+ "time"
+
+ "github.com/modelcontextprotocol/go-sdk/mcp"
+
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/ops"
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/record"
+)
+
+// MCPBridge connects to the backend's streamable HTTP MCP mount (/mcp or
+// /mcp/write) with a bearer PAT and exposes its tools to the model under the
+// mcp__traceway__ prefix, as Claude Code names them, plus list/read tools for
+// the server's knowledge resources.
+type MCPBridge struct {
+ Session *mcp.ClientSession
+ Instructions string
+ // DefaultProject fills project_id when the model omits it, the way the
+ // stdio server (traceway mcp) falls back to the CLI's current project.
+ DefaultProject string
+ tools []*mcp.Tool
+}
+
+type bearerRT struct{ token string }
+
+func (b bearerRT) RoundTrip(r *http.Request) (*http.Response, error) {
+ r = r.Clone(r.Context())
+ r.Header.Set("Authorization", "Bearer "+b.token)
+ return http.DefaultTransport.RoundTrip(r)
+}
+
+// ConnectMCP opens a session to url with token.
+func ConnectMCP(ctx context.Context, url, token, defaultProject string) (*MCPBridge, error) {
+ tr := &mcp.StreamableClientTransport{Endpoint: url, HTTPClient: &http.Client{Transport: bearerRT{token}, Timeout: 2 * time.Minute}, MaxRetries: 2}
+ cs, err := mcp.NewClient(&mcp.Implementation{Name: "twevals", Version: "1"}, nil).Connect(ctx, tr, nil)
+ if err != nil {
+ return nil, fmt.Errorf("connect %s: %w", url, err)
+ }
+ b := &MCPBridge{Session: cs, DefaultProject: defaultProject}
+ if ir := cs.InitializeResult(); ir != nil {
+ b.Instructions = ir.Instructions
+ }
+ for t, err := range cs.Tools(ctx, nil) {
+ if err != nil {
+ cs.Close()
+ return nil, fmt.Errorf("list tools: %w", err)
+ }
+ b.tools = append(b.tools, t)
+ }
+ return b, nil
+}
+
+// Close ends the session.
+func (b *MCPBridge) Close() { _ = b.Session.Close() }
+
+// Tools returns one Tool per server tool plus the resource tools.
+func (b *MCPBridge) Tools() []Tool {
+ var out []Tool
+ for _, t := range b.tools {
+ out = append(out, &mcpTool{b: b, t: t})
+ }
+ return append(out, &mcpResources{b: b, list: true}, &mcpResources{b: b})
+}
+
+// ToolNames lists the server's tool names (unprefixed).
+func (b *MCPBridge) ToolNames() []string {
+ var out []string
+ for _, t := range b.tools {
+ out = append(out, t.Name)
+ }
+ return out
+}
+
+type mcpTool struct {
+ b *MCPBridge
+ t *mcp.Tool
+}
+
+func (m *mcpTool) Spec() Spec {
+ params := map[string]any{"type": "object", "properties": map[string]any{}}
+ if m.t.InputSchema != nil {
+ raw, _ := json.Marshal(m.t.InputSchema)
+ _ = json.Unmarshal(raw, &params)
+ }
+ return Spec{Name: ops.MCPPrefix + m.t.Name, Description: m.t.Description, Parameters: params}
+}
+
+func (m *mcpTool) Call(ctx context.Context, raw json.RawMessage) Result {
+ args := map[string]any{}
+ if len(raw) > 0 && string(raw) != "null" {
+ if err := json.Unmarshal(raw, &args); err != nil {
+ return errResult("invalid arguments: %v", err)
+ }
+ }
+ if m.b.DefaultProject != "" && m.takesProject() {
+ if _, ok := args["project_id"]; !ok {
+ args["project_id"] = m.b.DefaultProject
+ }
+ }
+ res, err := m.b.Session.CallTool(ctx, &mcp.CallToolParams{Name: m.t.Name, Arguments: args})
+ op := ops.ParseMCP(m.t.Name, args)
+ if err != nil {
+ out := "error: " + err.Error()
+ return Result{Output: out, Ops: []record.OpRecord{{Op: op, OK: false, Output: out}}}
+ }
+ var b strings.Builder
+ for _, c := range res.Content {
+ switch tc := c.(type) {
+ case *mcp.TextContent:
+ b.WriteString(tc.Text)
+ default:
+ raw, _ := json.Marshal(c)
+ b.Write(raw)
+ }
+ }
+ text := b.String()
+ if text == "" && res.StructuredContent != nil {
+ raw, _ := json.Marshal(res.StructuredContent)
+ text = string(raw)
+ }
+ out, trunc := CapAt(text, MaxMCPOutput)
+ ok := !res.IsError
+ return Result{Output: out, OK: ok, Truncated: trunc, Ops: []record.OpRecord{{Op: op, OK: ok, Output: text, Exit: map[bool]int{true: 0, false: 1}[ok]}}}
+}
+
+func (m *mcpTool) takesProject() bool {
+ raw, _ := json.Marshal(m.t.InputSchema)
+ return strings.Contains(string(raw), `"project_id"`)
+}
+
+type mcpResources struct {
+ b *MCPBridge
+ list bool
+}
+
+func (r *mcpResources) Spec() Spec {
+ if r.list {
+ return Spec{Name: ops.MCPPrefix + "list_resources", Description: "List the Traceway MCP server's resources (its knowledge playbooks, traceway://knowledge/*).", Parameters: obj(map[string]any{})}
+ }
+ return Spec{Name: ops.MCPPrefix + "read_resource", Description: "Read one Traceway MCP resource by URI, e.g. traceway://knowledge/performance.",
+ Parameters: obj(map[string]any{"uri": strProp("Resource URI")}, "uri")}
+}
+
+func (r *mcpResources) Call(ctx context.Context, raw json.RawMessage) Result {
+ if r.list {
+ var b strings.Builder
+ for res, err := range r.b.Session.Resources(ctx, nil) {
+ if err != nil {
+ return errResult("%v", err)
+ }
+ fmt.Fprintf(&b, "%s %s: %s\n", res.URI, res.Name, res.Description)
+ }
+ return Result{Output: b.String(), OK: true, Ops: []record.OpRecord{{Op: ops.Op{Op: ops.OpListRes}, OK: true}}}
+ }
+ var in struct {
+ URI string `json:"uri"`
+ }
+ if err := decodeArgs(raw, &in); err != nil {
+ return errResult("%v", err)
+ }
+ res, err := r.b.Session.ReadResource(ctx, &mcp.ReadResourceParams{URI: in.URI})
+ op := ops.Op{Op: ops.OpReadResource, Args: map[string]any{"uri": in.URI}}
+ if err != nil {
+ return Result{Output: "error: " + err.Error(), Ops: []record.OpRecord{{Op: op}}}
+ }
+ var b strings.Builder
+ for _, c := range res.Contents {
+ b.WriteString(c.Text)
+ }
+ out, trunc := Cap(b.String())
+ return Result{Output: out, OK: true, Truncated: trunc, Ops: []record.OpRecord{{Op: op, OK: true}}}
+}
diff --git a/cli/test/evals/internal/tools/submit.go b/cli/test/evals/internal/tools/submit.go
index 2e9f7c70..09bb2f69 100644
--- a/cli/test/evals/internal/tools/submit.go
+++ b/cli/test/evals/internal/tools/submit.go
@@ -13,18 +13,27 @@ import (
"github.com/tracewayapp/traceway/cli/test/evals/internal/record"
)
-// SubmitPerf is the perf suite's final-answer tool.
-const SubmitPerf = "submit_perf_diagnosis"
+// The final-answer tools: perf, debug, and finish for write and setup.
+const (
+ SubmitPerf = "submit_perf_diagnosis"
+ SubmitDebug = "submit_debug_diagnosis"
+ Finish = "finish"
+)
// IsSubmit reports whether a tool is a final-answer tool.
-func IsSubmit(name string) bool { return name == SubmitPerf }
+func IsSubmit(name string) bool {
+ return name == SubmitPerf || name == SubmitDebug || name == Finish
+}
+
+// DebugCategories are submit_debug_diagnosis's category values.
+var DebugCategories = []string{"null_reference", "missing_key", "type_or_contract_mismatch", "race_or_duplicate", "timeout", "config", "resource_exhaustion", "logic", "unknown"}
// Submit is a final-answer tool. It validates the payload against its
// schema and the call ids it cites; a failure goes back to the model for
// repair, up to MaxRepairs times, after which the last payload is accepted
// as-is for grading.
type Submit struct {
- Kind string // the tool's name: SubmitPerf
+ Kind string // SubmitPerf | SubmitDebug | Finish
CallExists func(id string) (ok, exists bool)
MaxRepairs int
@@ -33,31 +42,63 @@ type Submit struct {
}
func (t *Submit) Spec() Spec {
- var rows []string
- for _, r := range checklist.Rows() {
- rows = append(rows, fmt.Sprintf("%s (%s)", r.ID, r.Title))
+ switch t.Kind {
+ case SubmitPerf:
+ var rows []string
+ for _, r := range checklist.Rows() {
+ rows = append(rows, fmt.Sprintf("%s (%s)", r.ID, r.Title))
+ }
+ ids := checklist.IDs()
+ item := map[string]any{"type": "string", "enum": ids}
+ return Spec{Name: SubmitPerf, Description: "Submit your final performance diagnosis. This ends the task. " +
+ "checklist_item is the bottleneck checklist row that explains the latency, or none_found when the latency is within expectations, or insufficient_evidence when the telemetry cannot tell. Rows: " + strings.Join(rows, "; ") + ". " +
+ "Every evidence item cites the id of an earlier tool call (the [call cNN] line at the top of its result).",
+ Parameters: obj(map[string]any{
+ "checklist_item": item,
+ "category": strProp("database, application, network, caching, infrastructure, or none"),
+ "locus": obj(map[string]any{
+ "endpoint": strProp("Endpoint name, e.g. GET /api/orders"),
+ "span": strProp("The dominating span or query"),
+ "service": strProp("Service name"),
+ "host": strProp("Host or instance, when the cause is host-specific"),
+ }),
+ "onset": map[string]any{"type": []string{"string", "null"}, "description": "RFC 3339 time the slowness started, or null"},
+ "evidence": evidenceSchema(),
+ "ruled_out": map[string]any{"type": "array", "items": obj(map[string]any{"checklist_item": item, "call_id": strProp("Call id"), "reason": strProp("Why it is ruled out")}, "checklist_item", "reason")},
+ "fix": strProp("The concrete fix"),
+ "confidence": map[string]any{"type": "string", "enum": []string{"low", "medium", "high"}},
+ "report": strProp("Your report to the user, in Markdown"),
+ }, "checklist_item", "locus", "evidence", "fix", "confidence", "report")}
+ case SubmitDebug:
+ return Spec{Name: SubmitDebug, Description: "Submit your final debugging diagnosis. This ends the task. Every evidence item cites the id of an earlier tool call (the [call cNN] line at the top of its result).",
+ Parameters: obj(map[string]any{
+ "issue_hash": strProp("The exception group hash (16 hex), if the bug has one"),
+ "occurrence_id": strProp("The occurrence you anchored on, if any"),
+ "category": map[string]any{"type": "string", "enum": DebugCategories},
+ "culprit": obj(map[string]any{
+ "file": strProp("Workspace-relative file"),
+ "symbol": strProp("Function or method"),
+ "line": intProp("Line number"),
+ }, "file"),
+ "evidence": evidenceSchema(),
+ "fix_summary": strProp("What you changed or propose to change"),
+ "files_changed": map[string]any{"type": "array", "items": map[string]any{"type": "string"}},
+ "needs_instrumentation": map[string]any{"type": "boolean", "description": "True when the telemetry cannot explain the failure and you added instrumentation instead of a fix"},
+ "confidence": map[string]any{"type": "string", "enum": []string{"low", "medium", "high"}},
+ "report": strProp("Your report to the user, in Markdown"),
+ }, "category", "culprit", "evidence", "fix_summary", "needs_instrumentation", "confidence", "report")}
+ default:
+ return Spec{Name: Finish, Description: "Finish the task and report what you did. This ends the task.",
+ Parameters: obj(map[string]any{
+ "summary": strProp("Your report to the user, in Markdown"),
+ "changes": map[string]any{"type": "array", "items": obj(map[string]any{
+ "action": strProp("What you changed, e.g. snooze alert rule"),
+ "target": strProp("The record, e.g. rule 12"),
+ "new_value": strProp("The new state"),
+ "undo": strProp("How to undo it"),
+ }, "action", "target")},
+ }, "summary", "changes")}
}
- ids := checklist.IDs()
- item := map[string]any{"type": "string", "enum": ids}
- return Spec{Name: SubmitPerf, Description: "Submit your final performance diagnosis. This ends the task. " +
- "checklist_item is the bottleneck checklist row that explains the latency, or none_found when the latency is within expectations, or insufficient_evidence when the telemetry cannot tell. Rows: " + strings.Join(rows, "; ") + ". " +
- "Every evidence item cites the id of an earlier tool call (the [call cNN] line at the top of its result).",
- Parameters: obj(map[string]any{
- "checklist_item": item,
- "category": strProp("database, application, network, caching, infrastructure, or none"),
- "locus": obj(map[string]any{
- "endpoint": strProp("Endpoint name, e.g. GET /api/orders"),
- "span": strProp("The dominating span or query"),
- "service": strProp("Service name"),
- "host": strProp("Host or instance, when the cause is host-specific"),
- }),
- "onset": map[string]any{"type": []string{"string", "null"}, "description": "RFC 3339 time the slowness started, or null"},
- "evidence": evidenceSchema(),
- "ruled_out": map[string]any{"type": "array", "items": obj(map[string]any{"checklist_item": item, "call_id": strProp("Call id"), "reason": strProp("Why it is ruled out")}, "checklist_item", "reason")},
- "fix": strProp("The concrete fix"),
- "confidence": map[string]any{"type": "string", "enum": []string{"low", "medium", "high"}},
- "report": strProp("Your report to the user, in Markdown"),
- }, "checklist_item", "locus", "evidence", "fix", "confidence", "report")}
}
func evidenceSchema() map[string]any {
@@ -129,19 +170,35 @@ func (t *Submit) validate(p map[string]any) []string {
}
}
}
- need("checklist_item", "locus", "evidence", "fix", "confidence", "report")
- if v := str("checklist_item"); v != "" && !checklist.IsID(v) {
- probs = append(probs, fmt.Sprintf("checklist_item %q is not one of the allowed values", v))
- }
- if rs, ok := p["ruled_out"].([]any); ok {
- for i, r := range rs {
- m, _ := r.(map[string]any)
- if v, _ := m["checklist_item"].(string); v != "" && !checklist.IsID(v) {
- probs = append(probs, fmt.Sprintf("ruled_out[%d].checklist_item %q is not an allowed value", i, v))
+ switch t.Kind {
+ case SubmitPerf:
+ need("checklist_item", "locus", "evidence", "fix", "confidence", "report")
+ if v := str("checklist_item"); v != "" && !checklist.IsID(v) {
+ probs = append(probs, fmt.Sprintf("checklist_item %q is not one of the allowed values", v))
+ }
+ if rs, ok := p["ruled_out"].([]any); ok {
+ for i, r := range rs {
+ m, _ := r.(map[string]any)
+ if v, _ := m["checklist_item"].(string); v != "" && !checklist.IsID(v) {
+ probs = append(probs, fmt.Sprintf("ruled_out[%d].checklist_item %q is not an allowed value", i, v))
+ }
+ }
+ }
+ checkEvidence()
+ case SubmitDebug:
+ need("category", "culprit", "evidence", "fix_summary", "needs_instrumentation", "confidence", "report")
+ if v := str("category"); v != "" && !slices.Contains(DebugCategories, v) {
+ probs = append(probs, fmt.Sprintf("category %q is not one of %v", v, DebugCategories))
+ }
+ if c, ok := p["culprit"].(map[string]any); ok {
+ if f, _ := c["file"].(string); f == "" {
+ probs = append(probs, "culprit.file is required")
}
}
+ checkEvidence()
+ default:
+ need("summary", "changes")
}
- checkEvidence()
if c := str("confidence"); c != "" && !slices.Contains([]string{"low", "medium", "high"}, c) {
probs = append(probs, "confidence must be low, medium or high")
}
diff --git a/cli/test/evals/internal/tools/tools.go b/cli/test/evals/internal/tools/tools.go
index 33de377c..4efc384f 100644
--- a/cli/test/evals/internal/tools/tools.go
+++ b/cli/test/evals/internal/tools/tools.go
@@ -1,8 +1,8 @@
// Package tools implements what the agent under test can call: a sandboxed
// shell (an interpreter with an allowlist, never the host shell), confined
-// file tools, skills, ask_user, and the structured submit tools. Every call
-// returns the operations it performed so the graders can check the
-// trajectory.
+// file tools, skills, an MCP bridge, ask_user, and the structured submit
+// tools. Every call returns the operations it performed so the graders can
+// map CLI and MCP runs onto the same trajectory.
package tools
import (
@@ -44,6 +44,11 @@ type Tool interface {
// MaxOutput caps what one shell or file call returns to the model.
const MaxOutput = 30000
+// MaxMCPOutput caps one MCP tool result. An MCP client cannot pipe a result
+// through jq, so the cap mirrors Claude Code's MCP output limit (about 25k
+// tokens) rather than the shell's.
+const MaxMCPOutput = 100000
+
// Cap truncates s to MaxOutput characters with a marker.
func Cap(s string) (string, bool) { return CapAt(s, MaxOutput) }
diff --git a/cli/test/evals/models.yaml b/cli/test/evals/models.yaml
index d6ff02e3..ac359142 100644
--- a/cli/test/evals/models.yaml
+++ b/cli/test/evals/models.yaml
@@ -1,15 +1,16 @@
-# Pinned OpenRouter models for the eval agents and the judge, for paid runs
-# through OpenRouter Chat Completions. This harness makes no paid calls: it
-# plays oracles with the fake model and only records these ids in the
-# results header.
+# Pinned OpenRouter models for the eval agents and the judge. Every paid call
+# goes through OpenRouter Chat Completions with one key (OPENROUTER_API_KEY).
+# Oracle runs use the fake model and only record these ids in the results
+# header.
#
# Never use a floating alias (~provider/model-latest): results must stay
# comparable over time, so a model change is a reviewed edit to this file.
# Prices and supported_parameters were checked against
-# GET https://openrouter.ai/api/v1/models on 2026-10-17.
+# GET https://openrouter.ai/api/v1/models on 2026-10-17. The preflight reads live
+# prices; fallback_price is used only when that list cannot be fetched.
#
-# estimate (a per-run token envelope) and fallback_price (USD per million
-# tokens) are for estimating a paid run's cost before it starts.
+# estimate is the per-run token envelope the preflight multiplies by live
+# prices until baseline.json carries measured averages.
agents:
opus:
diff --git a/cli/test/evals/presets.yaml b/cli/test/evals/presets.yaml
index f0b4c98b..dbe2d4aa 100644
--- a/cli/test/evals/presets.yaml
+++ b/cli/test/evals/presets.yaml
@@ -1,4 +1,4 @@
-# Case and arm selections for `twevals list --preset`. A --cases glob
+# Case and arm selections for `twevals list|plan|run --preset`. A --cases glob
# overrides the preset's case filter; --arms overrides its arm policy.
presets:
smoke:
@@ -9,8 +9,11 @@ presets:
description: Every case, each suite's primary arm only
arms: primary
full:
- description: Every case on every arm it lists (adds cli-noskill)
+ description: Every case on every arm it lists (adds cli-noskill, mcp, mcp-write)
arms: all
primary_arms:
perf: [cli+skill]
+ debug: [cli+skill]
+ write: [cli-write+skill]
+ setup: [cli+setup-skill]
diff --git a/cli/test/evals/testdata/hidden/debug-go-nil-regression/internal/invoices/zz_hidden_nil_test.go b/cli/test/evals/testdata/hidden/debug-go-nil-regression/internal/invoices/zz_hidden_nil_test.go
new file mode 100644
index 00000000..8f6b06e6
--- /dev/null
+++ b/cli/test/evals/testdata/hidden/debug-go-nil-regression/internal/invoices/zz_hidden_nil_test.go
@@ -0,0 +1,46 @@
+package invoices
+
+import (
+ "context"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+)
+
+type hiddenStore struct{ inv *Invoice }
+
+func (h hiddenStore) Invoice(context.Context, int64) (*Invoice, error) { return h.inv, nil }
+
+func TestHiddenRenderWithoutBillingAddress(t *testing.T) {
+ defer func() {
+ if r := recover(); r != nil {
+ t.Fatalf("render panics for a customer without a billing address: %v", r)
+ }
+ }()
+ v := render(&Invoice{Number: "INV-7", TotalCents: 990, Currency: "EUR", Customer: &Customer{Name: "Bo"}})
+ if v.CustomerName != "Bo" || v.Total != "9.90 EUR" {
+ t.Fatalf("unexpected view %+v", v)
+ }
+ if strings.Contains(v.BillingAddress, "<nil>") || strings.HasPrefix(strings.TrimSpace(v.BillingAddress), ",") {
+ t.Fatalf("a missing address must not render as %q", v.BillingAddress)
+ }
+}
+
+func TestHiddenGetWithoutBillingAddress(t *testing.T) {
+ h := NewHandler(hiddenStore{inv: &Invoice{Number: "INV-8", Currency: "EUR", Customer: &Customer{Name: "Cy"}}})
+ req := httptest.NewRequest(http.MethodGet, "/api/invoices/8", nil)
+ req.SetPathValue("id", "8")
+ rec := httptest.NewRecorder()
+ Recover(http.HandlerFunc(h.Get)).ServeHTTP(rec, req)
+ if rec.Code != http.StatusOK {
+ t.Fatalf("GET /api/invoices/8 for a customer without a billing address answered %d", rec.Code)
+ }
+}
+
+func TestHiddenRenderWithAddressUnchanged(t *testing.T) {
+ v := render(&Invoice{Currency: "EUR", Customer: &Customer{Name: "Ada", BillingAddress: &Address{Line1: "Main St 1", City: "Berlin", PostalCode: "10115", Country: "DE"}}})
+ if v.BillingAddress != "Main St 1, 10115 Berlin, DE" {
+ t.Fatalf("the address format changed: %q", v.BillingAddress)
+ }
+}
diff --git a/cli/test/evals/testdata/oracle/debug-go-nil-regression/render.go b/cli/test/evals/testdata/oracle/debug-go-nil-regression/render.go
new file mode 100644
index 00000000..1306174a
--- /dev/null
+++ b/cli/test/evals/testdata/oracle/debug-go-nil-regression/render.go
@@ -0,0 +1,49 @@
+package invoices
+
+import "fmt"
+
+// View is the invoice as the page shows it.
+type View struct {
+ Number string `json:"number"`
+ IssuedAt string `json:"issuedAt"`
+ CustomerName string `json:"customerName"`
+ BillingAddress string `json:"billingAddress"`
+ Lines []ViewLine `json:"lines"`
+ Total string `json:"total"`
+}
+
+// ViewLine is one rendered line.
+type ViewLine struct {
+ Description string `json:"description"`
+ Qty int `json:"qty"`
+ Amount string `json:"amount"`
+}
+
+func render(inv *Invoice) View {
+ v := View{
+ Number: inv.Number,
+ IssuedAt: inv.IssuedAt.Format("2 January 2006"),
+ CustomerName: inv.Customer.Name,
+ BillingAddress: formatAddress(inv.Customer.BillingAddress),
+ Total: money(inv.TotalCents, inv.Currency),
+ }
+ for _, l := range inv.Lines {
+ v.Lines = append(v.Lines, ViewLine{Description: l.Description, Qty: l.Qty, Amount: money(int64(l.Qty)*l.UnitCents, inv.Currency)})
+ }
+ return v
+}
+
+func formatAddress(a *Address) string {
+ if a == nil {
+ return ""
+ }
+ s := a.Line1
+ if a.Line2 != "" {
+ s += ", " + a.Line2
+ }
+ return fmt.Sprintf("%s, %s %s, %s", s, a.PostalCode, a.City, a.Country)
+}
+
+func money(cents int64, currency string) string {
+ return fmt.Sprintf("%d.%02d %s", cents/100, cents%100, currency)
+}
diff --git a/cli/test/evals/testdata/oracle/setup-go-nethttp/main.go b/cli/test/evals/testdata/oracle/setup-go-nethttp/main.go
new file mode 100644
index 00000000..9d79c73a
--- /dev/null
+++ b/cli/test/evals/testdata/oracle/setup-go-nethttp/main.go
@@ -0,0 +1,49 @@
+package main
+
+import (
+ "context"
+ "log"
+ "net/http"
+ "os"
+ "os/signal"
+ "time"
+
+ "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+)
+
+func main() {
+ ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
+ defer stop()
+ shutdown, err := setupTelemetry(ctx)
+ if err != nil {
+ log.Fatalf("telemetry: %v", err)
+ }
+ defer func() {
+ c, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ _ = shutdown(c)
+ }()
+
+ mux := http.NewServeMux()
+ // Wrap each handler, not the mux: otelhttp reads http.route from
+ // r.Pattern, which ServeMux sets only after it matched.
+ mux.Handle("GET /api/hello", otelhttp.NewHandler(http.HandlerFunc(hello), "hello"))
+ mux.Handle("GET /api/hello/{name}", otelhttp.NewHandler(http.HandlerFunc(helloName), "hello-name"))
+ addr := ":" + envOr("PORT", "8080")
+ srv := &http.Server{Addr: addr, Handler: mux}
+ go func() {
+ <-ctx.Done()
+ _ = srv.Shutdown(context.Background())
+ }()
+ log.Printf("listening on %s", addr)
+ if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+ log.Fatal(err)
+ }
+}
+
+func envOr(k, def string) string {
+ if v := os.Getenv(k); v != "" {
+ return v
+ }
+ return def
+}
diff --git a/cli/test/evals/testdata/oracle/setup-go-nethttp/telemetry.go b/cli/test/evals/testdata/oracle/setup-go-nethttp/telemetry.go
new file mode 100644
index 00000000..c2885d62
--- /dev/null
+++ b/cli/test/evals/testdata/oracle/setup-go-nethttp/telemetry.go
@@ -0,0 +1,43 @@
+package main
+
+import (
+ "context"
+ "os"
+ "strings"
+ "time"
+
+ "go.opentelemetry.io/otel"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
+ "go.opentelemetry.io/otel/propagation"
+ "go.opentelemetry.io/otel/sdk/resource"
+ sdktrace "go.opentelemetry.io/otel/sdk/trace"
+)
+
+// setupTelemetry exports traces to Traceway over OTLP/HTTP. The instance URL
+// and the project ingest token come from the environment, never the source.
+func setupTelemetry(ctx context.Context) (func(context.Context) error, error) {
+ base := strings.TrimRight(os.Getenv("TRACEWAY_URL"), "/")
+ exp, err := otlptracehttp.New(ctx,
+ otlptracehttp.WithEndpointURL(base+"/api/otel/v1/traces"),
+ otlptracehttp.WithHeaders(map[string]string{"Authorization": "Bearer " + os.Getenv("TRACEWAY_PROJECT_TOKEN")}),
+ )
+ if err != nil {
+ return nil, err
+ }
+ res, err := resource.New(ctx,
+ resource.WithAttributes(
+ attribute.String("service.name", envOr("OTEL_SERVICE_NAME", "hello-api")),
+ attribute.String("service.version", envOr("APP_VERSION", "dev")),
+ ),
+ resource.WithHost(),
+ resource.WithTelemetrySDK(),
+ )
+ if err != nil {
+ return nil, err
+ }
+ tp := sdktrace.NewTracerProvider(sdktrace.WithBatcher(exp, sdktrace.WithBatchTimeout(2*time.Second)), sdktrace.WithResource(res))
+ otel.SetTracerProvider(tp)
+ otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}))
+ return tp.Shutdown, nil
+}
diff --git a/cli/test/evals/testdata/repos/billing-api-go/.history/01/internal/invoices/render.go b/cli/test/evals/testdata/repos/billing-api-go/.history/01/internal/invoices/render.go
new file mode 100644
index 00000000..4120134c
--- /dev/null
+++ b/cli/test/evals/testdata/repos/billing-api-go/.history/01/internal/invoices/render.go
@@ -0,0 +1,36 @@
+package invoices
+
+import "fmt"
+
+// View is the invoice as the page shows it.
+type View struct {
+ Number string `json:"number"`
+ IssuedAt string `json:"issuedAt"`
+ CustomerName string `json:"customerName"`
+ Lines []ViewLine `json:"lines"`
+ Total string `json:"total"`
+}
+
+// ViewLine is one rendered line.
+type ViewLine struct {
+ Description string `json:"description"`
+ Qty int `json:"qty"`
+ Amount string `json:"amount"`
+}
+
+func render(inv *Invoice) View {
+ v := View{
+ Number: inv.Number,
+ IssuedAt: inv.IssuedAt.Format("2 January 2006"),
+ CustomerName: inv.Customer.Name,
+ Total: money(inv.TotalCents, inv.Currency),
+ }
+ for _, l := range inv.Lines {
+ v.Lines = append(v.Lines, ViewLine{Description: l.Description, Qty: l.Qty, Amount: money(int64(l.Qty)*l.UnitCents, inv.Currency)})
+ }
+ return v
+}
+
+func money(cents int64, currency string) string {
+ return fmt.Sprintf("%d.%02d %s", cents/100, cents%100, currency)
+}
diff --git a/cli/test/evals/testdata/repos/billing-api-go/.history/01/internal/invoices/render_test.go b/cli/test/evals/testdata/repos/billing-api-go/.history/01/internal/invoices/render_test.go
new file mode 100644
index 00000000..90f1bc4a
--- /dev/null
+++ b/cli/test/evals/testdata/repos/billing-api-go/.history/01/internal/invoices/render_test.go
@@ -0,0 +1,16 @@
+package invoices
+
+import (
+ "testing"
+ "time"
+)
+
+func TestRenderWithAddress(t *testing.T) {
+ inv := &Invoice{Number: "INV-1042", IssuedAt: time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC), TotalCents: 4990, Currency: "EUR",
+ Customer: &Customer{Name: "Ada", BillingAddress: &Address{Line1: "Main St 1", City: "Berlin", PostalCode: "10115", Country: "DE"}},
+ Lines: []Line{{Description: "Plan", Qty: 1, UnitCents: 4990}}}
+ v := render(inv)
+ if v.Total != "49.90 EUR" {
+ t.Fatalf("unexpected view %+v", v)
+ }
+}
diff --git a/cli/test/evals/testdata/repos/billing-api-go/README.md b/cli/test/evals/testdata/repos/billing-api-go/README.md
new file mode 100644
index 00000000..60b06de6
--- /dev/null
+++ b/cli/test/evals/testdata/repos/billing-api-go/README.md
@@ -0,0 +1,11 @@
+# billing-api
+
+Invoices for the storefront: the invoice page and invoice e-mails.
+Go, `net/http`, PostgreSQL. Telemetry goes to Traceway over OTLP.
+
+```
+go test ./...
+go run ./cmd/server
+```
+
+Releases are tagged in the commit message (`v1.8.2: ...`) and deployed from `main`.
diff --git a/cli/test/evals/testdata/repos/billing-api-go/cmd/server/main.go b/cli/test/evals/testdata/repos/billing-api-go/cmd/server/main.go
new file mode 100644
index 00000000..57b3a5d5
--- /dev/null
+++ b/cli/test/evals/testdata/repos/billing-api-go/cmd/server/main.go
@@ -0,0 +1,30 @@
+package main
+
+import (
+ "database/sql"
+ "log/slog"
+ "net/http"
+ "os"
+
+ "example.com/billing-api/internal/invoices"
+)
+
+func main() {
+ db, err := sql.Open("pgx", os.Getenv("DATABASE_URL"))
+ if err != nil {
+ slog.Error("open database", "err", err)
+ os.Exit(1)
+ }
+ h := invoices.NewHandler(invoices.NewStore(db))
+ mux := http.NewServeMux()
+ mux.HandleFunc("GET /api/invoices/{id}", h.Get)
+ addr := ":" + os.Getenv("PORT")
+ if addr == ":" {
+ addr = ":8080"
+ }
+ slog.Info("listening", "addr", addr)
+ if err := http.ListenAndServe(addr, invoices.Recover(mux)); err != nil {
+ slog.Error("serve", "err", err)
+ os.Exit(1)
+ }
+}
diff --git a/cli/test/evals/testdata/repos/billing-api-go/go.mod b/cli/test/evals/testdata/repos/billing-api-go/go.mod
new file mode 100644
index 00000000..62f397ef
--- /dev/null
+++ b/cli/test/evals/testdata/repos/billing-api-go/go.mod
@@ -0,0 +1,3 @@
+module example.com/billing-api
+
+go 1.26
diff --git a/cli/test/evals/testdata/repos/billing-api-go/history.yaml b/cli/test/evals/testdata/repos/billing-api-go/history.yaml
new file mode 100644
index 00000000..32fd273f
--- /dev/null
+++ b/cli/test/evals/testdata/repos/billing-api-go/history.yaml
@@ -0,0 +1,9 @@
+commits:
+- dir: .history/01
+ message: 'v1.8.1: invoice page totals in the customer''s currency'
+ age: 400h
+ author: Sam Okafor <[email protected]>
+head:
+ message: 'v1.8.2: show the billing address on the invoice page'
+ age: 30h40m
+ author: Rui Tanaka <[email protected]>
diff --git a/cli/test/evals/testdata/repos/billing-api-go/internal/invoices/handler.go b/cli/test/evals/testdata/repos/billing-api-go/internal/invoices/handler.go
new file mode 100644
index 00000000..c115faad
--- /dev/null
+++ b/cli/test/evals/testdata/repos/billing-api-go/internal/invoices/handler.go
@@ -0,0 +1,51 @@
+package invoices
+
+import (
+ "encoding/json"
+ "fmt"
+ "log/slog"
+ "net/http"
+ "runtime/debug"
+ "strconv"
+)
+
+// Handler serves invoices.
+type Handler struct{ store Store }
+
+// NewHandler returns a Handler.
+func NewHandler(s Store) *Handler { return &Handler{store: s} }
+
+// Get serves GET /api/invoices/{id}.
+func (h *Handler) Get(w http.ResponseWriter, r *http.Request) {
+ id, err := strconv.ParseInt(r.PathValue("id"), 10, 64)
+ if err != nil {
+ http.Error(w, "bad id", http.StatusBadRequest)
+ return
+ }
+ inv, err := h.store.Invoice(r.Context(), id)
+ if err != nil {
+ http.Error(w, "internal error", http.StatusInternalServerError)
+ return
+ }
+ if inv == nil {
+ http.NotFound(w, r)
+ return
+ }
+ view := render(inv)
+ w.Header().Set("Content-Type", "application/json")
+ _ = json.NewEncoder(w).Encode(view)
+}
+
+// Recover turns a panic into a 500 and records it on the request span
+// (span.RecordError with the stack in production builds).
+func Recover(next http.Handler) http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ defer func() {
+ if v := recover(); v != nil {
+ slog.Error("panic serving request", "path", r.URL.Path, "panic", fmt.Sprint(v), "stack", string(debug.Stack()))
+ http.Error(w, "internal error", http.StatusInternalServerError)
+ }
+ }()
+ next.ServeHTTP(w, r)
+ })
+}
diff --git a/cli/test/evals/testdata/repos/billing-api-go/internal/invoices/model.go b/cli/test/evals/testdata/repos/billing-api-go/internal/invoices/model.go
new file mode 100644
index 00000000..f02b0e31
--- /dev/null
+++ b/cli/test/evals/testdata/repos/billing-api-go/internal/invoices/model.go
@@ -0,0 +1,41 @@
+// Package invoices serves invoice pages.
+package invoices
+
+import "time"
+
+// Invoice is a billed order.
+type Invoice struct {
+ ID int64
+ Number string
+ IssuedAt time.Time
+ Customer *Customer
+ Lines []Line
+ TotalCents int64
+ Currency string
+}
+
+// Customer is who the invoice is billed to.
+type Customer struct {
+ ID int64
+ Name string
+ Email string
+ // BillingAddress is optional: customers who only buy digital goods
+ // never enter one.
+ BillingAddress *Address
+}
+
+// Address is a postal address.
+type Address struct {
+ Line1 string
+ Line2 string
+ City string
+ PostalCode string
+ Country string
+}
+
+// Line is one invoice line.
+type Line struct {
+ Description string
+ Qty int
+ UnitCents int64
+}
diff --git a/cli/test/evals/testdata/repos/billing-api-go/internal/invoices/render.go b/cli/test/evals/testdata/repos/billing-api-go/internal/invoices/render.go
new file mode 100644
index 00000000..9ee39406
--- /dev/null
+++ b/cli/test/evals/testdata/repos/billing-api-go/internal/invoices/render.go
@@ -0,0 +1,46 @@
+package invoices
+
+import "fmt"
+
+// View is the invoice as the page shows it.
+type View struct {
+ Number string `json:"number"`
+ IssuedAt string `json:"issuedAt"`
+ CustomerName string `json:"customerName"`
+ BillingAddress string `json:"billingAddress"`
+ Lines []ViewLine `json:"lines"`
+ Total string `json:"total"`
+}
+
+// ViewLine is one rendered line.
+type ViewLine struct {
+ Description string `json:"description"`
+ Qty int `json:"qty"`
+ Amount string `json:"amount"`
+}
+
+func render(inv *Invoice) View {
+ v := View{
+ Number: inv.Number,
+ IssuedAt: inv.IssuedAt.Format("2 January 2006"),
+ CustomerName: inv.Customer.Name,
+ BillingAddress: formatAddress(inv.Customer.BillingAddress),
+ Total: money(inv.TotalCents, inv.Currency),
+ }
+ for _, l := range inv.Lines {
+ v.Lines = append(v.Lines, ViewLine{Description: l.Description, Qty: l.Qty, Amount: money(int64(l.Qty)*l.UnitCents, inv.Currency)})
+ }
+ return v
+}
+
+func formatAddress(a *Address) string {
+ s := a.Line1
+ if a.Line2 != "" {
+ s += ", " + a.Line2
+ }
+ return fmt.Sprintf("%s, %s %s, %s", s, a.PostalCode, a.City, a.Country)
+}
+
+func money(cents int64, currency string) string {
+ return fmt.Sprintf("%d.%02d %s", cents/100, cents%100, currency)
+}
diff --git a/cli/test/evals/testdata/repos/billing-api-go/internal/invoices/render_test.go b/cli/test/evals/testdata/repos/billing-api-go/internal/invoices/render_test.go
new file mode 100644
index 00000000..32f402b1
--- /dev/null
+++ b/cli/test/evals/testdata/repos/billing-api-go/internal/invoices/render_test.go
@@ -0,0 +1,16 @@
+package invoices
+
+import (
+ "testing"
+ "time"
+)
+
+func TestRenderWithAddress(t *testing.T) {
+ inv := &Invoice{Number: "INV-1042", IssuedAt: time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC), TotalCents: 4990, Currency: "EUR",
+ Customer: &Customer{Name: "Ada", BillingAddress: &Address{Line1: "Main St 1", City: "Berlin", PostalCode: "10115", Country: "DE"}},
+ Lines: []Line{{Description: "Plan", Qty: 1, UnitCents: 4990}}}
+ v := render(inv)
+ if v.BillingAddress != "Main St 1, 10115 Berlin, DE" || v.Total != "49.90 EUR" {
+ t.Fatalf("unexpected view %+v", v)
+ }
+}
diff --git a/cli/test/evals/testdata/repos/billing-api-go/internal/invoices/store.go b/cli/test/evals/testdata/repos/billing-api-go/internal/invoices/store.go
new file mode 100644
index 00000000..91375dab
--- /dev/null
+++ b/cli/test/evals/testdata/repos/billing-api-go/internal/invoices/store.go
@@ -0,0 +1,42 @@
+package invoices
+
+import (
+ "context"
+ "database/sql"
+)
+
+// Store loads invoices.
+type Store interface {
+ Invoice(ctx context.Context, id int64) (*Invoice, error)
+}
+
+// SQLStore loads invoices from PostgreSQL.
+type SQLStore struct{ db *sql.DB }
+
+// NewStore returns a Store backed by db.
+func NewStore(db *sql.DB) *SQLStore { return &SQLStore{db: db} }
+
+// Invoice loads one invoice with its customer and lines.
+func (s *SQLStore) Invoice(ctx context.Context, id int64) (*Invoice, error) {
+ inv := &Invoice{Customer: &Customer{}}
+ err := s.db.QueryRowContext(ctx, `SELECT i.id, i.number, i.issued_at, i.total_cents, i.currency, c.id, c.name, c.email
+ FROM invoices i JOIN customers c ON c.id = i.customer_id WHERE i.id = $1`, id).
+ Scan(&inv.ID, &inv.Number, &inv.IssuedAt, &inv.TotalCents, &inv.Currency, &inv.Customer.ID, &inv.Customer.Name, &inv.Customer.Email)
+ if err == sql.ErrNoRows {
+ return nil, nil
+ }
+ if err != nil {
+ return nil, err
+ }
+ var a Address
+ err = s.db.QueryRowContext(ctx, `SELECT line1, line2, city, postal_code, country FROM billing_addresses WHERE customer_id = $1`, inv.Customer.ID).
+ Scan(&a.Line1, &a.Line2, &a.City, &a.PostalCode, &a.Country)
+ switch {
+ case err == sql.ErrNoRows:
+ case err != nil:
+ return nil, err
+ default:
+ inv.Customer.BillingAddress = &a
+ }
+ return inv, nil
+}
diff --git a/cli/test/evals/testdata/sandbox-image/Dockerfile b/cli/test/evals/testdata/sandbox-image/Dockerfile
new file mode 100644
index 00000000..c89fbfca
--- /dev/null
+++ b/cli/test/evals/testdata/sandbox-image/Dockerfile
@@ -0,0 +1,19 @@
+# Sandbox for the setup suite's shell and for running hidden tests against a
+# model's patch. Toolchains only: no credentials ever enter this image.
+# NODE_VERSION follows the repo's .nvmrc (see scripts/check-node-pins.sh);
+# GO_VERSION follows cli/test/evals/go.mod.
+ARG NODE_VERSION=26
+ARG GO_VERSION=1.26
+FROM golang:${GO_VERSION}-bookworm AS go
+
+FROM node:${NODE_VERSION}-bookworm
+COPY --from=go /usr/local/go /usr/local/go
+ENV PATH=/usr/local/go/bin:$PATH
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends python3 python3-pip python3-venv curl git jq ca-certificates \
+ && rm -rf /var/lib/apt/lists/* \
+ && git config --system user.email "[email protected]" \
+ && git config --system user.name "Dana Developer" \
+ && git config --system --add safe.directory '*'
+ENV GOTOOLCHAIN=local GOFLAGS=-mod=mod PIP_BREAK_SYSTEM_PACKAGES=1 npm_config_update_notifier=false
+WORKDIR /work
diff --git a/cli/test/evals/testdata/setup-templates/hello-go/Dockerfile b/cli/test/evals/testdata/setup-templates/hello-go/Dockerfile
new file mode 100644
index 00000000..f25b13cd
--- /dev/null
+++ b/cli/test/evals/testdata/setup-templates/hello-go/Dockerfile
@@ -0,0 +1,9 @@
+FROM golang:1.26 AS build
+WORKDIR /src
+COPY . .
+RUN go build -o /hello-api .
+
+FROM gcr.io/distroless/base-debian12
+COPY --from=build /hello-api /hello-api
+ENV PORT=8080
+ENTRYPOINT ["/hello-api"]
diff --git a/cli/test/evals/testdata/setup-templates/hello-go/README.md b/cli/test/evals/testdata/setup-templates/hello-go/README.md
new file mode 100644
index 00000000..99ffc09c
--- /dev/null
+++ b/cli/test/evals/testdata/setup-templates/hello-go/README.md
@@ -0,0 +1,11 @@
+# hello-api
+
+A small greeting API used by the marketing site. Go, standard library only.
+
+```
+PORT=8080 go run .
+curl localhost:8080/api/hello
+curl localhost:8080/api/hello/ada
+```
+
+Deployed as a single container (see `Dockerfile`).
diff --git a/cli/test/evals/testdata/setup-templates/hello-go/go.mod b/cli/test/evals/testdata/setup-templates/hello-go/go.mod
new file mode 100644
index 00000000..2f43fe7f
--- /dev/null
+++ b/cli/test/evals/testdata/setup-templates/hello-go/go.mod
@@ -0,0 +1,3 @@
+module example.com/hello-api
+
+go 1.26
diff --git a/cli/test/evals/testdata/setup-templates/hello-go/handlers.go b/cli/test/evals/testdata/setup-templates/hello-go/handlers.go
new file mode 100644
index 00000000..fb10cf3f
--- /dev/null
+++ b/cli/test/evals/testdata/setup-templates/hello-go/handlers.go
@@ -0,0 +1,29 @@
+package main
+
+import (
+ "encoding/json"
+ "net/http"
+ "strings"
+)
+
+type greeting struct {
+ Message string `json:"message"`
+}
+
+func hello(w http.ResponseWriter, r *http.Request) {
+ writeJSON(w, greeting{Message: "Hello, world!"})
+}
+
+func helloName(w http.ResponseWriter, r *http.Request) {
+ name := strings.TrimSpace(r.PathValue("name"))
+ if name == "" {
+ http.Error(w, "name is required", http.StatusBadRequest)
+ return
+ }
+ writeJSON(w, greeting{Message: "Hello, " + name + "!"})
+}
+
+func writeJSON(w http.ResponseWriter, v any) {
+ w.Header().Set("Content-Type", "application/json")
+ _ = json.NewEncoder(w).Encode(v)
+}
diff --git a/cli/test/evals/testdata/setup-templates/hello-go/handlers_test.go b/cli/test/evals/testdata/setup-templates/hello-go/handlers_test.go
new file mode 100644
index 00000000..cd920bae
--- /dev/null
+++ b/cli/test/evals/testdata/setup-templates/hello-go/handlers_test.go
@@ -0,0 +1,18 @@
+package main
+
+import (
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+)
+
+func TestHelloName(t *testing.T) {
+ req := httptest.NewRequest(http.MethodGet, "/api/hello/ada", nil)
+ req.SetPathValue("name", "ada")
+ rec := httptest.NewRecorder()
+ helloName(rec, req)
+ if !strings.Contains(rec.Body.String(), "Hello, ada!") {
+ t.Fatal(rec.Body.String())
+ }
+}
diff --git a/cli/test/evals/testdata/setup-templates/hello-go/main.go b/cli/test/evals/testdata/setup-templates/hello-go/main.go
new file mode 100644
index 00000000..e25c5ae5
--- /dev/null
+++ b/cli/test/evals/testdata/setup-templates/hello-go/main.go
@@ -0,0 +1,23 @@
+package main
+
+import (
+ "log"
+ "net/http"
+ "os"
+)
+
+func main() {
+ mux := http.NewServeMux()
+ mux.HandleFunc("GET /api/hello", hello)
+ mux.HandleFunc("GET /api/hello/{name}", helloName)
+ addr := ":" + envOr("PORT", "8080")
+ log.Printf("listening on %s", addr)
+ log.Fatal(http.ListenAndServe(addr, mux))
+}
+
+func envOr(k, def string) string {
+ if v := os.Getenv(k); v != "" {
+ return v
+ }
+ return def
+}