The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
Cleanup
narrow-adventure72bf8890f893
diff --git a/.github/workflows/cli-evals.yml b/.github/workflows/cli-evals.ymlindex 8d9b4e3d..4ec98a5e 100644--- a/.github/workflows/cli-evals.yml+++ b/.github/workflows/cli-evals.yml@@ -7,8 +7,9 @@ name: CLI Evals# It is the one exception to the label-gated CI pattern: every run spends real# money on OpenRouter, so a person dispatches it, reads the preflight estimate# in the job summary, and approves the paid job at the `evals` environment-# gate (the only place OPENROUTER_API_KEY lives). `dry_run` runs everything-# except the paid job for $0. The workflow never commits; update baseline.json+# gate (the only place OPENROUTER_API_KEY lives; the preflight refuses a paid+# run unless that environment has a required reviewer). `dry_run` runs+# everything except the paid and report jobs for $0. The workflow never commits; update baseline.json# yourself with `twevals baseline update`. See cli/test/evals/README.md.on:workflow_dispatch:@@ -41,7 +42,7 @@ on:max_cost_usd:type: stringrequired: true- description: "hard cap for the whole dispatch in USD, no default"+ description: "cap for the whole dispatch in USD, split per shard, no default"dry_run:type: booleandefault: false@@ -64,6 +65,10 @@ jobs:preflight:name: Preflight ($0)runs-on: ubuntu-latest+ permissions:+ contents: read+ # Reads the evals environment's protection rules (the gate check below).+ actions: readoutputs:matrix: ${{ steps.plan.outputs.matrix }}defaults:@@ -91,6 +96,22 @@ jobs:echo "::error::runs must be 1, 2 or 3 (got '$RUNS')"; exit 1fi+ # GitHub creates a missing environment on first use with no protection+ # rules, so a job naming `environment: evals` alone is no gate. Refuse+ # the paid job unless the environment exists with a required reviewer.+ - name: Check the evals gate+ if: ${{ !inputs.dry_run }}+ env:+ GH_TOKEN: ${{ github.token }}+ run: |+ reviewers=$(gh api "repos/$GITHUB_REPOSITORY/environments/evals" \+ --jq '[.protection_rules[]? | select(.type == "required_reviewers") | (.reviewers // [])[]] | length') || {+ echo "::error::cannot read the evals environment; create it with a required reviewer before a paid run"; exit 1+ }+ if [ "${reviewers:-0}" -lt 1 ]; then+ echo "::error::the evals environment has no required reviewer; add one (Settings > Environments > evals) before a paid run"; exit 1+ fi+- name: Plan and estimateid: planenv:@@ -139,7 +160,7 @@ jobs:SUITES: ${{ inputs.suites }}run: |if [ -n "$CASES" ]; then- go run ./cmd/twevals oracle -j 3 $(echo "$CASES" | tr ',' ' ')+ go run ./cmd/twevals oracle -j 3 "$CASES"elif [ -n "$SUITES" ]; thengo run ./cmd/twevals oracle -j 3 --suites "$SUITES"else@@ -164,6 +185,8 @@ jobs:steps:- name: Checkoutuses: actions/checkout@v7+ with:+ persist-credentials: false- name: Setup Gouses: actions/setup-go@v7@@ -196,7 +219,7 @@ jobs:find results -type f \( -name '*.json' -o -name '*.jsonl' -o -name '*.md' -o -name '*.log' -o -name '.env*' -o -name '*.txt' \) -print0 \| xargs -0 -r sed -i -E \-e 's/sk-or-[A-Za-z0-9_-]+/sk-or-[REDACTED]/g' \- -e 's/(tww|twp|tws|twa|twk)_[A-Za-z0-9_-]+/\1_[REDACTED]/g' \+ -e 's/(tww|twp|twr|tws|twa|twk)_[A-Za-z0-9_-]+/\1_[REDACTED]/g' \-e 's/eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/[REDACTED_JWT]/g'# Backend databases, CLI credential stores and env files the agent# wrote hold the run's tokens; never upload them.diff --git a/.github/workflows/eval-guard.yml b/.github/workflows/eval-guard.ymlnew file mode 100644index 00000000..4d48b5b1--- /dev/null+++ b/.github/workflows/eval-guard.yml@@ -0,0 +1,48 @@+name: Eval Guard++# Enforces the manual-only rule of the skill and MCP evals on every change to a+# workflow, a composite action or the CLI justfile: the paid eval workflow keeps+# workflow_dispatch as its only trigger, nothing else dispatches it or uses its+# environment, and no check/test/default recipe runs it. cli/test/evals is its+# own Go module, so the CLI workflow's `go test ./...` never reaches these+# tests. Free: no model calls. Equivalent local command:+# cd cli/test/evals && go test ./internal/guard/+on:+ push:+ branches: [main]+ paths:+ - ".github/workflows/**"+ - ".github/actions/**"+ - "cli/justfile"+ - "cli/test/evals/**"+ pull_request:+ types: [labeled]+ paths:+ - ".github/workflows/**"+ - ".github/actions/**"+ - "cli/justfile"+ - "cli/test/evals/**"+ workflow_dispatch:++permissions:+ contents: read++jobs:+ guard:+ if: github.event_name != 'pull_request' || github.event.label.name == 'ci'+ runs-on: ubuntu-latest+ defaults:+ run:+ working-directory: cli/test/evals+ steps:+ - name: Checkout+ uses: actions/checkout@v7++ - name: Setup Go+ uses: actions/setup-go@v7+ with:+ go-version-file: cli/test/evals/go.mod+ cache-dependency-path: cli/test/evals/go.sum++ - name: Manual-only guard+ run: go test ./internal/guard/diff --git a/CLAUDE.md b/CLAUDE.mdindex 96f772f2..ec6c26d6 100644--- a/CLAUDE.md+++ b/CLAUDE.md@@ -37,7 +37,7 @@ Traceway is an error tracking and monitoring platform consisting of:| CLI | `cd cli && just eval-harness` | Eval harness tests plus every case's oracle trajectory, fake model only ($0) || CLI | `cd cli && just eval-plan --preset smoke --max-cost-usd 30` | Eval matrix and live-price cost estimate ($0) || CLI | `cd cli && just eval-dispatch -f preset=smoke -f max_cost_usd=30` | Starts a paid eval run in the CLI Evals workflow; it still waits for approval at the `evals` environment gate |-| CLI | `cd cli && just eval-local ...` | Paid local eval run, for harness development only (typed `yes` confirmation, required `--max-cost-usd`) |+| CLI | `cd cli && just eval-local ...` | Paid local eval run, for harness development only (`yes` typed on a terminal, refused without one; required `--max-cost-usd`) || Any | `nix develop` (or direnv via `.envrc`) | Dev shell with the Go and Node versions read from `backend/go.mod` and `.nvmrc`, plus `just`, `golangci-lint`, `govulncheck` (`flake.nix`; shells `default`, `backend`, `frontend`, `oxc`). Put `JWT_SECRET` in the gitignored `.envrc.local` || Any | `./scripts/check-node-pins.sh` | Asserts every Dockerfile's `ARG NODE_VERSION` default matches `.nvmrc`; run by `release-traceway.yml` before it builds |@@ -45,7 +45,7 @@ Traceway is an error tracking and monitoring platform consisting of:Set `JWT_SECRET` (min 32 characters) before running the backend, including the All-in-One image. Former public signing keys are rejected. The All-in-One shell script leaves `.env` loading to the backend so Docker-provided values win. Embedded `Run(opts...)` never reads `JWT_SECRET`, since it runs inside the host app's process where that name usually belongs to the host: it takes `WithJWTSecret`, else `TRACEWAY_JWT_SECRET`, else a random key per start; set a stable secret to preserve dashboard sessions across embedded restarts. The All-in-One image's supervisord sends the backend's output to the container's stdout/stderr, so `docker logs` shows a startup rejection. `SQLITE_PATH` sets the database location, defaulting to `./traceway.db` in the working directory.-CI on pull requests is opt-in. `backend.yml`, `backend-vulncheck.yml`, `cli.yml`, `cli-lint.yml`, `cli-contract.yml`, `frontend.yml` and `plugin-scanner.yml` listen for `pull_request: types: [labeled]` and every job is gated on `github.event.label.name == 'ci'`, so a PR runs nothing until a maintainer applies the `ci` label (only collaborators can), and applying any other label runs nothing either. Each application validates the PR as it is at that moment: to validate later pushes, remove and re-add the label. The same workflows still run on pushes to `main` that touch their paths (`backend/**`, `cli/**`, `frontend/**`, `skills/traceway/**` for `cli.yml`, `.nvmrc` for `frontend.yml`, their own workflow file; `plugin-scanner.yml` runs on every push to `main` because it scans the whole repo), on their daily schedules (the two vulnchecks), and from the Actions tab via `workflow_dispatch`. `cli-evals.yml` (the skill and MCP evals) is the one exception to this pattern: its **only** trigger is `workflow_dispatch`, a person starts every run by hand, the paid shard job waits for a required reviewer on the `evals` environment (the only place `OPENROUTER_API_KEY` lives), and `max_cost_usd` is a required input with no default. It must never get a push, pull request, label, schedule, `workflow_run`, `workflow_call` or `repository_dispatch` trigger, no other workflow may call it, and no `just check`/`test`/`default` recipe may run it; `cli/test/evals/internal/guard/triggers_test.go` fails the build if any of that changes, and `twevals run` itself refuses every CI context but a dispatched, approved `cli-evals.yml` job. Validate edits to it (or Dependabot bumps of its actions) with `dry_run: true`, which costs nothing.+CI on pull requests is opt-in. `backend.yml`, `backend-vulncheck.yml`, `cli.yml`, `cli-lint.yml`, `cli-contract.yml`, `eval-guard.yml`, `frontend.yml` and `plugin-scanner.yml` listen for `pull_request: types: [labeled]` and every job is gated on `github.event.label.name == 'ci'`, so a PR runs nothing until a maintainer applies the `ci` label (only collaborators can), and applying any other label runs nothing either. Each application validates the PR as it is at that moment: to validate later pushes, remove and re-add the label. The same workflows still run on pushes to `main` that touch their paths (`backend/**`, `cli/**`, `frontend/**`, `skills/traceway/**` for `cli.yml`, `.nvmrc` for `frontend.yml`, `.github/workflows/**`, `.github/actions/**`, `cli/justfile` and `cli/test/evals/**` for `eval-guard.yml`, their own workflow file; `plugin-scanner.yml` runs on every push to `main` because it scans the whole repo), on their daily schedules (the two vulnchecks), and from the Actions tab via `workflow_dispatch`. `cli-evals.yml` (the skill and MCP evals) is the one exception to this pattern: its **only** trigger is `workflow_dispatch`, a person starts every run by hand, the paid shard job waits for a required reviewer on the `evals` environment (the only place `OPENROUTER_API_KEY` lives, never a repository or organization secret; GitHub creates a missing environment without protection, so the preflight refuses a paid run unless `evals` exists with a required reviewer), and `max_cost_usd` is a required input with no default. It must never get a push, pull request, label, schedule, `workflow_run`, `workflow_call` or `repository_dispatch` trigger, no other workflow or composite action may call, dispatch (by file, name or `just eval-dispatch`) or use its environment, and no `just check`/`test`/`default` recipe may run it; `cli/test/evals/internal/guard/triggers_test.go` fails if any of that changes. `cli/test/evals` is its own Go module, which `cli.yml`'s `go test ./...` never reaches, so `eval-guard.yml` runs that test. `twevals run` itself refuses every CI context but a dispatched, approved `cli-evals.yml` job, and locally it needs `yes` typed on a terminal. Validate edits to it (or Dependabot bumps of its actions) with `dry_run: true`, which costs nothing but skips the paid and report jobs, so it does not exercise `download-artifact` or the shard matrix.Action versions are watched by `.github/dependabot.yml` (the `github-actions` ecosystem only; npm and Go modules are deliberately not watched there). It opens a weekly PR per group — `actions/*` in one, every third-party action in the other — and deliberately does **not** auto-apply `ci`, so a bot PR passes the same collaborator gate as any other. Label the first-party group to validate it; `golangci/golangci-lint-action` is reachable the same way via `cli-lint.yml`. The rest of the third-party group lands on `release-*.yml`, `benchmark-*.yml` and `traceway-autofix.yml`, which no label reaches — and `workflow_dispatch` is **not** a dry run on the release path: dispatching `release-docs.yml` or `release-website.yml` deploys to Cloudflare for real, and the other `release-*.yml` workflows publish real assets, so dispatching one from an unreviewed bot branch ships it. Validate those by reading the action's release notes instead, and dispatch only where the target is disposable. `traceway-autofix.yml` is the in-repo caller of the reusable `autofix.yml` (composite actions `.github/actions/autofix-prepare` and `autofix-publish` around the agent step; `docs/pages/learn/auto-fix.mdx` documents the contract); its `workflow_dispatch` takes an `issue_number` and runs the whole loop from the dispatched branch, which is the one pre-merge validation path for the `claude-code-action` pin — the most privileged action here — but it runs the agent for real, so dispatch it only at a scratch issue. Note too that labelling a PR which touches no gated workflow yields zero checks, which reads like a pass but is silence. Every updatable pin is a bare major tag, so Dependabot only ever proposes major bumps here. The exception is `plugin-scanner.yml`, which pins both actions to commit SHAs because the HOL scanner scores unpinned actions down; Dependabot bumps those SHAs along with their version comments. It uses the scanner config in `.plugin-scanner.toml`, whose `ignore_paths` cover fixtures, demos and build output plus three reviewed production files (the read-only token mask in both `project.repository.go` copies and `KindPassword` in `cli/internal/state/state.go`). The HOL registry's own scan ignores that file and still fails on syntax-highlighter strings in the committed `backend/static/frontend` bundle; that is accepted.diff --git a/cli/justfile b/cli/justfileindex 95eeecde..7bda88b4 100644--- a/cli/justfile+++ b/cli/justfile@@ -39,7 +39,8 @@ eval-plan *args:eval-dispatch *args:gh workflow run cli-evals.yml {{args}}-# Paid, local, for harness development only: asks you to type yes.+# Paid, local, for harness development only: asks you to type yes on a+# terminal, and refuses to run without one.eval-local *args:cd test/evals && go run ./cmd/twevals run {{args}}diff --git a/cli/test/evals/README.md b/cli/test/evals/README.mdindex 52bb7c2a..3fdc8947 100644--- a/cli/test/evals/README.md+++ b/cli/test/evals/README.md@@ -28,26 +28,42 @@ harness, with the same tools, prompts and budgets (`models.yaml`).Run workflow (or `gh workflow run cli-evals.yml`). The free preflight jobwrites the matrix and a live-price estimate into the job summary. Thepaid job waits for a required reviewer on the `evals` environment, which- is the only place `OPENROUTER_API_KEY` exists.+ is the only place `OPENROUTER_API_KEY` exists. Store the key there only,+ never as a repository or organization secret: GitHub creates a missing+ environment on first use with no reviewer, so the preflight refuses a paid+ run unless `evals` exists with a required reviewer.2. **Nothing runs automatically.** `cli-evals.yml` has exactly one trigger,`workflow_dispatch`. There is no push, pull request, label, schedule,`workflow_run`, `workflow_call` or `repository_dispatch` trigger. No otherworkflow calls it, and no `just` recipe in `check`/`test`/`default` runs it.3. **Enforced in code:**- `internal/guard/triggers_test.go` fails if the workflow gains a trigger, a- paid job loses `environment: evals`, or another workflow references it or- starts `twevals run`.+ paid job loses `environment: evals`, the approval flag moves out of that+ job, another workflow or composite action references it (by file, by+ name, through `just eval-dispatch` or the key's name), uses the `evals`+ environment or starts `twevals run`, or a recipe reachable from `just+ check`/`test`/`default` runs or dispatches the evals. Because this module+ is its own Go module, the CLI workflow never runs it; `eval-guard.yml`+ does, on every change to a workflow, an action, the justfile or this+ module (label-gated like the other workflows).- `twevals run` and `twevals resume` start in CI only inside a dispatched`cli-evals.yml` job that set `TWEVALS_APPROVED=1` (the gated job does).They refuse any other GitHub event, any other workflow, and any other CIsystem.- - `--max-cost-usd` is required, with no default. The workflow splits it- into hard per-shard caps.+ - `--max-cost-usd` is required, with no default, and must be a finite+ number above 0. The workflow splits it into per-shard caps. Each run+ reserves its share when it starts, so the runs in flight are never+ promised more than is left, and a run stops before a model call that+ would likely cross its share. The cap can still be passed by the error in+ that estimate (about one turn's growth per run in flight) and the judge's+ few cents.- `go test ./...` can never call OpenRouter: the real client requires`guard.Live()`, which is false under `go test`.-4. **Local paid runs are for harness development only.** On a terminal you- must type `yes` after reading the preflight. Without a terminal, `--yes` is- required. Recorded results (checkpoints, baselines) come from the workflow.+4. **Local paid runs are for harness development only.** You must type `yes`+ on a terminal after reading the preflight. Without a terminal (a script, or+ an agent's shell) a local paid run is refused, and `--yes` does not stand in+ for the typed confirmation; it only serves the dispatched workflow. Recorded+ results (checkpoints, baselines) come from the workflow.## Running it@@ -68,7 +84,7 @@ the `evals` gate. `just eval-dispatch -f preset=smoke -f max_cost_usd=30` from| `suites`, `arms` | Narrow the selection || `models` | `opus,astra`, `opus` or `astra` || `runs` | Repeats per case, 1-3 |-| `max_cost_usd` | Hard cap for the whole dispatch. Required |+| `max_cost_usd` | Cap for the whole dispatch, split per shard (see the policy above). Required || `dry_run` | Preflight, harness tests and every oracle only. Costs $0. Use it to validate workflow edits and Dependabot bumps || `judge` | Score reports with the (non-gating) LLM judge || `compare_baseline` | Compare against `baseline.json`; a regression turns the report job red |@@ -85,22 +101,28 @@ From `cli/` (each recipe runs only when you type it):| `just eval-harness` | `go test ./...` plus every oracle trajectory, with the fake model. Free || `just eval-plan --preset core --max-cost-usd 200` | The preflight table and cost estimate. Free || `just eval-env perf-db-n-plus-1` | Boots and seeds one case, then prints the dashboard URL, a login and a read PAT so you can look at the data by eye |-| `just eval-local --cases perf-db-n-plus-1 --models opus --max-cost-usd 5` | A paid local run, for harness development only. It asks you to type `yes` |+| `just eval-local --cases perf-db-n-plus-1 --models opus --max-cost-usd 5` | A paid local run, for harness development only. It asks you to type `yes` on a terminal || `just eval-dispatch ...` | `gh workflow run cli-evals.yml ...` |Directly from `cli/test/evals`, `go run ./cmd/twevals help` lists everycommand. `twevals oracle <id>...` (or `--all`, or `--suites S`) plays a case'soracle on every arm it lists and writes `results.json` and `summary.md` to`--out` (default `.cache/oracle/<timestamp>/`).-`twevals report <dir>...` merges shard results. `twevals compare <dir>`+`twevals report <dir>... --out DIR` merges shard results (`--out` is+required for more than one directory, and a run that appears twice is an+error). `twevals compare <dir>`applies the regression rule. `twevals baseline update <dir>` rewrites`baseline.json` for you to commit.Requirements: Go (the `go`/`toolchain` lines of `go.mod`), git, and Docker forthe setup suite. Docker also runs the hidden tests of a model's patch with no-network. Without Docker, setup runs of a real model fail, and hidden tests of-a real model's patch are skipped unless `TWEVALS_HOST_TESTS=1`. Oracle runs-(our own code) use the host when Docker is missing.+network and an empty module cache, so a fixture with hidden tests must not+need module dependencies. Without Docker, setup runs of a real model fail, and hidden tests of+a real model's patch are skipped unless `TWEVALS_HOST_TESTS=1` (the diff+checks, such as touching the culprit, still gate). Oracle runs (our own code)+use the host when Docker is missing. The setup oracles fetch their pinned+OpenTelemetry modules, so `oracle --all` needs the Go module proxy or a warm+module cache.### Exit codes@@ -121,8 +143,8 @@ averages: 100-400k cumulative input tokens (about 70% cached), 5-15k output.`twevals plan` prints the real numbers from live OpenRouter prices, includingthe cache-write price and long-prompt tiers. Controls: the required cap, the-preflight, the gate, per-shard caps, a check before every run, `dry_run`, and-the record/replay cache (`--cache record|replay`, `EVAL_CACHE`).+preflight, the gate, per-shard caps, a reservation per run and a check before+every model call, and `dry_run`.## Reading the report@@ -136,8 +158,11 @@ the record/replay cache (`--cache record|replay`, `EVAL_CACHE`).- the top failing graders and the worst cases;- one line per run, with its first failure.-A run is `pass`, `fail`, `budget_exhausted` (counted as a fail),-`infra_error` or `cost_ceiling`; the last two are excluded from pass rates.+A run is `pass`, `fail`, `budget_exhausted` (the case's own turn, call,+token or time budget ran out; counted as a fail), `infra_error` or+`cost_ceiling` (not started, or stopped at its share of `--max-cost-usd`);+the last two are excluded from pass rates, and `twevals resume` runs them+again.Each run directory, `runs/<case>/<arm>/<model>/r<N>/` (with `+` in the armwritten as `_`, so `cli_skill`), holds:@@ -156,7 +181,7 @@ Graders:| `trajectory` | yes | Required, alternative, forbidden and ordered operations, and thrash limits. CLI and MCP calls map to the same operations (`internal/ops`) || `grounding` | yes | Cited calls exist, succeeded and were not truncated. Their output contains the deciding signal. No id in the answer was invented || `state` | yes | After write and setup runs: the backend and the workspace are in the expected state |-| `hidden_tests` | per case | Hidden tests copied in after the run, and whether the diff touches the culprit |+| `hidden_tests` | per case | Hidden tests copied in after the run, and whether the diff (against the workspace before the run) touches the culprit || `setup_verify` | yes | The instrumented app builds, starts against the backend, and serves traffic || `judge` | no | The rubric, scored by Haiku. It does not gate until it agrees with human labels on at least 85% of runs |@@ -169,8 +194,10 @@ envelope. To update it, download the `eval-report` artifact, runin a normal PR.The regression rule: a suite's pass rate drops more than 10 percentage points-for a model × arm, or a case at 2 of 3 or better in the baseline now passes 0-of N.+for a model × arm, or a case that passed at least 2 runs, at 2 of 3 or better,+in the baseline now passes 0 of N. With one run per case the suite rule is the+only one that can fire, and a suite of few cases moves in large steps (one+failure of 4 runs is 25 points).## Adding a case@@ -316,7 +343,9 @@ See `perf-db-n-plus-1.yaml` for the full shape. The keys:in `internal/grade/state.go`), `expect.code`, `expect.report`.- `judge.rubric` and `judge.expected`. Both are shown only to the judge.- `hooks`: harness actions after a matching call, such as editing a- post-mortem behind the agent's back.+ post-mortem behind the agent's back (`edit_post_mortem`, `stub_healthy`,+ `stub_unhealthy`, `sleep`, `approve_setup_draft`). A hook that fails makes+ the run an `infra_error`, since the case did not happen as written.- `setup`: the env given to the agent's container shell (`@token:<project>`,`@url`, `@port`), and the verify step.@@ -375,7 +404,7 @@ internal/checklist/ the bottleneck rows of performance.md, the perf answer eninternal/env/ backend subprocess, registration, members, PATs, CLI logins, the monitor stubinternal/seed/ the DSL, OTLP and /api/report writers, dashboard objects, read-back verificationinternal/scenarios/ one seed function per case-internal/agent/ OpenRouter client, the loop, prompt caching, transcripts, record/replay, the fake model+internal/agent/ OpenRouter client, the loop, the cost ledger, prompt caching, transcripts, the fake modelinternal/tools/ the sandboxed shell (mvdan.cc/sh, in-process coreutils, gojq, goawk), files, skills, MCP bridge, ask_user, submitinternal/ops/ CLI/MCP → operation mapping, cross-checked against tool-map.mdinternal/grade/ answer, trajectory, grounding, state, hidden tests, judge, leakagediff --git a/cli/test/evals/cases/debug/debug-android-npe.yaml b/cli/test/evals/cases/debug/debug-android-npe.yamlindex 3bfb8b59..e776baf9 100644--- a/cli/test/evals/cases/debug/debug-android-npe.yaml+++ b/cli/test/evals/cases/debug/debug-android-npe.yaml@@ -34,6 +34,7 @@ expect:max_identical_repeats: 2code:static: true+ gating: truetouches_culprit: truemust_change: [app/src/main/java/com/example/pinboard/feed/FeedAdapter.kt]must_not_change: [app/src/main/java/com/example/pinboard/media/FullImageActivity.kt]diff --git a/cli/test/evals/cases/debug/debug-flutter-null-check.yaml b/cli/test/evals/cases/debug/debug-flutter-null-check.yamlindex 6b345cef..b301f665 100644--- a/cli/test/evals/cases/debug/debug-flutter-null-check.yaml+++ b/cli/test/evals/cases/debug/debug-flutter-null-check.yaml@@ -36,6 +36,7 @@ expect:max_identical_repeats: 2code:static: true+ gating: truetouches_culprit: truemust_change: [lib/features/product/widgets/product_hero_carousel.dart]must_not_change: [lib/core/api_client.dart]diff --git a/cli/test/evals/cases/debug/debug-go-nil-regression.yaml b/cli/test/evals/cases/debug/debug-go-nil-regression.yamlindex 4ea247c4..157295fc 100644--- a/cli/test/evals/cases/debug/debug-go-nil-regression.yaml+++ b/cli/test/evals/cases/debug/debug-go-nil-regression.yaml@@ -25,9 +25,6 @@ expect:calls:required:- {op: exceptions.show, args: {hash: "{{seed.hash}}"}}- any_of:- - [{op: "read_file:internal/invoices/render.go"}]- - [{op: "search"}]forbidden:- {op: "*", write: true}order:diff --git a/cli/test/evals/cases/perf/perf-db-pool-exhaustion.yaml b/cli/test/evals/cases/perf/perf-db-pool-exhaustion.yamlindex 1b840fc1..cc16bbb3 100644--- a/cli/test/evals/cases/perf/perf-db-pool-exhaustion.yaml+++ b/cli/test/evals/cases/perf/perf-db-pool-exhaustion.yaml@@ -51,7 +51,7 @@ oracle:traceway endpoints chart --metric-type p99 --since 2d --interval-minutes 60 --output json| jq '[.series[] | select(.endpoint == "POST /api/checkout" or .endpoint == "GET /api/cart") | {endpoint, t: .timestamp, p99_ms: (.value | round)}]'- shell: >-- traceway logs query --since 24h --min-severity 13 --page-size 8 --output json+ traceway logs query --since 3d --min-severity 13 --page-size 8 --output json| jq '{data: [.data[] | {timestamp, severityText, traceId, body, exception: .logAttributes["exception.message"]}]}'- shell: >-traceway exceptions list --since 3d --output jsondiff --git a/cli/test/evals/cases/setup/setup-go-nethttp.yaml b/cli/test/evals/cases/setup/setup-go-nethttp.yamlindex e243db35..91fe8cdf 100644--- a/cli/test/evals/cases/setup/setup-go-nethttp.yaml+++ b/cli/test/evals/cases/setup/setup-go-nethttp.yaml@@ -29,9 +29,10 @@ expect:path: go.modcontains: ["go.opentelemetry.io/otel"]- kind: file- path: "*.go"+ path: "**/*.go"contains: ["TRACEWAY_PROJECT_TOKEN|OTEL_EXPORTER_OTLP_HEADERS", "service\\.name|ServiceName|OTEL_SERVICE_NAME"]- kind: no_token_literal+ - kind: no_token_printed- kind: endpoint_seenendpoint: "GET /api/hello"- kind: endpoint_seen@@ -53,7 +54,7 @@ oracle:- read_file: go.mod- write_file: {path: telemetry.go, from: oracle/setup-go-nethttp/telemetry.go}- write_file: {path: main.go, from: oracle/setup-go-nethttp/main.go}- - shell: go get go.opentelemetry.io/otel go.opentelemetry.io/otel/sdk go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp && go mod tidy && go vet ./... && go test ./...+ - shell: go get go.opentelemetry.io/[email protected] go.opentelemetry.io/otel/[email protected] go.opentelemetry.io/otel/exporters/otlp/otlptrace/[email protected] go.opentelemetry.io/contrib/instrumentation/net/http/[email protected] && go mod tidy && go vet ./... && go test ./...- finish:summary: |hello-api now exports traces to Traceway over OTLP/HTTP: telemetry.go sets up the SDK with an exporter to $TRACEWAY_URL/api/otel/v1/traces and the bearer token from $TRACEWAY_PROJECT_TOKEN (nothing is hard-coded), service.name hello-api, and the W3C propagator. Each route handler is wrapped with otelhttp so Traceway records the route pattern (GET /api/hello/{name}) instead of one row per URL. The app builds and its tests pass.diff --git a/cli/test/evals/cases/write/write-alert-snooze.yaml b/cli/test/evals/cases/write/write-alert-snooze.yamlindex 24149999..34af92b4 100644--- a/cli/test/evals/cases/write/write-alert-snooze.yaml+++ b/cli/test/evals/cases/write/write-alert-snooze.yaml@@ -28,8 +28,13 @@ expect:count: 1calls:required:- - {op: alerts.list}- {op: alerts.snooze, args: {id: "{{seed.ruleId}}"}}+ # Finding the rule: the rule list, or the page it keeps opening (a page+ # carries its rule's id and name).+ any_of:+ - [{op: alerts.list}]+ - [{op: pages.list}]+ - [{op: pages.show}]forbidden:- {op: alerts.disable}- {op: alerts.enable}@@ -37,15 +42,13 @@ expect:- {op: pages.resolve}- {op: exceptions.archive}- {op: alerts.snooze, args: {id: "{{seed.otherRuleId}}"}}- order:- - [alerts.list, alerts.snooze]max_failed_calls: 3report:contains:- "(snooz|mute|silenc)"- "(\\d{1,2}:\\d{2}|60 ?min|an hour|1 hour|one hour)"judge:- expected: The agent finds the checkout error-rate rule with alerts list, snoozes it for about 60 minutes (not disable), leaves the p95 rule and the open page alone, and reports when the snooze ends and how to undo it.+ expected: The agent finds the checkout error-rate rule (in the alert rules, or from the page it keeps opening), snoozes it for about 60 minutes (not disable), leaves the p95 rule and the open page alone, and reports when the snooze ends and how to undo it.rubric:- Snoozes (rather than disables) the error-rate rule, for about an hour.- Leaves the other rule and the page unchanged.diff --git a/cli/test/evals/cmd/twevals/commands_ops.go b/cli/test/evals/cmd/twevals/commands_ops.goindex 7fe73dd2..31c944ee 100644--- a/cli/test/evals/cmd/twevals/commands_ops.go+++ b/cli/test/evals/cmd/twevals/commands_ops.go@@ -21,12 +21,16 @@ import ("github.com/tracewayapp/traceway/cli/test/evals/internal/seed")-func mergeResume(prev, now []*runner.Result) []*runner.Result {+func mergeResume(prev, now []*runner.Result) ([]*runner.Result, float64) {byKey := map[string]*runner.Result{}for _, r := range prev {byKey[r.Key()] = r}+ replaced := 0.0for _, r := range now {+ if old, ok := byKey[r.Key()]; ok {+ replaced += old.Metrics.CostUSD + old.Metrics.JudgeCostUSD+ }byKey[r.Key()] = r}out := make([]*runner.Result, 0, len(byKey))@@ -34,7 +38,7 @@ func mergeResume(prev, now []*runner.Result) []*runner.Result {out = append(out, r)}sort.Slice(out, func(i, j int) bool { return out[i].Key() < out[j].Key() })- return out+ return out, replaced}// sandboxImage builds the container image when the selection needs it (the@@ -104,7 +108,7 @@ func cmdEnvUp(ctx context.Context, args []string) error {exposed, _ := json.MarshalIndent(sc.Exposed(), "", " ")pat, _ := env.CreatePAT(ctx, inst.Backend, inst.Admin.JWT, "env up", "read", 0)fmt.Printf("Case %s is seeded (%+v).\n\nDashboard: %s\nLogin: %s / %s\nRead PAT: %s\n %s login --url %s --token %s\n\nseed.json:\n%s\n\nCtrl-C to stop. Data dir: %s\n",- c.ID, sc.Stats(), inst.Backend.URL, env.AdminEmail, env.AdminPassword, pat, bins.CLI, inst.Backend.URL, pat, exposed, dir)+ c.ID, sc.Stats(), inst.Backend.URL, inst.Admin.Email, inst.Admin.Password, pat, bins.CLI, inst.Backend.URL, pat, exposed, dir)<-ctx.Done()return nil}@@ -136,7 +140,17 @@ func cmdReport(args []string) error {}var res *report.Resultsif *merge || len(parts) > 1 {- res = report.Merge(parts)+ if *out == "" {+ return &exitError{2, "merging several run directories needs --out (writing into the first one would overwrite that shard's results)"}+ }+ merged, notes, err := report.Merge(parts)+ if err != nil {+ return &exitError{2, err.Error()}+ }+ for _, n := range notes {+ fmt.Fprintln(os.Stderr, "note:", n)+ }+ res = merged} else {res = parts[0]}diff --git a/cli/test/evals/cmd/twevals/main.go b/cli/test/evals/cmd/twevals/main.goindex b145163c..eec95d66 100644--- a/cli/test/evals/cmd/twevals/main.go+++ b/cli/test/evals/cmd/twevals/main.go@@ -3,7 +3,7 @@// Manual only. Paid runs happen in the CLI Evals workflow, which a person// starts by hand (workflow_dispatch) and approves at the evals environment// gate. `twevals run` refuses every other CI context, and locally it needs a-// typed confirmation. `twevals oracle` and `twevals plan` are free.+// confirmation typed on a terminal. `twevals oracle` and `twevals plan` are free.package mainimport (@@ -21,14 +21,14 @@ Free:twevals list [--preset P] [--suites S] [--tags T] [--cases GLOB] [--arms A]twevals plan [selection] --models opus,astra [--runs N] --max-cost-usd X [--shards] [--out plan.json] [--summary FILE]twevals oracle [--all | <case-id>... | --suites S] [--arms A] [-j N] [--keep] [--out DIR]- twevals env up <case-id> [--arm A]+ twevals env up <case-id>twevals report <run-dir>... [--merge] [--out DIR]twevals compare <run-dir> [baseline.json]twevals baseline update <run-dir> [--out baseline.json]Paid (OpenRouter; guarded, see README):- twevals run [selection] [--shard SUITE] --models opus,astra --max-cost-usd X [--runs N] [-j N] [--keep] [--cache record|replay|off] [--judge=true] [--yes]- twevals resume <run-dir> --max-cost-usd X [--yes]+ twevals run [selection] [--shard SUITE] --models opus,astra --max-cost-usd X [--runs N] [-j N] [--keep] [--judge=true]+ twevals resume <run-dir> --max-cost-usd XSelection flags: --preset smoke|core|full, --cases (comma-separated ids or globs; overrides the preset), --suites, --tags, --arms.list defaults to --preset full, plan and run to --preset smoke. oracle writes results.json and summary.md to --outdiff --git a/cli/test/evals/cmd/twevals/run.go b/cli/test/evals/cmd/twevals/run.goindex 78c72bf4..87735131 100644--- a/cli/test/evals/cmd/twevals/run.go+++ b/cli/test/evals/cmd/twevals/run.go@@ -160,8 +160,8 @@ func cmdPlan(ctx context.Context, args []string) error {summary := fs.String("summary", "", "append the preflight table to this file (e.g. $GITHUB_STEP_SUMMARY)")prices := fs.String("prices", "", "read prices from a saved /api/v1/models response instead of fetching")_ = fs.Parse(args)- if *maxCost <= 0 {- return &exitError{2, "--max-cost-usd is required and must be greater than 0 (there is no default)"}+ if err := guard.CheckCap(*maxCost); err != nil {+ return &exitError{2, err.Error()}}ws, err := load()if err != nil {@@ -207,7 +207,6 @@ type runConfig struct {Models string `json:"models"`Runs int `json:"runs"`Judge bool `json:"judge"`- Cache string `json:"cache"`Jobs int `json:"jobs"`}@@ -221,9 +220,8 @@ func cmdRun(ctx context.Context, args []string, resumeDir string) error {maxCost := fs.Float64("max-cost-usd", 0, "hard cap for this invocation (required, no default)")jobs := fs.Int("j", 4, "parallel runs")keep := fs.Bool("keep", false, "keep every run directory (backend db, workspace)")- cache := fs.String("cache", envOr("EVAL_CACHE", agent.CacheOff), "model response cache: record | replay | off")judge := fs.Bool("judge", true, "score reports with the LLM judge (non-gating)")- yes := fs.Bool("yes", false, "proceed without a terminal (CI); a local terminal still asks")+ yes := fs.Bool("yes", false, "the dispatched workflow's confirmation; a local run always asks you to type yes")out := fs.String("out", "", "results directory (default results/<run-id>)")_ = fs.Parse(args)@@ -231,7 +229,7 @@ func cmdRun(ctx context.Context, args []string, resumeDir string) error {if err != nil {return err}- cfg := runConfig{Selection: sf.selection(), Models: *models, Runs: *runs, Judge: *judge, Cache: *cache, Jobs: *jobs}+ cfg := runConfig{Selection: sf.selection(), Models: *models, Runs: *runs, Judge: *judge, Jobs: *jobs}cfg.Selection.Shard = *shardoutDir := *outskip := map[string]bool{}@@ -267,9 +265,6 @@ func cmdRun(ctx context.Context, args []string, resumeDir string) error {return &exitError{2, err.Error()}}var chatter agent.Chatter = client- if cfg.Cache != agent.CacheOff {- chatter = &agent.CachedChatter{Inner: client, Dir: filepath.Join(ws.root, ".cache", "models"), Mode: cfg.Cache}- }runID := time.Now().UTC().Format("20060102-150405")if outDir == "" {outDir = filepath.Join(ws.root, "results", runID)@@ -289,11 +284,16 @@ func cmdRun(ctx context.Context, args []string, resumeDir string) error {if err != nil {return err}+ priced := chosen+ if cfg.Judge && ws.models.Judge != nil {+ priced = append(slices.Clone(chosen), ws.models.Judge)+ }+ prices := pricesByKey(ctx, ws, priced)opt := runner.Options{EvalsRoot: ws.root, RepoRoot: ws.repo, OutDir: outDir, Bins: bins, Models: chosen, Runs: cfg.Runs, Jobs: cfg.Jobs,Keep: *keep, Chat: func(*catalog.Model) agent.Chatter { return chatter }, Ledger: ledger, Log: os.Stderr, Skip: skip,- Prices: pricesByKey(ctx, ws, chosen), AllowHostTests: os.Getenv("TWEVALS_HOST_TESTS") == "1"}+ Prices: prices, AllowHostTests: os.Getenv("TWEVALS_HOST_TESTS") == "1"}if cfg.Judge {- opt.Judge = &grade.Judge{Chat: chatter, Model: ws.models.Judge, Ledger: ledger}+ opt.Judge = &grade.Judge{Chat: chatter, Model: ws.models.Judge, Ledger: ledger, Price: prices[ws.models.Judge.Key]}}opt.SandboxImage = sandboxImage(ctx, ws, units)header := report.NewHeader(ws.repo, runID, ws.models, bins.Hash, os.Args)@@ -301,7 +301,15 @@ func cmdRun(ctx context.Context, args []string, resumeDir string) error {results := runner.Execute(ctx, opt, runner.Jobs(units, chosen, cfg.Runs))if resumeDir != "" {if prev, err := report.Load(resumeDir); err == nil {- results = mergeResume(prev.Runs, results)+ // The caps and the spend of every invocation add up: the runs a+ // resume ran again keep what they cost the first time.+ var replaced float64+ results, replaced = mergeResume(prev.Runs, results)+ header.MaxCostUSD += prev.Header.MaxCostUSD+ header.ReplacedSpendUSD = prev.Header.ReplacedSpendUSD + replaced+ if !prev.Header.Started.IsZero() {+ header.Started = prev.Header.Started+ }}}header.Finished = time.Now().UTC()@@ -324,7 +332,7 @@ func partialExit(results []*runner.Result) error {}}if ceiling > 0 {- return &exitError{2, fmt.Sprintf("partial: %d runs not started at the cost ceiling (continue with twevals resume)", ceiling)}+ return &exitError{2, fmt.Sprintf("partial: %d runs were stopped or not started at the cost ceiling (continue with twevals resume)", ceiling)}}if len(results) > 0 && float64(infra)/float64(len(results)) > 0.10 {return &exitError{2, fmt.Sprintf("partial: %d of %d runs hit infra errors", infra, len(results))}@@ -348,10 +356,3 @@ func pricesByKey(ctx context.Context, ws *workspace, models []*catalog.Model) ma}return out}--func envOr(k, def string) string {- if v := os.Getenv(k); v != "" {- return v- }- return def-}diff --git a/cli/test/evals/internal/agent/agent_test.go b/cli/test/evals/internal/agent/agent_test.goindex 20af7d63..25c72eae 100644--- a/cli/test/evals/internal/agent/agent_test.go+++ b/cli/test/evals/internal/agent/agent_test.go@@ -144,10 +144,15 @@ func TestLoopBudgets(t *testing.T) {}c := cfg(&scripted{resps: []*Response{toolResp("echo", `{}`, 2), toolResp("echo", `{}`, 2)}})- c.RunCap = 1- if run := Run(context.Background(), c); run.StopReason != StopBudget || !strings.Contains(run.StopDetail, "cost cap") {+ ledger := NewLedger(10)+ c.Hold = ledger.Reserve(3)+ run = Run(context.Background(), c)+ if run.StopReason != StopCeiling || !strings.Contains(run.StopDetail, "cost cap") {t.Fatalf("run cap: %s %s", run.StopReason, run.StopDetail)}+ if run.Turns != 1 || ledger.Spent() != 2 {+ t.Fatalf("the run must stop before a second $2 call would cross its $3 cap: %d turns, $%v", run.Turns, ledger.Spent())+ }c = cfg(&scripted{resps: []*Response{toolResp("echo", `{}`, 2)}})c.Ledger = NewLedger(1)c.Ledger.Add("other", 1)diff --git a/cli/test/evals/internal/agent/budget.go b/cli/test/evals/internal/agent/budget.goindex c8019d98..a95e3b48 100644--- a/cli/test/evals/internal/agent/budget.go+++ b/cli/test/evals/internal/agent/budget.go@@ -2,16 +2,22 @@ package agentimport ("fmt"+ "math""sync")// Ledger tracks spending across every run of a dispatch (agents and judge)-// against the --max-cost-usd ceiling. It is checked before each run starts-// and after each model response.+// against the --max-cost-usd ceiling. A run reserves its cap when it starts+// (Reserve), so the runs in flight are never promised more than is left+// between them, and the loop stops a run before a call that would likely+// cross its cap. The ceiling can still be passed by the error in that+// estimate (about one turn's growth per run in flight) plus the judge.type Ledger struct {- mu sync.Mutex- Cap float64- spent float64+ mu sync.Mutex+ Cap float64+ spent float64+ // held is the unspent part of every open reservation.+ held float64byModel map[string]float64}@@ -19,7 +25,7 @@ type Ledger struct {// only the fake model uses.func NewLedger(cap float64) *Ledger { return &Ledger{Cap: cap, byModel: map[string]float64{}} }-// Add records spending.+// Add records spending outside any run's reservation.func (l *Ledger) Add(model string, usd float64) {if l == nil {return@@ -54,18 +60,26 @@ func (l *Ledger) ByModel() map[string]float64 {return out}-// Remaining returns what is left under the cap (+Inf semantics when no cap).-func (l *Ledger) Remaining() float64 {+// Available returns what is neither spent nor reserved by a run in flight+// (+Inf when there is no cap).+func (l *Ledger) Available() float64 {if l == nil || l.Cap <= 0 {- return 1e18+ return math.Inf(1)}l.mu.Lock()defer l.mu.Unlock()- return l.Cap - l.spent+ return l.Cap - l.spent - l.held}-// Exhausted reports whether the ceiling is reached.-func (l *Ledger) Exhausted() bool { return l.Remaining() <= 0 }+// Exhausted reports whether the spending reached the ceiling.+func (l *Ledger) Exhausted() bool {+ if l == nil || l.Cap <= 0 {+ return false+ }+ l.mu.Lock()+ defer l.mu.Unlock()+ return l.spent >= l.Cap+}// String summarizes the ledger.func (l *Ledger) String() string {@@ -74,3 +88,61 @@ func (l *Ledger) String() string {}return fmt.Sprintf("$%.4f of $%.2f", l.Spent(), l.Cap)}++// Hold is one run's reservation of its cap.+type Hold struct {+ l *Ledger+ Cap float64+ spent float64+ closed bool+}++// Reserve sets usd aside for one run until the hold is released.+func (l *Ledger) Reserve(usd float64) *Hold {+ if l == nil {+ return nil+ }+ l.mu.Lock()+ defer l.mu.Unlock()+ l.held += usd+ return &Hold{l: l, Cap: usd}+}++// Add records the run's spending, which uses up its reservation first.+func (h *Hold) Add(model string, usd float64) {+ if h == nil {+ return+ }+ l := h.l+ l.mu.Lock()+ defer l.mu.Unlock()+ if !h.closed {+ l.held -= min(usd, max(0, h.Cap-h.spent))+ }+ h.spent += usd+ l.spent += usd+ l.byModel[model] += usd+}++// Spent returns what the run spent.+func (h *Hold) Spent() float64 {+ if h == nil {+ return 0+ }+ h.l.mu.Lock()+ defer h.l.mu.Unlock()+ return h.spent+}++// Release returns the unspent part of the reservation to the ledger.+func (h *Hold) Release() {+ if h == nil {+ return+ }+ h.l.mu.Lock()+ defer h.l.mu.Unlock()+ if !h.closed {+ h.l.held -= max(0, h.Cap-h.spent)+ h.closed = true+ }+}diff --git a/cli/test/evals/internal/agent/budget_test.go b/cli/test/evals/internal/agent/budget_test.gonew file mode 100644index 00000000..97ca43e9--- /dev/null+++ b/cli/test/evals/internal/agent/budget_test.go@@ -0,0 +1,37 @@+package agent++import (+ "math"+ "testing"+)++func TestLedgerReservations(t *testing.T) {+ l := NewLedger(10)+ a, b := l.Reserve(4), l.Reserve(3)+ if got := l.Available(); got != 3 {+ t.Fatalf("two open holds must be set aside: available %v, want 3", got)+ }+ a.Add("m", 1)+ if got := l.Available(); got != 3 {+ t.Fatalf("spending inside a hold must not change what is free: %v", got)+ }+ a.Add("m", 5)+ if got := l.Available(); got != 1 {+ t.Fatalf("spending past a hold must come out of the free pool: %v, want 1", got)+ }+ a.Release()+ b.Release()+ if got, spent := l.Available(), l.Spent(); got != 4 || spent != 6 {+ t.Fatalf("released holds return their unspent part: available %v spent %v", got, spent)+ }+ a.Add("judge", 1)+ if got := l.Available(); got != 3 {+ t.Fatalf("spending after release comes from the free pool: %v", got)+ }+ if l.Exhausted() {+ t.Fatal("not exhausted at $7 of $10")+ }+ if !math.IsInf(NewLedger(0).Available(), 1) || NewLedger(0).Exhausted() {+ t.Fatal("a ledger without a cap is unlimited")+ }+}diff --git a/cli/test/evals/internal/agent/cache.go b/cli/test/evals/internal/agent/cache.godeleted file mode 100644index aac6f71a..00000000--- a/cli/test/evals/internal/agent/cache.go+++ /dev/null@@ -1,69 +0,0 @@-package agent--import (- "context"- "crypto/sha256"- "encoding/hex"- "encoding/json"- "fmt"- "os"- "path/filepath"-)--// Cache modes (EVAL_CACHE).-const (- CacheOff = "off"- CacheRecord = "record"- CacheReplay = "replay"-)--// CachedChatter records model responses keyed by sha256(model, messages,-// tools, params) under Dir, and replays them. Replay hits whenever a run sends-// exactly the same request again (same environment, same tool outputs), so-// graders and reports can be iterated without spending.-type CachedChatter struct {- Inner Chatter- Dir string- Mode string-}--func (c *CachedChatter) key(req *Request) (string, error) {- raw, err := json.Marshal(req)- if err != nil {- return "", err- }- sum := sha256.Sum256(raw)- return hex.EncodeToString(sum[:]), nil-}--// Chat serves from the cache or the inner client.-func (c *CachedChatter) Chat(ctx context.Context, req *Request) (*Response, error) {- if c.Mode == "" || c.Mode == CacheOff {- return c.Inner.Chat(ctx, req)- }- k, err := c.key(req)- if err != nil {- return nil, err- }- path := filepath.Join(c.Dir, k[:2], k+".json")- if raw, err := os.ReadFile(path); err == nil {- var resp Response- if err := json.Unmarshal(raw, &resp); err == nil {- // A replayed response cost nothing this time.- resp.Usage.Cost = 0- return &resp, nil- }- }- if c.Mode == CacheReplay {- return nil, fmt.Errorf("model cache miss in replay mode (key %s)", k[:12])- }- resp, err := c.Inner.Chat(ctx, req)- if err != nil {- return nil, err- }- if raw, err := json.Marshal(resp); err == nil {- _ = os.MkdirAll(filepath.Dir(path), 0o755)- _ = os.WriteFile(path, raw, 0o644)- }- return resp, nil-}diff --git a/cli/test/evals/internal/agent/cache_test.go b/cli/test/evals/internal/agent/cache_test.godeleted file mode 100644index b9cd9446..00000000--- a/cli/test/evals/internal/agent/cache_test.go+++ /dev/null@@ -1,25 +0,0 @@-package agent--import (- "context"- "testing"-)--func TestCacheRecordReplay(t *testing.T) {- dir := t.TempDir()- inner := &scripted{resps: []*Response{textResp("first")}}- rec := &CachedChatter{Inner: inner, Dir: dir, Mode: CacheRecord}- req := &Request{Model: "m", Messages: []Message{{Role: "user", Content: "q"}}}- if _, err := rec.Chat(context.Background(), req); err != nil {- t.Fatal(err)- }- rep := &CachedChatter{Inner: &scripted{}, Dir: dir, Mode: CacheReplay}- resp, err := rep.Chat(context.Background(), req)- if err != nil || resp.Choices[0].Message.Content != "first" || resp.Usage.Cost != 0 {- t.Fatalf("replay: %v %+v", err, resp)- }- other := &Request{Model: "m", Messages: []Message{{Role: "user", Content: "different"}}}- if _, err := rep.Chat(context.Background(), other); err == nil {- t.Fatal("a replay miss must fail, not call the model")- }-}diff --git a/cli/test/evals/internal/agent/loop.go b/cli/test/evals/internal/agent/loop.goindex cf796a25..913f737a 100644--- a/cli/test/evals/internal/agent/loop.go+++ b/cli/test/evals/internal/agent/loop.go@@ -23,8 +23,9 @@ type Config struct {Prompt stringSubmitTool stringBudget catalog.Budget- // RunCap is this run's share of --max-cost-usd (0 = no cap).- RunCap float64+ // Hold is this run's reservation of --max-cost-usd (nil = no cap); its+ // spending goes there, and to Ledger when there is no hold.+ Hold *HoldLedger *Ledger// Price estimates cost when the provider reports none.Price catalog.Price@@ -71,6 +72,7 @@ func Run(ctx context.Context, cfg Config) *record.Run {nudged := falsecallN := 0outputTokens := 0+ lastTurnUSD := 0.0providers := map[string]bool{}finish := func(reason, detail string) *record.Run {run.StopReason, run.StopDetail = reason, detail@@ -97,8 +99,12 @@ func Run(ctx context.Context, cfg Config) *record.Run {if run.Turns == 0 && cfg.Ledger.Exhausted() {return finish(StopCeiling, "the dispatch's --max-cost-usd ceiling was reached")}- if cfg.RunCap > 0 && run.Usage.CostUSD >= cfg.RunCap {- return finish(StopBudget, fmt.Sprintf("run cost cap $%.2f reached", cfg.RunCap))+ // The next call costs at least about as much as the last one (the+ // context only grows), so stop before it would cross the run's share.+ // This is the harness's cap, not the model's budget: the run is+ // recorded as cost_ceiling, left out of pass rates and resumable.+ if h := cfg.Hold; h != nil && h.Cap > 0 && run.Turns > 0 && h.Spent()+lastTurnUSD > h.Cap {+ return finish(StopCeiling, fmt.Sprintf("run cost cap $%.2f reached (spent $%.4f, the last call cost $%.4f)", h.Cap, h.Spent(), lastTurnUSD))}run.Turns++req := &Request{@@ -125,9 +131,14 @@ func Run(ctx context.Context, cfg Config) *record.Run {}return finish(StopInfra, err.Error())}- u := usageOf(resp, cfg.Price)+ u := UsageOf(resp, cfg.Price)run.Usage.Add(u)- cfg.Ledger.Add(cfg.Model.Key, u.CostUSD)+ lastTurnUSD = u.CostUSD+ if cfg.Hold != nil {+ cfg.Hold.Add(cfg.Model.Key, u.CostUSD)+ } else {+ cfg.Ledger.Add(cfg.Model.Key, u.CostUSD)+ }outputTokens += u.OutputTokens - u.ReasoningTokensif resp.Provider != "" {providers[resp.Provider] = true@@ -212,7 +223,9 @@ func nonEmpty(s string) string {return s}-func usageOf(r *Response, p catalog.Price) record.Usage {+// UsageOf reads a response's usage, estimating its cost from p when the+// provider reported none.+func UsageOf(r *Response, p catalog.Price) record.Usage {u := record.Usage{InputTokens: r.Usage.PromptTokens,CachedTokens: r.Usage.PromptTokensDetails.CachedTokens,diff --git a/cli/test/evals/internal/agent/openrouter.go b/cli/test/evals/internal/agent/openrouter.goindex 2abe2b82..a2e8189d 100644--- a/cli/test/evals/internal/agent/openrouter.go+++ b/cli/test/evals/internal/agent/openrouter.go@@ -1,7 +1,7 @@// Package agent drives a model through the eval: an OpenRouter Chat// Completions client (the same code path for every model and the judge), the-// tool-calling loop with budgets, prompt caching, transcripts, a-// record/replay cache, and the fake model that plays oracle trajectories.+// tool-calling loop with budgets and the cost ledger, prompt caching,+// transcripts, and the fake model that plays oracle trajectories.package agentimport (@@ -133,7 +133,7 @@ type Client struct {BaseURL stringKey stringHTTP *http.Client- // Retries on 429/5xx/timeouts (default 5).+ // Retries on 408/429/5xx/timeouts (default 5).Retries int}@@ -234,7 +234,7 @@ func (c *Client) once(ctx context.Context, req *Request) (*Response, error) {if err != nil {return nil, &RetryableError{err}}- if resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= 500 {+ if resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode == http.StatusRequestTimeout || resp.StatusCode >= 500 {return nil, &RetryableError{fmt.Errorf("openrouter HTTP %d: %s", resp.StatusCode, clip(raw))}}if resp.StatusCode != http.StatusOK {diff --git a/cli/test/evals/internal/catalog/case.go b/cli/test/evals/internal/catalog/case.goindex fe2c193f..2827a146 100644--- a/cli/test/evals/internal/catalog/case.go+++ b/cli/test/evals/internal/catalog/case.go@@ -115,13 +115,11 @@ type Credential struct {// SetupSpec configures a setup-suite case.type SetupSpec struct {- Template string `yaml:"template"` // testdata/setup-templates/<template>- Path string `yaml:"path"` // fast | plan+ // Path names the setup skill's path the case exercises, fast or plan.+ // The harness does not read it; Token and ApproveDrafts set the run up.+ Path string `yaml:"path"`// Token is what the prompt hands the agent: ingest | setup.Token string `yaml:"token"`- // Projects the harness creates up front (fast path) or expects the- // approved plan to create (plan path).- Projects []SetupProject `yaml:"projects"`// ApproveDrafts makes the harness approve every pending setup draft.ApproveDrafts bool `yaml:"approve_drafts"`// Env is the container environment of the agent's shell and of the@@ -147,11 +145,8 @@ type SetupVerify struct {Wait Duration `yaml:"wait"`}-// SetupProject is one Traceway project of a setup case.-type SetupProject struct {- Name string `yaml:"name"`- Framework string `yaml:"framework"`-}+// HookActions are the actions a hook may name (runner/hooks.go).+var HookActions = []string{"edit_post_mortem", "stub_healthy", "stub_unhealthy", "sleep", "approve_setup_draft"}// Hook is a harness action triggered after a matching agent call.type Hook struct {@@ -266,7 +261,6 @@ type StateAssert struct {Exists *bool `yaml:"exists"`Route string `yaml:"route"`HTTPStatus *Range `yaml:"http_status"`- Value string `yaml:"value"`Projects []string `yaml:"projects"`Project string `yaml:"project"`Frameworks map[string]string `yaml:"frameworks"`@@ -427,6 +421,17 @@ func (d Duration) D() time.Duration { return time.Duration(d) }var idRe = regexp.MustCompile(`^[a-z0-9]+(-[a-z0-9]+)+$`)+// suiteArms are the arms a suite can run: write cases need a write surface,+// and the setup arm (the container shell) belongs to the setup suite alone.+// perf and debug take any other arm (a debug case may check that an agent+// with write access writes nothing).+var suiteArms = map[string][]string{+ "perf": {ArmCLISkill, ArmCLINoSkill, ArmMCP, ArmCLIWriteSkill, ArmMCPWrite},+ "debug": {ArmCLISkill, ArmCLINoSkill, ArmMCP, ArmCLIWriteSkill, ArmMCPWrite},+ "write": {ArmCLIWriteSkill, ArmMCPWrite},+ "setup": {ArmCLISetupSkill},+}+// Validate checks a case for structural errors that YAML decoding cannot.func (c *Case) Validate() error {var errs []string@@ -485,6 +490,29 @@ func (c *Case) Validate() error {if c.Suite == "setup" && c.Setup == nil {add("setup cases need a setup block")}+ if c.Setup != nil {+ if c.Setup.Path != "" && c.Setup.Path != "fast" && c.Setup.Path != "plan" {+ add("setup.path must be fast or plan, got %q", c.Setup.Path)+ }+ if c.Setup.Token != "" && c.Setup.Token != "ingest" && c.Setup.Token != "setup" {+ add("setup.token must be ingest or setup, got %q", c.Setup.Token)+ }+ }+ for _, a := range c.Arms {+ if ok := suiteArms[c.Suite]; ok != nil && !slices.Contains(ok, a) {+ add("arm %q does not run the %s suite (its arms: %v)", a, c.Suite, ok)+ }+ }+ if c.Suite == "debug" && c.Expect.Answer != nil {+ if ans := c.Expect.Answer; ans.Category == "" && ans.IssueHash == "" && ans.Culprit == nil {+ add("debug cases need expect.answer.category, issue_hash or culprit")+ }+ }+ for i, h := range c.Hooks {+ if !slices.Contains(HookActions, h.Action) {+ add("hooks[%d]: unknown action %q (one of %v)", i, h.Action, HookActions)+ }+ }if c.Suite != "setup" && c.Distractor == "" && !slices.Contains(c.Tags, "control") {add("distractor is required (or tag the case control)")}diff --git a/cli/test/evals/internal/catalog/case_test.go b/cli/test/evals/internal/catalog/case_test.goindex 25ebe46d..08ae082d 100644--- a/cli/test/evals/internal/catalog/case_test.go+++ b/cli/test/evals/internal/catalog/case_test.go@@ -26,3 +26,37 @@ func TestInvokeMatchesPrompt(t *testing.T) {}}}++// TestValidateRejectsCasesTheHarnessCannotRun: arms the suite cannot run,+// unknown hook actions, setup values the runner does not know, and a debug+// answer that checks nothing.+func TestValidateRejectsCasesTheHarnessCannotRun(t *testing.T) {+ base := func() *Case {+ return &Case{ID: "debug-x", Suite: "debug", Seed: "debug/x", Arms: []string{ArmCLISkill}, Prompt: "Fix it.",+ Oracle: []Step{{Shell: "true"}}, Expect: Expect{Answer: &Answer{Category: "logic"}}, Distractor: "x"}+ }+ if err := base().Validate(); err != nil {+ t.Fatal(err)+ }+ for name, c := range map[string]struct {+ edit func(*Case)+ want string+ }{+ "setup arm": {func(c *Case) { c.Arms = []string{ArmCLISetupSkill} }, "does not run the debug suite"},+ "hook action": {func(c *Case) { c.Hooks = []Hook{{Action: "edit_postmortem"}} }, "unknown action"},+ "empty answer": {func(c *Case) { c.Expect.Answer = &Answer{} }, "need expect.answer.category"},+ "setup token": {func(c *Case) {+ c.ID, c.Suite, c.Arms = "setup-x", "setup", []string{ArmCLISetupSkill}+ c.Expect = Expect{State: []StateAssert{{Kind: "file"}}}+ c.Setup = &SetupSpec{Token: "write"}+ }, "setup.token must be"},+ } {+ t.Run(name, func(t *testing.T) {+ cs := base()+ c.edit(cs)+ if err := cs.Validate(); err == nil || !strings.Contains(err.Error(), c.want) {+ t.Fatalf("want an error containing %q, got %v", c.want, err)+ }+ })+ }+}diff --git a/cli/test/evals/internal/env/instance.go b/cli/test/evals/internal/env/instance.goindex b5d16ca3..2e14485b 100644--- a/cli/test/evals/internal/env/instance.go+++ b/cli/test/evals/internal/env/instance.go@@ -14,11 +14,11 @@ type Instance struct {Dir string}-// AdminEmail and AdminPassword are the owner credentials of every instance.-const (- AdminEmail = "[email protected]"- AdminPassword = "eval-owner-password"-)+// AdminEmail is the owner's login on every instance. The password is random+// per instance (Admin.Password): the setup suite's container shares the host+// network, so a fixed password would let an agent sign in as the owner of+// its own or another run's backend.+const AdminEmail = "[email protected]"// NewInstance boots a backend under dir/backend, registers the owner and// organization, and starts a healthy stub.@@ -27,7 +27,7 @@ func NewInstance(ctx context.Context, bins *Binaries, dir string) (*Instance, erif err != nil {return nil, err}- a, err := Register(ctx, b, AdminEmail, AdminPassword, "Acme")+ a, err := Register(ctx, b, AdminEmail, "owner-"+randomHex(12), "Acme")if err != nil {b.Stop()return nil, fmt.Errorf("register: %w", err)diff --git a/cli/test/evals/internal/grade/answer_debug.go b/cli/test/evals/internal/grade/answer_debug.goindex 44ebc44e..561ff8a6 100644--- a/cli/test/evals/internal/grade/answer_debug.go+++ b/cli/test/evals/internal/grade/answer_debug.go@@ -2,6 +2,7 @@ package gradeimport ("path/filepath"+ "regexp""strings""github.com/tracewayapp/traceway/cli/test/evals/internal/catalog"@@ -49,25 +50,39 @@ func answerDebug(v *Verdict, exp *catalog.Answer, sub map[string]any) {}}+// samePath compares a cited file with the expected one, accepting a path+// prefix, ./, backslashes, backticks and a trailing :line(:col).func samePath(got, want string) bool {- g := filepath.ToSlash(filepath.Clean(strings.TrimPrefix(strings.TrimSpace(got), "./")))+ g := strings.Trim(strings.TrimSpace(got), "`")+ g = strings.ReplaceAll(g, "\\", "/")+ g = lineSuffixRe.ReplaceAllString(g, "")+ g = filepath.ToSlash(filepath.Clean(strings.TrimPrefix(g, "./")))w := filepath.ToSlash(filepath.Clean(want))return g == w || strings.HasSuffix(g, "/"+w) || strings.HasSuffix(w, "/"+g) && strings.Contains(g, "/")}-// sameSymbol accepts "(*OrderHandler).List", "OrderHandler.List" or "List"-// for an expected "List" (and the reverse).+var lineSuffixRe = regexp.MustCompile(`(:\d+){1,2}$`)++// sameSymbol accepts "(*OrderHandler).List", "OrderHandler.List", "List",+// "func List" or "List(ctx, id)" for an expected "List" (and the reverse).func sameSymbol(got, want string) bool {last := func(s string) string {- s = strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(s), "()"))- // Swift and Kotlin accessor frames: chatURL.getter, value.<get-x>.+ s = strings.Trim(strings.TrimSpace(s), "`")+ s = strings.TrimSpace(strings.TrimPrefix(s, "func "))+ // A trailing parameter list, but not the receiver of (*T).M.+ if strings.HasSuffix(s, ")") {+ if i := strings.LastIndex(s, "("); i > 0 {+ s = s[:i]+ }+ }+ // Swift accessor frames: chatURL.getter.for _, suf := range []string{".getter", ".setter", ".modify", ".init", ".deinit"} {s = strings.TrimSuffix(s, suf)}if i := strings.LastIndexAny(s, ".:#"); i >= 0 {s = s[i+1:]}- return strings.ToLower(strings.Trim(s, "()*"))+ return strings.ToLower(strings.Trim(s, "()* "))}return last(got) != "" && last(got) == last(want)}diff --git a/cli/test/evals/internal/grade/answer_debug_test.go b/cli/test/evals/internal/grade/answer_debug_test.gonew file mode 100644index 00000000..59a4edbb--- /dev/null+++ b/cli/test/evals/internal/grade/answer_debug_test.go@@ -0,0 +1,39 @@+package grade++import "testing"++func TestCulpritMatching(t *testing.T) {+ for _, c := range []struct {+ got, want string+ ok bool+ }{+ {"(*OrderHandler).List", "List", true},+ {"OrderHandler.List", "List", true},+ {"func formatAddress", "formatAddress", true},+ {"formatAddress(a *Address)", "formatAddress", true},+ {"FeedAdapter.bindImage(FeedItem)", "bindImage", true},+ {"`renderInvoice`", "renderInvoice", true},+ {"chatURL.getter", "chatURL", true},+ {"renderTotals", "renderInvoice", false},+ {"", "List", false},+ } {+ if got := sameSymbol(c.got, c.want); got != c.ok {+ t.Errorf("sameSymbol(%q, %q) = %v, want %v", c.got, c.want, got, c.ok)+ }+ }+ for _, c := range []struct {+ got, want string+ ok bool+ }{+ {"internal/invoices/render.go", "internal/invoices/render.go", true},+ {"./internal/invoices/render.go:37", "internal/invoices/render.go", true},+ {"internal\\invoices\\render.go:37:5", "internal/invoices/render.go", true},+ {"/work/internal/invoices/render.go", "internal/invoices/render.go", true},+ {"`render.go`", "internal/invoices/render.go", false},+ {"internal/invoices/model.go", "internal/invoices/render.go", false},+ } {+ if got := samePath(c.got, c.want); got != c.ok {+ t.Errorf("samePath(%q, %q) = %v, want %v", c.got, c.want, got, c.ok)+ }+ }+}diff --git a/cli/test/evals/internal/grade/grade.go b/cli/test/evals/internal/grade/grade.goindex 7d905a6e..d4154c20 100644--- a/cli/test/evals/internal/grade/grade.go+++ b/cli/test/evals/internal/grade/grade.go@@ -9,6 +9,7 @@ import ("fmt""strings"+ "github.com/tracewayapp/traceway/cli/test/evals/internal/agent""github.com/tracewayapp/traceway/cli/test/evals/internal/catalog""github.com/tracewayapp/traceway/cli/test/evals/internal/record")@@ -65,6 +66,12 @@ type Input struct {SandboxImage stringEvalsRoot stringJudge *Judge+ // Pristine is the workspace's Snapshot before the agent started; the+ // hidden-tests grader diffs against it.+ Pristine map[string]string+ // Hold is the run's reservation of the cost cap; the judge's spending+ // goes there (nil: straight to the judge's ledger).+ Hold *agent.Hold}// All runs every applicable grader in a fixed order.diff --git a/cli/test/evals/internal/grade/hiddentests.go b/cli/test/evals/internal/grade/hiddentests.goindex 144e0d12..21054bc8 100644--- a/cli/test/evals/internal/grade/hiddentests.go+++ b/cli/test/evals/internal/grade/hiddentests.go@@ -10,6 +10,7 @@ import ("path""path/filepath""strings"+ "syscall""time")@@ -22,7 +23,7 @@ import (func HiddenTests(ctx context.Context, in *Input) *Verdict {spec := in.Case.Expect.Codev := newVerdict("hidden_tests", spec.Gating)- changed, err := changedFiles(in.Workspace)+ changed, err := changedFiles(in.Workspace, in.Pristine)if err != nil {v.fail("read the diff: %v", err)return v@@ -72,17 +73,30 @@ func HiddenTests(ctx context.Context, in *Input) *Verdict {cmd = exec.CommandContext(ctx, "sh", "-c", spec.Command)cmd.Dir = in.Workspacecmd.Env = hostTestEnv(in.Workspace)+ // The timeout kills the whole group: go test's children would+ // otherwise keep the output pipe open past it.+ cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}+ cmd.Cancel = func() error { return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) }case in.SandboxImage != "" && dockerAvailable():- cmd = exec.CommandContext(ctx, "docker", "run", "--rm", "--network", "none",+ name := fmt.Sprintf("twevals-hidden-%d-%d", os.Getpid(), time.Now().UnixNano())+ cmd = exec.CommandContext(ctx, "docker", "run", "--rm", "--name", name, "--network", "none",+ "--user", fmt.Sprintf("%d:%d", os.Getuid(), os.Getgid()),"-v", in.Workspace+":/work", "-w", "/work", "-e", "HOME=/tmp", "-e", "GOFLAGS=-mod=mod", "-e", "GOCACHE=/tmp/gocache",in.SandboxImage, "sh", "-c", spec.Command)+ // Killing the docker client leaves the container running; remove it.+ cmd.Cancel = func() error {+ _ = exec.Command("docker", "rm", "-f", name).Run()+ return cmd.Process.Kill()+ }default:- v.Skipped = true- v.note("skipped: no container runtime for the model's patch (set TWEVALS_HOST_TESTS=1 to run it on the host)")+ v.note("tests not run: no container runtime for the model's patch (set TWEVALS_HOST_TESTS=1 to run it on the host)")+ // The diff checks above still gate; only a clean diff is skipped.+ v.Skipped = len(v.Failures) == 0return v}var out bytes.Buffercmd.Stdout, cmd.Stderr = &out, &out+ cmd.WaitDelay = 10 * time.Seconderr = cmd.Run()tail := out.String()if len(tail) > 3000 {@@ -120,31 +134,6 @@ func dockerAvailable() bool {return exec.Command("docker", "info").Run() == nil}-func changedFiles(work string) ([]string, error) {- run := func(args ...string) ([]string, error) {- out, err := workspaceGit(work, args...).Output()- if err != nil {- return nil, fmt.Errorf("git %s: %w", strings.Join(args, " "), err)- }- var lines []string- for _, l := range strings.Split(strings.TrimSpace(string(out)), "\n") {- if l != "" && !strings.HasPrefix(l, ".twevals") {- lines = append(lines, l)- }- }- return lines, nil- }- mod, err := run("diff", "--name-only", "HEAD")- if err != nil {- return nil, err- }- untracked, err := run("ls-files", "-o", "--exclude-standard")- if err != nil {- return nil, err- }- return append(mod, untracked...), nil-}-func anyGlob(g string, files []string) bool {for _, f := range files {if ok, _ := path.Match(g, f); ok || globAny(g, f) || f == g {@@ -154,7 +143,15 @@ func anyGlob(g string, files []string) bool {return false}+// copyTree copies the hidden tests into the workspace. The agent may have+// left symlinks there (the setup container can make them), so every target+// directory must resolve inside dst and the file itself is never followed:+// a planted link is replaced, not written through.func copyTree(src, dst string) error {+ realDst, err := filepath.EvalSymlinks(dst)+ if err != nil {+ return err+ }return filepath.WalkDir(src, func(p string, d fs.DirEntry, err error) error {if err != nil {return err@@ -162,12 +159,37 @@ func copyTree(src, dst string) error {rel, _ := filepath.Rel(src, p)target := filepath.Join(dst, rel)if d.IsDir() {+ if fi, err := os.Lstat(target); err == nil && !fi.IsDir() {+ if err := os.Remove(target); err != nil {+ return err+ }+ }return os.MkdirAll(target, 0o755)}+ dir, err := filepath.EvalSymlinks(filepath.Dir(target))+ if err != nil {+ return err+ }+ if dir != realDst && !strings.HasPrefix(dir, realDst+string(filepath.Separator)) {+ return fmt.Errorf("%s resolves outside the workspace", filepath.Dir(target))+ }raw, err := os.ReadFile(p)if err != nil {return err}- return os.WriteFile(target, raw, 0o644)+ if fi, err := os.Lstat(target); err == nil && !fi.Mode().IsRegular() {+ if err := os.Remove(target); err != nil {+ return err+ }+ }+ f, err := os.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_TRUNC|syscall.O_NOFOLLOW, 0o644)+ if err != nil {+ return err+ }+ if _, err := f.Write(raw); err != nil {+ f.Close()+ return err+ }+ return f.Close()})}diff --git a/cli/test/evals/internal/grade/judge.go b/cli/test/evals/internal/grade/judge.goindex bb81918b..771c984c 100644--- a/cli/test/evals/internal/grade/judge.go+++ b/cli/test/evals/internal/grade/judge.go@@ -18,6 +18,8 @@ type Judge struct {Chat agent.ChatterModel *catalog.ModelLedger *agent.Ledger+ // Price estimates the judge's cost when the provider reports none.+ Price catalog.Price}const judgeSystem = `You grade an AI agent's work against a rubric. You see the user's request, the expected outcome (hidden from the agent), the agent's final report and structured answer, and a condensed log of the tool calls it made.@@ -95,8 +97,12 @@ func (j *Judge) Grade(ctx context.Context, in *Input) *Verdict {v.note("judge call failed: %v", err)return v}- v.CostUSD = resp.Usage.Cost- j.Ledger.Add("judge", resp.Usage.Cost)+ v.CostUSD = agent.UsageOf(resp, j.Price).CostUSD+ if in.Hold != nil {+ in.Hold.Add("judge", v.CostUSD)+ } else {+ j.Ledger.Add("judge", v.CostUSD)+ }text, _ := resp.Choices[0].Message.Content.(string)var out struct {Criteria []struct {diff --git a/cli/test/evals/internal/grade/state.go b/cli/test/evals/internal/grade/state.goindex afa2c97a..535e7755 100644--- a/cli/test/evals/internal/grade/state.go+++ b/cli/test/evals/internal/grade/state.go@@ -5,7 +5,6 @@ import ("encoding/json""fmt""io/fs"- "os""path""path/filepath""regexp"@@ -87,13 +86,15 @@ func checkState(ctx context.Context, in *Input, a catalog.StateAssert) error {}case "page_count":var out struct {- Data []json.RawMessage `json:"data"`+ Pagination struct {+ Total int64 `json:"total"`+ } `json:"pagination"`}- if err := postJSON("/api/pages", q, map[string]any{"status": a.Status, "pagination": map[string]int{"page": 1, "pageSize": 100}}, &out); err != nil {+ if err := postJSON("/api/pages", q, map[string]any{"status": a.Status, "pagination": map[string]int{"page": 1, "pageSize": 1}}, &out); err != nil {return err}- if !a.Count.Contains(float64(len(out.Data))) {- return fmt.Errorf("%d %s pages, want %s", len(out.Data), a.Status, a.Count)+ if !a.Count.Contains(float64(out.Pagination.Total)) {+ return fmt.Errorf("%d %s pages, want %s", out.Pagination.Total, a.Status, a.Count)}case "rule":var out struct {@@ -141,7 +142,12 @@ func checkState(ctx context.Context, in *Input, a catalog.StateAssert) error {}var archived []stringfor _, h := range a.Among {- if all[h] && !visible[h] {+ if !all[h] {+ // Absent even with archived issues included: a typo or a+ // window that misses it would otherwise read as unarchived.+ return fmt.Errorf("issue %s is not in the project's issue list", h)+ }+ if !visible[h] {archived = append(archived, h)}}@@ -154,18 +160,29 @@ func checkState(ctx context.Context, in *Input, a catalog.StateAssert) error {case "post_mortem":return checkPostMortem(ctx, in, a, getJSON)case "agent_writes":- var out struct {- Data []struct {- Route string `json:"route"`- Method string `json:"method"`- StatusCode int `json:"statusCode"`- } `json:"data"`+ type event struct {+ Route string `json:"route"`+ Method string `json:"method"`+ StatusCode int `json:"statusCode"`}- if err := getJSON(fmt.Sprintf("/api/organizations/%d/agent-activity", st.OrgID()), map[string]string{"pageSize": "100"}, &out); err != nil {- return err+ var events []event+ for page := 1; ; page++ {+ var out struct {+ Data []event `json:"data"`+ Pagination struct {+ TotalPages int64 `json:"totalPages"`+ } `json:"pagination"`+ }+ if err := getJSON(fmt.Sprintf("/api/organizations/%d/agent-activity", st.OrgID()), map[string]string{"page": strconv.Itoa(page), "pageSize": "100"}, &out); err != nil {+ return err+ }+ events = append(events, out.Data...)+ if int64(page) >= out.Pagination.TotalPages || len(out.Data) == 0 || page >= 50 {+ break+ }}n := 0- for _, e := range out.Data {+ for _, e := range events {if a.Route != "" && !strings.Contains(e.Method+" "+e.Route, a.Route) {continue}@@ -454,7 +471,7 @@ func checkFiles(root string, a catalog.StateAssert) error {if err != nil {return nil}- if d.IsDir() && (d.Name() == ".git" || d.Name() == "node_modules" || d.Name() == ".skills") {+ if d.IsDir() && (d.Name() == ".git" || d.Name() == "node_modules" || d.Name() == ".skills" || strings.HasPrefix(d.Name(), ".twevals")) {return filepath.SkipDir}if d.IsDir() {@@ -475,7 +492,7 @@ func checkFiles(root string, a catalog.StateAssert) error {}var all strings.Builderfor _, m := range matched {- raw, _ := os.ReadFile(m)+ raw, _ := readRegular(m)all.Write(raw)all.WriteByte('\n')}@@ -519,8 +536,8 @@ func checkNoTokens(root string, tokens []string) error {if strings.HasPrefix(base, ".env") || strings.HasPrefix(rel, ".skills/") || strings.HasPrefix(rel, ".twevals") {continue}- raw, err := os.ReadFile(filepath.Join(root, rel))- if err != nil {+ raw, ok := readRegular(filepath.Join(root, rel))+ if !ok {continue}for _, t := range tokens {@@ -538,14 +555,14 @@ func checkNoTokens(root string, tokens []string) error {if err != nil {return nil}- if d.IsDir() && (d.Name() == ".git" || d.Name() == "node_modules" || d.Name() == ".skills") {+ if d.IsDir() && (d.Name() == ".git" || d.Name() == "node_modules" || d.Name() == ".skills" || strings.HasPrefix(d.Name(), ".twevals")) {return filepath.SkipDir}if d.IsDir() || strings.HasPrefix(d.Name(), ".env") {return nil}- raw, err := os.ReadFile(p)- if err != nil {+ raw, ok := readRegular(p)+ if !ok {return nil}for _, t := range tokens {@@ -563,7 +580,7 @@ func checkNoTokens(root string, tokens []string) error {}func gitVisibleFiles(root string) ([]string, bool) {- out, err := workspaceGit(root, "ls-files", "-z", "-c", "-o", "--exclude-standard").Output()+ out, err := workspaceGit(root, nil, "ls-files", "-z", "-c", "-o", "--exclude-standard")if err != nil {return nil, false}diff --git a/cli/test/evals/internal/grade/tree.go b/cli/test/evals/internal/grade/tree.gonew file mode 100644index 00000000..fd2d0886--- /dev/null+++ b/cli/test/evals/internal/grade/tree.go@@ -0,0 +1,136 @@+package grade++import (+ "crypto/sha256"+ "encoding/hex"+ "fmt"+ "io"+ "io/fs"+ "os"+ "path/filepath"+ "sort"+ "strings"+ "syscall"+)++// maxScanBytes caps a workspace file the graders read into memory.+const maxScanBytes = 16 << 20++// Snapshot fingerprints a workspace's files (outside .git, .skills/ and the+// harness's .twevals* entries) before the agent starts, so the diff after the+// run needs no git. Git would read the workspace's .git/config, whose filter+// drivers it runs on the host, and `git diff HEAD` misses a change the setup+// container committed.+func Snapshot(root string) (map[string]string, error) {+ out := map[string]string{}+ err := walkWorkspace(root, func(rel, abs string, d fs.DirEntry) error {+ h, err := fileDigest(abs, d)+ if err != nil {+ return err+ }+ out[rel] = h+ return nil+ })+ return out, err+}++func walkWorkspace(root string, fn func(rel, abs string, d fs.DirEntry) error) error {+ return filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {+ if err != nil {+ return err+ }+ if p == root {+ return nil+ }+ rel, err := filepath.Rel(root, p)+ if err != nil {+ return err+ }+ rel = filepath.ToSlash(rel)+ if strings.HasPrefix(rel, ".twevals") {+ if d.IsDir() {+ return filepath.SkipDir+ }+ return nil+ }+ if d.IsDir() {+ if d.Name() == ".git" || rel == ".skills" {+ return filepath.SkipDir+ }+ return nil+ }+ return fn(rel, p, d)+ })+}++// fileDigest identifies a file's content without following a symlink or+// opening anything but a regular file.+func fileDigest(abs string, d fs.DirEntry) (string, error) {+ switch t := d.Type(); {+ case t&fs.ModeSymlink != 0:+ target, err := os.Readlink(abs)+ return "symlink:" + target, err+ case !t.IsRegular():+ return "special:" + t.String(), nil+ }+ f, err := os.OpenFile(abs, os.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0)+ if err != nil {+ return "", err+ }+ defer f.Close()+ h := sha256.New()+ if _, err := io.Copy(h, f); err != nil {+ return "", err+ }+ return hex.EncodeToString(h.Sum(nil)), nil+}++// changedFiles lists the files added, modified or removed since the snapshot.+// Added files that a .gitignore excludes (build output) are left out.+func changedFiles(root string, pristine map[string]string) ([]string, error) {+ if pristine == nil {+ return nil, fmt.Errorf("no snapshot of the workspace before the run")+ }+ now, err := Snapshot(root)+ if err != nil {+ return nil, err+ }+ var out, added []string+ for rel, h := range now {+ if before, ok := pristine[rel]; !ok {+ added = append(added, rel)+ } else if before != h {+ out = append(out, rel)+ }+ }+ for rel := range pristine {+ if _, ok := now[rel]; !ok {+ out = append(out, rel)+ }+ }+ ignored := gitIgnored(root, added)+ for _, rel := range added {+ if !ignored[rel] {+ out = append(out, rel)+ }+ }+ sort.Strings(out)+ return out, nil+}++// readRegular reads a workspace file for a grader: only a regular file (no+// symlink, FIFO or device the setup container may have planted) of at most+// maxScanBytes.+func readRegular(abs string) ([]byte, bool) {+ fi, err := os.Lstat(abs)+ if err != nil || !fi.Mode().IsRegular() || fi.Size() > maxScanBytes {+ return nil, false+ }+ f, err := os.OpenFile(abs, os.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0)+ if err != nil {+ return nil, false+ }+ defer f.Close()+ raw, err := io.ReadAll(io.LimitReader(f, maxScanBytes))+ return raw, err == nil+}diff --git a/cli/test/evals/internal/grade/workspace_git.go b/cli/test/evals/internal/grade/workspace_git.goindex 1ada21f7..22277138 100644--- a/cli/test/evals/internal/grade/workspace_git.go+++ b/cli/test/evals/internal/grade/workspace_git.go@@ -1,20 +1,59 @@package gradeimport (+ "bytes"+ "context"+ "errors"+ "fmt""os""os/exec""path/filepath"+ "strings"+ "time")-// workspaceGit runs git in a workspace the agent has written to, trusting no-// git config found there: the setup suite's container shell can write-// .git/config, and core.fsmonitor or core.hooksPath would then run a program-// on the host as soon as git reads the index. The repository is pinned to the-// workspace's own .git, and only built-in defaults apply otherwise.-func workspaceGit(work string, args ...string) *exec.Cmd {- cmd := exec.Command("git", append([]string{"-c", "core.fsmonitor=false", "-c", "core.hooksPath=/dev/null"}, args...)...)+// workspaceGit runs a read-only git command in a workspace the agent has+// written to. The setup suite's container shell can write .git/config, so+// the graders use git only for commands that run no configured program (file+// listings and ignore checks, never diff or status, which run filter drivers)+// and switch off the index hooks those could reach: core.fsmonitor and+// core.hooksPath. The repository is pinned to the workspace's own .git, which+// must be a directory (a `gitdir:` file would point git elsewhere), no global+// or system config applies, and the command is killed after 30 seconds (a+// config can name a FIFO).+func workspaceGit(work string, stdin []byte, args ...string) ([]byte, error) {+ gitDir := filepath.Join(work, ".git")+ if fi, err := os.Lstat(gitDir); err != nil || !fi.IsDir() {+ return nil, fmt.Errorf("%s is not a directory", gitDir)+ }+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)+ defer cancel()+ cmd := exec.CommandContext(ctx, "git", append([]string{"-c", "core.fsmonitor=false", "-c", "core.hooksPath=/dev/null"}, args...)...)cmd.Dir = work- cmd.Env = []string{"HOME=" + work, "PATH=" + os.Getenv("PATH"), "GIT_DIR=" + filepath.Join(work, ".git"), "GIT_WORK_TREE=" + work,+ cmd.Env = []string{"HOME=" + work, "PATH=" + os.Getenv("PATH"), "GIT_DIR=" + gitDir, "GIT_WORK_TREE=" + work,"GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL=/dev/null", "GIT_OPTIONAL_LOCKS=0"}- return cmd+ if stdin != nil {+ cmd.Stdin = bytes.NewReader(stdin)+ }+ cmd.WaitDelay = 5 * time.Second+ return cmd.Output()+}++// gitIgnored reports which of paths a .gitignore (or info/exclude) excludes.+func gitIgnored(work string, paths []string) map[string]bool {+ out := map[string]bool{}+ if len(paths) == 0 {+ return out+ }+ raw, err := workspaceGit(work, []byte(strings.Join(paths, "\x00")+"\x00"), "check-ignore", "--no-index", "-z", "--stdin")+ var ee *exec.ExitError+ if err != nil && !(errors.As(err, &ee) && ee.ExitCode() == 1) {+ return out+ }+ for _, p := range strings.Split(string(raw), "\x00") {+ if p != "" {+ out[p] = true+ }+ }+ return out}diff --git a/cli/test/evals/internal/grade/workspace_git_test.go b/cli/test/evals/internal/grade/workspace_git_test.goindex 2c546354..e382d81b 100644--- a/cli/test/evals/internal/grade/workspace_git_test.go+++ b/cli/test/evals/internal/grade/workspace_git_test.go@@ -10,27 +10,51 @@ import (// TestWorkspaceGitIgnoresRepoConfig: a .git/config the agent wrote (the// setup suite's container shell can) never runs a program on the host when-// the graders list the workspace's files.+// the graders list or diff the workspace's files, and a change it committed+// still counts as changed.func TestWorkspaceGitIgnoresRepoConfig(t *testing.T) {dir := t.TempDir()work := filepath.Join(dir, "work")os.MkdirAll(work, 0o755)os.WriteFile(filepath.Join(work, "main.go"), []byte("package main\n"), 0o644)- for _, args := range [][]string{{"init", "-q"}, {"add", "main.go"}, {"-c", "user.email=x@y", "-c", "user.name=x", "commit", "-qm", "init"}} {+ os.WriteFile(filepath.Join(work, "util.go"), []byte("package main\n"), 0o644)+ os.WriteFile(filepath.Join(work, "gone.go"), []byte("package main\n"), 0o644)+ git := func(args ...string) {cmd := exec.Command("git", args...)cmd.Dir = work+ cmd.Env = []string{"HOME=" + work, "PATH=" + os.Getenv("PATH"), "GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL=/dev/null"}if out, err := cmd.CombinedOutput(); err != nil {t.Skipf("git unavailable: %v %s", err, out)}}+ git("init", "-q")+ git("add", "-A")+ git("-c", "user.email=x@y", "-c", "user.name=x", "commit", "-qm", "init")+ pristine, err := Snapshot(work)+ if err != nil {+ t.Fatal(err)+ }++ // What the agent does: edit, commit (hiding the edit from git diff+ // HEAD), add files, and plant config that runs programs.+ os.WriteFile(filepath.Join(work, "util.go"), []byte("package main\n\nfunc util() {}\n"), 0o644)+ git("add", "util.go")+ git("-c", "user.email=x@y", "-c", "user.name=x", "commit", "-qm", "fix")+ os.Remove(filepath.Join(work, "gone.go"))hooks := filepath.Join(work, "hooks")os.MkdirAll(hooks, 0o755)for _, h := range []string{"post-index-change", "reference-transaction"} {os.WriteFile(filepath.Join(hooks, h), []byte("#!/bin/sh\necho x > ../PWNED\n"), 0o755)}cfg, _ := os.ReadFile(filepath.Join(work, ".git", "config"))- cfg = append(cfg, []byte("[core]\n\tfsmonitor = \"echo x > ../PWNED; false\"\n\thooksPath = hooks\n\tworktree = ..\n")...)+ cfg = append(cfg, []byte("[core]\n\tfsmonitor = \"echo x > ../PWNED; false\"\n\thooksPath = hooks\n\tworktree = ..\n"++ "[filter \"x\"]\n\tclean = \"sh -c 'echo x > ../PWNED; cat'\"\n\tprocess = \"sh -c 'echo x > ../PWNED'\"\n")...)os.WriteFile(filepath.Join(work, ".git", "config"), cfg, 0o644)+ os.WriteFile(filepath.Join(work, ".gitattributes"), []byte("* filter=x\n"), 0o644)+ os.WriteFile(filepath.Join(work, ".git", "info", "attributes"), []byte("* filter=x\n"), 0o644)+ os.WriteFile(filepath.Join(work, ".gitignore"), []byte("build/\n"), 0o644)+ os.MkdirAll(filepath.Join(work, "build"), 0o755)+ os.WriteFile(filepath.Join(work, "build", "out.js"), []byte("x"), 0o644)os.WriteFile(filepath.Join(work, "new.go"), []byte("package main\n"), 0o644)os.WriteFile(filepath.Join(work, "main.go"), []byte("package main\n\nfunc main() {}\n"), 0o644)@@ -38,13 +62,61 @@ func TestWorkspaceGitIgnoresRepoConfig(t *testing.T) {if !ok || !slices.Contains(files, "main.go") || !slices.Contains(files, "new.go") || slices.Contains(files, "work/main.go") {t.Errorf("gitVisibleFiles = %v, %v", files, ok)}- changed, err := changedFiles(work)- if err != nil || !slices.Contains(changed, "main.go") || !slices.Contains(changed, "new.go") {- t.Errorf("changedFiles = %v, %v", changed, err)+ changed, err := changedFiles(work, pristine)+ if err != nil {+ t.Fatal(err)+ }+ for _, want := range []string{"main.go", "util.go", "gone.go", "new.go"} {+ if !slices.Contains(changed, want) {+ t.Errorf("changedFiles = %v, missing %s", changed, want)+ }+ }+ if slices.Contains(changed, "build/out.js") {+ t.Errorf("changedFiles = %v: ignored build output must not count", changed)}for _, p := range []string{filepath.Join(dir, "PWNED"), filepath.Join(work, "PWNED")} {if _, err := os.Stat(p); err == nil {t.Fatalf("git ran a program from the workspace's config (%s)", p)}}++ // A .git file pointing at another repository is refused.+ os.Rename(filepath.Join(work, ".git"), filepath.Join(dir, "elsewhere"))+ os.WriteFile(filepath.Join(work, ".git"), []byte("gitdir: "+filepath.Join(dir, "elsewhere")+"\n"), 0o644)+ if _, ok := gitVisibleFiles(work); ok {+ t.Fatal("a gitdir: file must not be followed")+ }+}++// TestGraderReadsSkipPlantedFiles: symlinks and FIFOs in the workspace are+// never read through, and the hidden-test copy does not write through a+// planted link.+func TestGraderReadsSkipPlantedFiles(t *testing.T) {+ dir := t.TempDir()+ work := filepath.Join(dir, "work")+ outside := filepath.Join(dir, "outside")+ os.MkdirAll(work, 0o755)+ os.MkdirAll(outside, 0o755)+ os.WriteFile(filepath.Join(outside, "secret"), []byte("token-123"), 0o644)+ os.Symlink(filepath.Join(outside, "secret"), filepath.Join(work, "leak.go"))+ if _, ok := readRegular(filepath.Join(work, "leak.go")); ok {+ t.Fatal("a symlink must not be read through")+ }+ if err := checkNoTokens(work, []string{"token-123"}); err != nil {+ t.Fatalf("a symlink to a file outside the workspace is not a leak in it: %v", err)+ }++ src := filepath.Join(dir, "hidden")+ os.MkdirAll(filepath.Join(src, "pkg"), 0o755)+ os.WriteFile(filepath.Join(src, "pkg", "h_test.go"), []byte("package pkg\n"), 0o644)+ os.Symlink(outside, filepath.Join(work, "pkg"))+ if err := copyTree(src, work); err != nil {+ t.Fatal(err)+ }+ if _, err := os.Stat(filepath.Join(outside, "h_test.go")); err == nil {+ t.Fatal("the hidden-test copy wrote through a symlinked directory")+ }+ if fi, err := os.Lstat(filepath.Join(work, "pkg", "h_test.go")); err != nil || !fi.Mode().IsRegular() {+ t.Fatalf("the hidden test must land in the workspace: %v", err)+ }}diff --git a/cli/test/evals/internal/guard/manual.go b/cli/test/evals/internal/guard/manual.goindex 0a3df9fa..7f22147d 100644--- a/cli/test/evals/internal/guard/manual.go+++ b/cli/test/evals/internal/guard/manual.go@@ -1,7 +1,7 @@// Package guard enforces the manual-only policy in code: paid// commands start in CI only inside a hand-dispatched cli-evals.yml run behind-// the evals environment gate, locally only after a typed confirmation, and-// always under a mandatory cost cap. The real OpenRouter client asks Live()+// the evals environment gate, locally only after a confirmation typed on a+// terminal, and always under a mandatory cost cap. The real OpenRouter client asks Live()// before every request, and Live() is false inside `go test`.package guard@@ -10,6 +10,7 @@ import ("errors""fmt""io"+ "math""strings""sync/atomic""testing"@@ -41,9 +42,11 @@ var otherCI = []string{type Request struct {Command string // run | resumeMaxCostUSD float64- Yes bool- StdinTTY bool- Getenv func(string) string+ // Yes is the workflow's --yes. It never replaces the typed confirmation+ // of a local run.+ Yes bool+ StdinTTY bool+ Getenv func(string) string// Preflight is printed before the local confirmation prompt.Preflight stringIn io.Reader@@ -58,8 +61,8 @@ type Decision struct {// Decide applies the policy without side effects or prompts.func Decide(r Request) (Decision, error) {- if r.MaxCostUSD <= 0 {- return Decision{}, errors.New("--max-cost-usd is required for paid commands and must be greater than 0 (there is no default)")+ if err := CheckCap(r.MaxCostUSD); err != nil {+ return Decision{}, err}env := r.Getenvif env("GITHUB_ACTIONS") == "true" {@@ -83,14 +86,22 @@ func Decide(r Request) (Decision, error) {}}if !r.StdinTTY {- if !r.Yes {- return Decision{}, errors.New("stdin is not a terminal: refusing a paid run without --yes (local paid runs are for harness development only)")- }- return Decision{}, nil+ // A script or an agent's shell has no terminal: a local paid run+ // needs a person to type yes, and --yes does not stand in for one.+ return Decision{}, errors.New("stdin is not a terminal: a local paid run needs you to type yes on a terminal (--yes is for the dispatched workflow only)")}return Decision{Confirm: true}, nil}+// CheckCap validates a --max-cost-usd value: a finite number above 0. NaN+// and Inf parse as floats and would otherwise lift every cap.+func CheckCap(usd float64) error {+ if math.IsNaN(usd) || math.IsInf(usd, 0) || usd <= 0 {+ return fmt.Errorf("--max-cost-usd is required for paid commands and must be a finite number greater than 0 (got %v; there is no default)", usd)+ }+ return nil+}+// Authorize applies the policy, prompts for the typed confirmation when one is// needed, and unlocks Live() for the rest of the process.func Authorize(r Request) error {diff --git a/cli/test/evals/internal/guard/manual_test.go b/cli/test/evals/internal/guard/manual_test.goindex 75337078..d813a686 100644--- a/cli/test/evals/internal/guard/manual_test.go+++ b/cli/test/evals/internal/guard/manual_test.go@@ -1,6 +1,7 @@package guardimport (+ "math""strings""testing")@@ -47,8 +48,11 @@ func TestDecide(t *testing.T) {{name: "buildkite", env: map[string]string{"BUILDKITE": "true"}, yes: true, cost: 10, wantErr: "BUILDKITE"},{name: "gitlab", env: map[string]string{"GITLAB_CI": "true"}, yes: true, cost: 10, wantErr: "GITLAB_CI"},{name: "jenkins", env: map[string]string{"JENKINS_URL": "http://ci"}, yes: true, cost: 10, wantErr: "JENKINS_URL"},- {name: "local non-tty without yes", env: map[string]string{}, cost: 10, wantErr: "without --yes"},- {name: "local non-tty with yes", env: map[string]string{}, yes: true, cost: 10, want: Decision{}},+ {name: "NaN cap", env: dispatched, yes: true, cost: math.NaN(), wantErr: "finite number"},+ {name: "Inf cap", env: dispatched, yes: true, cost: math.Inf(1), wantErr: "finite number"},+ {name: "negative cap", env: map[string]string{}, tty: true, cost: -1, wantErr: "--max-cost-usd is required"},+ {name: "local non-tty without yes", env: map[string]string{}, cost: 10, wantErr: "not a terminal"},+ {name: "local non-tty with yes", env: map[string]string{}, yes: true, cost: 10, wantErr: "not a terminal"},{name: "local tty asks", env: map[string]string{}, tty: true, cost: 10, want: Decision{Confirm: true}},{name: "local tty asks even with yes", env: map[string]string{}, tty: true, yes: true, cost: 10, want: Decision{Confirm: true}},}@@ -72,7 +76,8 @@ func TestDecide(t *testing.T) {}func TestLiveIsNeverOpenUnderGoTest(t *testing.T) {- err := Authorize(Request{Command: "run", MaxCostUSD: 1, Yes: true, Getenv: envOf(map[string]string{})})+ var out strings.Builder+ err := Authorize(Request{Command: "run", MaxCostUSD: 1, StdinTTY: true, Getenv: envOf(map[string]string{}), In: strings.NewReader("yes\n"), Out: &out})if err != nil {t.Fatal(err)}diff --git a/cli/test/evals/internal/guard/triggers.go b/cli/test/evals/internal/guard/triggers.goindex ffb6e2ee..9000ced6 100644--- a/cli/test/evals/internal/guard/triggers.go+++ b/cli/test/evals/internal/guard/triggers.go@@ -10,55 +10,205 @@ import ("gopkg.in/yaml.v3")-// paidRunRe matches a command line that starts a paid eval run.-var paidRunRe = regexp.MustCompile(`twevals(\s+|\S*\s+)(run|resume)\b|eval-local`)+// paidRunRe matches a command line that starts a paid eval run, also when a+// backslash continues it onto the next line.+var paidRunRe = regexp.MustCompile(`twevals\S*[\s\\]+(run|resume)\b|eval-local`)++// dispatchRe matches the other ways to start the evals workflow or reach its+// key: the recipe that dispatches it and the secret's name.+var dispatchRe = regexp.MustCompile(`eval-dispatch|OPENROUTER_API_KEY`)// CheckWorkflows enforces the dispatch-only rule over a .github/workflows// directory: cli-evals.yml exists, its `on:` is exactly workflow_dispatch, the// approval flag is set only by a job behind the evals environment, and no-// other workflow references it or starts a paid run.+// other workflow (or composite action next to it) references it, dispatches+// it, uses the evals environment, or starts a paid run.func CheckWorkflows(dir string) error {evalsPath := filepath.Join(dir, filepath.Base(WorkflowFile))raw, err := os.ReadFile(evalsPath)if err != nil {return fmt.Errorf("%s must exist: %w", WorkflowFile, err)}- if err := checkEvalsWorkflow(raw); err != nil {+ evalsName, err := checkEvalsWorkflow(raw)+ if err != nil {return fmt.Errorf("%s: %w", WorkflowFile, err)}+ var files []stringentries, err := os.ReadDir(dir)if err != nil {return err}for _, e := range entries {- name := e.Name()- if e.IsDir() || name == filepath.Base(WorkflowFile) || !(strings.HasSuffix(name, ".yml") || strings.HasSuffix(name, ".yaml")) {- continue+ if !e.IsDir() && e.Name() != filepath.Base(WorkflowFile) && isYAML(e.Name()) {+ files = append(files, filepath.Join(dir, e.Name()))+ }+ }+ actions := filepath.Join(dir, "..", "actions")+ _ = filepath.WalkDir(actions, func(p string, d os.DirEntry, err error) error {+ if err == nil && !d.IsDir() && isYAML(d.Name()) {+ files = append(files, p)}- other, err := os.ReadFile(filepath.Join(dir, name))+ return nil+ })+ for _, path := range files {+ name, _ := filepath.Rel(filepath.Dir(dir), path)+ other, err := os.ReadFile(path)if err != nil {return err}text := string(other)- if strings.Contains(text, "cli-evals.yml") || strings.Contains(text, "cli-evals.yaml") {- return fmt.Errorf("%s references cli-evals.yml; no workflow may call or dispatch the evals", name)+ if strings.Contains(text, "cli-evals.yml") || strings.Contains(text, "cli-evals.yaml") ||+ (evalsName != "" && strings.Contains(text, evalsName)) || dispatchRe.MatchString(text) {+ return fmt.Errorf("%s references the evals workflow, its dispatch recipe or its key; no workflow may call or dispatch the evals", name)}if paidRunRe.MatchString(text) || strings.Contains(text, "TWEVALS_APPROVED") {return fmt.Errorf("%s starts a paid eval run; only %s may, and only when dispatched by hand", name, WorkflowFile)}+ if usesEvalsEnvironment(other) {+ return fmt.Errorf("%s uses the evals environment, which holds the OpenRouter key; only %s may", name, WorkflowFile)+ }}return nil}-func checkEvalsWorkflow(raw []byte) error {+var (+ justHeaderRe = regexp.MustCompile(`^@?([A-Za-z_][A-Za-z0-9_-]*)[^:=]*:([^=].*|)$`)+ justAliasRe = regexp.MustCompile(`^alias\s+([A-Za-z0-9_-]+)\s*:=\s*([A-Za-z0-9_-]+)`)+ justCallRe = regexp.MustCompile(`\bjust\s+(?:-\S+\s+)*([A-Za-z_][A-Za-z0-9_-]*)`)+)++// CheckJustfile enforces that no recipe reachable from default, check or+// test (through dependencies or a `just <recipe>` line) starts a paid eval+// run or dispatches the evals workflow.+func CheckJustfile(path string) error {+ raw, err := os.ReadFile(path)+ if err != nil {+ return err+ }+ type recipe struct {+ deps []string+ body string+ }+ recipes := map[string]*recipe{}+ alias := map[string]string{}+ var cur *recipe+ for _, line := range strings.Split(string(raw), "\n") {+ if line != "" && line[0] != ' ' && line[0] != '\t' {+ cur = nil+ if m := justAliasRe.FindStringSubmatch(line); m != nil {+ alias[m[1]] = m[2]+ continue+ }+ if strings.HasPrefix(line, "#") || strings.HasPrefix(line, "[") {+ continue+ }+ if m := justHeaderRe.FindStringSubmatch(line); m != nil {+ cur = &recipe{}+ recipes[m[1]] = cur+ for _, d := range strings.Fields(m[2]) {+ if d == "&&" {+ continue+ }+ cur.deps = append(cur.deps, strings.SplitN(d, "(", 2)[0])+ }+ }+ continue+ }+ if cur != nil {+ cur.body += line + "\n"+ }+ }+ for _, root := range []string{"default", "check", "test"} {+ seen := map[string]bool{}+ var walk func(name string) error+ walk = func(name string) error {+ if a, ok := alias[name]; ok {+ name = a+ }+ r, ok := recipes[name]+ if seen[name] || !ok {+ return nil+ }+ seen[name] = true+ if name == "eval-local" || name == "eval-dispatch" || paidRunRe.MatchString(r.body) || dispatchRe.MatchString(r.body) ||+ strings.Contains(r.body, "gh workflow run") {+ return fmt.Errorf("just %s reaches the recipe %s, which starts a paid eval run or dispatches the evals", root, name)+ }+ next := r.deps+ for _, m := range justCallRe.FindAllStringSubmatch(r.body, -1) {+ next = append(next, m[1])+ }+ for _, d := range next {+ if err := walk(d); err != nil {+ return err+ }+ }+ return nil+ }+ if err := walk(root); err != nil {+ return err+ }+ }+ return nil+}++func isYAML(name string) bool {+ return strings.HasSuffix(name, ".yml") || strings.HasSuffix(name, ".yaml")+}++// usesEvalsEnvironment reports whether a workflow has a job with+// environment: evals.+func usesEvalsEnvironment(raw []byte) bool {+ var doc yaml.Node+ if yaml.Unmarshal(raw, &doc) != nil || len(doc.Content) == 0 {+ return false+ }+ jobs := mapValue(doc.Content[0], "jobs")+ if jobs == nil || jobs.Kind != yaml.MappingNode {+ return false+ }+ for i := 1; i < len(jobs.Content); i += 2 {+ if environmentOf(jobs.Content[i]) == "evals" {+ return true+ }+ }+ return false+}++func environmentOf(job *yaml.Node) string {+ envNode := mapValue(job, "environment")+ if envNode == nil {+ return ""+ }+ if envNode.Kind == yaml.ScalarNode {+ return envNode.Value+ }+ if n := mapValue(envNode, "name"); n != nil {+ return n.Value+ }+ return ""+}++// checkEvalsWorkflow checks cli-evals.yml and returns its name.+func checkEvalsWorkflow(raw []byte) (string, error) {var doc yaml.Nodeif err := yaml.Unmarshal(raw, &doc); err != nil {- return err+ return "", err}if len(doc.Content) == 0 || doc.Content[0].Kind != yaml.MappingNode {- return fmt.Errorf("not a YAML mapping")+ return "", fmt.Errorf("not a YAML mapping")}root := doc.Content[0]+ if err := noDuplicateKeys(root); err != nil {+ return "", err+ }+ name := ""+ if n := mapValue(root, "name"); n != nil && n.Kind == yaml.ScalarNode {+ name = n.Value+ }+ if env := mapValue(root, "env"); env != nil && strings.Contains(encode(env), "TWEVALS_APPROVED") {+ return "", fmt.Errorf("the workflow-level env sets TWEVALS_APPROVED; only the job behind environment: evals may")+ }on := mapValue(root, "on")if on == nil {// YAML 1.1 readers turn a bare `on` key into true; accept that@@ -66,61 +216,73 @@ func checkEvalsWorkflow(raw []byte) error {on = mapValue(root, "true")}if on == nil {- return fmt.Errorf("has no on: block")+ return "", fmt.Errorf("has no on: block")}switch on.Kind {case yaml.ScalarNode:if on.Value != "workflow_dispatch" {- return fmt.Errorf("on: must be exactly workflow_dispatch, got %q", on.Value)+ return "", fmt.Errorf("on: must be exactly workflow_dispatch, got %q", on.Value)}case yaml.SequenceNode:if len(on.Content) != 1 || on.Content[0].Value != "workflow_dispatch" {- return fmt.Errorf("on: must be exactly [workflow_dispatch]")+ return "", fmt.Errorf("on: must be exactly [workflow_dispatch]")}case yaml.MappingNode:+ if err := noDuplicateKeys(on); err != nil {+ return "", err+ }var keys []stringfor i := 0; i < len(on.Content); i += 2 {keys = append(keys, on.Content[i].Value)}if len(keys) != 1 || keys[0] != "workflow_dispatch" {- return fmt.Errorf("on: must have exactly one trigger, workflow_dispatch; found %v", keys)+ return "", fmt.Errorf("on: must have exactly one trigger, workflow_dispatch; found %v", keys)}default:- return fmt.Errorf("on: has an unexpected shape")+ return "", fmt.Errorf("on: has an unexpected shape")}jobs := mapValue(root, "jobs")if jobs == nil || jobs.Kind != yaml.MappingNode {- return fmt.Errorf("has no jobs")+ return "", fmt.Errorf("has no jobs")}gated := 0for i := 0; i < len(jobs.Content); i += 2 {- name, job := jobs.Content[i].Value, jobs.Content[i+1]- var buf strings.Builder- enc := yaml.NewEncoder(&buf)- _ = enc.Encode(job)- body := buf.String()+ jobName, job := jobs.Content[i].Value, jobs.Content[i+1]+ body := encode(job)approves := strings.Contains(body, "TWEVALS_APPROVED")runsPaid := paidRunRe.MatchString(body)if !approves && !runsPaid {continue}- envNode := mapValue(job, "environment")- envName := ""- if envNode != nil {- if envNode.Kind == yaml.ScalarNode {- envName = envNode.Value- } else if n := mapValue(envNode, "name"); n != nil {- envName = n.Value- }- }- if envName != "evals" {- return fmt.Errorf("job %q starts a paid run or sets TWEVALS_APPROVED without environment: evals", name)+ if environmentOf(job) != "evals" {+ return "", fmt.Errorf("job %q starts a paid run or sets TWEVALS_APPROVED without environment: evals", jobName)}gated++}if gated == 0 {- return fmt.Errorf("no job runs the evals behind environment: evals")+ return "", fmt.Errorf("no job runs the evals behind environment: evals")+ }+ return name, nil+}++func encode(n *yaml.Node) string {+ var buf strings.Builder+ enc := yaml.NewEncoder(&buf)+ _ = enc.Encode(n)+ return buf.String()+}++// noDuplicateKeys rejects a mapping that repeats a key: a reader that keeps+// the last one would see a different workflow than this check does.+func noDuplicateKeys(m *yaml.Node) error {+ seen := map[string]bool{}+ for i := 0; i+1 < len(m.Content); i += 2 {+ k := m.Content[i].Value+ if seen[k] {+ return fmt.Errorf("the key %q appears twice", k)+ }+ seen[k] = true}return nil}diff --git a/cli/test/evals/internal/guard/triggers_test.go b/cli/test/evals/internal/guard/triggers_test.goindex d50f03bd..190795ce 100644--- a/cli/test/evals/internal/guard/triggers_test.go+++ b/cli/test/evals/internal/guard/triggers_test.go@@ -17,6 +17,46 @@ func TestEvalsWorkflowIsDispatchOnly(t *testing.T) {}}+// TestJustfileNeverRunsPaidEvals holds cli/justfile to the same rule: no+// recipe reachable from default, check or test runs or dispatches the evals.+func TestJustfileNeverRunsPaidEvals(t *testing.T) {+ if err := CheckJustfile(filepath.Join("..", "..", "..", "..", "justfile")); err != nil {+ t.Fatal(err)+ }+}++func TestCheckJustfileRejectsPaidRecipes(t *testing.T) {+ base := "default: test\n\ntest:\n go test ./...\n\neval-local *args:\n cd test/evals && go run ./cmd/twevals run {{args}}\n\neval-dispatch *args:\n gh workflow run cli-evals.yml {{args}}\n"+ cases := map[string]struct {+ file string+ want string+ }{+ "good": {file: base},+ "dependency": {file: strings.Replace(base, "test:\n", "test: eval-local\n", 1), want: "reaches the recipe eval-local"},+ "nested just": {file: strings.Replace(base, " go test ./...\n", " go test ./...\n just eval-dispatch -f max_cost_usd=5\n", 1), want: "reaches the recipe test"},+ "inline paid run": {file: base + "\ncheck: lint\n\nlint:\n cd test/evals && go run ./cmd/twevals \\\n run --max-cost-usd 5\n", want: "reaches the recipe lint"},+ "alias": {file: base + "\nalias el := eval-local\n\ncheck: el\n", want: "reaches the recipe eval-local"},+ }+ for name, c := range cases {+ t.Run(name, func(t *testing.T) {+ path := filepath.Join(t.TempDir(), "justfile")+ if err := os.WriteFile(path, []byte(c.file), 0o644); err != nil {+ t.Fatal(err)+ }+ err := CheckJustfile(path)+ if c.want == "" {+ if err != nil {+ t.Fatal(err)+ }+ return+ }+ if err == nil || !strings.Contains(err.Error(), c.want) {+ t.Fatalf("want error containing %q, got %v", c.want, err)+ }+ })+ }+}+func TestCheckWorkflowsRejectsAutomation(t *testing.T) {good := `name: CLI Evalson:@@ -49,13 +89,23 @@ jobs:"workflow_call": {evals: strings.Replace(good, "on:\n", "on:\n workflow_call:\n", 1), want: "exactly one trigger"},"list of triggers": {evals: strings.Replace(good, "on:\n workflow_dispatch:\n inputs:\n max_cost_usd: {type: string, required: true}\n", "on: [workflow_dispatch, push]\n", 1), want: "exactly [workflow_dispatch]"},"ungated paid job": {evals: strings.Replace(good, " environment: evals\n", "", 1), want: "without environment: evals"},- "caller": {evals: good, others: map[string]string{"ci.yml": "jobs:\n x:\n uses: ./.github/workflows/cli-evals.yml\n"}, want: "references cli-evals.yml"},- "dispatcher": {evals: good, others: map[string]string{"nightly.yml": "on: schedule\njobs:\n x:\n steps:\n - run: gh workflow run cli-evals.yml\n"}, want: "references cli-evals.yml"},+ "caller": {evals: good, others: map[string]string{"ci.yml": "jobs:\n x:\n uses: ./.github/workflows/cli-evals.yml\n"}, want: "references the evals workflow"},+ "dispatcher": {evals: good, others: map[string]string{"nightly.yml": "on: schedule\njobs:\n x:\n steps:\n - run: gh workflow run cli-evals.yml\n"}, want: "references the evals workflow"},"other paid run": {evals: good, others: map[string]string{"cli.yml": "jobs:\n x:\n steps:\n - run: cd cli/test/evals && go run ./cmd/twevals run --yes\n"}, want: "starts a paid eval run"},+ "continued line": {evals: strings.Replace(good, " - run: go run ./cmd/twevals plan --max-cost-usd 1\n",+ " - run: |\n go run ./cmd/twevals \\\n run --yes --max-cost-usd 1\n", 1), want: "without environment: evals"},+ "workflow env": {evals: strings.Replace(good, "permissions:\n", "env:\n TWEVALS_APPROVED: \"1\"\npermissions:\n", 1), want: "workflow-level env"},+ "duplicate on": {evals: good + "on: push\n", want: "appears twice"},+ "by name": {evals: good, others: map[string]string{"nightly.yml": "on: schedule\njobs:\n x:\n steps:\n - run: gh workflow run \"CLI Evals\" -f max_cost_usd=5\n"}, want: "references the evals workflow"},+ "by recipe": {evals: good, others: map[string]string{"nightly.yml": "on: schedule\njobs:\n x:\n steps:\n - run: cd cli && just eval-dispatch -f max_cost_usd=5\n"}, want: "references the evals workflow"},+ "evals env": {evals: good, others: map[string]string{"other.yml": "on: push\njobs:\n x:\n environment:\n name: evals\n steps:\n - run: echo\n"}, want: "uses the evals environment"},}for name, c := range cases {t.Run(name, func(t *testing.T) {- dir := t.TempDir()+ dir := filepath.Join(t.TempDir(), "workflows")+ if err := os.MkdirAll(dir, 0o755); err != nil {+ t.Fatal(err)+ }if err := os.WriteFile(filepath.Join(dir, "cli-evals.yml"), []byte(c.evals), 0o644); err != nil {t.Fatal(err)}diff --git a/cli/test/evals/internal/report/compare.go b/cli/test/evals/internal/report/compare.goindex 0b8c2e25..a459b41a 100644--- a/cli/test/evals/internal/report/compare.go+++ b/cli/test/evals/internal/report/compare.go@@ -101,8 +101,10 @@ type Regression struct {}// Compare applies the regression rule: a suite's pass rate drops-// more than 10 percentage points for a model × arm, or a case at 2 of 3 or-// better in the baseline now passes 0 of N.+// more than 10 percentage points for a model × arm, or a case that passed at+// least 2 runs at a rate of 2 of 3 or better in the baseline now passes 0 of N+// (a single passing run is not enough history to call a single failure a+// regression).func Compare(r *Results, b *Baseline) []Regression {a := Aggregate(r.Runs)var out []Regression@@ -121,7 +123,7 @@ func Compare(r *Results, b *Baseline) []Regression {if !ok || before.Total == 0 {continue}- if before.Rate >= 2.0/3.0-1e-9 && now.Pass == 0 && now.Total > 0 {+ if before.Pass >= 2 && before.Rate >= 2.0/3.0-1e-9 && now.Pass == 0 && now.Total > 0 {out = append(out, Regression{Kind: "case", Key: k, Before: before.Rate, After: 0,Detail: fmt.Sprintf("%s passed %d/%d in the baseline and 0/%d now", k, before.Pass, before.Total, now.Total)})}diff --git a/cli/test/evals/internal/report/compare_test.go b/cli/test/evals/internal/report/compare_test.goindex 18077df6..2e27e292 100644--- a/cli/test/evals/internal/report/compare_test.go+++ b/cli/test/evals/internal/report/compare_test.go@@ -23,6 +23,12 @@ func TestCompare(t *testing.T) {if regs := Compare(&Results{Runs: base.casesAsRuns()}, base); len(regs) != 0 {t.Fatalf("no change must not regress: %+v", regs)}+ once := NewBaseline(&Results{Runs: []*runner.Result{res("perf-b", "cli+skill", "opus", 1, "pass", 0), res("perf-c", "cli+skill", "opus", 1, "pass", 0)}})+ for _, r := range Compare(&Results{Runs: []*runner.Result{res("perf-b", "cli+skill", "opus", 1, "fail", 0), res("perf-c", "cli+skill", "opus", 1, "pass", 0)}}, once) {+ if r.Kind == "case" {+ t.Fatalf("a case that passed once in the baseline is not a case regression after one failure: %+v", r)+ }+ }}// casesAsRuns rebuilds runs matching the baseline exactly.diff --git a/cli/test/evals/internal/report/markdown.go b/cli/test/evals/internal/report/markdown.goindex 0a94fab7..c056a195 100644--- a/cli/test/evals/internal/report/markdown.go+++ b/cli/test/evals/internal/report/markdown.go@@ -38,7 +38,10 @@ func Markdown(r *Results) string {// An oracle run costs nothing, so its summary leaves money out.paid := !h.Oracleif paid {- fmt.Fprintf(&b, "- **Spend:** $%.2f (judge $%.2f) of a $%.2f cap\n", a.SpendUSD, a.JudgeUSD, h.MaxCostUSD)+ fmt.Fprintf(&b, "- **Spend:** $%.2f (judge $%.2f) of a $%.2f cap\n", a.SpendUSD+h.ReplacedSpendUSD, a.JudgeUSD, h.MaxCostUSD)+ if h.ReplacedSpendUSD > 0 {+ fmt.Fprintf(&b, " (including $%.2f spent on runs that a resume ran again)\n", h.ReplacedSpendUSD)+ }}var st []stringfor k, v := range a.Statuses {diff --git a/cli/test/evals/internal/report/results.go b/cli/test/evals/internal/report/results.goindex 0bd0bb74..3d372be4 100644--- a/cli/test/evals/internal/report/results.go+++ b/cli/test/evals/internal/report/results.go@@ -37,8 +37,10 @@ type Header struct {Started time.Time `json:"started"`Finished time.Time `json:"finished"`MaxCostUSD float64 `json:"maxCostUsd"`- Oracle bool `json:"oracle,omitempty"`- CI map[string]string `json:"ci,omitempty"`+ // ReplacedSpendUSD is what the runs a resume ran again had cost.+ ReplacedSpendUSD float64 `json:"replacedSpendUsd,omitempty"`+ Oracle bool `json:"oracle,omitempty"`+ CI map[string]string `json:"ci,omitempty"`}// Results is results.json.@@ -151,21 +153,41 @@ func Load(dir string) (*Results, error) {return r, nil}-// Merge combines shard results under one header.-func Merge(parts []*Results) *Results {+// Merge combines shard results under one header. A run that appears in+// two parts (a merged directory passed with its own shards, or the same shard+// twice) is an error, and parts from different commits, models or binaries+// are reported in the returned notes.+func Merge(parts []*Results) (*Results, []string, error) {out := &Results{}- total := 0.0+ total, replaced := 0.0, 0.0+ from := map[string]int{}+ var notes []stringfor i, p := range parts {if i == 0 {out.Header = p.Header+ } else {+ h := out.Header+ for _, d := range [][3]string{{"commit", h.GitSHA, p.Header.GitSHA}, {"models", h.ModelsDigest, p.Header.ModelsDigest}, {"binaries", h.BinariesHash, p.Header.BinariesHash}} {+ if d[1] != d[2] {+ notes = append(notes, fmt.Sprintf("part %d (%s) has a different %s than part 1 (%s): %q vs %q", i+1, p.Header.RunID, d[0], h.RunID, d[2], d[1]))+ }+ }}total += p.Header.MaxCostUSD+ replaced += p.Header.ReplacedSpendUSDif p.Header.Finished.After(out.Header.Finished) {out.Header.Finished = p.Header.Finished}+ for _, r := range p.Runs {+ if j, ok := from[r.Key()]; ok {+ return nil, nil, fmt.Errorf("run %s is in part %d and part %d: pass each shard once, and not a merged directory together with its shards", r.Key(), j+1, i+1)+ }+ from[r.Key()] = i+ }out.Runs = append(out.Runs, p.Runs...)}out.Header.MaxCostUSD = total+ out.Header.ReplacedSpendUSD = replacedsort.SliceStable(out.Runs, func(i, j int) bool { return out.Runs[i].Key() < out.Runs[j].Key() })- return out+ return out, notes, nil}diff --git a/cli/test/evals/internal/runner/container.go b/cli/test/evals/internal/runner/container.goindex 6a7b4da2..d52d1766 100644--- a/cli/test/evals/internal/runner/container.go+++ b/cli/test/evals/internal/runner/container.go@@ -13,6 +13,7 @@ import ("sort""strconv""strings"+ "sync""syscall""time"@@ -57,6 +58,11 @@ type ContainerShell struct {host booldir stringpath string+ // pgids are the host-mode process groups, killed on Close so a+ // background process the commands started cannot outlive the shell.+ mu sync.Mutex+ pgids []int+ closed bool}// workdir is where the repository is mounted for the shell.@@ -67,9 +73,9 @@ func (c *ContainerShell) workdir() string {return "/work"}-func setupEnv(c *catalog.Case, inst *env.Instance, projects []env.Project, port int) []string {+func setupEnv(c *catalog.Case, inst *env.Instance, projects []env.Project, port int) ([]string, error) {if c.Setup == nil {- return nil+ return nil, nil}var out []stringkeys := make([]string, 0, len(c.Setup.Env))@@ -86,15 +92,19 @@ func setupEnv(c *catalog.Case, inst *env.Instance, projects []env.Project, portv = strconv.Itoa(port)case strings.HasPrefix(v, "@token:"):name := strings.TrimPrefix(v, "@token:")+ found := falsefor _, p := range projects {if p.Name == name {- v = p.Token+ v, found = p.Token, true}}+ if !found {+ return nil, fmt.Errorf("setup.env %s names project %q, which the seed did not create", k, name)+ }}out = append(out, k+"="+v)}- return out+ return out, nil}func startContainerShell(ctx context.Context, opt Options, work string, surf surface) (*ContainerShell, error) {@@ -127,9 +137,19 @@ func startContainerShell(ctx context.Context, opt Options, work string, surf surreturn &ContainerShell{name: name, timeout: 10 * time.Minute, env: surf.containerEnv}, nil}-// Close removes the container.+// Close removes the container (on the host: kills every process group the+// shell started). It may be called more than once.func (c *ContainerShell) Close() {+ c.mu.Lock()+ defer c.mu.Unlock()+ if c.closed {+ return+ }+ c.closed = trueif c.host {+ for _, g := range c.pgids {+ _ = syscall.Kill(-g, syscall.SIGKILL)+ }return}_ = exec.Command("docker", "rm", "-f", c.name).Run()@@ -178,7 +198,15 @@ func (c *ContainerShell) Exec(ctx context.Context, cmdline string, timeout time.}var out bytes.Buffercmd.Stdout, cmd.Stderr = &out, &out- err := cmd.Run()+ err := cmd.Start()+ if err == nil {+ if c.host {+ c.mu.Lock()+ c.pgids = append(c.pgids, cmd.Process.Pid)+ c.mu.Unlock()+ }+ err = cmd.Wait()+ }code := 0if err != nil {if ee, ok := err.(*exec.ExitError); ok {@@ -273,6 +301,14 @@ func verifySetup(ctx context.Context, opt Options, c *catalog.Case, work string,for _, f := range vs.EnvFiles {source += fmt.Sprintf("if [ -f %s ]; then set -a; . ./%s; set +a; fi; ", f, f)}+ // The harness's variables (URL, token, PORT) win over an env file the+ // agent wrote: the readiness check polls PORT.+ if source != "" {+ for _, e := range cs.env {+ k, v, _ := strings.Cut(e, "=")+ source += "export " + k + "=" + shellQuoteArg(v) + "; "+ }+ }if vs.Build != "" {out, code := cs.Exec(ctx, cd+" && "+vs.Build, 10*time.Minute)if code != 0 {diff --git a/cli/test/evals/internal/runner/execute_test.go b/cli/test/evals/internal/runner/execute_test.gonew file mode 100644index 00000000..f63a40c7--- /dev/null+++ b/cli/test/evals/internal/runner/execute_test.go@@ -0,0 +1,94 @@+package runner++import (+ "context"+ "fmt"+ "math"+ "sync"+ "testing"++ "github.com/tracewayapp/traceway/cli/test/evals/internal/agent"+ "github.com/tracewayapp/traceway/cli/test/evals/internal/catalog"+)++// fakeRuns replaces runOne: each run spends spend(i) through its hold and+// records the cap it was given.+func fakeRuns(t *testing.T, spend func(n int) float64) (caps *[]float64) {+ t.Helper()+ var mu sync.Mutex+ var got []float64+ n := 0+ orig := startRun+ startRun = func(_ context.Context, _ Options, j Job, hold *agent.Hold) *Result {+ mu.Lock()+ got = append(got, hold.Cap)+ i := n+ n+++ mu.Unlock()+ hold.Add("m", min(spend(i), hold.Cap))+ return &Result{Case: j.Unit.Case.ID, Arm: j.Unit.Arm, Model: j.Model.Key, RunIdx: j.Idx, Status: "pass"}+ }+ t.Cleanup(func() { startRun = orig })+ return &got+}++func jobsN(n int) []Job {+ m := &catalog.Model{Key: "m"}+ var out []Job+ for i := 1; i <= n; i++ {+ out = append(out, Job{Unit: catalog.Unit{Case: &catalog.Case{ID: fmt.Sprintf("perf-c%d", i), Suite: "perf"}, Arm: "cli+skill"}, Model: m, Idx: 1})+ }+ return out+}++// TestExecuteSharesTheCapAmongTheRunsLeft: two runs at -j 4 each get half the+// shard's cap, not a quarter.+func TestExecuteSharesTheCapAmongTheRunsLeft(t *testing.T) {+ caps := fakeRuns(t, func(int) float64 { return 0.5 })+ ledger := agent.NewLedger(3.75)+ res := Execute(context.Background(), Options{Jobs: 4, Ledger: ledger}, jobsN(2))+ if len(res) != 2 {+ t.Fatalf("%d results", len(res))+ }+ for _, c := range *caps {+ if math.Abs(c-1.875) > 1e-9 {+ t.Fatalf("caps %v, want 1.875 each", *caps)+ }+ }+}++// TestExecuteNeverPromisesMoreThanTheCap: with every run spending its whole+// reservation, the total stays at the cap and the runs that no longer fit+// are recorded as cost_ceiling, not started.+func TestExecuteNeverPromisesMoreThanTheCap(t *testing.T) {+ fakeRuns(t, func(int) float64 { return 100 })+ ledger := agent.NewLedger(5)+ res := Execute(context.Background(), Options{Jobs: 4, Ledger: ledger}, jobsN(12))+ if ledger.Spent() > 5+1e-9 {+ t.Fatalf("spent $%.2f of a $5 cap", ledger.Spent())+ }+ ceiling := 0+ for _, r := range res {+ if r.Status == agent.StopCeiling {+ ceiling+++ }+ }+ if len(res) != 12 || ceiling == 0 {+ t.Fatalf("%d results, %d at the ceiling", len(res), ceiling)+ }+}++// TestExecuteResumeSharesAmongTheRemainingRuns: skipped (already finished)+// runs do not shrink the share of the ones a resume still has to do.+func TestExecuteResumeSharesAmongTheRemainingRuns(t *testing.T) {+ caps := fakeRuns(t, func(int) float64 { return 0.1 })+ jobs := jobsN(60)+ skip := map[string]bool{}+ for _, j := range jobs[2:] {+ skip[RunKey(j.Unit.Case.ID, j.Unit.Arm, j.Model.Key, j.Idx)] = true+ }+ res := Execute(context.Background(), Options{Jobs: 4, Ledger: agent.NewLedger(5), Skip: skip}, jobs)+ if len(res) != 2 || len(*caps) != 2 || (*caps)[0] < 2.5-1e-9 {+ t.Fatalf("%d results, caps %v: each of the 2 remaining runs should get at least half of $5", len(res), *caps)+ }+}diff --git a/cli/test/evals/internal/runner/hooks.go b/cli/test/evals/internal/runner/hooks.goindex da7306f4..0b4899f2 100644--- a/cli/test/evals/internal/runner/hooks.go+++ b/cli/test/evals/internal/runner/hooks.go@@ -23,6 +23,9 @@ type hookRunner struct {pid stringsink *seed.LiveSinkmu sync.Mutex+ // errs are hooks that failed: the run did not happen as the case+ // describes, so it is an infra error rather than the model's result.+ errs []string}func newHookRunner(c *catalog.Case, inst *env.Instance, primary env.Project, sink *seed.LiveSink) *hookRunner {@@ -50,11 +53,20 @@ func (h *hookRunner) after(ctx context.Context, cl *record.Call) {continue}h.counts[i]++- h.run(ctx, hk)+ if err := h.run(ctx, hk); err != nil {+ h.errs = append(h.errs, fmt.Sprintf("%s after %s: %v", hk.Action, cl.ID, err))+ }}}-func (h *hookRunner) run(ctx context.Context, hk catalog.Hook) {+// errors returns the hooks that failed.+func (h *hookRunner) errors() []string {+ h.mu.Lock()+ defer h.mu.Unlock()+ return append([]string(nil), h.errs...)+}++func (h *hookRunner) run(ctx context.Context, hk catalog.Hook) error {api := h.inst.Backend.API().WithBearer(h.inst.Admin.JWT)arg := func(k string) string { return fmt.Sprint(hk.Args[k]) }switch hk.Action {@@ -62,22 +74,45 @@ func (h *hookRunner) run(ctx context.Context, hk catalog.Hook) {// A teammate edits the document between the agent's read and write.// The PUT replaces the whole document (title required), so the// title, tags and incident link are carried over from a fresh read.- body := map[string]any{"contentMd": arg("content")}- if cur, err := api.Do(ctx, http.MethodGet, "/api/post-mortems/"+arg("id"), env.ProjectQuery(h.pid), nil); err == nil && cur.OK() {- m := cur.Map()- body["title"], body["tags"], body["incidentId"] = m["title"], m["tags"], m["incidentId"]+ path := "/api/post-mortems/" + arg("id")+ cur, err := api.Do(ctx, http.MethodGet, path, env.ProjectQuery(h.pid), nil)+ if err != nil {+ return err+ }+ if !cur.OK() {+ return fmt.Errorf("GET %s: HTTP %d", path, cur.Status)+ }+ m := cur.Map()+ body := map[string]any{"contentMd": arg("content"), "title": m["title"], "tags": m["tags"], "incidentId": m["incidentId"]}+ res, err := api.Do(ctx, http.MethodPut, path, env.ProjectQuery(h.pid), body)+ if err != nil {+ return err+ }+ if !res.OK() {+ return fmt.Errorf("PUT %s: HTTP %d", path, res.Status)}- _, _ = api.Do(ctx, http.MethodPut, "/api/post-mortems/"+arg("id"), env.ProjectQuery(h.pid), body)case "stub_healthy":h.inst.Stub.SetHealthy(true)case "stub_unhealthy":h.inst.Stub.SetHealthy(false)case "sleep":- n, _ := strconv.Atoi(arg("seconds"))- time.Sleep(time.Duration(n) * time.Second)+ n, err := strconv.Atoi(arg("seconds"))+ if err != nil {+ return fmt.Errorf("seconds: %w", err)+ }+ select {+ case <-ctx.Done():+ return ctx.Err()+ case <-time.After(time.Duration(n) * time.Second):+ }case "approve_setup_draft":- _, _ = env.ApprovePendingDraft(ctx, h.inst.Backend, h.inst.Admin)+ if _, err := env.ApprovePendingDraft(ctx, h.inst.Backend, h.inst.Admin); err != nil {+ return err+ }+ default:+ return fmt.Errorf("unknown hook action %q", hk.Action)}+ return nil}// startDraftApprover approves pending setup drafts while the run lasts, asdiff --git a/cli/test/evals/internal/runner/runner.go b/cli/test/evals/internal/runner/runner.goindex d1139af6..873b5847 100644--- a/cli/test/evals/internal/runner/runner.go+++ b/cli/test/evals/internal/runner/runner.go@@ -12,6 +12,7 @@ import ("path/filepath""strings""sync"+ "sync/atomic""time""github.com/tracewayapp/traceway/cli/test/evals/internal/agent"@@ -110,46 +111,74 @@ func Jobs(units []catalog.Unit, models []*catalog.Model, runs int) []Job {return out}-// Execute runs every job with up to Jobs in parallel. Before each run starts-// it checks the ledger: at the ceiling, in-flight runs finish, nothing new-// starts, and the remaining runs are recorded as cost_ceiling.+// Execute runs every job with up to Jobs in parallel. With a ceiling, each+// run reserves its cap from the ledger as it starts: up to three times its+// even share of --max-cost-usd, and no more than an even split of what is+// neither spent nor reserved among the runs that can still be in flight. A run+// whose split falls below its even share waits for a run in flight to return+// what it did not use; with nothing in flight it is not started and is+// recorded as cost_ceiling. After an interrupt nothing new starts.func Execute(ctx context.Context, opt Options, jobs []Job) []*Result {par := opt.Jobsif par <= 0 {par = 1}+ var todo []int+ for i, j := range jobs {+ if !opt.Skip[RunKey(j.Unit.Case.ID, j.Unit.Arm, j.Model.Key, j.Idx)] {+ todo = append(todo, i)+ }+ }results := make([]*Result, len(jobs))sem := make(chan struct{}, par)+ finished := make(chan struct{}, len(todo))+ var inFlight atomic.Int32var wg sync.WaitGroup- perRun := 0.0- if opt.Ledger != nil && opt.Ledger.Cap > 0 && len(jobs) > 0 {- perRun = opt.Ledger.Cap / float64(len(jobs)) * 3- }- for i, j := range jobs {- key := RunKey(j.Unit.Case.ID, j.Unit.Arm, j.Model.Key, j.Idx)- if opt.Skip[key] {- continue+ capped := opt.Ledger != nil && opt.Ledger.Cap > 0 && len(todo) > 0+ even, perRun := 0.0, 0.0+ if capped {+ even = opt.Ledger.Cap / float64(len(todo))+ perRun = even * 3+ }+ for n, i := range todo {+ j := jobs[i]+ notStarted := func(status, why string) {+ results[i] = &Result{Case: j.Unit.Case.ID, Suite: j.Unit.Case.Suite, Arm: j.Unit.Arm, Model: j.Model.Key, ModelID: j.Model.ID, RunIdx: j.Idx,+ Status: status, Error: "not started: " + why}}sem <- struct{}{}- if opt.Ledger != nil && opt.Ledger.Exhausted() {+ if ctx.Err() != nil {<-sem- results[i] = &Result{Case: j.Unit.Case.ID, Suite: j.Unit.Case.Suite, Arm: j.Unit.Arm, Model: j.Model.Key, ModelID: j.Model.ID, RunIdx: j.Idx,- Status: agent.StopCeiling, Error: "not started: the --max-cost-usd ceiling was reached"}+ notStarted(agent.StopInfra, "interrupted")continue}+ var hold *agent.Hold+ if capped {+ slots := float64(min(par, len(todo)-n))+ share := min(perRun, opt.Ledger.Available()/slots)+ for share < even && inFlight.Load() > 0 && ctx.Err() == nil {+ <-finished+ share = min(perRun, opt.Ledger.Available()/slots)+ }+ if share < even || ctx.Err() != nil {+ <-sem+ notStarted(agent.StopCeiling, "the --max-cost-usd ceiling was reached")+ continue+ }+ hold = opt.Ledger.Reserve(share)+ }+ inFlight.Add(1)wg.Add(1)- go func(i int, j Job) {+ go func(i int, j Job, hold *agent.Hold) {defer wg.Done()- defer func() { <-sem }()- // Each run may use up to three times its even share, but no more- // than its slice of what is left, so the runs in flight cannot- // together overshoot the ceiling by much.- cap := perRun- if opt.Ledger != nil && cap > 0 {- cap = min(cap, opt.Ledger.Remaining()/float64(par))- }- results[i] = runOne(ctx, opt, j, cap)- }(i, j)+ defer func() {+ hold.Release()+ inFlight.Add(-1)+ finished <- struct{}{}+ <-sem+ }()+ results[i] = startRun(ctx, opt, j, hold)+ }(i, j, hold)}wg.Wait()var out []*Result@@ -161,6 +190,9 @@ func Execute(ctx context.Context, opt Options, jobs []Job) []*Result {return out}+// startRun is runOne; the scheduling tests replace it.+var startRun = runOne+func logf(opt Options, format string, a ...any) {if opt.Log != nil {fmt.Fprintf(opt.Log, format+"\n", a...)@@ -191,7 +223,7 @@ func submitFor(suite string) string {}}-func runOne(ctx context.Context, opt Options, j Job, runCap float64) (res *Result) {+func runOne(ctx context.Context, opt Options, j Job, hold *agent.Hold) (res *Result) {c := j.Unit.Casearm := j.Unit.Armres = &Result{Case: c.ID, Suite: c.Suite, Arm: arm, Model: j.Model.Key, ModelID: j.Model.ID, RunIdx: j.Idx}@@ -283,12 +315,20 @@ func runOne(ctx context.Context, opt Options, j Job, runCap float64) (res *Resulif err := installSkills(opt.RepoRoot, work, surf.skills); err != nil {return infra(err)}+ var pristine map[string]string+ if resolved.Expect.Code != nil {+ if pristine, err = grade.Snapshot(work); err != nil {+ return infra(err)+ }+ }if surf.setupShell {port, err := env.FreePort()if err != nil {return infra(err)}- surf.containerEnv = setupEnv(resolved, inst, projects, port)+ if surf.containerEnv, err = setupEnv(resolved, inst, projects, port); err != nil {+ return infra(err)+ }}sb := &tools.Sandbox{Root: work, ReadOnly: []string{".skills"}}@@ -308,12 +348,14 @@ func runOne(ctx context.Context, opt Options, j Job, runCap float64) (res *Resul}return false, false}+ var agentShell *ContainerShellif surf.setupShell {cs, err := startContainerShell(ctx, opt, work, surf)if err != nil {return infra(err)}defer cs.Close()+ agentShell = cstoolset = append(toolset, cs)} else {sh := &tools.Shell{Sandbox: sb, Git: true, Denied: deniedFor(arm)}@@ -381,7 +423,7 @@ func runOne(ctx context.Context, opt Options, j Job, runCap float64) (res *Resul}run := agent.Run(ctx, agent.Config{Model: model, Chat: chat, Tools: toolset, System: system, Prompt: prompt, SubmitTool: surf.submit,- Budget: resolved.Budget, RunCap: runCap, Ledger: opt.Ledger, Price: opt.Prices[j.Model.Key], Transcript: tr,+ Budget: resolved.Budget, Hold: hold, Ledger: opt.Ledger, Price: opt.Prices[j.Model.Key], Transcript: tr,AfterCall: func(ctx context.Context, cl *record.Call) {callsMu.Lock()calls = append(calls, cl)@@ -392,6 +434,11 @@ func runOne(ctx context.Context, opt Options, j Job, runCap float64) (res *ResulSkillLoaded: func() []string { return loaded },})writeJSON(filepath.Join(dir, "run.json"), run)+ if agentShell != nil {+ // Whatever the agent left running (its own copy of the app on PORT)+ // must be gone before the verify step starts the app afresh.+ agentShell.Close()+ }res.Providers = run.Providersres.Metrics = metricsOf(run)@@ -400,6 +447,9 @@ func runOne(ctx context.Context, opt Options, j Job, runCap float64) (res *Resulres.Error = "oracle: " + strings.Join(errs, "; ")}}+ if errs := hooks.errors(); len(errs) > 0 {+ return infra(fmt.Errorf("hook failed: %s", strings.Join(errs, "; ")))+ }switch run.StopReason {case agent.StopInfra:res.Status, res.Error = agent.StopInfra, run.StopDetail@@ -412,7 +462,8 @@ func runOne(ctx context.Context, opt Options, j Job, runCap float64) (res *Resulin := &grade.Input{Case: resolved, Arm: arm, Run: run, Seed: jsonify(exposed), Workspace: work,State: &stateReader{inst: inst, projects: projects, primary: primary}, FakeModel: opt.Oracle,- AllowHostTests: opt.AllowHostTests, SandboxImage: opt.SandboxImage, EvalsRoot: opt.EvalsRoot, Judge: opt.Judge,+ AllowHostTests: opt.AllowHostTests, SandboxImage: opt.SandboxImage, EvalsRoot: opt.EvalsRoot, Judge: opt.Judge, Hold: hold,+ Pristine: pristine,}if resolved.Setup != nil && resolved.Setup.Verify != nil {if v := verifySetup(ctx, opt, resolved, work, inst, projects, surf); v != nil {diff --git a/cli/test/evals/internal/suite/suite_test.go b/cli/test/evals/internal/suite/suite_test.goindex 8b4c7092..4d72039b 100644--- a/cli/test/evals/internal/suite/suite_test.go+++ b/cli/test/evals/internal/suite/suite_test.go@@ -69,7 +69,11 @@ func TestCaseReferences(t *testing.T) {t.Error("perf answers need checklist_item or any_of")}}- for arm, steps := range map[string][]catalog.Step{"oracle": c.Oracle} {+ byArm := map[string][]catalog.Step{"oracle": c.Oracle}+ for arm, steps := range c.OracleByArm {+ byArm["oracle_by_arm."+arm] = steps+ }+ for arm, steps := range byArm {for i, s := range steps {if s.Submit != nil {item, _ := s.Submit["checklist_item"].(string)diff --git a/cli/test/evals/internal/tools/coreutils.go b/cli/test/evals/internal/tools/coreutils.goindex 08b09d57..849f8261 100644--- a/cli/test/evals/internal/tools/coreutils.go+++ b/cli/test/evals/internal/tools/coreutils.go@@ -3,6 +3,7 @@ package toolsimport ("bufio""bytes"+ "context""fmt""io""io/fs"@@ -24,6 +25,8 @@ import (type uctx struct {hc interp.HandlerContextrun *shellRun+ // ctx carries the command's timeout (nil for jq and awk).+ ctx context.Context}func (u *uctx) errf(format string, a ...any) { fmt.Fprintf(u.hc.Stderr, format+"\n", a...) }diff --git a/cli/test/evals/internal/tools/coreutils_more.go b/cli/test/evals/internal/tools/coreutils_more.goindex ee2acb40..9ec59097 100644--- a/cli/test/evals/internal/tools/coreutils_more.go+++ b/cli/test/evals/internal/tools/coreutils_more.go@@ -1,6 +1,7 @@package toolsimport (+ "context""fmt""io/fs""os"@@ -187,12 +188,14 @@ func uFind(u *uctx, args []string) int {roots = []string{"."}}var name, pathGlob, typ string+ fold := falsemaxDepth := -1for ; i < len(args); i++ {switch args[i] {case "-name", "-iname":if i+1 < len(args) {name = args[i+1]+ fold = args[i] == "-iname"i++}case "-path":@@ -205,6 +208,10 @@ func uFind(u *uctx, args []string) int {typ = args[i+1]i++}+ if typ != "f" && typ != "d" && typ != "l" {+ u.errf("find: unsupported -type %s (f, d or l)", typ)+ return 1+ }case "-maxdepth":if i+1 < len(args) {maxDepth, _ = strconv.Atoi(args[i+1])@@ -240,12 +247,31 @@ func uFind(u *uctx, args []string) int {if d.IsDir() && d.Name() == ".git" {return filepath.SkipDir}- shown := filepath.Join(root, rel)- if typ == "f" && d.IsDir() || typ == "d" && !d.IsDir() {- return nil+ // As GNU find prints it: ./x under ".", the root itself as given.+ shown := root+ if rel != "." {+ shown = strings.TrimSuffix(root, "/") + "/" + filepath.ToSlash(rel)+ }+ switch typ {+ case "f":+ if !d.Type().IsRegular() {+ return nil+ }+ case "d":+ if !d.IsDir() {+ return nil+ }+ case "l":+ if d.Type()&fs.ModeSymlink == 0 {+ return nil+ }}if name != "" {- if ok, _ := filepath.Match(name, d.Name()); !ok {+ pat, base := name, d.Name()+ if fold {+ pat, base = strings.ToLower(pat), strings.ToLower(base)+ }+ if ok, _ := filepath.Match(pat, base); !ok {return nil}}@@ -296,7 +322,15 @@ func uSleep(u *uctx, args []string) int {if d > 30 {d = 30}- time.Sleep(time.Duration(d * float64(time.Second)))+ ctx := u.ctx+ if ctx == nil {+ ctx = context.Background()+ }+ select {+ case <-ctx.Done():+ return 124+ case <-time.After(time.Duration(d * float64(time.Second))):+ }return 0}@@ -367,16 +401,25 @@ func uSed(u *uctx, args []string) int {case 'q':quit = truecase 's':- repl := c.repl- if c.global {- line = c.re.ReplaceAllString(line, repl)- } else if loc := c.re.FindStringSubmatchIndex(line); loc != nil {- var b []byte- b = c.re.ExpandString(b, repl, line, loc)- line = line[:loc[0]] + string(b) + line[loc[1]:]- } else {+ // The Nth match (default the first), and with g every match+ // from it on.+ locs := c.re.FindAllStringSubmatchIndex(line, -1)+ from := max(c.nth, 1) - 1+ if from >= len(locs) {continue}+ to := from + 1+ if c.global {+ to = len(locs)+ }+ var b []byte+ prev := 0+ for _, loc := range locs[from:to] {+ b = append(b, line[prev:loc[0]]...)+ b = c.re.ExpandString(b, c.repl, line, loc)+ prev = loc[1]+ }+ line = string(append(b, line[prev:]...))if c.print {out = append(out, line)}@@ -410,6 +453,8 @@ type sedCmd struct {repl stringglobal boolprint bool+ // nth is the s command's numeric flag (0 = none).+ nth int}func (a sedAddr) hit(n int, last bool, line string) bool {@@ -521,14 +566,16 @@ func parseSed(script string, ere bool) ([]sedCmd, error) {flags = parts[2]}k := 0- for k < len(flags) && strings.ContainsRune("gpi0123456789", rune(flags[k])) {- switch flags[k] {- case 'g':+ for k < len(flags) && strings.ContainsRune("gpiI0123456789", rune(flags[k])) {+ switch f := flags[k]; {+ case f == 'g':c.global = true- case 'p':+ case f == 'p':c.print = true- case 'i':+ case f == 'i' || f == 'I':c.re = regexp.MustCompile("(?i)" + c.re.String())+ default:+ c.nth = c.nth*10 + int(f-'0')}k++}@@ -541,17 +588,22 @@ func parseSed(script string, ere bool) ([]sedCmd, error) {}// splitUnescaped splits s on delim (not preceded by a backslash) into at most-// n parts; the last part keeps the remainder.+// n parts, unescaping the delimiter in all but the last; the last part keeps+// the remainder verbatim (it holds the rest of the script).func splitUnescaped(s string, delim byte, n int) []string {var parts []stringvar b strings.Builderfor i := 0; i < len(s); i++ {+ if len(parts) == n-1 {+ b.WriteString(s[i:])+ break+ }if s[i] == '\\' && i+1 < len(s) && s[i+1] == delim {b.WriteByte(delim)i++continue}- if s[i] == delim && len(parts) < n-1 {+ if s[i] == delim {parts = append(parts, b.String())b.Reset()continuediff --git a/cli/test/evals/internal/tools/coreutils_more_test.go b/cli/test/evals/internal/tools/coreutils_more_test.goindex a6573073..fbbdf9ec 100644--- a/cli/test/evals/internal/tools/coreutils_more_test.go+++ b/cli/test/evals/internal/tools/coreutils_more_test.go@@ -1,15 +1,25 @@package toolsimport (+ "os"+ "path/filepath""strings""testing")func TestMoreUtilities(t *testing.T) {- s, _ := newShell(t)+ s, work := newShell(t)+ os.WriteFile(filepath.Join(work, "Notes.MD"), []byte("x\n"), 0o644)+ os.Symlink("notes.txt", filepath.Join(work, "link.txt"))cases := []struct{ cmd, want string }{{`sed -n '2p' notes.txt`, "beta 1"},{`sed 's/alpha/ALPHA/' notes.txt | head -1`, "ALPHA 3"},+ {`echo 'beta beta beta' | sed 's/beta/B/2'`, "beta B beta"},+ {`echo 'beta beta beta' | sed 's/beta/B/2g'`, "beta B B"},+ {`echo 'a x/y' | sed -e 's/a/b/;s/x\/y/z/'`, "b z"},+ {`find . -iname 'notes.md'`, "./Notes.MD"},+ {`find . -type l`, "./link.txt"},+ {`find . -maxdepth 1 -type f -name 'link*'`, ""},{`seq 3 | xargs echo`, "1 2 3"},{`date -u -d '2026-01-02T03:04:05Z' +%Y-%m-%d`, "2026-01-02"},}diff --git a/cli/test/evals/internal/tools/files.go b/cli/test/evals/internal/tools/files.goindex cf8d6750..4dab5d28 100644--- a/cli/test/evals/internal/tools/files.go+++ b/cli/test/evals/internal/tools/files.go@@ -180,6 +180,11 @@ func (t *Search) Call(_ context.Context, raw json.RawMessage) Result {if d.IsDir() {return nil}+ // Only regular files: a symlink the setup container planted could+ // point anywhere on the host.+ if !d.Type().IsRegular() {+ return nil+ }if in.Glob != "" {if ok, _ := path.Match(in.Glob, d.Name()); !ok {return nildiff --git a/cli/test/evals/internal/tools/files_test.go b/cli/test/evals/internal/tools/files_test.gonew file mode 100644index 00000000..eaad26d3--- /dev/null+++ b/cli/test/evals/internal/tools/files_test.go@@ -0,0 +1,27 @@+package tools++import (+ "context"+ "encoding/json"+ "os"+ "path/filepath"+ "strings"+ "testing"+)++// TestSearchSkipsSymlinks: the setup container shares the workspace and can+// plant a symlink to a host file; the host-side search never reads through+// it.+func TestSearchSkipsSymlinks(t *testing.T) {+ root := t.TempDir()+ work := filepath.Join(root, "work")+ os.MkdirAll(work, 0o755)+ os.WriteFile(filepath.Join(root, "secret"), []byte("TOKEN=host-secret\n"), 0o644)+ os.Symlink(filepath.Join(root, "secret"), filepath.Join(work, "leak.txt"))+ os.WriteFile(filepath.Join(work, "app.txt"), []byte("TOKEN=from-env\n"), 0o644)+ raw, _ := json.Marshal(map[string]string{"pattern": "TOKEN"})+ r := (&Search{Sandbox: &Sandbox{Root: work}}).Call(context.Background(), raw)+ if strings.Contains(r.Output, "host-secret") || !strings.Contains(r.Output, "from-env") {+ t.Fatalf("search output: %q", r.Output)+ }+}diff --git a/cli/test/evals/internal/tools/mcpbridge.go b/cli/test/evals/internal/tools/mcpbridge.goindex ca9841ff..078873cc 100644--- a/cli/test/evals/internal/tools/mcpbridge.go+++ b/cli/test/evals/internal/tools/mcpbridge.go@@ -35,9 +35,19 @@ func (b bearerRT) RoundTrip(r *http.Request) (*http.Response, error) {return http.DefaultTransport.RoundTrip(r)}-// ConnectMCP opens a session to url with token.+// mcpCallTimeout bounds one MCP request. It is a per-call context, not an+// http.Client timeout: a client timeout would also cut the session's+// streams, and go-sdk gives up a session after a few reconnects that deliver+// nothing.+const mcpCallTimeout = 2 * time.Minute++// ConnectMCP opens a session to url with token. The bridge only makes+// requests and lists the tools once, so it opens no standalone SSE stream for+// server-initiated messages.func ConnectMCP(ctx context.Context, url, token, defaultProject string) (*MCPBridge, error) {- tr := &mcp.StreamableClientTransport{Endpoint: url, HTTPClient: &http.Client{Transport: bearerRT{token}, Timeout: 2 * time.Minute}, MaxRetries: 2}+ tr := &mcp.StreamableClientTransport{Endpoint: url, HTTPClient: &http.Client{Transport: bearerRT{token}}, MaxRetries: 2, DisableStandaloneSSE: true}+ ctx, cancel := context.WithTimeout(ctx, mcpCallTimeout)+ defer cancel()cs, err := mcp.NewClient(&mcp.Implementation{Name: "twevals", Version: "1"}, nil).Connect(ctx, tr, nil)if err != nil {return nil, fmt.Errorf("connect %s: %w", url, err)@@ -103,6 +113,8 @@ func (m *mcpTool) Call(ctx context.Context, raw json.RawMessage) Result {args["project_id"] = m.b.DefaultProject}}+ ctx, cancel := context.WithTimeout(ctx, mcpCallTimeout)+ defer cancel()res, err := m.b.Session.CallTool(ctx, &mcp.CallToolParams{Name: m.t.Name, Arguments: args})op := ops.ParseMCP(m.t.Name, args)if err != nil {@@ -148,6 +160,8 @@ func (r *mcpResources) Spec() Spec {}func (r *mcpResources) Call(ctx context.Context, raw json.RawMessage) Result {+ ctx, cancel := context.WithTimeout(ctx, mcpCallTimeout)+ defer cancel()if r.list {var b strings.Builderfor res, err := range r.b.Session.Resources(ctx, nil) {diff --git a/cli/test/evals/internal/tools/shell.go b/cli/test/evals/internal/tools/shell.goindex 8f8ed220..61b95f5d 100644--- a/cli/test/evals/internal/tools/shell.go+++ b/cli/test/evals/internal/tools/shell.go@@ -165,7 +165,7 @@ func (r *shellRun) exec(ctx context.Context, args []string) error {return exitStatus(r.xargs(ctx, hc, args[1:]))}if f, ok := coreutils[name]; ok {- return exitStatus(f(&uctx{hc: hc, run: r}, args[1:]))+ return exitStatus(f(&uctx{hc: hc, run: r, ctx: ctx}, args[1:]))}msg := fmt.Sprintf("%s: command not found in this sandbox", name)if why, ok := r.s.Denied[name]; ok {diff --git a/cli/test/evals/internal/tools/submit.go b/cli/test/evals/internal/tools/submit.goindex 09bb2f69..a8805359 100644--- a/cli/test/evals/internal/tools/submit.go+++ b/cli/test/evals/internal/tools/submit.go@@ -190,14 +190,33 @@ func (t *Submit) validate(p map[string]any) []string {if v := str("category"); v != "" && !slices.Contains(DebugCategories, v) {probs = append(probs, fmt.Sprintf("category %q is not one of %v", v, DebugCategories))}- if c, ok := p["culprit"].(map[string]any); ok {- if f, _ := c["file"].(string); f == "" {+ if raw, present := p["culprit"]; present {+ if c, ok := raw.(map[string]any); !ok {+ probs = append(probs, "culprit must be an object with file, symbol and line")+ } else if f, _ := c["file"].(string); f == "" {probs = append(probs, "culprit.file is required")}}checkEvidence()default:need("summary", "changes")+ if _, present := p["summary"]; present && strings.TrimSpace(str("summary")) == "" {+ probs = append(probs, "summary must be a non-empty string")+ }+ if raw, present := p["changes"]; present {+ if cs, ok := raw.([]any); !ok {+ probs = append(probs, "changes must be an array (empty when you changed nothing)")+ } else {+ for i, c := range cs {+ m, _ := c.(map[string]any)+ a, _ := m["action"].(string)+ tg, _ := m["target"].(string)+ if a == "" || tg == "" {+ probs = append(probs, fmt.Sprintf("changes[%d] needs an action and a target", i))+ }+ }+ }+ }}if c := str("confidence"); c != "" && !slices.Contains([]string{"low", "medium", "high"}, c) {probs = append(probs, "confidence must be low, medium or high")diff --git a/cli/test/evals/models.yaml b/cli/test/evals/models.yamlindex ac359142..172f71f5 100644--- a/cli/test/evals/models.yaml+++ b/cli/test/evals/models.yaml@@ -7,7 +7,8 @@# comparable over time, so a model change is a reviewed edit to this file.# Prices and supported_parameters were checked against# GET https://openrouter.ai/api/v1/models on 2026-10-17. The preflight reads live-# prices; fallback_price is used only when that list cannot be fetched.+# prices; fallback_price is used when that list cannot be fetched or does not+# list the pinned id.## estimate is the per-run token envelope the preflight multiplies by live# prices until baseline.json carries measured averages.diff --git a/scripts/check-node-pins.sh b/scripts/check-node-pins.shindex 3c295aed..925ea9e9 100755--- a/scripts/check-node-pins.sh+++ b/scripts/check-node-pins.sh@@ -17,8 +17,12 @@ if [[ ! $pinned =~ ^[0-9]+$ ]]; thenexit 1fi+# The eval harness's sandbox image (cli/test/evals) is built locally and in+# the CLI Evals workflow, not by release-traceway.yml, but it follows .nvmrc too.+files=(Dockerfile Dockerfile.minimal Dockerfile.sqlite Dockerfile.duckdb Dockerfile.browser cli/test/evals/testdata/sandbox-image/Dockerfile)+status=0-for f in Dockerfile Dockerfile.minimal Dockerfile.sqlite Dockerfile.duckdb Dockerfile.browser; do+for f in "${files[@]}"; dofound=$(sed -n 's/^ARG NODE_VERSION=\([0-9]*\).*/\1/p' "$f")if [[ -z $found ]]; thenecho "check-node-pins: $f has no 'ARG NODE_VERSION=' default" >&2@@ -30,7 +34,7 @@ for f in Dockerfile Dockerfile.minimal Dockerfile.sqlite Dockerfile.duckdb Dockedone# A literal node: tag left behind would silently ignore both .nvmrc and the ARG.-if literal=$(grep -rnE '^FROM .*node:[0-9]' Dockerfile Dockerfile.minimal Dockerfile.sqlite Dockerfile.duckdb Dockerfile.browser); then+if literal=$(grep -nE '^FROM .*node:[0-9]' "${files[@]}"); thenecho "check-node-pins: hardcoded node tag, use \${NODE_VERSION}:" >&2echo "$literal" >&2status=1