The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
evals: report thrash limits instead of failing on them; fix five grader bugs
narrow-adventuree06317c9f53e
diff --git a/cli/test/evals/README.md b/cli/test/evals/README.mdindex e89e0fd2..f9661cdf 100644--- a/cli/test/evals/README.md+++ b/cli/test/evals/README.md@@ -183,9 +183,10 @@ Graders:| Grader | Gates | Checks ||---|---|---|-| `answer` | yes | The structured answer: checklist row (and not the distractor), locus, onset; or the debug category, culprit file, symbol and line (±3) |-| `trajectory` | yes | Required, alternative, forbidden and ordered operations, and thrash limits. CLI and MCP calls map to the same operations (`internal/ops`) |-| `grounding` | yes | Cited calls exist, succeeded and were not truncated. Their output contains the deciding signal. No id in the answer was invented |+| `answer` | yes | The structured answer: checklist row (and not the distractor), locus (text after the endpoint's route, such as "(and every route behind the guard)", is ignored), onset; or the debug category, culprit file, symbol and line (±3) |+| `trajectory` | yes | Required, alternative, forbidden and ordered operations. CLI and MCP calls map to the same operations (`internal/ops`) |+| `efficiency` | no | The thrash limits in `expect.calls`: `max_calls`, `max_failed_calls` and `max_identical_repeats`. An operation counts once per tool call, so one shell loop is one call, and a `git` op is keyed by its argv. Re-running a call to check a JSON shape is not a wrong answer, so these are counted in the summary's Efficiency table instead of failing the run. The case `budget` still stops a run that spins |+| `grounding` | yes | Cited calls exist, succeeded and were not truncated. A shell call succeeded when one of its operations (a `traceway` or `git` command) did, even if a later command in it (a `jq`) failed; then only what the model saw counts. Their output contains the deciding signal. No id in the answer was invented || `state` | yes | After write and setup runs: the backend and the workspace are in the expected state || `hidden_tests` | per case | Hidden tests copied in after the run, and whether the diff (against the workspace before the run) touches the culprit || `setup_verify` | yes | The instrumented app builds, starts against the backend, and serves traffic |@@ -308,12 +309,6 @@ Backend facts that bite (each one cost a case author an afternoon):is the projects' default. An endpoint seeded on one never appears and`Verify` fails with "endpoints never became visible"; a failing one (status400 or more) is kept.-- `endpoints list --order-by p95|lastSeen` does not sort: the CLI sends- `p95`/`lastSeen`, the SQLite/DuckDB repositories only know- `p95_duration`/`last_seen`, and the list falls back to impact order. Keep- a case's endpoint count at ten or fewer so the slow one is on the first page- whatever the order. This is a product bug, not a harness one; the evals- should catch the skill's reliance on it.- A log body search over more than 24 hours answers 422 unless another filter(service, severity, trace) narrows it.- MCP results are JSON and escape `&`, `<` and `>`; grounding compares themdiff --git a/cli/test/evals/internal/grade/answer.go b/cli/test/evals/internal/grade/answer.goindex 56b8b616..91d02dc4 100644--- a/cli/test/evals/internal/grade/answer.go+++ b/cli/test/evals/internal/grade/answer.go@@ -38,7 +38,7 @@ func Answer(in *Input) *Verdict {if locus == nil {locus = map[string]any{}}- if exp.Locus.Endpoint != "" && !sameEndpoint(str(locus, "endpoint"), exp.Locus.Endpoint) {+ if exp.Locus.Endpoint != "" && !answerEndpoint(str(locus, "endpoint"), exp.Locus.Endpoint) {v.fail("locus.endpoint %q, want %q", str(locus, "endpoint"), exp.Locus.Endpoint)}if exp.Locus.SpanContains != "" && !containsFold(str(locus, "span"), exp.Locus.SpanContains) {@@ -123,6 +123,26 @@ func sameEndpoint(got, want string) bool {return g == wr || strings.HasSuffix(g, " "+w)}+// answerEndpoint is sameEndpoint for an answer, which may qualify the right+// endpoint: "GET /api/me (and every route behind the same guard)". Text after+// the route is ignored; the route itself must still match.+func answerEndpoint(got, want string) bool {+ return sameEndpoint(got, want) || sameEndpoint(withoutQualifier(got), want)+}++// withoutQualifier cuts an endpoint answer after its route, the first field+// that starts with "/", and at a "(" inside that field.+func withoutQualifier(s string) string {+ fields := strings.Fields(s)+ for i, f := range fields {+ if strings.HasPrefix(strings.Trim(f, "`\"'"), "/") {+ route, _, _ := strings.Cut(f, "(")+ return strings.Join(append(fields[:i:i], route), " ")+ }+ }+ return s+}+func containsFold(s, sub string) bool {return strings.Contains(strings.ToLower(s), strings.ToLower(sub))}diff --git a/cli/test/evals/internal/grade/grade.go b/cli/test/evals/internal/grade/grade.goindex 70c14458..5f321c65 100644--- a/cli/test/evals/internal/grade/grade.go+++ b/cli/test/evals/internal/grade/grade.go@@ -1,7 +1,7 @@// Package grade scores one run. Deterministic graders gate the result:-// answer, trajectory, grounding, state and hidden tests. The LLM judge scores-// report quality and does not gate until it is calibrated against human-// labels.+// answer, trajectory, grounding, state and hidden tests. Efficiency (the+// thrash limits) is reported and does not gate. The LLM judge scores report+// quality and does not gate until it is calibrated against human labels.package gradeimport (@@ -86,7 +86,7 @@ func All(ctx context.Context, in *Input) []*Verdict {out = append(out, Answer(in))}if in.Case.Expect.Calls != nil {- out = append(out, Trajectory(in.Run, in.Case.Expect.Calls))+ out = append(out, Trajectory(in.Run, in.Case.Expect.Calls), Efficiency(in.Run, in.Case.Expect.Calls))}if in.Case.Suite == "perf" || in.Case.Suite == "debug" || len(in.Case.Expect.Grounding) > 0 {out = append(out, Grounding(in))diff --git a/cli/test/evals/internal/grade/grade_test.go b/cli/test/evals/internal/grade/grade_test.goindex 1603d64d..ec4b77ef 100644--- a/cli/test/evals/internal/grade/grade_test.go+++ b/cli/test/evals/internal/grade/grade_test.go@@ -59,41 +59,7 @@ func TestTrajectory(t *testing.T) {r.Calls = r.Calls[:2]r.Calls = append(r.Calls, call("c03", true, "{}", "traceway metrics query --name mem.used"))}, "none of these alternatives"},- "thrash": {func(r *record.Run, s *catalog.Trajectory) {- s.MaxIdenticalRepeats = 1- r.Calls = append(r.Calls, call("c09", true, "{}", "traceway endpoints list --since 24h --output json"))- }, "same call ran 2 times"},- "thrash, read in chunks of the same window": {func(r *record.Run, s *catalog.Trajectory) {- s.MaxIdenticalRepeats = 1- read := func(id string) *record.Call {- return &record.Call{ID: id, Tool: "read_file", OK: true, Ops: []record.OpRecord{{Op: ops.Op{Op: ops.OpReadFile, Args: map[string]any{"path": "big.go", "offset": 2001}}, OK: true}}}- }- r.Calls = append(r.Calls, read("c11"), read("c12"))- }, "same call ran 2 times"},- "too many failures": {func(r *record.Run, s *catalog.Trajectory) {- s.MaxFailedCalls = 0- s.MaxFailedCalls = 1- r.Calls = append(r.Calls, call("c10", false, "x", "traceway logs query"))- }, "failed tool calls"},- }- // Reading a file in chunks and changing one file several times are not- // thrashing.- chunked := &record.Run{Calls: append([]*record.Call{}, run.Calls...)}- for i, off := range []int{0, 2001, 4001} {- args := map[string]any{"path": "big.go"}- if off > 0 {- args["offset"] = off- }- chunked.Calls = append(chunked.Calls,- &record.Call{ID: fmt.Sprintf("r%d", i), Tool: "read_file", OK: true, Ops: []record.OpRecord{{Op: ops.Op{Op: ops.OpReadFile, Args: args}, OK: true}}},- &record.Call{ID: fmt.Sprintf("e%d", i), Tool: "edit_file", OK: true, Ops: []record.OpRecord{{Op: ops.Op{Op: ops.OpEditFile, Args: map[string]any{"path": "big.go"}}, OK: true}}})- }- repeats := *spec- repeats.MaxIdenticalRepeats = 1- if v := Trajectory(chunked, &repeats); !v.Pass {- t.Fatalf("chunked reads and repeated edits counted as thrashing:\n%s", failures(v))}-for name, c := range cases {t.Run(name, func(t *testing.T) {r := &record.Run{Calls: append([]*record.Call{}, run.Calls...)}@@ -107,6 +73,71 @@ func TestTrajectory(t *testing.T) {}}+func TestEfficiency(t *testing.T) {+ base := []*record.Call{+ call("c01", true, "{}", "traceway endpoints list --since 24h"),+ call("c02", false, "error", "traceway metrics query --name system.cpu.utilization"),+ }+ limits := &catalog.Trajectory{MaxIdenticalRepeats: 1, MaxFailedCalls: 1, MaxCalls: 10}+ if v := Efficiency(&record.Run{Calls: base}, limits); !v.Pass || v.Gating {+ t.Fatalf("pass=%v gating=%v:\n%s", v.Pass, v.Gating, failures(v))+ }+ read := func(id string, off int) *record.Call {+ args := map[string]any{"path": "big.go"}+ if off > 0 {+ args["offset"] = off+ }+ return &record.Call{ID: id, Tool: "read_file", OK: true, Ops: []record.OpRecord{{Op: ops.Op{Op: ops.OpReadFile, Args: args}, OK: true}}}+ }+ edit := func(id string) *record.Call {+ return &record.Call{ID: id, Tool: "edit_file", OK: true, Ops: []record.OpRecord{{Op: ops.Op{Op: ops.OpEditFile, Args: map[string]any{"path": "big.go"}}, OK: true}}}+ }+ git := func(id string, argv string) *record.Call {+ a := strings.Fields(argv)+ return &record.Call{ID: id, Tool: "shell", OK: true, Ops: []record.OpRecord{{Op: ops.Op{Op: ops.OpGit, Args: map[string]any{"subcommand": a[1]}}, Argv: a, OK: true}}}+ }+ var many []*record.Call+ for i := range 9 {+ many = append(many, call(fmt.Sprintf("x%d", i), true, "{}", fmt.Sprintf("traceway endpoints list --since %dh", i+1)))+ }+ cases := map[string]struct {+ extra []*record.Call+ want string // "" = no limit exceeded+ }{+ "same call twice": {[]*record.Call{call("c09", true, "{}", "traceway endpoints list --since 24h --output json")}, "same call ran 2 times"},+ "same read window twice": {[]*record.Call{read("c11", 2001), read("c12", 2001)}, "same call ran 2 times"},+ "too many failures": {[]*record.Call{call("c10", false, "x", "traceway logs query")}, "failed tool calls"},+ "too many calls": {many, "more than max_calls"},+ // Reading a file in chunks and changing one file several times are+ // not thrashing.+ "chunked reads and edits": {[]*record.Call{read("r0", 0), edit("e0"), read("r1", 2001), edit("e1"), read("r2", 4001), edit("e2")}, ""},+ // One shell loop runs the same command on every iteration.+ "one loop": {[]*record.Call{call("c20", true, "{}", "traceway endpoints list --search /api/x", "traceway endpoints list --search /api/x", "traceway endpoints list --search /api/x")}, ""},+ // git records only the subcommand; different commits are different questions.+ "git show of different commits": {[]*record.Call{git("g1", "git show 1a2b3c4"), git("g2", "git show 5d6e7f8"), git("g3", "git show 9a8b7c6")}, ""},+ "git show of one commit twice": {[]*record.Call{git("g1", "git show 1a2b3c4"), git("g2", "git show 1a2b3c4")}, "same call ran 2 times"},+ }+ for name, c := range cases {+ t.Run(name, func(t *testing.T) {+ v := Efficiency(&record.Run{Calls: append(append([]*record.Call{}, base...), c.extra...)}, limits)+ if v.Gating {+ t.Fatal("efficiency must not gate")+ }+ switch {+ case c.want == "" && !v.Pass:+ t.Fatalf("want no limit exceeded, got:\n%s", failures(v))+ case c.want != "" && (v.Pass || !strings.Contains(failures(v), c.want)):+ t.Fatalf("want %q, got pass=%v:\n%s", c.want, v.Pass, failures(v))+ }+ })+ }+ // A run over every limit still passes when nothing gating failed.+ over := &record.Run{Calls: append(append([]*record.Call{}, base...), call("c09", true, "{}", "traceway endpoints list --since 24h"), call("c10", false, "x", "traceway logs query"))}+ if !Passed([]*Verdict{Trajectory(over, limits), Efficiency(over, limits)}) {+ t.Fatal("the thrash limits failed the run")+ }+}+func perfInput(sub map[string]any, calls ...*record.Call) *Input {return &Input{Case: &catalog.Case{ID: "perf-x", Suite: "perf", Expect: catalog.Expect{@@ -153,6 +184,22 @@ func TestSameEndpoint(t *testing.T) {t.Errorf("sameEndpoint(%q, %q) = %v", c.got, c.want, got)}}+ for _, c := range []struct {+ got, want string+ same bool+ }{+ {"GET /api/me (and every other route via the global APP_GUARD)", "GET /api/me", true},+ {"GET /api/me(global guard)", "GET /api/me", true},+ {"`GET /api/me` for every request", "GET /api/me", true},+ {"/api/me, via the guard", "GET /api/me", false},+ {"GET /api/me/settings (the guard)", "GET /api/me", false},+ {"POST /api/me (the guard)", "GET /api/me", false},+ {"every route (GET /api/me included)", "GET /api/me", false},+ } {+ if got := answerEndpoint(c.got, c.want); got != c.same {+ t.Errorf("answerEndpoint(%q, %q) = %v", c.got, c.want, got)+ }+ }}func TestGrounding(t *testing.T) {@@ -164,6 +211,10 @@ func TestGrounding(t *testing.T) {if v := Grounding(perfInput(sub("c01", ""), c1)); !v.Pass {t.Fatalf("grounded answer failed: %s", failures(v))}+ // The traceway command printed its data; only a jq later in the call exited non-zero.+ if v := Grounding(perfInput(sub("c01", ""), jqFailed("c01", out+"\njq: error: Cannot iterate over null", out))); !v.Pass {+ t.Fatalf("a trailing jq error failed a cited call: %s", failures(v))+ }tests := map[string]struct {in *Inputwant string@@ -177,6 +228,9 @@ func TestGrounding(t *testing.T) {"invented id, submit recorded": {perfInput(sub("c01", "and trace 4bf92f3577b34da6a3ce929d0e0e4736"), c1,&record.Call{ID: "c02", Tool: "submit_perf_diagnosis", OK: true, Args: json.RawMessage(`{"report":"trace 4bf92f3577b34da6a3ce929d0e0e4736"}`)}), "no tool returned"},"missing key": {perfInput(sub("c02", ""), c1, call("c02", true, `{"data":[]}`, "traceway endpoints list")), "contains"},+ // A failed jq after a traceway command leaves only what the model saw:+ // the raw traceway output never reached it.+ "key only in raw output, jq failed": {perfInput(sub("c01", ""), jqFailed("c01", "jq: error: Cannot index array", out)), "contains"},}for name, tc := range tests {t.Run(name, func(t *testing.T) {@@ -188,6 +242,14 @@ func TestGrounding(t *testing.T) {}}+// jqFailed is a shell call whose traceway command succeeded and printed raw,+// and whose exit status is a later jq's failure.+func jqFailed(id, seen, raw string) *record.Call {+ c := call(id, true, raw, "traceway endpoints show 6f9619ff-8b86-d011-b42d-00c04fc964ff --recorded-at 2026-10-17T10:00:00Z")+ c.OK, c.Output = false, seen+ return c+}+func TestLeakageLint(t *testing.T) {c := &catalog.Case{Prompt: "Why is checkout slow?", Expect: catalog.Expect{Answer: &catalog.Answer{ChecklistItem: "cache_stampede_thundering_herd"}}}if got := Lint(c, []string{"GET catalog:v3", "SELECT id FROM products WHERE id = $1"}); len(got) != 0 {diff --git a/cli/test/evals/internal/grade/grounding.go b/cli/test/evals/internal/grade/grounding.goindex 58741f5c..f39d29a3 100644--- a/cli/test/evals/internal/grade/grounding.go+++ b/cli/test/evals/internal/grade/grounding.go@@ -12,7 +12,8 @@ import (// Grounding checks that the evidence is real:// - every evidence.call_id names a call that exists, succeeded and was-// not truncated;+// not truncated (a shell call succeeded when one of its operations did,+// even if a later command in it, such as jq, failed);// - every grounding key appears in the output of at least one cited call// (what the model saw, or the raw traceway output inside that call);// - every UUID, 32-hex trace id or 16-hex hash in the answer appears in@@ -36,7 +37,7 @@ func Grounding(in *Input) *Verdict {v.fail("evidence[%d] cites %q, which is not a call", i, id)case c.Rejected:v.fail("evidence[%d] cites %s, which was never executed", i, id)- case !c.OK || !anyOpOK(c):+ case !anyOpOK(c) || (len(c.Ops) == 0 && !c.OK):v.fail("evidence[%d] cites %s, which failed", i, id)case c.Truncated:v.fail("evidence[%d] cites %s, whose output was truncated", i, id)@@ -66,13 +67,20 @@ func Grounding(in *Input) *Verdict {}// callContains looks for a grounding key in what the model saw and in the-// raw output of the call's successful operations. MCP results are JSON, which-// escapes &, < and > (\u0026, \u003c, \u003e), so those are compared-// unescaped.+// raw output of the call's successful operations. When a later command in the+// call failed, only what the model saw counts: the raw output may never have+// reached it. MCP results are JSON, which escapes &, < and > (\u0026, \u003c,+// \u003e), so those are compared unescaped.func callContains(c *record.Call, key string) bool {- if anyOpOK(c) && strings.Contains(unescapeJSON(c.Output), key) {+ if !anyOpOK(c) {+ return false+ }+ if strings.Contains(unescapeJSON(c.Output), key) {return true}+ if !c.OK {+ return false+ }for _, op := range c.Ops {if op.OK && strings.Contains(unescapeJSON(op.Output), key) {return truediff --git a/cli/test/evals/internal/grade/trajectory.go b/cli/test/evals/internal/grade/trajectory.goindex c913bab1..da31e3ea 100644--- a/cli/test/evals/internal/grade/trajectory.go+++ b/cli/test/evals/internal/grade/trajectory.go@@ -21,7 +21,10 @@ import (// any_of at least one group is fully satisfied// forbidden no operation matches// order the first match of A precedes the first match of B-// max_calls, max_failed_calls, max_identical_repeats thrash detection+//+// The thrash limits (max_calls, max_failed_calls, max_identical_repeats) are+// reported by Efficiency instead: checking a JSON shape twice is not a wrong+// answer. The case budget still stops a run that spins.//// An op pattern is an exact op, "*", a prefix ("endpoints.*"), or a file op// with a path glob ("edit_file:handlers/*.go").@@ -80,6 +83,15 @@ func Trajectory(run *record.Run, t *catalog.Trajectory) *Verdict {v.fail("%s (%s) came after %s (%s)", pair[0], all[a].Call.ID, pair[1], all[b].Call.ID)}}+ return v+}++// Efficiency checks the thrash limits. It does not gate: a run that repeats+// or retries calls still passes on its answer, evidence and writes, and the+// limits it went over are listed here for the report.+func Efficiency(run *record.Run, t *catalog.Trajectory) *Verdict {+ v := newVerdict("efficiency", false)+ all := run.AllOps()executed := 0failed := 0for _, c := range run.Calls {@@ -99,13 +111,25 @@ func Trajectory(run *record.Run, t *catalog.Trajectory) *Verdict {}if t.MaxIdenticalRepeats > 0 {// Thrashing is asking the same question again. Changing a local file- // (several edits, or appending to one file) is not.+ // (several edits, or appending to one file) is not, and neither is+ // one shell loop that runs the same command on each iteration: an+ // operation counts once per tool call.counts := map[string]int{}- for _, op := range all {- if op.Op.Op == ops.OpWriteFile || op.Op.Op == ops.OpEditFile {+ for _, c := range run.Calls {+ if c.Rejected {continue}- counts[opKey(op.OpRecord)]+++ seen := map[string]bool{}+ for i := range c.Ops {+ op := &c.Ops[i]+ if op.Op.Op == ops.OpWriteFile || op.Op.Op == ops.OpEditFile {+ continue+ }+ if k := opKey(op); !seen[k] {+ seen[k] = true+ counts[k]+++ }+ }}keys := make([]string, 0, len(counts))for k := range counts {@@ -282,7 +306,12 @@ func opString(op *record.OpRecord) string {}// opKey identifies a call for repeat detection, ignoring output-format flags.+// A git op records only its subcommand, so its argv is the key: three+// `git show` calls of different commits are three questions.func opKey(op *record.OpRecord) string {+ if op.Op.Op == ops.OpGit && len(op.Argv) > 0 {+ return op.Op.Op + " " + strings.Join(op.Argv, " ")+ }args := map[string]any{}for k, v := range op.Args {if k == "output" || k == "fields" {diff --git a/cli/test/evals/internal/report/aggregate.go b/cli/test/evals/internal/report/aggregate.goindex 0bb5aeca..4313a4da 100644--- a/cli/test/evals/internal/report/aggregate.go+++ b/cli/test/evals/internal/report/aggregate.go@@ -61,6 +61,19 @@ type Aggregates struct {JudgeUSD float64 `json:"judgeUsd"`Statuses map[string]int `json:"statuses"`TopFailing []GraderCount `json:"topFailingGraders"`+ // Efficiency is keyed "model|arm". It sits beside the pass rate and is+ // never part of it.+ Efficiency map[string]*Effort `json:"efficiency"`+}++// Effort is how hard the runs of one model and arm worked.+type Effort struct {+ Runs int `json:"runs"`+ AvgToolCalls float64 `json:"avgToolCalls"`+ AvgFailedCalls float64 `json:"avgFailedCalls"`+ // OverLimit counts runs that went over a case's max_calls,+ // max_failed_calls or max_identical_repeats.+ OverLimit int `json:"overLimit"`}// GraderCount counts failing graders.@@ -77,7 +90,7 @@ func Counted(r *runner.Result) bool {// Aggregate computes pass rates and deltas.func Aggregate(runs []*runner.Result) Aggregates {- a := Aggregates{Cases: map[string]*Rate{}, Suites: map[string]*Rate{}, Cost: map[string]*Costs{}, Statuses: map[string]int{}}+ a := Aggregates{Cases: map[string]*Rate{}, Suites: map[string]*Rate{}, Cost: map[string]*Costs{}, Statuses: map[string]int{}, Efficiency: map[string]*Effort{}}graderFails := map[string]int{}cacheIn, cacheHit := map[string]int{}, map[string]int{}for _, r := range runs {@@ -115,12 +128,28 @@ func Aggregate(runs []*runner.Result) Aggregates {c.AvgInputTokens += float64(r.Metrics.InputTokens)c.AvgOutputTokens += float64(r.Metrics.OutputTokens)}+ ekey := r.Model + "|" + r.Arm+ e := a.Efficiency[ekey]+ if e == nil {+ e = &Effort{}+ a.Efficiency[ekey] = e+ }+ e.Runs+++ e.AvgToolCalls += float64(r.Metrics.ToolCalls)+ e.AvgFailedCalls += float64(r.Metrics.FailedCalls)for _, g := range r.Graders {if g.Gating && !g.Skipped && !g.Pass {graderFails[g.Grader]++}+ if g.Grader == "efficiency" && !g.Pass {+ e.OverLimit+++ }}}+ for _, e := range a.Efficiency {+ e.AvgToolCalls /= float64(e.Runs)+ e.AvgFailedCalls /= float64(e.Runs)+ }for m, c := range a.Cost {if c.PassingRuns > 0 {n := float64(c.PassingRuns)diff --git a/cli/test/evals/internal/report/markdown.go b/cli/test/evals/internal/report/markdown.goindex c056a195..be49b5d3 100644--- a/cli/test/evals/internal/report/markdown.go+++ b/cli/test/evals/internal/report/markdown.go@@ -75,6 +75,14 @@ func Markdown(r *Results) string {fmt.Fprintf(&b, "| %s | %s | %s | %.0f%% | %.0f%% | %+.0fpp |\n", d.Suite, d.A, d.B, d.RateA*100, d.RateB*100, d.Delta)}}+ if len(a.Efficiency) > 0 {+ b.WriteString("\n## Efficiency (reported, not part of the pass rate)\n\n| Model | Arm | Runs | Avg tool calls | Avg failed calls | Over a call limit |\n|---|---|---|---|---|---|\n")+ for _, k := range sortedKeys(a.Efficiency) {+ m, arm, _ := strings.Cut(k, "|")+ e := a.Efficiency[k]+ fmt.Fprintf(&b, "| %s | %s | %d | %.1f | %.1f | %d |\n", m, arm, e.Runs, e.AvgToolCalls, e.AvgFailedCalls, e.OverLimit)+ }+ }if paid {b.WriteString("\n## Cost (averages over passing runs)\n\n| Model | Passing runs | Avg cost | Avg tool calls | Avg input tokens | Avg output tokens | Cache hit rate | Total spend |\n|---|---|---|---|---|---|---|---|\n")for _, m := range sortedKeys(a.Cost) {diff --git a/cli/test/evals/internal/report/report_test.go b/cli/test/evals/internal/report/report_test.goindex bf286f87..48236051 100644--- a/cli/test/evals/internal/report/report_test.go+++ b/cli/test/evals/internal/report/report_test.go@@ -5,6 +5,7 @@ import ("strings""testing"+ "github.com/tracewayapp/traceway/cli/test/evals/internal/grade""github.com/tracewayapp/traceway/cli/test/evals/internal/runner")@@ -36,6 +37,30 @@ func TestAggregate(t *testing.T) {}}+// TestEfficiency: going over a thrash limit is counted beside the pass rate,+// not in it.+func TestEfficiency(t *testing.T) {+ over := res("perf-a", "cli+skill", "opus", 1, "pass", 0.5)+ over.Metrics.FailedCalls = 4+ over.Graders = []*grade.Verdict{{Grader: "answer", Pass: true, Gating: true}, {Grader: "efficiency", Pass: false, Failures: []string{"the same call ran 3 times"}}}+ runs := []*runner.Result{over, res("perf-b", "cli+skill", "opus", 1, "pass", 0.5)}+ a := Aggregate(runs)+ if r := a.Suites["perf|opus|cli+skill"]; r.Pass != 2 {+ t.Fatalf("an efficiency miss changed the pass rate: %+v", r)+ }+ e := a.Efficiency["opus|cli+skill"]+ if e == nil || e.Runs != 2 || e.OverLimit != 1 || e.AvgFailedCalls != 2 || len(a.TopFailing) != 0 {+ t.Fatalf("efficiency %+v, top failing %+v", e, a.TopFailing)+ }+ out := Markdown(&Results{Runs: runs, Aggregates: a})+ if !strings.Contains(out, "| opus | cli+skill | 2 | 10.0 | 2.0 | 1 |") {+ t.Errorf("summary lacks the efficiency row:\n%s", out)+ }+ if strings.Contains(out, "efficiency:") {+ t.Errorf("an efficiency miss was reported as a run's failure:\n%s", out)+ }+}+// TestOracleSummary: an oracle run costs nothing, so its summary names no// spend, and each run's transcript is listed at the path it was written to.func TestOracleSummary(t *testing.T) {diff --git a/cli/test/evals/internal/tools/submit.go b/cli/test/evals/internal/tools/submit.goindex a8805359..39092d20 100644--- a/cli/test/evals/internal/tools/submit.go+++ b/cli/test/evals/internal/tools/submit.go@@ -173,6 +173,11 @@ func (t *Submit) validate(p map[string]any) []string {switch t.Kind {case SubmitPerf:need("checklist_item", "locus", "evidence", "fix", "confidence", "report")+ if raw, present := p["locus"]; present {+ if _, ok := raw.(map[string]any); !ok {+ probs = append(probs, "locus must be an object with endpoint, span, service and host")+ }+ }if v := str("checklist_item"); v != "" && !checklist.IsID(v) {probs = append(probs, fmt.Sprintf("checklist_item %q is not one of the allowed values", v))}@@ -221,5 +226,39 @@ func (t *Submit) validate(p map[string]any) []string {if c := str("confidence"); c != "" && !slices.Contains([]string{"low", "medium", "high"}, c) {probs = append(probs, "confidence must be low, medium or high")}+ for _, f := range markupFields(p, "") {+ probs = append(probs, f+" contains tool-call markup (<parameter name=...>); send every field as plain JSON")+ }return probs}++// markupFields lists the string fields that carry tool-call markup: a model+// that serialized a nested object as its own tool-call syntax, which leaves+// the object a string and its fields flattened to the top level.+func markupFields(v any, at string) []string {+ var out []string+ switch t := v.(type) {+ case string:+ if strings.Contains(t, "<parameter name=") {+ out = append(out, at)+ }+ case map[string]any:+ keys := make([]string, 0, len(t))+ for k := range t {+ keys = append(keys, k)+ }+ slices.Sort(keys)+ for _, k := range keys {+ name := k+ if at != "" {+ name = at + "." + k+ }+ out = append(out, markupFields(t[k], name)...)+ }+ case []any:+ for i, x := range t {+ out = append(out, markupFields(x, fmt.Sprintf("%s[%d]", at, i))...)+ }+ }+ return out+}diff --git a/cli/test/evals/internal/tools/submit_test.go b/cli/test/evals/internal/tools/submit_test.gonew file mode 100644index 00000000..c4ca35f4--- /dev/null+++ b/cli/test/evals/internal/tools/submit_test.go@@ -0,0 +1,52 @@+package tools++import (+ "context"+ "encoding/json"+ "strings"+ "testing"+)++func perfPayload(locus any) map[string]any {+ return map[string]any{+ "checklist_item": "disk_io_bottleneck",+ "locus": locus,+ "evidence": []any{map[string]any{"call_id": "c01", "claim": "iowait"}},+ "fix": "stop the rebuild job",+ "confidence": "high",+ "report": "Disk contention on pg-bookings-1.",+ }+}++func TestSubmitPerfRejectsMalformedLocus(t *testing.T) {+ good := perfPayload(map[string]any{"endpoint": "GET /api/availability", "service": "bookings-api", "host": "pg-bookings-1"})+ sub := &Submit{Kind: SubmitPerf, MaxRepairs: 2}+ if probs := sub.validate(good); len(probs) != 0 {+ t.Fatalf("a well-formed submission was rejected: %v", probs)+ }++ // The shape Opus 5.5 sent in 5 of 107 diagnoses: locus serialized as+ // tool-call markup, its fields flattened to the top level.+ bad := perfPayload("\n <parameter name=\"service\">bookings-api")+ bad["endpoint"] = "GET /api/availability"+ bad["host"] = "pg-bookings-1"+ raw, _ := json.Marshal(bad)+ res := sub.Call(context.Background(), raw)+ if res.OK || res.Final {+ t.Fatalf("malformed locus was accepted: %+v", res)+ }+ for _, want := range []string{"locus must be an object", "locus contains tool-call markup"} {+ if !strings.Contains(res.Output, want) {+ t.Errorf("repair message lacks %q:\n%s", want, res.Output)+ }+ }+}++func TestSubmitRejectsMarkupInNestedFields(t *testing.T) {+ p := perfPayload(map[string]any{"endpoint": "GET /api/x"})+ p["evidence"] = []any{map[string]any{"call_id": "c01", "claim": `<parameter name="claim">x`}}+ probs := (&Submit{Kind: SubmitPerf}).validate(p)+ if len(probs) != 1 || !strings.HasPrefix(probs[0], "evidence[0].claim contains tool-call markup") {+ t.Fatalf("got %v", probs)+ }+}