Skip to content

The only tool you need to know what is happening and how to fix it.

evals: report thrash limits instead of failing on them; fix five grader bugs

narrow-adventuree06317c9f53e
diff --git a/cli/test/evals/README.md b/cli/test/evals/README.md
index e89e0fd2..f9661cdf 100644
--- a/cli/test/evals/README.md
+++ b/cli/test/evals/README.md
@@ -183,9 +183,10 @@ Graders:
| Grader | Gates | Checks |
|---|---|---|
-| `answer` | yes | The structured answer: checklist row (and not the distractor), locus, onset; or the debug category, culprit file, symbol and line (±3) |
-| `trajectory` | yes | Required, alternative, forbidden and ordered operations, and thrash limits. CLI and MCP calls map to the same operations (`internal/ops`) |
-| `grounding` | yes | Cited calls exist, succeeded and were not truncated. Their output contains the deciding signal. No id in the answer was invented |
+| `answer` | yes | The structured answer: checklist row (and not the distractor), locus (text after the endpoint's route, such as "(and every route behind the guard)", is ignored), onset; or the debug category, culprit file, symbol and line (±3) |
+| `trajectory` | yes | Required, alternative, forbidden and ordered operations. CLI and MCP calls map to the same operations (`internal/ops`) |
+| `efficiency` | no | The thrash limits in `expect.calls`: `max_calls`, `max_failed_calls` and `max_identical_repeats`. An operation counts once per tool call, so one shell loop is one call, and a `git` op is keyed by its argv. Re-running a call to check a JSON shape is not a wrong answer, so these are counted in the summary's Efficiency table instead of failing the run. The case `budget` still stops a run that spins |
+| `grounding` | yes | Cited calls exist, succeeded and were not truncated. A shell call succeeded when one of its operations (a `traceway` or `git` command) did, even if a later command in it (a `jq`) failed; then only what the model saw counts. Their output contains the deciding signal. No id in the answer was invented |
| `state` | yes | After write and setup runs: the backend and the workspace are in the expected state |
| `hidden_tests` | per case | Hidden tests copied in after the run, and whether the diff (against the workspace before the run) touches the culprit |
| `setup_verify` | yes | The instrumented app builds, starts against the backend, and serves traffic |
@@ -308,12 +309,6 @@ Backend facts that bite (each one cost a case author an afternoon):
is the projects' default. An endpoint seeded on one never appears and
`Verify` fails with "endpoints never became visible"; a failing one (status
400 or more) is kept.
-- `endpoints list --order-by p95|lastSeen` does not sort: the CLI sends
- `p95`/`lastSeen`, the SQLite/DuckDB repositories only know
- `p95_duration`/`last_seen`, and the list falls back to impact order. Keep
- a case's endpoint count at ten or fewer so the slow one is on the first page
- whatever the order. This is a product bug, not a harness one; the evals
- should catch the skill's reliance on it.
- A log body search over more than 24 hours answers 422 unless another filter
(service, severity, trace) narrows it.
- MCP results are JSON and escape `&`, `<` and `>`; grounding compares them
diff --git a/cli/test/evals/internal/grade/answer.go b/cli/test/evals/internal/grade/answer.go
index 56b8b616..91d02dc4 100644
--- a/cli/test/evals/internal/grade/answer.go
+++ b/cli/test/evals/internal/grade/answer.go
@@ -38,7 +38,7 @@ func Answer(in *Input) *Verdict {
if locus == nil {
locus = map[string]any{}
}
- if exp.Locus.Endpoint != "" && !sameEndpoint(str(locus, "endpoint"), exp.Locus.Endpoint) {
+ if exp.Locus.Endpoint != "" && !answerEndpoint(str(locus, "endpoint"), exp.Locus.Endpoint) {
v.fail("locus.endpoint %q, want %q", str(locus, "endpoint"), exp.Locus.Endpoint)
}
if exp.Locus.SpanContains != "" && !containsFold(str(locus, "span"), exp.Locus.SpanContains) {
@@ -123,6 +123,26 @@ func sameEndpoint(got, want string) bool {
return g == wr || strings.HasSuffix(g, " "+w)
}
+// answerEndpoint is sameEndpoint for an answer, which may qualify the right
+// endpoint: "GET /api/me (and every route behind the same guard)". Text after
+// the route is ignored; the route itself must still match.
+func answerEndpoint(got, want string) bool {
+ return sameEndpoint(got, want) || sameEndpoint(withoutQualifier(got), want)
+}
+
+// withoutQualifier cuts an endpoint answer after its route, the first field
+// that starts with "/", and at a "(" inside that field.
+func withoutQualifier(s string) string {
+ fields := strings.Fields(s)
+ for i, f := range fields {
+ if strings.HasPrefix(strings.Trim(f, "`\"'"), "/") {
+ route, _, _ := strings.Cut(f, "(")
+ return strings.Join(append(fields[:i:i], route), " ")
+ }
+ }
+ return s
+}
+
func containsFold(s, sub string) bool {
return strings.Contains(strings.ToLower(s), strings.ToLower(sub))
}
diff --git a/cli/test/evals/internal/grade/grade.go b/cli/test/evals/internal/grade/grade.go
index 70c14458..5f321c65 100644
--- a/cli/test/evals/internal/grade/grade.go
+++ b/cli/test/evals/internal/grade/grade.go
@@ -1,7 +1,7 @@
// Package grade scores one run. Deterministic graders gate the result:
-// answer, trajectory, grounding, state and hidden tests. The LLM judge scores
-// report quality and does not gate until it is calibrated against human
-// labels.
+// answer, trajectory, grounding, state and hidden tests. Efficiency (the
+// thrash limits) is reported and does not gate. The LLM judge scores report
+// quality and does not gate until it is calibrated against human labels.
package grade
import (
@@ -86,7 +86,7 @@ func All(ctx context.Context, in *Input) []*Verdict {
out = append(out, Answer(in))
}
if in.Case.Expect.Calls != nil {
- out = append(out, Trajectory(in.Run, in.Case.Expect.Calls))
+ out = append(out, Trajectory(in.Run, in.Case.Expect.Calls), Efficiency(in.Run, in.Case.Expect.Calls))
}
if in.Case.Suite == "perf" || in.Case.Suite == "debug" || len(in.Case.Expect.Grounding) > 0 {
out = append(out, Grounding(in))
diff --git a/cli/test/evals/internal/grade/grade_test.go b/cli/test/evals/internal/grade/grade_test.go
index 1603d64d..ec4b77ef 100644
--- a/cli/test/evals/internal/grade/grade_test.go
+++ b/cli/test/evals/internal/grade/grade_test.go
@@ -59,41 +59,7 @@ func TestTrajectory(t *testing.T) {
r.Calls = r.Calls[:2]
r.Calls = append(r.Calls, call("c03", true, "{}", "traceway metrics query --name mem.used"))
}, "none of these alternatives"},
- "thrash": {func(r *record.Run, s *catalog.Trajectory) {
- s.MaxIdenticalRepeats = 1
- r.Calls = append(r.Calls, call("c09", true, "{}", "traceway endpoints list --since 24h --output json"))
- }, "same call ran 2 times"},
- "thrash, read in chunks of the same window": {func(r *record.Run, s *catalog.Trajectory) {
- s.MaxIdenticalRepeats = 1
- read := func(id string) *record.Call {
- return &record.Call{ID: id, Tool: "read_file", OK: true, Ops: []record.OpRecord{{Op: ops.Op{Op: ops.OpReadFile, Args: map[string]any{"path": "big.go", "offset": 2001}}, OK: true}}}
- }
- r.Calls = append(r.Calls, read("c11"), read("c12"))
- }, "same call ran 2 times"},
- "too many failures": {func(r *record.Run, s *catalog.Trajectory) {
- s.MaxFailedCalls = 0
- s.MaxFailedCalls = 1
- r.Calls = append(r.Calls, call("c10", false, "x", "traceway logs query"))
- }, "failed tool calls"},
- }
- // Reading a file in chunks and changing one file several times are not
- // thrashing.
- chunked := &record.Run{Calls: append([]*record.Call{}, run.Calls...)}
- for i, off := range []int{0, 2001, 4001} {
- args := map[string]any{"path": "big.go"}
- if off > 0 {
- args["offset"] = off
- }
- chunked.Calls = append(chunked.Calls,
- &record.Call{ID: fmt.Sprintf("r%d", i), Tool: "read_file", OK: true, Ops: []record.OpRecord{{Op: ops.Op{Op: ops.OpReadFile, Args: args}, OK: true}}},
- &record.Call{ID: fmt.Sprintf("e%d", i), Tool: "edit_file", OK: true, Ops: []record.OpRecord{{Op: ops.Op{Op: ops.OpEditFile, Args: map[string]any{"path": "big.go"}}, OK: true}}})
- }
- repeats := *spec
- repeats.MaxIdenticalRepeats = 1
- if v := Trajectory(chunked, &repeats); !v.Pass {
- t.Fatalf("chunked reads and repeated edits counted as thrashing:\n%s", failures(v))
}
-
for name, c := range cases {
t.Run(name, func(t *testing.T) {
r := &record.Run{Calls: append([]*record.Call{}, run.Calls...)}
@@ -107,6 +73,71 @@ func TestTrajectory(t *testing.T) {
}
}
+func TestEfficiency(t *testing.T) {
+ base := []*record.Call{
+ call("c01", true, "{}", "traceway endpoints list --since 24h"),
+ call("c02", false, "error", "traceway metrics query --name system.cpu.utilization"),
+ }
+ limits := &catalog.Trajectory{MaxIdenticalRepeats: 1, MaxFailedCalls: 1, MaxCalls: 10}
+ if v := Efficiency(&record.Run{Calls: base}, limits); !v.Pass || v.Gating {
+ t.Fatalf("pass=%v gating=%v:\n%s", v.Pass, v.Gating, failures(v))
+ }
+ read := func(id string, off int) *record.Call {
+ args := map[string]any{"path": "big.go"}
+ if off > 0 {
+ args["offset"] = off
+ }
+ return &record.Call{ID: id, Tool: "read_file", OK: true, Ops: []record.OpRecord{{Op: ops.Op{Op: ops.OpReadFile, Args: args}, OK: true}}}
+ }
+ edit := func(id string) *record.Call {
+ return &record.Call{ID: id, Tool: "edit_file", OK: true, Ops: []record.OpRecord{{Op: ops.Op{Op: ops.OpEditFile, Args: map[string]any{"path": "big.go"}}, OK: true}}}
+ }
+ git := func(id string, argv string) *record.Call {
+ a := strings.Fields(argv)
+ return &record.Call{ID: id, Tool: "shell", OK: true, Ops: []record.OpRecord{{Op: ops.Op{Op: ops.OpGit, Args: map[string]any{"subcommand": a[1]}}, Argv: a, OK: true}}}
+ }
+ var many []*record.Call
+ for i := range 9 {
+ many = append(many, call(fmt.Sprintf("x%d", i), true, "{}", fmt.Sprintf("traceway endpoints list --since %dh", i+1)))
+ }
+ cases := map[string]struct {
+ extra []*record.Call
+ want string // "" = no limit exceeded
+ }{
+ "same call twice": {[]*record.Call{call("c09", true, "{}", "traceway endpoints list --since 24h --output json")}, "same call ran 2 times"},
+ "same read window twice": {[]*record.Call{read("c11", 2001), read("c12", 2001)}, "same call ran 2 times"},
+ "too many failures": {[]*record.Call{call("c10", false, "x", "traceway logs query")}, "failed tool calls"},
+ "too many calls": {many, "more than max_calls"},
+ // Reading a file in chunks and changing one file several times are
+ // not thrashing.
+ "chunked reads and edits": {[]*record.Call{read("r0", 0), edit("e0"), read("r1", 2001), edit("e1"), read("r2", 4001), edit("e2")}, ""},
+ // One shell loop runs the same command on every iteration.
+ "one loop": {[]*record.Call{call("c20", true, "{}", "traceway endpoints list --search /api/x", "traceway endpoints list --search /api/x", "traceway endpoints list --search /api/x")}, ""},
+ // git records only the subcommand; different commits are different questions.
+ "git show of different commits": {[]*record.Call{git("g1", "git show 1a2b3c4"), git("g2", "git show 5d6e7f8"), git("g3", "git show 9a8b7c6")}, ""},
+ "git show of one commit twice": {[]*record.Call{git("g1", "git show 1a2b3c4"), git("g2", "git show 1a2b3c4")}, "same call ran 2 times"},
+ }
+ for name, c := range cases {
+ t.Run(name, func(t *testing.T) {
+ v := Efficiency(&record.Run{Calls: append(append([]*record.Call{}, base...), c.extra...)}, limits)
+ if v.Gating {
+ t.Fatal("efficiency must not gate")
+ }
+ switch {
+ case c.want == "" && !v.Pass:
+ t.Fatalf("want no limit exceeded, got:\n%s", failures(v))
+ case c.want != "" && (v.Pass || !strings.Contains(failures(v), c.want)):
+ t.Fatalf("want %q, got pass=%v:\n%s", c.want, v.Pass, failures(v))
+ }
+ })
+ }
+ // A run over every limit still passes when nothing gating failed.
+ over := &record.Run{Calls: append(append([]*record.Call{}, base...), call("c09", true, "{}", "traceway endpoints list --since 24h"), call("c10", false, "x", "traceway logs query"))}
+ if !Passed([]*Verdict{Trajectory(over, limits), Efficiency(over, limits)}) {
+ t.Fatal("the thrash limits failed the run")
+ }
+}
+
func perfInput(sub map[string]any, calls ...*record.Call) *Input {
return &Input{
Case: &catalog.Case{ID: "perf-x", Suite: "perf", Expect: catalog.Expect{
@@ -153,6 +184,22 @@ func TestSameEndpoint(t *testing.T) {
t.Errorf("sameEndpoint(%q, %q) = %v", c.got, c.want, got)
}
}
+ for _, c := range []struct {
+ got, want string
+ same bool
+ }{
+ {"GET /api/me (and every other route via the global APP_GUARD)", "GET /api/me", true},
+ {"GET /api/me(global guard)", "GET /api/me", true},
+ {"`GET /api/me` for every request", "GET /api/me", true},
+ {"/api/me, via the guard", "GET /api/me", false},
+ {"GET /api/me/settings (the guard)", "GET /api/me", false},
+ {"POST /api/me (the guard)", "GET /api/me", false},
+ {"every route (GET /api/me included)", "GET /api/me", false},
+ } {
+ if got := answerEndpoint(c.got, c.want); got != c.same {
+ t.Errorf("answerEndpoint(%q, %q) = %v", c.got, c.want, got)
+ }
+ }
}
func TestGrounding(t *testing.T) {
@@ -164,6 +211,10 @@ func TestGrounding(t *testing.T) {
if v := Grounding(perfInput(sub("c01", ""), c1)); !v.Pass {
t.Fatalf("grounded answer failed: %s", failures(v))
}
+ // The traceway command printed its data; only a jq later in the call exited non-zero.
+ if v := Grounding(perfInput(sub("c01", ""), jqFailed("c01", out+"\njq: error: Cannot iterate over null", out))); !v.Pass {
+ t.Fatalf("a trailing jq error failed a cited call: %s", failures(v))
+ }
tests := map[string]struct {
in *Input
want string
@@ -177,6 +228,9 @@ func TestGrounding(t *testing.T) {
"invented id, submit recorded": {perfInput(sub("c01", "and trace 4bf92f3577b34da6a3ce929d0e0e4736"), c1,
&record.Call{ID: "c02", Tool: "submit_perf_diagnosis", OK: true, Args: json.RawMessage(`{"report":"trace 4bf92f3577b34da6a3ce929d0e0e4736"}`)}), "no tool returned"},
"missing key": {perfInput(sub("c02", ""), c1, call("c02", true, `{"data":[]}`, "traceway endpoints list")), "contains"},
+ // A failed jq after a traceway command leaves only what the model saw:
+ // the raw traceway output never reached it.
+ "key only in raw output, jq failed": {perfInput(sub("c01", ""), jqFailed("c01", "jq: error: Cannot index array", out)), "contains"},
}
for name, tc := range tests {
t.Run(name, func(t *testing.T) {
@@ -188,6 +242,14 @@ func TestGrounding(t *testing.T) {
}
}
+// jqFailed is a shell call whose traceway command succeeded and printed raw,
+// and whose exit status is a later jq's failure.
+func jqFailed(id, seen, raw string) *record.Call {
+ c := call(id, true, raw, "traceway endpoints show 6f9619ff-8b86-d011-b42d-00c04fc964ff --recorded-at 2026-10-17T10:00:00Z")
+ c.OK, c.Output = false, seen
+ return c
+}
+
func TestLeakageLint(t *testing.T) {
c := &catalog.Case{Prompt: "Why is checkout slow?", Expect: catalog.Expect{Answer: &catalog.Answer{ChecklistItem: "cache_stampede_thundering_herd"}}}
if got := Lint(c, []string{"GET catalog:v3", "SELECT id FROM products WHERE id = $1"}); len(got) != 0 {
diff --git a/cli/test/evals/internal/grade/grounding.go b/cli/test/evals/internal/grade/grounding.go
index 58741f5c..f39d29a3 100644
--- a/cli/test/evals/internal/grade/grounding.go
+++ b/cli/test/evals/internal/grade/grounding.go
@@ -12,7 +12,8 @@ import (
// Grounding checks that the evidence is real:
// - every evidence.call_id names a call that exists, succeeded and was
-// not truncated;
+// not truncated (a shell call succeeded when one of its operations did,
+// even if a later command in it, such as jq, failed);
// - every grounding key appears in the output of at least one cited call
// (what the model saw, or the raw traceway output inside that call);
// - every UUID, 32-hex trace id or 16-hex hash in the answer appears in
@@ -36,7 +37,7 @@ func Grounding(in *Input) *Verdict {
v.fail("evidence[%d] cites %q, which is not a call", i, id)
case c.Rejected:
v.fail("evidence[%d] cites %s, which was never executed", i, id)
- case !c.OK || !anyOpOK(c):
+ case !anyOpOK(c) || (len(c.Ops) == 0 && !c.OK):
v.fail("evidence[%d] cites %s, which failed", i, id)
case c.Truncated:
v.fail("evidence[%d] cites %s, whose output was truncated", i, id)
@@ -66,13 +67,20 @@ func Grounding(in *Input) *Verdict {
}
// callContains looks for a grounding key in what the model saw and in the
-// raw output of the call's successful operations. MCP results are JSON, which
-// escapes &, < and > (\u0026, \u003c, \u003e), so those are compared
-// unescaped.
+// raw output of the call's successful operations. When a later command in the
+// call failed, only what the model saw counts: the raw output may never have
+// reached it. MCP results are JSON, which escapes &, < and > (\u0026, \u003c,
+// \u003e), so those are compared unescaped.
func callContains(c *record.Call, key string) bool {
- if anyOpOK(c) && strings.Contains(unescapeJSON(c.Output), key) {
+ if !anyOpOK(c) {
+ return false
+ }
+ if strings.Contains(unescapeJSON(c.Output), key) {
return true
}
+ if !c.OK {
+ return false
+ }
for _, op := range c.Ops {
if op.OK && strings.Contains(unescapeJSON(op.Output), key) {
return true
diff --git a/cli/test/evals/internal/grade/trajectory.go b/cli/test/evals/internal/grade/trajectory.go
index c913bab1..da31e3ea 100644
--- a/cli/test/evals/internal/grade/trajectory.go
+++ b/cli/test/evals/internal/grade/trajectory.go
@@ -21,7 +21,10 @@ import (
// any_of at least one group is fully satisfied
// forbidden no operation matches
// order the first match of A precedes the first match of B
-// max_calls, max_failed_calls, max_identical_repeats thrash detection
+//
+// The thrash limits (max_calls, max_failed_calls, max_identical_repeats) are
+// reported by Efficiency instead: checking a JSON shape twice is not a wrong
+// answer. The case budget still stops a run that spins.
//
// An op pattern is an exact op, "*", a prefix ("endpoints.*"), or a file op
// with a path glob ("edit_file:handlers/*.go").
@@ -80,6 +83,15 @@ func Trajectory(run *record.Run, t *catalog.Trajectory) *Verdict {
v.fail("%s (%s) came after %s (%s)", pair[0], all[a].Call.ID, pair[1], all[b].Call.ID)
}
}
+ return v
+}
+
+// Efficiency checks the thrash limits. It does not gate: a run that repeats
+// or retries calls still passes on its answer, evidence and writes, and the
+// limits it went over are listed here for the report.
+func Efficiency(run *record.Run, t *catalog.Trajectory) *Verdict {
+ v := newVerdict("efficiency", false)
+ all := run.AllOps()
executed := 0
failed := 0
for _, c := range run.Calls {
@@ -99,13 +111,25 @@ func Trajectory(run *record.Run, t *catalog.Trajectory) *Verdict {
}
if t.MaxIdenticalRepeats > 0 {
// Thrashing is asking the same question again. Changing a local file
- // (several edits, or appending to one file) is not.
+ // (several edits, or appending to one file) is not, and neither is
+ // one shell loop that runs the same command on each iteration: an
+ // operation counts once per tool call.
counts := map[string]int{}
- for _, op := range all {
- if op.Op.Op == ops.OpWriteFile || op.Op.Op == ops.OpEditFile {
+ for _, c := range run.Calls {
+ if c.Rejected {
continue
}
- counts[opKey(op.OpRecord)]++
+ seen := map[string]bool{}
+ for i := range c.Ops {
+ op := &c.Ops[i]
+ if op.Op.Op == ops.OpWriteFile || op.Op.Op == ops.OpEditFile {
+ continue
+ }
+ if k := opKey(op); !seen[k] {
+ seen[k] = true
+ counts[k]++
+ }
+ }
}
keys := make([]string, 0, len(counts))
for k := range counts {
@@ -282,7 +306,12 @@ func opString(op *record.OpRecord) string {
}
// opKey identifies a call for repeat detection, ignoring output-format flags.
+// A git op records only its subcommand, so its argv is the key: three
+// `git show` calls of different commits are three questions.
func opKey(op *record.OpRecord) string {
+ if op.Op.Op == ops.OpGit && len(op.Argv) > 0 {
+ return op.Op.Op + " " + strings.Join(op.Argv, " ")
+ }
args := map[string]any{}
for k, v := range op.Args {
if k == "output" || k == "fields" {
diff --git a/cli/test/evals/internal/report/aggregate.go b/cli/test/evals/internal/report/aggregate.go
index 0bb5aeca..4313a4da 100644
--- a/cli/test/evals/internal/report/aggregate.go
+++ b/cli/test/evals/internal/report/aggregate.go
@@ -61,6 +61,19 @@ type Aggregates struct {
JudgeUSD float64 `json:"judgeUsd"`
Statuses map[string]int `json:"statuses"`
TopFailing []GraderCount `json:"topFailingGraders"`
+ // Efficiency is keyed "model|arm". It sits beside the pass rate and is
+ // never part of it.
+ Efficiency map[string]*Effort `json:"efficiency"`
+}
+
+// Effort is how hard the runs of one model and arm worked.
+type Effort struct {
+ Runs int `json:"runs"`
+ AvgToolCalls float64 `json:"avgToolCalls"`
+ AvgFailedCalls float64 `json:"avgFailedCalls"`
+ // OverLimit counts runs that went over a case's max_calls,
+ // max_failed_calls or max_identical_repeats.
+ OverLimit int `json:"overLimit"`
}
// GraderCount counts failing graders.
@@ -77,7 +90,7 @@ func Counted(r *runner.Result) bool {
// Aggregate computes pass rates and deltas.
func Aggregate(runs []*runner.Result) Aggregates {
- a := Aggregates{Cases: map[string]*Rate{}, Suites: map[string]*Rate{}, Cost: map[string]*Costs{}, Statuses: map[string]int{}}
+ a := Aggregates{Cases: map[string]*Rate{}, Suites: map[string]*Rate{}, Cost: map[string]*Costs{}, Statuses: map[string]int{}, Efficiency: map[string]*Effort{}}
graderFails := map[string]int{}
cacheIn, cacheHit := map[string]int{}, map[string]int{}
for _, r := range runs {
@@ -115,12 +128,28 @@ func Aggregate(runs []*runner.Result) Aggregates {
c.AvgInputTokens += float64(r.Metrics.InputTokens)
c.AvgOutputTokens += float64(r.Metrics.OutputTokens)
}
+ ekey := r.Model + "|" + r.Arm
+ e := a.Efficiency[ekey]
+ if e == nil {
+ e = &Effort{}
+ a.Efficiency[ekey] = e
+ }
+ e.Runs++
+ e.AvgToolCalls += float64(r.Metrics.ToolCalls)
+ e.AvgFailedCalls += float64(r.Metrics.FailedCalls)
for _, g := range r.Graders {
if g.Gating && !g.Skipped && !g.Pass {
graderFails[g.Grader]++
}
+ if g.Grader == "efficiency" && !g.Pass {
+ e.OverLimit++
+ }
}
}
+ for _, e := range a.Efficiency {
+ e.AvgToolCalls /= float64(e.Runs)
+ e.AvgFailedCalls /= float64(e.Runs)
+ }
for m, c := range a.Cost {
if c.PassingRuns > 0 {
n := float64(c.PassingRuns)
diff --git a/cli/test/evals/internal/report/markdown.go b/cli/test/evals/internal/report/markdown.go
index c056a195..be49b5d3 100644
--- a/cli/test/evals/internal/report/markdown.go
+++ b/cli/test/evals/internal/report/markdown.go
@@ -75,6 +75,14 @@ func Markdown(r *Results) string {
fmt.Fprintf(&b, "| %s | %s | %s | %.0f%% | %.0f%% | %+.0fpp |\n", d.Suite, d.A, d.B, d.RateA*100, d.RateB*100, d.Delta)
}
}
+ if len(a.Efficiency) > 0 {
+ b.WriteString("\n## Efficiency (reported, not part of the pass rate)\n\n| Model | Arm | Runs | Avg tool calls | Avg failed calls | Over a call limit |\n|---|---|---|---|---|---|\n")
+ for _, k := range sortedKeys(a.Efficiency) {
+ m, arm, _ := strings.Cut(k, "|")
+ e := a.Efficiency[k]
+ fmt.Fprintf(&b, "| %s | %s | %d | %.1f | %.1f | %d |\n", m, arm, e.Runs, e.AvgToolCalls, e.AvgFailedCalls, e.OverLimit)
+ }
+ }
if paid {
b.WriteString("\n## Cost (averages over passing runs)\n\n| Model | Passing runs | Avg cost | Avg tool calls | Avg input tokens | Avg output tokens | Cache hit rate | Total spend |\n|---|---|---|---|---|---|---|---|\n")
for _, m := range sortedKeys(a.Cost) {
diff --git a/cli/test/evals/internal/report/report_test.go b/cli/test/evals/internal/report/report_test.go
index bf286f87..48236051 100644
--- a/cli/test/evals/internal/report/report_test.go
+++ b/cli/test/evals/internal/report/report_test.go
@@ -5,6 +5,7 @@ import (
"strings"
"testing"
+ "github.com/tracewayapp/traceway/cli/test/evals/internal/grade"
"github.com/tracewayapp/traceway/cli/test/evals/internal/runner"
)
@@ -36,6 +37,30 @@ func TestAggregate(t *testing.T) {
}
}
+// TestEfficiency: going over a thrash limit is counted beside the pass rate,
+// not in it.
+func TestEfficiency(t *testing.T) {
+ over := res("perf-a", "cli+skill", "opus", 1, "pass", 0.5)
+ over.Metrics.FailedCalls = 4
+ over.Graders = []*grade.Verdict{{Grader: "answer", Pass: true, Gating: true}, {Grader: "efficiency", Pass: false, Failures: []string{"the same call ran 3 times"}}}
+ runs := []*runner.Result{over, res("perf-b", "cli+skill", "opus", 1, "pass", 0.5)}
+ a := Aggregate(runs)
+ if r := a.Suites["perf|opus|cli+skill"]; r.Pass != 2 {
+ t.Fatalf("an efficiency miss changed the pass rate: %+v", r)
+ }
+ e := a.Efficiency["opus|cli+skill"]
+ if e == nil || e.Runs != 2 || e.OverLimit != 1 || e.AvgFailedCalls != 2 || len(a.TopFailing) != 0 {
+ t.Fatalf("efficiency %+v, top failing %+v", e, a.TopFailing)
+ }
+ out := Markdown(&Results{Runs: runs, Aggregates: a})
+ if !strings.Contains(out, "| opus | cli+skill | 2 | 10.0 | 2.0 | 1 |") {
+ t.Errorf("summary lacks the efficiency row:\n%s", out)
+ }
+ if strings.Contains(out, "efficiency:") {
+ t.Errorf("an efficiency miss was reported as a run's failure:\n%s", out)
+ }
+}
+
// TestOracleSummary: an oracle run costs nothing, so its summary names no
// spend, and each run's transcript is listed at the path it was written to.
func TestOracleSummary(t *testing.T) {
diff --git a/cli/test/evals/internal/tools/submit.go b/cli/test/evals/internal/tools/submit.go
index a8805359..39092d20 100644
--- a/cli/test/evals/internal/tools/submit.go
+++ b/cli/test/evals/internal/tools/submit.go
@@ -173,6 +173,11 @@ func (t *Submit) validate(p map[string]any) []string {
switch t.Kind {
case SubmitPerf:
need("checklist_item", "locus", "evidence", "fix", "confidence", "report")
+ if raw, present := p["locus"]; present {
+ if _, ok := raw.(map[string]any); !ok {
+ probs = append(probs, "locus must be an object with endpoint, span, service and host")
+ }
+ }
if v := str("checklist_item"); v != "" && !checklist.IsID(v) {
probs = append(probs, fmt.Sprintf("checklist_item %q is not one of the allowed values", v))
}
@@ -221,5 +226,39 @@ func (t *Submit) validate(p map[string]any) []string {
if c := str("confidence"); c != "" && !slices.Contains([]string{"low", "medium", "high"}, c) {
probs = append(probs, "confidence must be low, medium or high")
}
+ for _, f := range markupFields(p, "") {
+ probs = append(probs, f+" contains tool-call markup (<parameter name=...>); send every field as plain JSON")
+ }
return probs
}
+
+// markupFields lists the string fields that carry tool-call markup: a model
+// that serialized a nested object as its own tool-call syntax, which leaves
+// the object a string and its fields flattened to the top level.
+func markupFields(v any, at string) []string {
+ var out []string
+ switch t := v.(type) {
+ case string:
+ if strings.Contains(t, "<parameter name=") {
+ out = append(out, at)
+ }
+ case map[string]any:
+ keys := make([]string, 0, len(t))
+ for k := range t {
+ keys = append(keys, k)
+ }
+ slices.Sort(keys)
+ for _, k := range keys {
+ name := k
+ if at != "" {
+ name = at + "." + k
+ }
+ out = append(out, markupFields(t[k], name)...)
+ }
+ case []any:
+ for i, x := range t {
+ out = append(out, markupFields(x, fmt.Sprintf("%s[%d]", at, i))...)
+ }
+ }
+ return out
+}
diff --git a/cli/test/evals/internal/tools/submit_test.go b/cli/test/evals/internal/tools/submit_test.go
new file mode 100644
index 00000000..c4ca35f4
--- /dev/null
+++ b/cli/test/evals/internal/tools/submit_test.go
@@ -0,0 +1,52 @@
+package tools
+
+import (
+ "context"
+ "encoding/json"
+ "strings"
+ "testing"
+)
+
+func perfPayload(locus any) map[string]any {
+ return map[string]any{
+ "checklist_item": "disk_io_bottleneck",
+ "locus": locus,
+ "evidence": []any{map[string]any{"call_id": "c01", "claim": "iowait"}},
+ "fix": "stop the rebuild job",
+ "confidence": "high",
+ "report": "Disk contention on pg-bookings-1.",
+ }
+}
+
+func TestSubmitPerfRejectsMalformedLocus(t *testing.T) {
+ good := perfPayload(map[string]any{"endpoint": "GET /api/availability", "service": "bookings-api", "host": "pg-bookings-1"})
+ sub := &Submit{Kind: SubmitPerf, MaxRepairs: 2}
+ if probs := sub.validate(good); len(probs) != 0 {
+ t.Fatalf("a well-formed submission was rejected: %v", probs)
+ }
+
+ // The shape Opus 5.5 sent in 5 of 107 diagnoses: locus serialized as
+ // tool-call markup, its fields flattened to the top level.
+ bad := perfPayload("\n <parameter name=\"service\">bookings-api")
+ bad["endpoint"] = "GET /api/availability"
+ bad["host"] = "pg-bookings-1"
+ raw, _ := json.Marshal(bad)
+ res := sub.Call(context.Background(), raw)
+ if res.OK || res.Final {
+ t.Fatalf("malformed locus was accepted: %+v", res)
+ }
+ for _, want := range []string{"locus must be an object", "locus contains tool-call markup"} {
+ if !strings.Contains(res.Output, want) {
+ t.Errorf("repair message lacks %q:\n%s", want, res.Output)
+ }
+ }
+}
+
+func TestSubmitRejectsMarkupInNestedFields(t *testing.T) {
+ p := perfPayload(map[string]any{"endpoint": "GET /api/x"})
+ p["evidence"] = []any{map[string]any{"call_id": "c01", "claim": `<parameter name="claim">x`}}
+ probs := (&Submit{Kind: SubmitPerf}).validate(p)
+ if len(probs) != 1 || !strings.HasPrefix(probs[0], "evidence[0].claim contains tool-call markup") {
+ t.Fatalf("got %v", probs)
+ }
+}