Skip to content

Git designed for AI agents.

graft-resolve bench: keep the answers, and judge them again without the model

Federico Elia144e1c00f488
diff --git a/tools/graft-agents/cmd/graft-resolve/bench.go b/tools/graft-agents/cmd/graft-resolve/bench.go
index b353f4d..d1262e5 100644
--- a/tools/graft-agents/cmd/graft-resolve/bench.go
+++ b/tools/graft-agents/cmd/graft-resolve/bench.go
@@ -40,8 +40,10 @@ func runBench(args []string, stdout, stderr io.Writer) int {
return benchMine(ctx, args[1:], stdout, stderr)
case "run":
return benchRun(ctx, args[1:], stdout, stderr)
+ case "rejudge":
+ return benchRejudge(ctx, args[1:], stdout, stderr)
}
- fmt.Fprintf(stderr, "graft-resolve bench: unknown subcommand %q, want mine or run\n", args[0])
+ fmt.Fprintf(stderr, "graft-resolve bench: unknown subcommand %q, want mine, run or rejudge\n", args[0])
return 2
}
@@ -124,6 +126,7 @@ func benchRun(ctx context.Context, args []string, stdout, stderr io.Writer) int
seed := fs.Int64("seed", 1, "seed of the sample")
label := fs.String("label", "", "name of the run in the report")
jsonOut := fs.String("json-out", "", "report as JSON")
+ answersDir := fs.String("answers-dir", "", "keep every accepted resolution here, for audit and bench rejudge (local: third-party code)")
mdOut := fs.String("md-out", "", "report as Markdown")
maxUSD := fs.Float64("max-usd", 0, "hard cap on the AI spend of the run in USD (required when the chain has an llm strategy)")
pricingSource := fs.String("pricing", "openrouter", "where the model's price comes from: openrouter (read at run time) or fixed (the two price flags)")
@@ -202,9 +205,16 @@ func benchRun(ctx context.Context, args []string, stdout, stderr io.Writer) int
}
cases := bench.Sample(manifest.Cases, *sample, *seed)
+ if *answersDir != "" {
+ if err := os.MkdirAll(*answersDir, 0o700); err != nil {
+ fmt.Fprintf(stderr, "graft-resolve bench run: %v\n", err)
+ return 1
+ }
+ }
runner := bench.Runner{
- Resolver: res,
- Clone: func(repo string) string { return bench.CloneDir(*clones, repo) },
+ Resolver: res,
+ AnswersDir: *answersDir,
+ Clone: func(repo string) string { return bench.CloneDir(*clones, repo) },
Progress: func(done, total int, r bench.Result) {
fmt.Fprintf(stderr, "[%d/%d] %s %s %s: %s %s\n", done, total, r.Family, r.Repo, r.Path, r.Outcome, r.Reason)
},
@@ -220,33 +230,88 @@ func benchRun(ctx context.Context, args []string, stdout, stderr io.Writer) int
report.SpendUSD, report.MaxUSD, report.Pricing = snap.SpentUSD, *maxUSD, pricing
}
- if *jsonOut != "" {
+ if err := writeReport(report, *jsonOut, *mdOut, stdout); err != nil {
+ fmt.Fprintf(stderr, "graft-resolve bench run: %v\n", err)
+ return 1
+ }
+ if runErr != nil {
+ fmt.Fprintf(stderr, "graft-resolve bench run: stopped after %d of %d cases: %v\n", len(results), len(cases), runErr)
+ return 1
+ }
+ return 0
+}
+
+// benchRejudge judges the stored answers of a run again, with no model call:
+//
+// graft-resolve bench rejudge --report-in FILE --manifest FILE --clones DIR --answers-dir DIR
+// [--json-out FILE] [--md-out FILE]
+//
+// The report keeps everything of the run (label, spend, pricing, the results that were not
+// accepted) and recomputes the outcome of every accepted resolution.
+func benchRejudge(ctx context.Context, args []string, stdout, stderr io.Writer) int {
+ fs := flag.NewFlagSet("graft-resolve bench rejudge", flag.ContinueOnError)
+ fs.SetOutput(stderr)
+ reportIn := fs.String("report-in", "", "JSON report of the run")
+ manifestPath := fs.String("manifest", "", "manifest of the run")
+ clones := fs.String("clones", "", "directory of the clones of the corpus repositories")
+ answersDir := fs.String("answers-dir", "", "answers kept by the run (bench run --answers-dir)")
+ jsonOut := fs.String("json-out", "", "report as JSON")
+ mdOut := fs.String("md-out", "", "report as Markdown")
+ if err := fs.Parse(args); err != nil {
+ return 2
+ }
+ if *reportIn == "" || *manifestPath == "" || *clones == "" || *answersDir == "" || fs.NArg() != 0 {
+ fmt.Fprintln(stderr, "graft-resolve bench rejudge: --report-in, --manifest, --clones and --answers-dir are required")
+ return 2
+ }
+ raw, err := os.ReadFile(*reportIn)
+ if err != nil {
+ fmt.Fprintf(stderr, "graft-resolve bench rejudge: %v\n", err)
+ return 1
+ }
+ var old bench.Report
+ if err := json.Unmarshal(raw, &old); err != nil {
+ fmt.Fprintf(stderr, "graft-resolve bench rejudge: %s: %v\n", *reportIn, err)
+ return 1
+ }
+ manifest, err := bench.LoadManifest(*manifestPath)
+ if err != nil {
+ fmt.Fprintf(stderr, "graft-resolve bench rejudge: %v\n", err)
+ return 1
+ }
+ results, err := bench.Rejudge(ctx, old.Results, manifest.Cases, func(repo string) string { return bench.CloneDir(*clones, repo) }, *answersDir)
+ if err != nil {
+ fmt.Fprintf(stderr, "graft-resolve bench rejudge: %v\n", err)
+ return 1
+ }
+ report := bench.NewReport(old.Label, results)
+ report.SpendUSD, report.MaxUSD, report.Pricing = old.SpendUSD, old.MaxUSD, old.Pricing
+ if err := writeReport(report, *jsonOut, *mdOut, stdout); err != nil {
+ fmt.Fprintf(stderr, "graft-resolve bench rejudge: %v\n", err)
+ return 1
+ }
+ return 0
+}
+
+// writeReport writes report as JSON to jsonOut and as Markdown to stdout and mdOut, when set.
+func writeReport(report bench.Report, jsonOut, mdOut string, stdout io.Writer) error {
+ if jsonOut != "" {
data, err := json.MarshalIndent(report, "", " ")
- if err == nil {
- err = os.WriteFile(*jsonOut, append(data, '\n'), 0o644)
- }
if err != nil {
- fmt.Fprintf(stderr, "graft-resolve bench run: %v\n", err)
- return 1
+ return err
+ }
+ if err := os.WriteFile(jsonOut, append(data, '\n'), 0o644); err != nil {
+ return err
}
}
out := stdout
- if *mdOut != "" {
- f, err := os.Create(*mdOut)
+ if mdOut != "" {
+ f, err := os.Create(mdOut)
if err != nil {
- fmt.Fprintf(stderr, "graft-resolve bench run: %v\n", err)
- return 1
+ return err
}
defer f.Close()
out = io.MultiWriter(stdout, f)
}
- if err := report.WriteMarkdown(out); err != nil {
- fmt.Fprintf(stderr, "graft-resolve bench run: %v\n", err)
- return 1
- }
- if runErr != nil {
- fmt.Fprintf(stderr, "graft-resolve bench run: stopped after %d of %d cases: %v\n", len(results), len(cases), runErr)
- return 1
- }
- return 0
+ return report.WriteMarkdown(out)
}
diff --git a/tools/graft-agents/cmd/graft-resolve/bench_budget_test.go b/tools/graft-agents/cmd/graft-resolve/bench_budget_test.go
index a2fd51e..c766372 100644
--- a/tools/graft-agents/cmd/graft-resolve/bench_budget_test.go
+++ b/tools/graft-agents/cmd/graft-resolve/bench_budget_test.go
@@ -128,6 +128,58 @@ func TestBenchRunRefusesUnreadablePrice(t *testing.T) {
}
}
+// TestBenchAnswersAndRejudge keeps the answers of a run, then judges them again without the
+// model: the same answer gives the same outcome, and a changed answer is judged anew.
+func TestBenchAnswersAndRejudge(t *testing.T) {
+ work, _, clones, manifest := mineFixture(t)
+ cfg := modelConfig(t, work)
+ answers := filepath.Join(work, "answers")
+ reportIn := filepath.Join(work, "run.json")
+ var stdout, stderr bytes.Buffer
+ if code := run([]string{"bench", "run", "--config", cfg, "--manifest", manifest, "--clones", clones, "--max-usd", "1",
+ "--pricing", "fixed", "--input-usd-per-million", "1", "--output-usd-per-million", "1",
+ "--answers-dir", answers, "--json-out", reportIn}, nil, &stdout, &stderr); code != 0 {
+ t.Fatalf("run exited %d: %s", code, stderr.String())
+ }
+ var rep bench.Report
+ raw, _ := os.ReadFile(reportIn)
+ if err := json.Unmarshal(raw, &rep); err != nil {
+ t.Fatal(err)
+ }
+ if rep.Total.Correct != 1 || rep.Results[0].Answer != "write" {
+ t.Fatalf("run report %+v", rep)
+ }
+ stored, err := os.ReadFile(filepath.Join(answers, rep.Results[0].ID+".answer"))
+ if err != nil || string(stored) != "x = 2 + 3\n" {
+ t.Fatalf("stored answer %q, %v", stored, err)
+ }
+
+ rejudge := func() bench.Report {
+ t.Helper()
+ out := filepath.Join(t.TempDir(), "rejudged.json")
+ var stdout, stderr bytes.Buffer
+ if code := run([]string{"bench", "rejudge", "--report-in", reportIn, "--manifest", manifest, "--clones", clones,
+ "--answers-dir", answers, "--json-out", out}, nil, &stdout, &stderr); code != 0 {
+ t.Fatalf("rejudge exited %d: %s", code, stderr.String())
+ }
+ var r bench.Report
+ raw, _ := os.ReadFile(out)
+ if err := json.Unmarshal(raw, &r); err != nil {
+ t.Fatal(err)
+ }
+ return r
+ }
+ if r := rejudge(); r.Total.Correct != 1 || r.SpendUSD != rep.SpendUSD || r.Label != rep.Label {
+ t.Fatalf("rejudged %+v", r)
+ }
+ if err := os.WriteFile(filepath.Join(answers, rep.Results[0].ID+".answer"), []byte("x = 5\n"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if r := rejudge(); r.Total.Wrong != 1 || r.Total.Correct != 0 {
+ t.Fatalf("rejudged a changed answer %+v", r.Total)
+ }
+}
+
func TestBenchRunFixedPricing(t *testing.T) {
work, _, clones, manifest := mineFixture(t)
cfg := modelConfig(t, work)
diff --git a/tools/graft-agents/resolve/bench/run.go b/tools/graft-agents/resolve/bench/run.go
index da73718..7cf2e44 100644
--- a/tools/graft-agents/resolve/bench/run.go
+++ b/tools/graft-agents/resolve/bench/run.go
@@ -5,6 +5,8 @@ import (
"errors"
"fmt"
"math/rand"
+ "os"
+ "path/filepath"
"sort"
"strings"
"time"
@@ -37,17 +39,20 @@ const (
// Result is the outcome of one case.
type Result struct {
- ID string `json:"id"`
- Repo string `json:"repo"`
- Family string `json:"family"`
- Language string `json:"language"`
- Kind string `json:"kind"`
- Path string `json:"path"`
- Outcome string `json:"outcome"`
- Exact bool `json:"exact,omitempty"`
- Strategy string `json:"strategy,omitempty"`
- Reason string `json:"reason,omitempty"`
- Seconds float64 `json:"seconds"`
+ ID string `json:"id"`
+ Repo string `json:"repo"`
+ Family string `json:"family"`
+ Language string `json:"language"`
+ Kind string `json:"kind"`
+ Path string `json:"path"`
+ Outcome string `json:"outcome"`
+ Exact bool `json:"exact,omitempty"`
+ Strategy string `json:"strategy,omitempty"`
+ // Answer is the action of an accepted resolution ("write" or "delete"); with
+ // Runner.AnswersDir its content is stored for audit and Rejudge.
+ Answer string `json:"answer,omitempty"`
+ Reason string `json:"reason,omitempty"`
+ Seconds float64 `json:"seconds"`
// HumanSyntaxError is set when the human resolution of a case that was sent fails the
// syntax check itself: the check would reject the right answer there (a grammar gap, or a
// human merge that did not parse).
@@ -109,6 +114,11 @@ type Runner struct {
MaxFileBytes int64
// Progress, when set, is called after each case.
Progress func(done, total int, r Result)
+ // AnswersDir, when set, keeps every accepted resolution: <AnswersDir>/<case ID>.answer holds
+ // the resolved content (no file for a deletion), so that the judge can be audited and the
+ // same answers judged again (Rejudge) without new model calls. The directory holds
+ // third-party code and stays local.
+ AnswersDir string
}
// Run replays every case and returns the results in the order of cases.
@@ -216,14 +226,70 @@ func (r Runner) runCase(ctx context.Context, c Case, maxBytes int64) (Result, er
f := resp.Files[0]
res.Strategy = strings.TrimPrefix(resp.Resolver, "graft-resolve/")
- correct, exact := judge(f.Content, human, f.Action == protocol.ActionDelete, c.HumanBlob == nil)
+ res.Answer = f.Action
+ if r.AnswersDir != "" && f.Action != protocol.ActionDelete {
+ if err := os.WriteFile(answerPath(r.AnswersDir, c.ID), f.Content, 0o600); err != nil {
+ return res, err
+ }
+ }
+ judgeResult(&res, f.Content, f.Action == protocol.ActionDelete, human, c.HumanBlob == nil)
+ return res, nil
+}
+
+func answerPath(dir, id string) string { return filepath.Join(dir, id+".answer") }
+
+// judgeResult sets the outcome of an accepted resolution from the judge.
+func judgeResult(res *Result, content []byte, deleted bool, human []byte, humanDeleted bool) {
+ correct, exact := judge(content, human, deleted, humanDeleted)
res.Exact = exact
if correct {
res.Outcome = OutcomeCorrect
} else {
res.Outcome = OutcomeWrong
}
- return res, nil
+}
+
+// Rejudge judges the stored answers of results again (Runner.AnswersDir), with no model call:
+// the outcome of every accepted resolution (correct or wrong) is recomputed from its stored
+// answer and the human resolution; every other result is kept as it is. It is how a fix of the
+// judge is applied to a run that was already paid for.
+func Rejudge(ctx context.Context, results []Result, cases []Case, clone func(repo string) string, answersDir string) ([]Result, error) {
+ byID := make(map[string]Case, len(cases))
+ for _, c := range cases {
+ byID[c.ID] = c
+ }
+ out := make([]Result, len(results))
+ for i, res := range results {
+ out[i] = res
+ if res.Outcome != OutcomeCorrect && res.Outcome != OutcomeWrong {
+ continue
+ }
+ c, ok := byID[res.ID]
+ if !ok {
+ return nil, fmt.Errorf("rejudge: case %s is not in the manifest", res.ID)
+ }
+ if res.Answer == "" {
+ return nil, fmt.Errorf("rejudge: case %s has no stored answer (the run kept no answers)", res.ID)
+ }
+ g := gitRepo{dir: clone(c.Repo)}
+ var human []byte
+ if c.HumanBlob != nil {
+ var err error
+ if human, err = g.blob(ctx, c.HumanBlob.OID); err != nil {
+ return nil, err
+ }
+ }
+ deleted := res.Answer == protocol.ActionDelete
+ var content []byte
+ if !deleted {
+ var err error
+ if content, err = os.ReadFile(answerPath(answersDir, res.ID)); err != nil {
+ return nil, fmt.Errorf("rejudge: %w", err)
+ }
+ }
+ judgeResult(&out[i], content, deleted, human, c.HumanBlob == nil)
+ }
+ return out, nil
}
// classifyFailure returns the outcome of a file that the chain did not resolve: the last attempt