Git designed for AI agents.
graft-resolve bench: keep the answers, and judge them again without the model
Federico Elia144e1c00f488
diff --git a/tools/graft-agents/cmd/graft-resolve/bench.go b/tools/graft-agents/cmd/graft-resolve/bench.goindex b353f4d..d1262e5 100644--- a/tools/graft-agents/cmd/graft-resolve/bench.go+++ b/tools/graft-agents/cmd/graft-resolve/bench.go@@ -40,8 +40,10 @@ func runBench(args []string, stdout, stderr io.Writer) int {return benchMine(ctx, args[1:], stdout, stderr)case "run":return benchRun(ctx, args[1:], stdout, stderr)+ case "rejudge":+ return benchRejudge(ctx, args[1:], stdout, stderr)}- fmt.Fprintf(stderr, "graft-resolve bench: unknown subcommand %q, want mine or run\n", args[0])+ fmt.Fprintf(stderr, "graft-resolve bench: unknown subcommand %q, want mine, run or rejudge\n", args[0])return 2}@@ -124,6 +126,7 @@ func benchRun(ctx context.Context, args []string, stdout, stderr io.Writer) intseed := fs.Int64("seed", 1, "seed of the sample")label := fs.String("label", "", "name of the run in the report")jsonOut := fs.String("json-out", "", "report as JSON")+ answersDir := fs.String("answers-dir", "", "keep every accepted resolution here, for audit and bench rejudge (local: third-party code)")mdOut := fs.String("md-out", "", "report as Markdown")maxUSD := fs.Float64("max-usd", 0, "hard cap on the AI spend of the run in USD (required when the chain has an llm strategy)")pricingSource := fs.String("pricing", "openrouter", "where the model's price comes from: openrouter (read at run time) or fixed (the two price flags)")@@ -202,9 +205,16 @@ func benchRun(ctx context.Context, args []string, stdout, stderr io.Writer) int}cases := bench.Sample(manifest.Cases, *sample, *seed)+ if *answersDir != "" {+ if err := os.MkdirAll(*answersDir, 0o700); err != nil {+ fmt.Fprintf(stderr, "graft-resolve bench run: %v\n", err)+ return 1+ }+ }runner := bench.Runner{- Resolver: res,- Clone: func(repo string) string { return bench.CloneDir(*clones, repo) },+ Resolver: res,+ AnswersDir: *answersDir,+ Clone: func(repo string) string { return bench.CloneDir(*clones, repo) },Progress: func(done, total int, r bench.Result) {fmt.Fprintf(stderr, "[%d/%d] %s %s %s: %s %s\n", done, total, r.Family, r.Repo, r.Path, r.Outcome, r.Reason)},@@ -220,33 +230,88 @@ func benchRun(ctx context.Context, args []string, stdout, stderr io.Writer) intreport.SpendUSD, report.MaxUSD, report.Pricing = snap.SpentUSD, *maxUSD, pricing}- if *jsonOut != "" {+ if err := writeReport(report, *jsonOut, *mdOut, stdout); err != nil {+ fmt.Fprintf(stderr, "graft-resolve bench run: %v\n", err)+ return 1+ }+ if runErr != nil {+ fmt.Fprintf(stderr, "graft-resolve bench run: stopped after %d of %d cases: %v\n", len(results), len(cases), runErr)+ return 1+ }+ return 0+}++// benchRejudge judges the stored answers of a run again, with no model call:+//+// graft-resolve bench rejudge --report-in FILE --manifest FILE --clones DIR --answers-dir DIR+// [--json-out FILE] [--md-out FILE]+//+// The report keeps everything of the run (label, spend, pricing, the results that were not+// accepted) and recomputes the outcome of every accepted resolution.+func benchRejudge(ctx context.Context, args []string, stdout, stderr io.Writer) int {+ fs := flag.NewFlagSet("graft-resolve bench rejudge", flag.ContinueOnError)+ fs.SetOutput(stderr)+ reportIn := fs.String("report-in", "", "JSON report of the run")+ manifestPath := fs.String("manifest", "", "manifest of the run")+ clones := fs.String("clones", "", "directory of the clones of the corpus repositories")+ answersDir := fs.String("answers-dir", "", "answers kept by the run (bench run --answers-dir)")+ jsonOut := fs.String("json-out", "", "report as JSON")+ mdOut := fs.String("md-out", "", "report as Markdown")+ if err := fs.Parse(args); err != nil {+ return 2+ }+ if *reportIn == "" || *manifestPath == "" || *clones == "" || *answersDir == "" || fs.NArg() != 0 {+ fmt.Fprintln(stderr, "graft-resolve bench rejudge: --report-in, --manifest, --clones and --answers-dir are required")+ return 2+ }+ raw, err := os.ReadFile(*reportIn)+ if err != nil {+ fmt.Fprintf(stderr, "graft-resolve bench rejudge: %v\n", err)+ return 1+ }+ var old bench.Report+ if err := json.Unmarshal(raw, &old); err != nil {+ fmt.Fprintf(stderr, "graft-resolve bench rejudge: %s: %v\n", *reportIn, err)+ return 1+ }+ manifest, err := bench.LoadManifest(*manifestPath)+ if err != nil {+ fmt.Fprintf(stderr, "graft-resolve bench rejudge: %v\n", err)+ return 1+ }+ results, err := bench.Rejudge(ctx, old.Results, manifest.Cases, func(repo string) string { return bench.CloneDir(*clones, repo) }, *answersDir)+ if err != nil {+ fmt.Fprintf(stderr, "graft-resolve bench rejudge: %v\n", err)+ return 1+ }+ report := bench.NewReport(old.Label, results)+ report.SpendUSD, report.MaxUSD, report.Pricing = old.SpendUSD, old.MaxUSD, old.Pricing+ if err := writeReport(report, *jsonOut, *mdOut, stdout); err != nil {+ fmt.Fprintf(stderr, "graft-resolve bench rejudge: %v\n", err)+ return 1+ }+ return 0+}++// writeReport writes report as JSON to jsonOut and as Markdown to stdout and mdOut, when set.+func writeReport(report bench.Report, jsonOut, mdOut string, stdout io.Writer) error {+ if jsonOut != "" {data, err := json.MarshalIndent(report, "", " ")- if err == nil {- err = os.WriteFile(*jsonOut, append(data, '\n'), 0o644)- }if err != nil {- fmt.Fprintf(stderr, "graft-resolve bench run: %v\n", err)- return 1+ return err+ }+ if err := os.WriteFile(jsonOut, append(data, '\n'), 0o644); err != nil {+ return err}}out := stdout- if *mdOut != "" {- f, err := os.Create(*mdOut)+ if mdOut != "" {+ f, err := os.Create(mdOut)if err != nil {- fmt.Fprintf(stderr, "graft-resolve bench run: %v\n", err)- return 1+ return err}defer f.Close()out = io.MultiWriter(stdout, f)}- if err := report.WriteMarkdown(out); err != nil {- fmt.Fprintf(stderr, "graft-resolve bench run: %v\n", err)- return 1- }- if runErr != nil {- fmt.Fprintf(stderr, "graft-resolve bench run: stopped after %d of %d cases: %v\n", len(results), len(cases), runErr)- return 1- }- return 0+ return report.WriteMarkdown(out)}diff --git a/tools/graft-agents/cmd/graft-resolve/bench_budget_test.go b/tools/graft-agents/cmd/graft-resolve/bench_budget_test.goindex a2fd51e..c766372 100644--- a/tools/graft-agents/cmd/graft-resolve/bench_budget_test.go+++ b/tools/graft-agents/cmd/graft-resolve/bench_budget_test.go@@ -128,6 +128,58 @@ func TestBenchRunRefusesUnreadablePrice(t *testing.T) {}}+// TestBenchAnswersAndRejudge keeps the answers of a run, then judges them again without the+// model: the same answer gives the same outcome, and a changed answer is judged anew.+func TestBenchAnswersAndRejudge(t *testing.T) {+ work, _, clones, manifest := mineFixture(t)+ cfg := modelConfig(t, work)+ answers := filepath.Join(work, "answers")+ reportIn := filepath.Join(work, "run.json")+ var stdout, stderr bytes.Buffer+ if code := run([]string{"bench", "run", "--config", cfg, "--manifest", manifest, "--clones", clones, "--max-usd", "1",+ "--pricing", "fixed", "--input-usd-per-million", "1", "--output-usd-per-million", "1",+ "--answers-dir", answers, "--json-out", reportIn}, nil, &stdout, &stderr); code != 0 {+ t.Fatalf("run exited %d: %s", code, stderr.String())+ }+ var rep bench.Report+ raw, _ := os.ReadFile(reportIn)+ if err := json.Unmarshal(raw, &rep); err != nil {+ t.Fatal(err)+ }+ if rep.Total.Correct != 1 || rep.Results[0].Answer != "write" {+ t.Fatalf("run report %+v", rep)+ }+ stored, err := os.ReadFile(filepath.Join(answers, rep.Results[0].ID+".answer"))+ if err != nil || string(stored) != "x = 2 + 3\n" {+ t.Fatalf("stored answer %q, %v", stored, err)+ }++ rejudge := func() bench.Report {+ t.Helper()+ out := filepath.Join(t.TempDir(), "rejudged.json")+ var stdout, stderr bytes.Buffer+ if code := run([]string{"bench", "rejudge", "--report-in", reportIn, "--manifest", manifest, "--clones", clones,+ "--answers-dir", answers, "--json-out", out}, nil, &stdout, &stderr); code != 0 {+ t.Fatalf("rejudge exited %d: %s", code, stderr.String())+ }+ var r bench.Report+ raw, _ := os.ReadFile(out)+ if err := json.Unmarshal(raw, &r); err != nil {+ t.Fatal(err)+ }+ return r+ }+ if r := rejudge(); r.Total.Correct != 1 || r.SpendUSD != rep.SpendUSD || r.Label != rep.Label {+ t.Fatalf("rejudged %+v", r)+ }+ if err := os.WriteFile(filepath.Join(answers, rep.Results[0].ID+".answer"), []byte("x = 5\n"), 0o600); err != nil {+ t.Fatal(err)+ }+ if r := rejudge(); r.Total.Wrong != 1 || r.Total.Correct != 0 {+ t.Fatalf("rejudged a changed answer %+v", r.Total)+ }+}+func TestBenchRunFixedPricing(t *testing.T) {work, _, clones, manifest := mineFixture(t)cfg := modelConfig(t, work)diff --git a/tools/graft-agents/resolve/bench/run.go b/tools/graft-agents/resolve/bench/run.goindex da73718..7cf2e44 100644--- a/tools/graft-agents/resolve/bench/run.go+++ b/tools/graft-agents/resolve/bench/run.go@@ -5,6 +5,8 @@ import ("errors""fmt""math/rand"+ "os"+ "path/filepath""sort""strings""time"@@ -37,17 +39,20 @@ const (// Result is the outcome of one case.type Result struct {- ID string `json:"id"`- Repo string `json:"repo"`- Family string `json:"family"`- Language string `json:"language"`- Kind string `json:"kind"`- Path string `json:"path"`- Outcome string `json:"outcome"`- Exact bool `json:"exact,omitempty"`- Strategy string `json:"strategy,omitempty"`- Reason string `json:"reason,omitempty"`- Seconds float64 `json:"seconds"`+ ID string `json:"id"`+ Repo string `json:"repo"`+ Family string `json:"family"`+ Language string `json:"language"`+ Kind string `json:"kind"`+ Path string `json:"path"`+ Outcome string `json:"outcome"`+ Exact bool `json:"exact,omitempty"`+ Strategy string `json:"strategy,omitempty"`+ // Answer is the action of an accepted resolution ("write" or "delete"); with+ // Runner.AnswersDir its content is stored for audit and Rejudge.+ Answer string `json:"answer,omitempty"`+ Reason string `json:"reason,omitempty"`+ Seconds float64 `json:"seconds"`// HumanSyntaxError is set when the human resolution of a case that was sent fails the// syntax check itself: the check would reject the right answer there (a grammar gap, or a// human merge that did not parse).@@ -109,6 +114,11 @@ type Runner struct {MaxFileBytes int64// Progress, when set, is called after each case.Progress func(done, total int, r Result)+ // AnswersDir, when set, keeps every accepted resolution: <AnswersDir>/<case ID>.answer holds+ // the resolved content (no file for a deletion), so that the judge can be audited and the+ // same answers judged again (Rejudge) without new model calls. The directory holds+ // third-party code and stays local.+ AnswersDir string}// Run replays every case and returns the results in the order of cases.@@ -216,14 +226,70 @@ func (r Runner) runCase(ctx context.Context, c Case, maxBytes int64) (Result, erf := resp.Files[0]res.Strategy = strings.TrimPrefix(resp.Resolver, "graft-resolve/")- correct, exact := judge(f.Content, human, f.Action == protocol.ActionDelete, c.HumanBlob == nil)+ res.Answer = f.Action+ if r.AnswersDir != "" && f.Action != protocol.ActionDelete {+ if err := os.WriteFile(answerPath(r.AnswersDir, c.ID), f.Content, 0o600); err != nil {+ return res, err+ }+ }+ judgeResult(&res, f.Content, f.Action == protocol.ActionDelete, human, c.HumanBlob == nil)+ return res, nil+}++func answerPath(dir, id string) string { return filepath.Join(dir, id+".answer") }++// judgeResult sets the outcome of an accepted resolution from the judge.+func judgeResult(res *Result, content []byte, deleted bool, human []byte, humanDeleted bool) {+ correct, exact := judge(content, human, deleted, humanDeleted)res.Exact = exactif correct {res.Outcome = OutcomeCorrect} else {res.Outcome = OutcomeWrong}- return res, nil+}++// Rejudge judges the stored answers of results again (Runner.AnswersDir), with no model call:+// the outcome of every accepted resolution (correct or wrong) is recomputed from its stored+// answer and the human resolution; every other result is kept as it is. It is how a fix of the+// judge is applied to a run that was already paid for.+func Rejudge(ctx context.Context, results []Result, cases []Case, clone func(repo string) string, answersDir string) ([]Result, error) {+ byID := make(map[string]Case, len(cases))+ for _, c := range cases {+ byID[c.ID] = c+ }+ out := make([]Result, len(results))+ for i, res := range results {+ out[i] = res+ if res.Outcome != OutcomeCorrect && res.Outcome != OutcomeWrong {+ continue+ }+ c, ok := byID[res.ID]+ if !ok {+ return nil, fmt.Errorf("rejudge: case %s is not in the manifest", res.ID)+ }+ if res.Answer == "" {+ return nil, fmt.Errorf("rejudge: case %s has no stored answer (the run kept no answers)", res.ID)+ }+ g := gitRepo{dir: clone(c.Repo)}+ var human []byte+ if c.HumanBlob != nil {+ var err error+ if human, err = g.blob(ctx, c.HumanBlob.OID); err != nil {+ return nil, err+ }+ }+ deleted := res.Answer == protocol.ActionDelete+ var content []byte+ if !deleted {+ var err error+ if content, err = os.ReadFile(answerPath(answersDir, res.ID)); err != nil {+ return nil, fmt.Errorf("rejudge: %w", err)+ }+ }+ judgeResult(&out[i], content, deleted, human, c.HumanBlob == nil)+ }+ return out, nil}// classifyFailure returns the outcome of a file that the chain did not resolve: the last attempt