Git designed for AI agents.
graft-resolve bench: sample by language, report by language and by kind
Federico Elia56a7e1e1de31
diff --git a/tools/graft-agents/resolve/bench/bench_test.go b/tools/graft-agents/resolve/bench/bench_test.goindex 4f95ac8..43bf29f 100644--- a/tools/graft-agents/resolve/bench/bench_test.go+++ b/tools/graft-agents/resolve/bench/bench_test.go@@ -3,6 +3,7 @@ package benchimport ("context""errors"+ "fmt""os""os/exec""path/filepath"@@ -262,6 +263,56 @@ func TestNotSentReason(t *testing.T) {}}+// TestSampleSeparatesRustFromJava mirrors the mined corpus: few Rust and Python cases, many of the+// others. Rust and Python are taken whole and the rest is split evenly over TS/JS, Go and Java.+func TestSampleSeparatesRustFromJava(t *testing.T) {+ var cases []Case+ add := func(n int, family, path string) {+ for i := 0; i < n; i++ {+ cases = append(cases, Case{ID: fmt.Sprintf("%s-%s-%d", family, path, i), Family: family, Path: path})+ }+ }+ add(983, FamilyTSJS, "a.ts")+ add(96, FamilyPython, "a.py")+ add(457, FamilyGo, "a.go")+ add(22, FamilyRustJava, "a.rs")+ add(1625, FamilyRustJava, "A.java")++ counts := map[string]int{}+ for _, c := range Sample(cases, 600, 1) {+ counts[LanguageOf(c.Path)]+++ }+ if counts["rust"] != 22 || counts["python"] != 96 {+ t.Fatalf("counts %v: Rust and Python must be taken whole", counts)+ }+ rest := 600 - 22 - 96+ for _, l := range []string{"typescript", "go", "java"} {+ if counts[l] < rest/3 || counts[l] > rest/3+1 {+ t.Fatalf("counts %v: the rest must be split evenly (%d each)", counts, rest/3)+ }+ }+}++func TestReportByLanguageAndKind(t *testing.T) {+ rep := NewReport("test", []Result{+ {Family: FamilyRustJava, Language: "rust", Kind: protocol.KindContent, Outcome: OutcomeCorrect},+ {Family: FamilyRustJava, Language: "java", Kind: protocol.KindContent, Outcome: OutcomeWrong},+ {Family: FamilyRustJava, Language: "java", Kind: protocol.KindModifyDelete, Outcome: OutcomeAbstained},+ })+ if rep.Languages["java"].Cases != 2 || rep.Kinds[protocol.KindContent].ErrorRate() != 0.5 || rep.Kinds[protocol.KindModifyDelete].Abstained != 1 {+ t.Fatalf("languages %+v, kinds %+v", rep.Languages, rep.Kinds)+ }+ var b strings.Builder+ if err := rep.WriteMarkdown(&b); err != nil {+ t.Fatal(err)+ }+ for _, want := range []string{"| language | cases", "| rust (low n) | 1 |", "| kind | cases", "| content (low n) | 2 | 1 (0) | 1 |", "| modify-delete (low n) | 1 |"} {+ if !strings.Contains(b.String(), want) {+ t.Fatalf("markdown lacks %q:\n%s", want, b.String())+ }+ }+}+func TestSampleIsStratifiedAndStable(t *testing.T) {var cases []Casefor i := 0; i < 30; i++ {diff --git a/tools/graft-agents/resolve/bench/corpus.go b/tools/graft-agents/resolve/bench/corpus.goindex 3a5038f..f9cfa4f 100644--- a/tools/graft-agents/resolve/bench/corpus.go+++ b/tools/graft-agents/resolve/bench/corpus.go@@ -25,6 +25,38 @@ const ()// familyOf returns the family of a path by its extension, or "" for other files.+// LanguageOf returns the language of a benchmark path, by extension: typescript, javascript,+// python, go, rust or java; "" for any other path.+func LanguageOf(p string) string {+ switch strings.ToLower(path.Ext(p)) {+ case ".ts", ".tsx", ".mts", ".cts":+ return "typescript"+ case ".js", ".jsx", ".mjs", ".cjs":+ return "javascript"+ case ".py", ".pyi":+ return "python"+ case ".go":+ return "go"+ case ".rs":+ return "rust"+ case ".java":+ return "java"+ }+ return ""+}++// stratumOf returns the sampling stratum of a case: its family, except that Rust and Java,+// one family, are two strata, so that the few Rust cases are not drowned by the Java ones.+func stratumOf(c Case) string {+ if c.Family == FamilyRustJava {+ if LanguageOf(c.Path) == "rust" {+ return "rust"+ }+ return "java"+ }+ return c.Family+}+func familyOf(p string) string {switch strings.ToLower(path.Ext(p)) {case ".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs":diff --git a/tools/graft-agents/resolve/bench/report.go b/tools/graft-agents/resolve/bench/report.goindex 3c68209..551bc94 100644--- a/tools/graft-agents/resolve/bench/report.go+++ b/tools/graft-agents/resolve/bench/report.go@@ -67,6 +67,11 @@ type Report struct {Label string `json:"label"`Total Counts `json:"total"`Families map[string]Counts `json:"families"`+ // Languages and Kinds count the same results by language (bench.LanguageOf) and by kind of+ // conflict: the llm strategy declines a modify-delete file, so the content row is what the+ // model attempted.+ Languages map[string]Counts `json:"languages"`+ Kinds map[string]Counts `json:"kinds"`// NotSent counts the cases that were not sent, by reason.NotSent map[string]int `json:"not_sent"`// SpendUSD is the AI spend of the run in USD, when a budget was used: the usage the provider@@ -78,14 +83,23 @@ type Report struct {Results []Result `json:"results"`}+// LowN is the number of cases below which a row is marked as low-n in the Markdown report.+const LowN = 30+// NewReport aggregates results.func NewReport(label string, results []Result) Report {- rep := Report{Label: label, Families: map[string]Counts{}, NotSent: map[string]int{}, Results: results}+ rep := Report{Label: label, Families: map[string]Counts{}, Languages: map[string]Counts{}, Kinds: map[string]Counts{},+ NotSent: map[string]int{}, Results: results}for _, r := range results {rep.Total.add(r)- fc := rep.Families[r.Family]- fc.add(r)- rep.Families[r.Family] = fc+ for _, g := range []struct {+ m map[string]Counts+ key string+ }{{rep.Families, r.Family}, {rep.Languages, r.Language}, {rep.Kinds, r.Kind}} {+ c := g.m[g.key]+ c.add(r)+ g.m[g.key] = c+ }if r.Outcome == OutcomeNotSent {rep.NotSent[r.Reason]++}@@ -125,6 +139,34 @@ func (rep Report) WriteMarkdown(w io.Writer) error {row(f, rep.Families[f])}row("total", rep.Total)++ // The same results by language and by kind of conflict.+ for _, group := range []struct {+ title string+ m map[string]Counts+ }{{"language", rep.Languages}, {"kind", rep.Kinds}} {+ if len(group.m) == 0 {+ continue+ }+ fmt.Fprintf(&b, "\n| %s | cases | correct (exact) | wrong | rejected by checks | not sent | model error | not run (cap) | abstained | error rate |\n", group.title)+ b.WriteString("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n")+ keys := make([]string, 0, len(group.m))+ for k := range group.m {+ keys = append(keys, k)+ }+ sort.Strings(keys)+ for _, k := range keys {+ name := k+ if name == "" {+ name = "other"+ }+ if group.m[k].Cases < LowN {+ name += " (low n)"+ }+ row(name, group.m[k])+ }+ }+ fmt.Fprintf(&b, "\nRows marked low n have fewer than %d cases; their rates are not significant. The llm strategy declines a modify-delete or mode conflict (a side is absent), which goes to the agent as abstained: the content row is what the model attempted.\n", LowN)b.WriteString("\nThe error rate is wrong / (correct + wrong): the share of the results that would land and differ from the human resolution. Correct means equal after normalizing line endings and trailing whitespace; exact means byte-identical.\n")syntaxParts := make([]string, 0, len(families))for _, f := range families {diff --git a/tools/graft-agents/resolve/bench/run.go b/tools/graft-agents/resolve/bench/run.goindex f393442..da73718 100644--- a/tools/graft-agents/resolve/bench/run.go+++ b/tools/graft-agents/resolve/bench/run.go@@ -40,6 +40,8 @@ type Result struct {ID string `json:"id"`Repo string `json:"repo"`Family string `json:"family"`+ Language string `json:"language"`+ Kind string `json:"kind"`Path string `json:"path"`Outcome string `json:"outcome"`Exact bool `json:"exact,omitempty"`@@ -55,10 +57,14 @@ type Result struct {// Sample returns n cases of cases, stratified by family: each family's cases are shuffled with the// seed and the families are taken in turn, so that the sample is the same for the same manifest and// seed. n <= 0 takes every case.+//+// The strata are the families, with Rust and Java apart (stratumOf): a stratum with fewer cases+// than its share is taken whole, and the rest is split evenly over the others.func Sample(cases []Case, n int, seed int64) []Case {byFamily := map[string][]Case{}for _, c := range cases {- byFamily[c.Family] = append(byFamily[c.Family], c)+ s := stratumOf(c)+ byFamily[s] = append(byFamily[s], c)}families := make([]string, 0, len(byFamily))for f := range byFamily {@@ -130,7 +136,7 @@ func (r Runner) Run(ctx context.Context, cases []Case) ([]Result, error) {}func (r Runner) runCase(ctx context.Context, c Case, maxBytes int64) (Result, error) {- res := Result{ID: c.ID, Repo: c.Repo, Family: c.Family, Path: c.Path}+ res := Result{ID: c.ID, Repo: c.Repo, Family: c.Family, Language: LanguageOf(c.Path), Kind: c.Kind, Path: c.Path}g := gitRepo{dir: r.Clone(c.Repo)}sides := map[string]*protocol.Side{}