Skip to content

Git designed for AI agents.

graft-resolve bench: sample by language, report by language and by kind

Federico Elia56a7e1e1de31
diff --git a/tools/graft-agents/resolve/bench/bench_test.go b/tools/graft-agents/resolve/bench/bench_test.go
index 4f95ac8..43bf29f 100644
--- a/tools/graft-agents/resolve/bench/bench_test.go
+++ b/tools/graft-agents/resolve/bench/bench_test.go
@@ -3,6 +3,7 @@ package bench
import (
"context"
"errors"
+ "fmt"
"os"
"os/exec"
"path/filepath"
@@ -262,6 +263,56 @@ func TestNotSentReason(t *testing.T) {
}
}
+// TestSampleSeparatesRustFromJava mirrors the mined corpus: few Rust and Python cases, many of the
+// others. Rust and Python are taken whole and the rest is split evenly over TS/JS, Go and Java.
+func TestSampleSeparatesRustFromJava(t *testing.T) {
+ var cases []Case
+ add := func(n int, family, path string) {
+ for i := 0; i < n; i++ {
+ cases = append(cases, Case{ID: fmt.Sprintf("%s-%s-%d", family, path, i), Family: family, Path: path})
+ }
+ }
+ add(983, FamilyTSJS, "a.ts")
+ add(96, FamilyPython, "a.py")
+ add(457, FamilyGo, "a.go")
+ add(22, FamilyRustJava, "a.rs")
+ add(1625, FamilyRustJava, "A.java")
+
+ counts := map[string]int{}
+ for _, c := range Sample(cases, 600, 1) {
+ counts[LanguageOf(c.Path)]++
+ }
+ if counts["rust"] != 22 || counts["python"] != 96 {
+ t.Fatalf("counts %v: Rust and Python must be taken whole", counts)
+ }
+ rest := 600 - 22 - 96
+ for _, l := range []string{"typescript", "go", "java"} {
+ if counts[l] < rest/3 || counts[l] > rest/3+1 {
+ t.Fatalf("counts %v: the rest must be split evenly (%d each)", counts, rest/3)
+ }
+ }
+}
+
+func TestReportByLanguageAndKind(t *testing.T) {
+ rep := NewReport("test", []Result{
+ {Family: FamilyRustJava, Language: "rust", Kind: protocol.KindContent, Outcome: OutcomeCorrect},
+ {Family: FamilyRustJava, Language: "java", Kind: protocol.KindContent, Outcome: OutcomeWrong},
+ {Family: FamilyRustJava, Language: "java", Kind: protocol.KindModifyDelete, Outcome: OutcomeAbstained},
+ })
+ if rep.Languages["java"].Cases != 2 || rep.Kinds[protocol.KindContent].ErrorRate() != 0.5 || rep.Kinds[protocol.KindModifyDelete].Abstained != 1 {
+ t.Fatalf("languages %+v, kinds %+v", rep.Languages, rep.Kinds)
+ }
+ var b strings.Builder
+ if err := rep.WriteMarkdown(&b); err != nil {
+ t.Fatal(err)
+ }
+ for _, want := range []string{"| language | cases", "| rust (low n) | 1 |", "| kind | cases", "| content (low n) | 2 | 1 (0) | 1 |", "| modify-delete (low n) | 1 |"} {
+ if !strings.Contains(b.String(), want) {
+ t.Fatalf("markdown lacks %q:\n%s", want, b.String())
+ }
+ }
+}
+
func TestSampleIsStratifiedAndStable(t *testing.T) {
var cases []Case
for i := 0; i < 30; i++ {
diff --git a/tools/graft-agents/resolve/bench/corpus.go b/tools/graft-agents/resolve/bench/corpus.go
index 3a5038f..f9cfa4f 100644
--- a/tools/graft-agents/resolve/bench/corpus.go
+++ b/tools/graft-agents/resolve/bench/corpus.go
@@ -25,6 +25,38 @@ const (
)
// familyOf returns the family of a path by its extension, or "" for other files.
+// LanguageOf returns the language of a benchmark path, by extension: typescript, javascript,
+// python, go, rust or java; "" for any other path.
+func LanguageOf(p string) string {
+ switch strings.ToLower(path.Ext(p)) {
+ case ".ts", ".tsx", ".mts", ".cts":
+ return "typescript"
+ case ".js", ".jsx", ".mjs", ".cjs":
+ return "javascript"
+ case ".py", ".pyi":
+ return "python"
+ case ".go":
+ return "go"
+ case ".rs":
+ return "rust"
+ case ".java":
+ return "java"
+ }
+ return ""
+}
+
+// stratumOf returns the sampling stratum of a case: its family, except that Rust and Java,
+// one family, are two strata, so that the few Rust cases are not drowned by the Java ones.
+func stratumOf(c Case) string {
+ if c.Family == FamilyRustJava {
+ if LanguageOf(c.Path) == "rust" {
+ return "rust"
+ }
+ return "java"
+ }
+ return c.Family
+}
+
func familyOf(p string) string {
switch strings.ToLower(path.Ext(p)) {
case ".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs":
diff --git a/tools/graft-agents/resolve/bench/report.go b/tools/graft-agents/resolve/bench/report.go
index 3c68209..551bc94 100644
--- a/tools/graft-agents/resolve/bench/report.go
+++ b/tools/graft-agents/resolve/bench/report.go
@@ -67,6 +67,11 @@ type Report struct {
Label string `json:"label"`
Total Counts `json:"total"`
Families map[string]Counts `json:"families"`
+ // Languages and Kinds count the same results by language (bench.LanguageOf) and by kind of
+ // conflict: the llm strategy declines a modify-delete file, so the content row is what the
+ // model attempted.
+ Languages map[string]Counts `json:"languages"`
+ Kinds map[string]Counts `json:"kinds"`
// NotSent counts the cases that were not sent, by reason.
NotSent map[string]int `json:"not_sent"`
// SpendUSD is the AI spend of the run in USD, when a budget was used: the usage the provider
@@ -78,14 +83,23 @@ type Report struct {
Results []Result `json:"results"`
}
+// LowN is the number of cases below which a row is marked as low-n in the Markdown report.
+const LowN = 30
+
// NewReport aggregates results.
func NewReport(label string, results []Result) Report {
- rep := Report{Label: label, Families: map[string]Counts{}, NotSent: map[string]int{}, Results: results}
+ rep := Report{Label: label, Families: map[string]Counts{}, Languages: map[string]Counts{}, Kinds: map[string]Counts{},
+ NotSent: map[string]int{}, Results: results}
for _, r := range results {
rep.Total.add(r)
- fc := rep.Families[r.Family]
- fc.add(r)
- rep.Families[r.Family] = fc
+ for _, g := range []struct {
+ m map[string]Counts
+ key string
+ }{{rep.Families, r.Family}, {rep.Languages, r.Language}, {rep.Kinds, r.Kind}} {
+ c := g.m[g.key]
+ c.add(r)
+ g.m[g.key] = c
+ }
if r.Outcome == OutcomeNotSent {
rep.NotSent[r.Reason]++
}
@@ -125,6 +139,34 @@ func (rep Report) WriteMarkdown(w io.Writer) error {
row(f, rep.Families[f])
}
row("total", rep.Total)
+
+ // The same results by language and by kind of conflict.
+ for _, group := range []struct {
+ title string
+ m map[string]Counts
+ }{{"language", rep.Languages}, {"kind", rep.Kinds}} {
+ if len(group.m) == 0 {
+ continue
+ }
+ fmt.Fprintf(&b, "\n| %s | cases | correct (exact) | wrong | rejected by checks | not sent | model error | not run (cap) | abstained | error rate |\n", group.title)
+ b.WriteString("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n")
+ keys := make([]string, 0, len(group.m))
+ for k := range group.m {
+ keys = append(keys, k)
+ }
+ sort.Strings(keys)
+ for _, k := range keys {
+ name := k
+ if name == "" {
+ name = "other"
+ }
+ if group.m[k].Cases < LowN {
+ name += " (low n)"
+ }
+ row(name, group.m[k])
+ }
+ }
+ fmt.Fprintf(&b, "\nRows marked low n have fewer than %d cases; their rates are not significant. The llm strategy declines a modify-delete or mode conflict (a side is absent), which goes to the agent as abstained: the content row is what the model attempted.\n", LowN)
b.WriteString("\nThe error rate is wrong / (correct + wrong): the share of the results that would land and differ from the human resolution. Correct means equal after normalizing line endings and trailing whitespace; exact means byte-identical.\n")
syntaxParts := make([]string, 0, len(families))
for _, f := range families {
diff --git a/tools/graft-agents/resolve/bench/run.go b/tools/graft-agents/resolve/bench/run.go
index f393442..da73718 100644
--- a/tools/graft-agents/resolve/bench/run.go
+++ b/tools/graft-agents/resolve/bench/run.go
@@ -40,6 +40,8 @@ type Result struct {
ID string `json:"id"`
Repo string `json:"repo"`
Family string `json:"family"`
+ Language string `json:"language"`
+ Kind string `json:"kind"`
Path string `json:"path"`
Outcome string `json:"outcome"`
Exact bool `json:"exact,omitempty"`
@@ -55,10 +57,14 @@ type Result struct {
// Sample returns n cases of cases, stratified by family: each family's cases are shuffled with the
// seed and the families are taken in turn, so that the sample is the same for the same manifest and
// seed. n <= 0 takes every case.
+//
+// The strata are the families, with Rust and Java apart (stratumOf): a stratum with fewer cases
+// than its share is taken whole, and the rest is split evenly over the others.
func Sample(cases []Case, n int, seed int64) []Case {
byFamily := map[string][]Case{}
for _, c := range cases {
- byFamily[c.Family] = append(byFamily[c.Family], c)
+ s := stratumOf(c)
+ byFamily[s] = append(byFamily[s], c)
}
families := make([]string, 0, len(byFamily))
for f := range byFamily {
@@ -130,7 +136,7 @@ func (r Runner) Run(ctx context.Context, cases []Case) ([]Result, error) {
}
func (r Runner) runCase(ctx context.Context, c Case, maxBytes int64) (Result, error) {
- res := Result{ID: c.ID, Repo: c.Repo, Family: c.Family, Path: c.Path}
+ res := Result{ID: c.ID, Repo: c.Repo, Family: c.Family, Language: LanguageOf(c.Path), Kind: c.Kind, Path: c.Path}
g := gitRepo{dir: r.Clone(c.Repo)}
sides := map[string]*protocol.Side{}