Skip to content

Git designed for AI agents.

graft-resolve: the benchmark miner, runner and report (WP1, part 1)

Federico Elia2893dfcba0cd
diff --git a/tools/graft-agents/resolve/bench/bench_test.go b/tools/graft-agents/resolve/bench/bench_test.go
new file mode 100644
index 0000000..52351fd
--- /dev/null
+++ b/tools/graft-agents/resolve/bench/bench_test.go
@@ -0,0 +1,281 @@
+package bench
+
+import (
+ "context"
+ "errors"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/feder-cr/git-ai/tools/graft-agents/protocol"
+ "github.com/feder-cr/git-ai/tools/graft-agents/resolve"
+)
+
+// fixture is a repository with crafted merges.
+type fixture struct {
+ t *testing.T
+ dir string
+ now int64
+}
+
+func newFixture(t *testing.T) *fixture {
+ t.Helper()
+ if _, err := exec.LookPath("git"); err != nil {
+ t.Skip("git is not installed")
+ }
+ f := &fixture{t: t, dir: t.TempDir(), now: 1767225600}
+ f.git("init", "-q", "-b", "main")
+ return f
+}
+
+// git runs git in the fixture with a fixed identity and date, given on the command line so that no
+// configuration file is touched.
+func (f *fixture) git(args ...string) string {
+ f.t.Helper()
+ f.now += 60
+ cmd := exec.Command("git", append([]string{"-C", f.dir, "-c", "user.name=Bench", "-c", "[email protected]",
+ "-c", "commit.gpgsign=false", "-c", "init.defaultBranch=main"}, args...)...)
+ date := time.Unix(f.now, 0).UTC().Format(time.RFC3339)
+ cmd.Env = append(os.Environ(), "GIT_AUTHOR_DATE="+date, "GIT_COMMITTER_DATE="+date, "GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_SYSTEM=/dev/null")
+ out, err := cmd.CombinedOutput()
+ if err != nil {
+ f.t.Fatalf("git %s: %v: %s", strings.Join(args, " "), err, out)
+ }
+ return strings.TrimSpace(string(out))
+}
+
+func (f *fixture) write(name, content string) {
+ f.t.Helper()
+ p := filepath.Join(f.dir, name)
+ if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
+ f.t.Fatal(err)
+ }
+ if err := os.WriteFile(p, []byte(content), 0o644); err != nil {
+ f.t.Fatal(err)
+ }
+}
+
+func (f *fixture) commit(msg string, files map[string]string) string {
+ f.t.Helper()
+ for name, content := range files {
+ f.write(name, content)
+ }
+ f.git("add", "-A")
+ f.git("commit", "-q", "-m", msg)
+ return f.git("rev-parse", "HEAD")
+}
+
+// conflictMerge merges branch into the current branch, which conflicts, and commits the human
+// resolution.
+func (f *fixture) conflictMerge(branch string, resolved map[string]string) string {
+ f.t.Helper()
+ cmd := exec.Command("git", "-C", f.dir, "-c", "user.name=Bench", "-c", "[email protected]", "merge", "--no-ff", "-q", branch)
+ cmd.Env = append(os.Environ(), "GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_SYSTEM=/dev/null")
+ if out, err := cmd.CombinedOutput(); err == nil {
+ f.t.Fatalf("merge of %s did not conflict: %s", branch, out)
+ }
+ return f.commit("Merge "+branch, resolved)
+}
+
+const (
+ goBase = "package a\n\nfunc A() int {\n\treturn 1\n}\n"
+ goOurs = "package a\n\nfunc A() int {\n\treturn 2\n}\n"
+ goTheirs = "package a\n\nfunc A() int {\n\treturn 3\n}\n"
+ goHuman = "package a\n\nfunc A() int {\n\treturn 2 + 3\n}\n"
+)
+
+// buildHistory creates: a conflicting merge of a.go resolved by hand (one case), a conflicting
+// merge of b.py where the human took one side (not a case), a conflict in notes.txt (no family),
+// and a clean merge.
+func buildHistory(f *fixture) (mergeA string) {
+ f.commit("base", map[string]string{"a.go": goBase, "b.py": "x = 1\n", "notes.txt": "one\n"})
+
+ f.git("checkout", "-q", "-b", "feature")
+ f.commit("feature: A returns 3", map[string]string{"a.go": goTheirs})
+ f.git("checkout", "-q", "main")
+ f.commit("main: A returns 2", map[string]string{"a.go": goOurs})
+ mergeA = f.conflictMerge("feature", map[string]string{"a.go": goHuman})
+
+ f.git("checkout", "-q", "-b", "feature2")
+ f.commit("feature2", map[string]string{"b.py": "x = 3\n", "notes.txt": "three\n"})
+ f.git("checkout", "-q", "main")
+ f.commit("main2", map[string]string{"b.py": "x = 2\n", "notes.txt": "two\n"})
+ f.conflictMerge("feature2", map[string]string{"b.py": "x = 2\n", "notes.txt": "two and three\n"})
+
+ f.git("checkout", "-q", "-b", "feature3")
+ f.commit("feature3", map[string]string{"c.go": "package c\n"})
+ f.git("checkout", "-q", "main")
+ f.commit("main3", map[string]string{"d.go": "package d\n"})
+ f.git("merge", "--no-ff", "-q", "-m", "clean merge", "feature3")
+ return mergeA
+}
+
+func TestMineKeepsRealResolutions(t *testing.T) {
+ f := newFixture(t)
+ mergeA := buildHistory(f)
+ head := f.git("rev-parse", "HEAD")
+
+ cases, stats, err := Mine(context.Background(), Repo{Name: "fixture", Commit: head, License: "MIT"}, f.dir, MineOptions{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(cases) != 1 {
+ t.Fatalf("got %d cases, want 1: %+v (stats %+v)", len(cases), cases, stats)
+ }
+ c := cases[0]
+ if c.Merge != mergeA || c.Path != "a.go" || c.Family != FamilyGo || c.Kind != protocol.KindContent || c.License != "MIT" {
+ t.Fatalf("unexpected case %+v", c)
+ }
+ if c.BaseBlob == nil || c.OursBlob == nil || c.TheirsBlob == nil || c.HumanBlob == nil {
+ t.Fatalf("missing sides in %+v", c)
+ }
+ want := MineStats{Merges: 3, ConflictingMerges: 2, ConflictingFiles: 3, OtherFamily: 1, OneSide: 1, Kept: 1}
+ if stats != want {
+ t.Fatalf("stats %+v, want %+v", stats, want)
+ }
+
+ // The manifest replays identically.
+ file := filepath.Join(t.TempDir(), "manifest.json")
+ if err := SaveManifest(file, Manifest{Version: ManifestVersion, Cases: cases}); err != nil {
+ t.Fatal(err)
+ }
+ m, err := LoadManifest(file)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(m.Cases) != 1 || m.Cases[0].ID != c.ID || *m.Cases[0].HumanBlob != *c.HumanBlob {
+ t.Fatalf("manifest round trip changed the case: %+v", m.Cases)
+ }
+}
+
+// fixedStrategy answers every file with content, or fails with err.
+type fixedStrategy struct {
+ content []byte
+ err error
+}
+
+func (s fixedStrategy) Resolve(_ context.Context, _ resolve.File, _ protocol.Request) (resolve.Resolution, bool, error) {
+ if s.err != nil {
+ return resolve.Resolution{}, false, s.err
+ }
+ return resolve.Resolution{Action: protocol.ActionWrite, Mode: protocol.ModeFile, Content: s.content}, true, nil
+}
+
+func TestRunnerOutcomes(t *testing.T) {
+ f := newFixture(t)
+ buildHistory(f)
+ head := f.git("rev-parse", "HEAD")
+ cases, _, err := Mine(context.Background(), Repo{Name: "fixture", Commit: head, License: "MIT"}, f.dir, MineOptions{})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ for _, tc := range []struct {
+ name string
+ strategy resolve.Strategy
+ want string
+ exact bool
+ }{
+ {"correct", fixedStrategy{content: []byte(goHuman)}, OutcomeCorrect, true},
+ {"correct after normalizing", fixedStrategy{content: []byte(strings.ReplaceAll(goHuman, "\n", " \r\n") + "\n\n")}, OutcomeCorrect, false},
+ {"wrong", fixedStrategy{content: []byte(goOurs)}, OutcomeWrong, false},
+ {"rejected by the marker rule", fixedStrategy{content: []byte("<<<<<<< ours\nx\n=======\ny\n>>>>>>> theirs\n")}, OutcomeRejected, false},
+ {"model error", fixedStrategy{err: errors.New("provider: 500")}, OutcomeModelError, false},
+ {"budget", fixedStrategy{err: resolve.ErrBudgetExhausted}, OutcomeNotRun, false},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ res, err := resolve.NewResolver([]resolve.Binding{{Name: "fake", Strategy: tc.strategy}}, nil, time.Minute)
+ if err != nil {
+ t.Fatal(err)
+ }
+ results, err := Runner{Resolver: res, Clone: func(string) string { return f.dir }}.Run(context.Background(), cases)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(results) != 1 || results[0].Outcome != tc.want || results[0].Exact != tc.exact {
+ t.Fatalf("results %+v, want outcome %s exact %v", results, tc.want, tc.exact)
+ }
+ })
+ }
+}
+
+func TestNotSentReason(t *testing.T) {
+ big := strings.Repeat("x", 3000)
+ for _, tc := range []struct {
+ path string
+ side string
+ generated bool
+ want string
+ }{
+ {"a.go", "package a\n", false, ""},
+ {"a.bin", "a\x00b", false, NotSentBinary},
+ {"big.go", strings.Repeat("a\n", 600), false, NotSentLarge},
+ {"web/package-lock.json", "{}", false, NotSentLockfile},
+ {"go.sum", "x\n", false, NotSentLockfile},
+ {"gen.go", "// Code generated by protoc-gen-go. DO NOT EDIT.\npackage gen\n", false, NotSentGenerated},
+ {"x.ts", "// @generated\n", false, NotSentGenerated},
+ {"y.py", "x = 1\n", true, NotSentGenerated},
+ {"app.min.js", "a\n", false, NotSentMinified},
+ {"bundle.js", big + "\n", false, NotSentMinified},
+ {"code.js", strings.Repeat("let a = 1;\n", 400), false, ""},
+ } {
+ maxBytes := int64(protocol.DefaultMaxFileBytes)
+ if tc.path == "big.go" {
+ maxBytes = 100
+ }
+ got := notSentReason(tc.path, [][]byte{[]byte(tc.side)}, maxBytes, tc.generated)
+ if got != tc.want {
+ t.Errorf("%s: got %q, want %q", tc.path, got, tc.want)
+ }
+ }
+}
+
+func TestSampleIsStratifiedAndStable(t *testing.T) {
+ var cases []Case
+ for i := 0; i < 30; i++ {
+ family := []string{FamilyGo, FamilyPython, FamilyTSJS}[i%3]
+ cases = append(cases, Case{ID: string(rune('a'+i)) + family, Family: family})
+ }
+ a, b := Sample(cases, 9, 7), Sample(cases, 9, 7)
+ counts := map[string]int{}
+ for i := range a {
+ if a[i].ID != b[i].ID {
+ t.Fatalf("the sample is not stable: %v vs %v", a, b)
+ }
+ counts[a[i].Family]++
+ }
+ if counts[FamilyGo] != 3 || counts[FamilyPython] != 3 || counts[FamilyTSJS] != 3 {
+ t.Fatalf("the sample is not stratified: %v", counts)
+ }
+ if c := Sample(cases, 9, 8); c[0].ID == a[0].ID && c[1].ID == a[1].ID && c[2].ID == a[2].ID {
+ t.Fatalf("another seed gave the same sample")
+ }
+ if len(Sample(cases, 0, 1)) != 30 {
+ t.Fatal("n <= 0 must take every case")
+ }
+}
+
+func TestReportCounts(t *testing.T) {
+ rep := NewReport("test", []Result{
+ {Family: FamilyGo, Outcome: OutcomeCorrect, Exact: true},
+ {Family: FamilyGo, Outcome: OutcomeWrong},
+ {Family: FamilyPython, Outcome: OutcomeRejected},
+ {Family: FamilyPython, Outcome: OutcomeNotSent, Reason: NotSentGenerated},
+ })
+ if rep.Total.Cases != 4 || rep.Total.Accepted() != 2 || rep.Total.ErrorRate() != 0.5 {
+ t.Fatalf("totals %+v", rep.Total)
+ }
+ if rep.Families[FamilyGo].Correct != 1 || rep.NotSent[NotSentGenerated] != 1 {
+ t.Fatalf("families %+v, not sent %+v", rep.Families, rep.NotSent)
+ }
+ var b strings.Builder
+ if err := rep.WriteMarkdown(&b); err != nil {
+ t.Fatal(err)
+ }
+ if !strings.Contains(b.String(), "| total | 4 | 1 (1) | 1 | 1 | 1 |") {
+ t.Fatalf("markdown:\n%s", b.String())
+ }
+}
diff --git a/tools/graft-agents/resolve/bench/corpus.go b/tools/graft-agents/resolve/bench/corpus.go
new file mode 100644
index 0000000..8e4e45b
--- /dev/null
+++ b/tools/graft-agents/resolve/bench/corpus.go
@@ -0,0 +1,158 @@
+// Package bench measures the strategies of graft-resolve on real merge conflicts: it mines the
+// conflicts of the merge commits of pinned repositories (with the human resolution of each), and
+// replays them through a configured chain, judging every accepted result against the human one.
+//
+// Only a manifest of the mined cases is kept (repositories, commit and object IDs, paths,
+// licenses); the contents are read from the clones at run time, so no third-party code is ever
+// stored with the benchmark.
+package bench
+
+import (
+ "encoding/json"
+ "errors"
+ "fmt"
+ "os"
+ "path"
+ "strings"
+)
+
+// Families of the benchmark, by the extension of the conflicted path.
+const (
+ FamilyTSJS = "ts-js"
+ FamilyPython = "python"
+ FamilyGo = "go"
+ FamilyRustJava = "rust-java"
+)
+
+// familyOf returns the family of a path by its extension, or "" for other files.
+func familyOf(p string) string {
+ switch strings.ToLower(path.Ext(p)) {
+ case ".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs":
+ return FamilyTSJS
+ case ".py", ".pyi":
+ return FamilyPython
+ case ".go":
+ return FamilyGo
+ case ".rs", ".java":
+ return FamilyRustJava
+ }
+ return ""
+}
+
+// Repo is one repository of the corpus.
+type Repo struct {
+ // Name identifies the repository in the manifest and the report, for example
+ // "golang/go".
+ Name string `json:"name"`
+ // URL is the clone URL.
+ URL string `json:"url"`
+ // Commit pins the history: the merges reachable from it are mined.
+ Commit string `json:"commit"`
+ // License is the SPDX identifier of the repository's license. The corpus only lists
+ // permissively licensed repositories.
+ License string `json:"license"`
+ // MaxMerges bounds the merges that are replayed, newest first (0: the miner's default).
+ MaxMerges int `json:"max_merges,omitempty"`
+}
+
+// Corpus is the committed list of repositories.
+type Corpus struct {
+ Repos []Repo `json:"repos"`
+}
+
+// permissiveLicenses are the licenses a corpus repository may have.
+var permissiveLicenses = map[string]bool{
+ "MIT": true, "Apache-2.0": true, "BSD-2-Clause": true, "BSD-3-Clause": true,
+ "ISC": true, "PSF-2.0": true, "MIT OR Apache-2.0": true, "Apache-2.0 OR MIT": true,
+}
+
+// LoadCorpus reads and validates a corpus file.
+func LoadCorpus(file string) (Corpus, error) {
+ data, err := os.ReadFile(file)
+ if err != nil {
+ return Corpus{}, err
+ }
+ var c Corpus
+ dec := json.NewDecoder(strings.NewReader(string(data)))
+ dec.DisallowUnknownFields()
+ if err := dec.Decode(&c); err != nil {
+ return Corpus{}, fmt.Errorf("corpus %s: %w", file, err)
+ }
+ if len(c.Repos) == 0 {
+ return Corpus{}, errors.New("corpus lists no repositories")
+ }
+ names := map[string]bool{}
+ for _, r := range c.Repos {
+ switch {
+ case r.Name == "" || r.URL == "" || r.Commit == "":
+ return Corpus{}, fmt.Errorf("corpus repository %q needs a name, a URL and a commit", r.Name)
+ case names[r.Name]:
+ return Corpus{}, fmt.Errorf("corpus lists %q twice", r.Name)
+ case !permissiveLicenses[r.License]:
+ return Corpus{}, fmt.Errorf("corpus repository %q has license %q, not a permissive one", r.Name, r.License)
+ case strings.ContainsAny(r.Name, " \t\n") || strings.Contains(r.Name, ".."):
+ return Corpus{}, fmt.Errorf("corpus repository name %q is not a plain name", r.Name)
+ }
+ names[r.Name] = true
+ }
+ return c, nil
+}
+
+// Case is one conflicting file of one replayed merge, with the human resolution.
+type Case struct {
+ // ID is stable across runs: the repository, the merge commit and the path.
+ ID string `json:"id"`
+ Repo string `json:"repo"`
+ License string `json:"license"`
+ Family string `json:"family"`
+ // Merge is the merge commit; Ours and Theirs its first and second parent; Base their merge
+ // base.
+ Merge string `json:"merge"`
+ Ours string `json:"ours"`
+ Theirs string `json:"theirs"`
+ Base string `json:"base"`
+ Path string `json:"path"`
+ Kind string `json:"kind"`
+ // The blobs of each side, nil when the side is absent; Human is the merge commit's version,
+ // nil when the human resolution deleted the file.
+ BaseBlob *entry `json:"base_blob"`
+ OursBlob *entry `json:"ours_blob"`
+ TheirsBlob *entry `json:"theirs_blob"`
+ HumanBlob *entry `json:"human_blob"`
+}
+
+// Manifest is the result of mining: the cases and what was skipped.
+type Manifest struct {
+ Version int `json:"version"`
+ Cases []Case `json:"cases"`
+ // Stats holds the counts of the miner per repository.
+ Stats map[string]MineStats `json:"stats"`
+}
+
+// ManifestVersion is the format of the manifest.
+const ManifestVersion = 1
+
+// SaveManifest writes the manifest as indented JSON.
+func SaveManifest(file string, m Manifest) error {
+ data, err := json.MarshalIndent(m, "", " ")
+ if err != nil {
+ return err
+ }
+ return os.WriteFile(file, append(data, '\n'), 0o644)
+}
+
+// LoadManifest reads a manifest.
+func LoadManifest(file string) (Manifest, error) {
+ data, err := os.ReadFile(file)
+ if err != nil {
+ return Manifest{}, err
+ }
+ var m Manifest
+ if err := json.Unmarshal(data, &m); err != nil {
+ return Manifest{}, fmt.Errorf("manifest %s: %w", file, err)
+ }
+ if m.Version != ManifestVersion {
+ return Manifest{}, fmt.Errorf("manifest %s has version %d, want %d", file, m.Version, ManifestVersion)
+ }
+ return m, nil
+}
diff --git a/tools/graft-agents/resolve/bench/git.go b/tools/graft-agents/resolve/bench/git.go
new file mode 100644
index 0000000..e1703ee
--- /dev/null
+++ b/tools/graft-agents/resolve/bench/git.go
@@ -0,0 +1,150 @@
+package bench
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+ "os/exec"
+ "strings"
+)
+
+// gitRepo runs git in one clone of the corpus.
+type gitRepo struct {
+ dir string
+ // bin is the git program, "git" by default.
+ bin string
+}
+
+// run runs git with args and returns its stdout. A non-zero exit is an error that carries stderr,
+// unless the exit code is in okCodes, in which case it is returned as code.
+func (g gitRepo) run(ctx context.Context, stdin []byte, okCodes []int, args ...string) ([]byte, int, error) {
+ bin := g.bin
+ if bin == "" {
+ bin = "git"
+ }
+ cmd := exec.CommandContext(ctx, bin, append([]string{"-C", g.dir}, args...)...)
+ var stdout, stderr bytes.Buffer
+ cmd.Stdout, cmd.Stderr = &stdout, &stderr
+ if stdin != nil {
+ cmd.Stdin = bytes.NewReader(stdin)
+ }
+ err := cmd.Run()
+ if err == nil {
+ return stdout.Bytes(), 0, nil
+ }
+ var exit *exec.ExitError
+ if errors.As(err, &exit) {
+ for _, ok := range okCodes {
+ if exit.ExitCode() == ok {
+ return stdout.Bytes(), ok, nil
+ }
+ }
+ }
+ return nil, 0, fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String()))
+}
+
+// merges returns the merge commits reachable from rev with exactly two parents, newest first,
+// at most max of them (0 means no limit), each with its parents.
+func (g gitRepo) merges(ctx context.Context, rev string, max int) ([][3]string, error) {
+ args := []string{"rev-list", "--merges", "--parents"}
+ if max > 0 {
+ args = append(args, fmt.Sprintf("--max-count=%d", max))
+ }
+ out, _, err := g.run(ctx, nil, nil, append(args, rev)...)
+ if err != nil {
+ return nil, err
+ }
+
+ var merges [][3]string
+ for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
+ fields := strings.Fields(line)
+ if len(fields) != 3 {
+ // Octopus merges are not replayed: the resolver merges two sides.
+ continue
+ }
+ merges = append(merges, [3]string{fields[0], fields[1], fields[2]})
+ }
+ return merges, nil
+}
+
+// mergeBases returns the merge bases of a and b.
+func (g gitRepo) mergeBases(ctx context.Context, a, b string) ([]string, error) {
+ out, code, err := g.run(ctx, nil, []int{1}, "merge-base", "--all", a, b)
+ if err != nil {
+ return nil, err
+ }
+ if code == 1 {
+ return nil, nil
+ }
+ return strings.Fields(string(out)), nil
+}
+
+// conflicts replays the merge of ours and theirs and returns the paths that conflict, as git
+// merge-tree --write-tree reports them.
+func (g gitRepo) conflicts(ctx context.Context, ours, theirs string) ([]string, error) {
+ out, code, err := g.run(ctx, nil, []int{1}, "merge-tree", "--write-tree", "--name-only", "--no-messages", "-z", ours, theirs)
+ if err != nil {
+ return nil, err
+ }
+ if code == 0 {
+ return nil, nil
+ }
+
+ // The output is the tree OID, then the conflicted paths, each terminated by a NUL; an empty
+ // entry ends the list.
+ parts := strings.Split(string(out), "\x00")
+ var paths []string
+ seen := map[string]bool{}
+ for _, p := range parts[1:] {
+ if p == "" {
+ break
+ }
+ if !seen[p] {
+ seen[p] = true
+ paths = append(paths, p)
+ }
+ }
+ return paths, nil
+}
+
+// entry is a blob of a tree: its object ID and mode.
+type entry struct {
+ OID string `json:"oid"`
+ Mode string `json:"mode"`
+}
+
+// entryAt returns the blob at path in the tree of rev, or nil when the path is absent or not a
+// blob.
+func (g gitRepo) entryAt(ctx context.Context, rev, path string) (*entry, error) {
+ out, _, err := g.run(ctx, nil, nil, "ls-tree", "-z", rev, "--", path)
+ if err != nil {
+ return nil, err
+ }
+ line := strings.TrimSuffix(string(out), "\x00")
+ if line == "" {
+ return nil, nil
+ }
+ // "<mode> SP <type> SP <oid> TAB <path>"
+ meta, name, ok := strings.Cut(line, "\t")
+ if !ok || name != path {
+ return nil, nil
+ }
+ fields := strings.Fields(meta)
+ if len(fields) != 3 || fields[1] != "blob" {
+ return nil, nil
+ }
+ return &entry{OID: fields[2], Mode: fields[0]}, nil
+}
+
+// blob returns the content of a blob.
+func (g gitRepo) blob(ctx context.Context, oid string) ([]byte, error) {
+ out, _, err := g.run(ctx, nil, nil, "cat-file", "blob", oid)
+ return out, err
+}
+
+// message returns the full commit message of a commit.
+func (g gitRepo) message(ctx context.Context, oid string) (string, error) {
+ out, _, err := g.run(ctx, nil, nil, "log", "-1", "--format=%B", oid)
+ return strings.TrimRight(string(out), "\n"), err
+}
diff --git a/tools/graft-agents/resolve/bench/judge.go b/tools/graft-agents/resolve/bench/judge.go
new file mode 100644
index 0000000..bd37b8c
--- /dev/null
+++ b/tools/graft-agents/resolve/bench/judge.go
@@ -0,0 +1,135 @@
+package bench
+
+import (
+ "bytes"
+ "path"
+ "regexp"
+ "strings"
+)
+
+// normalize returns content with line endings as LF, trailing whitespace of every line removed
+// and trailing blank lines dropped: the equivalence by which a result is judged correct.
+func normalize(content []byte) []byte {
+ text := strings.ReplaceAll(string(content), "\r\n", "\n")
+ lines := strings.Split(text, "\n")
+ for i, line := range lines {
+ lines[i] = strings.TrimRight(line, " \t\r")
+ }
+ for len(lines) > 0 && lines[len(lines)-1] == "" {
+ lines = lines[:len(lines)-1]
+ }
+ return []byte(strings.Join(lines, "\n"))
+}
+
+// judge reports whether a result equals the human resolution after normalization, and whether it
+// is byte-identical. A nil content is a deletion.
+func judge(result, human []byte, resultDeleted, humanDeleted bool) (correct, exact bool) {
+ if resultDeleted || humanDeleted {
+ return resultDeleted == humanDeleted, resultDeleted == humanDeleted
+ }
+ return bytes.Equal(normalize(result), normalize(human)), bytes.Equal(result, human)
+}
+
+// Reasons why a file is never sent to the resolver, as the server decides (PLAN10, 3.4).
+const (
+ NotSentBinary = "binary"
+ NotSentLarge = "large"
+ NotSentLockfile = "lockfile"
+ NotSentGenerated = "generated"
+ NotSentMinified = "minified"
+)
+
+// lockfiles are the base names of the lockfiles that always go to the agent.
+var lockfiles = map[string]bool{
+ "package-lock.json": true, "yarn.lock": true, "pnpm-lock.yaml": true,
+ "poetry.lock": true, "Cargo.lock": true, "go.sum": true,
+}
+
+// generatedMarkers are looked for in the first lines of every side.
+var generatedMarkers = []*regexp.Regexp{
+ regexp.MustCompile(`Code generated .* DO NOT EDIT\.`),
+ regexp.MustCompile(`@generated`),
+ regexp.MustCompile(`DO NOT EDIT`),
+ regexp.MustCompile(`(?i)auto-generated`),
+}
+
+// generatedMarkerLines is the number of first lines searched for a marker.
+const generatedMarkerLines = 10
+
+// minifiedGlobs name minified files.
+var minifiedGlobs = []string{"*.min.js", "*.min.mjs", "*.min.css", "*.bundle.js"}
+
+// Minified thresholds: a side of more than minifiedMinBytes whose average line length is above
+// minifiedAverageLine, or with a line longer than minifiedLongestLine.
+const (
+ minifiedMinBytes = 2 << 10
+ minifiedAverageLine = 300
+ minifiedLongestLine = 5000
+)
+
+// notSentReason returns why the file at p with the given sides would not be sent to the resolver,
+// or "" when it would be. generated reports linguist-generated for the path.
+func notSentReason(p string, sides [][]byte, maxBytes int64, generated bool) string {
+ for _, s := range sides {
+ if bytes.IndexByte(s, 0) >= 0 {
+ return NotSentBinary
+ }
+ }
+ for _, s := range sides {
+ if int64(len(s)) > maxBytes {
+ return NotSentLarge
+ }
+ }
+ if lockfiles[path.Base(p)] {
+ return NotSentLockfile
+ }
+ if generated {
+ return NotSentGenerated
+ }
+ for _, s := range sides {
+ head := firstLines(s, generatedMarkerLines)
+ for _, m := range generatedMarkers {
+ if m.Match(head) {
+ return NotSentGenerated
+ }
+ }
+ }
+ base := path.Base(p)
+ for _, g := range minifiedGlobs {
+ if ok, _ := path.Match(g, base); ok {
+ return NotSentMinified
+ }
+ }
+ for _, s := range sides {
+ if looksMinified(s) {
+ return NotSentMinified
+ }
+ }
+ return ""
+}
+
+func firstLines(s []byte, n int) []byte {
+ end := 0
+ for i := 0; i < n; i++ {
+ j := bytes.IndexByte(s[end:], '\n')
+ if j < 0 {
+ return s
+ }
+ end += j + 1
+ }
+ return s[:end]
+}
+
+func looksMinified(s []byte) bool {
+ if len(s) <= minifiedMinBytes {
+ return false
+ }
+ lines := bytes.Split(s, []byte("\n"))
+ longest := 0
+ for _, l := range lines {
+ if len(l) > longest {
+ longest = len(l)
+ }
+ }
+ return longest > minifiedLongestLine || len(s)/len(lines) > minifiedAverageLine
+}
diff --git a/tools/graft-agents/resolve/bench/mine.go b/tools/graft-agents/resolve/bench/mine.go
new file mode 100644
index 0000000..75dca47
--- /dev/null
+++ b/tools/graft-agents/resolve/bench/mine.go
@@ -0,0 +1,145 @@
+package bench
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "fmt"
+
+ "github.com/feder-cr/git-ai/tools/graft-agents/protocol"
+)
+
+// DefaultMaxMerges bounds the merges replayed per repository when the corpus sets no bound.
+const DefaultMaxMerges = 3000
+
+// MineOptions bound the mining of one repository.
+type MineOptions struct {
+ // MaxCases bounds the cases kept per repository, newest merges first (0: no bound).
+ MaxCases int
+}
+
+// MineStats counts what the miner saw in one repository.
+type MineStats struct {
+ Merges int `json:"merges"`
+ ConflictingMerges int `json:"conflicting_merges"`
+ // SkippedMergeBases counts merges whose parents have no single merge base (criss-cross or
+ // unrelated histories): the resolver is given one base.
+ SkippedMergeBases int `json:"skipped_merge_bases"`
+ ConflictingFiles int `json:"conflicting_files"`
+ // OtherFamily counts conflicting files of no benchmark family.
+ OtherFamily int `json:"other_family"`
+ // OneSide counts conflicting files whose human resolution is one of the two sides, which
+ // measures nothing about merging.
+ OneSide int `json:"one_side"`
+ Kept int `json:"kept"`
+}
+
+// caseID returns the stable ID of a case.
+func caseID(repo, merge, p string) string {
+ sum := sha256.Sum256([]byte(repo + "\x00" + merge + "\x00" + p))
+ return hex.EncodeToString(sum[:8])
+}
+
+// Mine replays the merges of repo, cloned at dir, and returns the cases: the conflicting files of
+// a benchmark family whose human resolution differs from both sides.
+func Mine(ctx context.Context, repo Repo, dir string, opts MineOptions) ([]Case, MineStats, error) {
+ g := gitRepo{dir: dir}
+ maxMerges := repo.MaxMerges
+ if maxMerges <= 0 {
+ maxMerges = DefaultMaxMerges
+ }
+
+ merges, err := g.merges(ctx, repo.Commit, maxMerges)
+ if err != nil {
+ return nil, MineStats{}, err
+ }
+
+ var stats MineStats
+ var cases []Case
+ for _, m := range merges {
+ if opts.MaxCases > 0 && len(cases) >= opts.MaxCases {
+ break
+ }
+ stats.Merges++
+ merge, ours, theirs := m[0], m[1], m[2]
+
+ paths, err := g.conflicts(ctx, ours, theirs)
+ if err != nil {
+ return nil, stats, fmt.Errorf("merge %s: %w", merge, err)
+ }
+ if len(paths) == 0 {
+ continue
+ }
+ stats.ConflictingMerges++
+
+ bases, err := g.mergeBases(ctx, ours, theirs)
+ if err != nil {
+ return nil, stats, fmt.Errorf("merge %s: %w", merge, err)
+ }
+ if len(bases) != 1 {
+ stats.SkippedMergeBases++
+ continue
+ }
+ base := bases[0]
+
+ for _, p := range paths {
+ if opts.MaxCases > 0 && len(cases) >= opts.MaxCases {
+ break
+ }
+ stats.ConflictingFiles++
+ family := familyOf(p)
+ if family == "" {
+ stats.OtherFamily++
+ continue
+ }
+
+ c := Case{
+ ID: caseID(repo.Name, merge, p), Repo: repo.Name, License: repo.License, Family: family,
+ Merge: merge, Ours: ours, Theirs: theirs, Base: base, Path: p,
+ }
+ for _, side := range []struct {
+ rev string
+ dst **entry
+ }{{base, &c.BaseBlob}, {ours, &c.OursBlob}, {theirs, &c.TheirsBlob}, {merge, &c.HumanBlob}} {
+ e, err := g.entryAt(ctx, side.rev, p)
+ if err != nil {
+ return nil, stats, fmt.Errorf("merge %s, %s: %w", merge, p, err)
+ }
+ *side.dst = e
+ }
+ if c.OursBlob == nil && c.TheirsBlob == nil {
+ continue
+ }
+ if sameEntry(c.HumanBlob, c.OursBlob) || sameEntry(c.HumanBlob, c.TheirsBlob) {
+ stats.OneSide++
+ continue
+ }
+ c.Kind = kindOf(c)
+
+ stats.Kept++
+ cases = append(cases, c)
+ }
+ }
+ return cases, stats, nil
+}
+
+func sameEntry(a, b *entry) bool {
+ if a == nil || b == nil {
+ return a == nil && b == nil
+ }
+ return a.OID == b.OID && a.Mode == b.Mode
+}
+
+// kindOf returns the protocol kind of the conflict of a case, as the server computes it
+// (landConflictKind in internal/gitaly/service/operations/land_resolve.go).
+func kindOf(c Case) string {
+ switch {
+ case c.OursBlob == nil || c.TheirsBlob == nil:
+ return protocol.KindModifyDelete
+ case c.BaseBlob == nil:
+ return protocol.KindAddAdd
+ case c.OursBlob.OID == c.TheirsBlob.OID:
+ return protocol.KindMode
+ }
+ return protocol.KindContent
+}
diff --git a/tools/graft-agents/resolve/bench/report.go b/tools/graft-agents/resolve/bench/report.go
new file mode 100644
index 0000000..a0ed608
--- /dev/null
+++ b/tools/graft-agents/resolve/bench/report.go
@@ -0,0 +1,125 @@
+package bench
+
+import (
+ "fmt"
+ "io"
+ "sort"
+ "strings"
+)
+
+// Counts aggregates the outcomes of a set of cases.
+type Counts struct {
+ Cases int `json:"cases"`
+ Correct int `json:"correct"`
+ Exact int `json:"exact"`
+ Wrong int `json:"wrong"`
+ Rejected int `json:"rejected_by_checks"`
+ NotSent int `json:"not_sent"`
+ Model int `json:"model_error"`
+ NotRun int `json:"not_run_cap"`
+ Abstained int `json:"abstained"`
+}
+
+// Accepted is the number of results that would land: correct and wrong.
+func (c Counts) Accepted() int { return c.Correct + c.Wrong }
+
+// ErrorRate is wrong / accepted, the share of the landed AI resolutions that differ from the
+// human one; 0 when nothing was accepted.
+func (c Counts) ErrorRate() float64 {
+ if c.Accepted() == 0 {
+ return 0
+ }
+ return float64(c.Wrong) / float64(c.Accepted())
+}
+
+func (c *Counts) add(r Result) {
+ c.Cases++
+ switch r.Outcome {
+ case OutcomeCorrect:
+ c.Correct++
+ if r.Exact {
+ c.Exact++
+ }
+ case OutcomeWrong:
+ c.Wrong++
+ case OutcomeRejected:
+ c.Rejected++
+ case OutcomeNotSent:
+ c.NotSent++
+ case OutcomeModelError:
+ c.Model++
+ case OutcomeNotRun:
+ c.NotRun++
+ case OutcomeAbstained:
+ c.Abstained++
+ }
+}
+
+// Report is the outcome of a run: the totals, the counts per family and every result.
+type Report struct {
+ // Label names the run, for example the model.
+ Label string `json:"label"`
+ Total Counts `json:"total"`
+ Families map[string]Counts `json:"families"`
+ // NotSent counts the cases that were not sent, by reason.
+ NotSent map[string]int `json:"not_sent"`
+ // Spend is the AI spend of the run in USD, when a budget was used.
+ SpendUSD float64 `json:"spend_usd,omitempty"`
+ Results []Result `json:"results"`
+}
+
+// NewReport aggregates results.
+func NewReport(label string, results []Result) Report {
+ rep := Report{Label: label, Families: map[string]Counts{}, NotSent: map[string]int{}, Results: results}
+ for _, r := range results {
+ rep.Total.add(r)
+ fc := rep.Families[r.Family]
+ fc.add(r)
+ rep.Families[r.Family] = fc
+ if r.Outcome == OutcomeNotSent {
+ rep.NotSent[r.Reason]++
+ }
+ }
+ return rep
+}
+
+// WriteMarkdown writes the report as Markdown tables.
+func (rep Report) WriteMarkdown(w io.Writer) error {
+ var b strings.Builder
+ fmt.Fprintf(&b, "## graft-resolve bench: %s\n\n", rep.Label)
+ fmt.Fprintf(&b, "Cases run: %d", rep.Total.Cases)
+ if rep.SpendUSD > 0 {
+ fmt.Fprintf(&b, ", AI spend $%.2f", rep.SpendUSD)
+ }
+ b.WriteString(".\n\n")
+ b.WriteString("| family | cases | correct (exact) | wrong | rejected by checks | not sent | model error | not run (cap) | abstained | error rate |\n")
+ b.WriteString("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n")
+ families := make([]string, 0, len(rep.Families))
+ for f := range rep.Families {
+ families = append(families, f)
+ }
+ sort.Strings(families)
+ row := func(name string, c Counts) {
+ fmt.Fprintf(&b, "| %s | %d | %d (%d) | %d | %d | %d | %d | %d | %d | %.1f%% |\n",
+ name, c.Cases, c.Correct, c.Exact, c.Wrong, c.Rejected, c.NotSent, c.Model, c.NotRun, c.Abstained, 100*c.ErrorRate())
+ }
+ for _, f := range families {
+ row(f, rep.Families[f])
+ }
+ row("total", rep.Total)
+ b.WriteString("\nThe error rate is wrong / (correct + wrong): the share of the results that would land and differ from the human resolution. Correct means equal after normalizing line endings and trailing whitespace; exact means byte-identical.\n")
+ if len(rep.NotSent) > 0 {
+ reasons := make([]string, 0, len(rep.NotSent))
+ for r := range rep.NotSent {
+ reasons = append(reasons, r)
+ }
+ sort.Strings(reasons)
+ parts := make([]string, 0, len(reasons))
+ for _, r := range reasons {
+ parts = append(parts, fmt.Sprintf("%s %d", r, rep.NotSent[r]))
+ }
+ fmt.Fprintf(&b, "\nNot sent, by reason: %s.\n", strings.Join(parts, ", "))
+ }
+ _, err := io.WriteString(w, b.String())
+ return err
+}
diff --git a/tools/graft-agents/resolve/bench/run.go b/tools/graft-agents/resolve/bench/run.go
new file mode 100644
index 0000000..511f7f5
--- /dev/null
+++ b/tools/graft-agents/resolve/bench/run.go
@@ -0,0 +1,243 @@
+package bench
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "math/rand"
+ "sort"
+ "strings"
+ "time"
+
+ "github.com/feder-cr/git-ai/tools/graft-agents/protocol"
+ "github.com/feder-cr/git-ai/tools/graft-agents/resolve"
+)
+
+// Outcomes of a case.
+const (
+ // OutcomeCorrect: the chain resolved the file and the result equals the human resolution.
+ OutcomeCorrect = "correct"
+ // OutcomeWrong: the chain resolved the file, the checks accepted it, and it differs from the
+ // human resolution. A wrong answer lands without review.
+ OutcomeWrong = "wrong"
+ // OutcomeRejected: a result was rejected by the rules (markers, mode, size, syntax) or a
+ // configured check; the land would fail and the agent would get the package.
+ OutcomeRejected = "rejected_by_checks"
+ // OutcomeNotSent: the file is never sent to the resolver (binary, large, lockfile, generated,
+ // minified).
+ OutcomeNotSent = "not_sent"
+ // OutcomeModelError: a strategy failed (for a model, the call or its answer).
+ OutcomeModelError = "model_error"
+ // OutcomeNotRun: the AI budget of the run was spent before the case.
+ OutcomeNotRun = "not_run_cap"
+ // OutcomeAbstained: no strategy applied to the file.
+ OutcomeAbstained = "abstained"
+)
+
+// Result is the outcome of one case.
+type Result struct {
+ ID string `json:"id"`
+ Repo string `json:"repo"`
+ Family string `json:"family"`
+ Path string `json:"path"`
+ Outcome string `json:"outcome"`
+ Exact bool `json:"exact,omitempty"`
+ Strategy string `json:"strategy,omitempty"`
+ Reason string `json:"reason,omitempty"`
+ Seconds float64 `json:"seconds"`
+}
+
+// Sample returns n cases of cases, stratified by family: each family's cases are shuffled with the
+// seed and the families are taken in turn, so that the sample is the same for the same manifest and
+// seed. n <= 0 takes every case.
+func Sample(cases []Case, n int, seed int64) []Case {
+ byFamily := map[string][]Case{}
+ for _, c := range cases {
+ byFamily[c.Family] = append(byFamily[c.Family], c)
+ }
+ families := make([]string, 0, len(byFamily))
+ for f := range byFamily {
+ families = append(families, f)
+ }
+ sort.Strings(families)
+
+ rnd := rand.New(rand.NewSource(seed))
+ for _, f := range families {
+ group := byFamily[f]
+ sort.Slice(group, func(i, j int) bool { return group[i].ID < group[j].ID })
+ rnd.Shuffle(len(group), func(i, j int) { group[i], group[j] = group[j], group[i] })
+ }
+
+ if n <= 0 {
+ n = len(cases)
+ }
+ var sample []Case
+ for len(sample) < n {
+ took := false
+ for _, f := range families {
+ if len(byFamily[f]) == 0 || len(sample) >= n {
+ continue
+ }
+ sample = append(sample, byFamily[f][0])
+ byFamily[f] = byFamily[f][1:]
+ took = true
+ }
+ if !took {
+ break
+ }
+ }
+ return sample
+}
+
+// Runner replays cases through a resolver.
+type Runner struct {
+ Resolver *resolve.Resolver
+ // Clone returns the directory of the clone of a corpus repository.
+ Clone func(repo string) string
+ // MaxFileBytes is the server's max_file_bytes (protocol.DefaultMaxFileBytes when 0).
+ MaxFileBytes int64
+ // Progress, when set, is called after each case.
+ Progress func(done, total int, r Result)
+}
+
+// Run replays every case and returns the results in the order of cases.
+func (r Runner) Run(ctx context.Context, cases []Case) ([]Result, error) {
+ maxBytes := r.MaxFileBytes
+ if maxBytes <= 0 {
+ maxBytes = protocol.DefaultMaxFileBytes
+ }
+
+ results := make([]Result, 0, len(cases))
+ for i, c := range cases {
+ if err := ctx.Err(); err != nil {
+ return results, err
+ }
+ res, err := r.runCase(ctx, c, maxBytes)
+ if err != nil {
+ return results, fmt.Errorf("case %s (%s %s): %w", c.ID, c.Repo, c.Path, err)
+ }
+ results = append(results, res)
+ if r.Progress != nil {
+ r.Progress(i+1, len(cases), res)
+ }
+ }
+ return results, nil
+}
+
+func (r Runner) runCase(ctx context.Context, c Case, maxBytes int64) (Result, error) {
+ res := Result{ID: c.ID, Repo: c.Repo, Family: c.Family, Path: c.Path}
+ g := gitRepo{dir: r.Clone(c.Repo)}
+
+ sides := map[string]*protocol.Side{}
+ var contents [][]byte
+ for name, e := range map[string]*entry{"base": c.BaseBlob, "ours": c.OursBlob, "theirs": c.TheirsBlob} {
+ if e == nil {
+ continue
+ }
+ content, err := g.blob(ctx, e.OID)
+ if err != nil {
+ return res, err
+ }
+ sides[name] = &protocol.Side{OID: e.OID, Mode: e.Mode, Content: content}
+ contents = append(contents, content)
+ }
+ var human []byte
+ if c.HumanBlob != nil {
+ var err error
+ if human, err = g.blob(ctx, c.HumanBlob.OID); err != nil {
+ return res, err
+ }
+ }
+
+ generated, err := g.linguistGenerated(ctx, c.Ours, c.Path)
+ if err != nil {
+ return res, err
+ }
+ if reason := notSentReason(c.Path, contents, maxBytes, generated); reason != "" {
+ res.Outcome, res.Reason = OutcomeNotSent, reason
+ return res, nil
+ }
+
+ oursMessage, err := g.message(ctx, c.Ours)
+ if err != nil {
+ return res, err
+ }
+ theirsMessage, err := g.message(ctx, c.Theirs)
+ if err != nil {
+ return res, err
+ }
+
+ req := protocol.Request{
+ Version: protocol.Version,
+ Repository: protocol.Repository{Storage: "bench", RelativePath: c.Repo},
+ Branch: "bench",
+ Tip: c.Ours,
+ Commit: c.Theirs,
+ MergeBase: c.Base,
+ Messages: protocol.Messages{Tip: oursMessage, Commit: theirsMessage},
+ Actor: "bench",
+ Files: []protocol.FileConflict{{
+ Path: c.Path, Kind: c.Kind, Base: sides["base"], Ours: sides["ours"], Theirs: sides["theirs"],
+ }},
+ Limits: protocol.Limits{MaxFileBytes: maxBytes},
+ }
+
+ start := time.Now()
+ resp, err := r.Resolver.Resolve(ctx, req)
+ res.Seconds = time.Since(start).Seconds()
+ if err != nil {
+ var unresolved *resolve.UnresolvedError
+ if !errors.As(err, &unresolved) || len(unresolved.Failures) != 1 {
+ return res, err
+ }
+ res.Outcome, res.Strategy, res.Reason = classifyFailure(unresolved.Failures[0])
+ return res, nil
+ }
+
+ f := resp.Files[0]
+ res.Strategy = strings.TrimPrefix(resp.Resolver, "graft-resolve/")
+ correct, exact := judge(f.Content, human, f.Action == protocol.ActionDelete, c.HumanBlob == nil)
+ res.Exact = exact
+ if correct {
+ res.Outcome = OutcomeCorrect
+ } else {
+ res.Outcome = OutcomeWrong
+ }
+ return res, nil
+}
+
+// classifyFailure returns the outcome of a file that the chain did not resolve: the last attempt
+// decides, since the chain went on past the earlier ones.
+func classifyFailure(f resolve.Failure) (outcome, strategy, reason string) {
+ if len(f.Attempts) == 0 {
+ return OutcomeAbstained, "", strings.Join(f.Reasons, "; ")
+ }
+ last := f.Attempts[len(f.Attempts)-1]
+ reason = truncate(fmt.Sprint(last.Err), 512)
+ switch {
+ case errors.Is(last.Err, resolve.ErrBudgetExhausted):
+ return OutcomeNotRun, last.Strategy, reason
+ case last.Kind == resolve.AttemptRejected || last.Kind == resolve.AttemptCheckFailed:
+ return OutcomeRejected, last.Strategy, reason
+ }
+ return OutcomeModelError, last.Strategy, reason
+}
+
+func truncate(s string, n int) string {
+ if len(s) <= n {
+ return s
+ }
+ return s[:n]
+}
+
+// linguistGenerated reports whether the .gitattributes of rev's tree mark path as
+// linguist-generated.
+func (g gitRepo) linguistGenerated(ctx context.Context, rev, p string) (bool, error) {
+ out, _, err := g.run(ctx, nil, nil, "check-attr", "--source", rev, "linguist-generated", "--", p)
+ if err != nil {
+ return false, err
+ }
+ // "<path>: linguist-generated: <value>"
+ value := strings.TrimSpace(string(out[strings.LastIndex(string(out), ":")+1:]))
+ return value == "set" || value == "true", nil
+}
diff --git a/tools/graft-agents/resolve/resolve.go b/tools/graft-agents/resolve/resolve.go
index 7982c46..e54484c 100644
--- a/tools/graft-agents/resolve/resolve.go
+++ b/tools/graft-agents/resolve/resolve.go
@@ -203,8 +203,37 @@ func NewResolver(bindings []Binding, checks []Check, timeout time.Duration) (*Re
type Failure struct {
Path string
Reasons []string
+ // Attempts are the strategies that applied to the file and did not resolve
+ // it, in chain order, with what happened. Reasons holds the same in text.
+ Attempts []Attempt
+}
+
+// AttemptKind says why a strategy that applied to a file did not resolve it.
+type AttemptKind string
+
+const (
+ // AttemptFailed: the strategy returned an error (for a model, the call or
+ // its answer failed; errors.Is(Err, ErrBudgetExhausted) when the AI budget
+ // is spent).
+ AttemptFailed AttemptKind = "failed"
+ // AttemptRejected: the result broke a rule that every result must keep
+ // (conflict markers, mode, size, or its syntax).
+ AttemptRejected AttemptKind = "rejected"
+ // AttemptCheckFailed: a configured check rejected the result.
+ AttemptCheckFailed AttemptKind = "check_failed"
+)
+
+// Attempt is one strategy that applied to a file without resolving it.
+type Attempt struct {
+ Strategy string
+ Kind AttemptKind
+ Err error
}
+// ErrBudgetExhausted is returned by a strategy that does not call its model
+// because the AI budget is spent.
+var ErrBudgetExhausted = errors.New("AI budget exhausted")
+
// UnresolvedError reports files that stayed unresolved.
type UnresolvedError struct {
Failures []Failure
@@ -240,12 +269,12 @@ func (r *Resolver) Resolve(ctx context.Context, req protocol.Request) (protocol.
var failures []Failure
for _, f := range req.Files {
- out, name, reasons, err := r.resolveFile(ctx, req, f, maxBytes)
+ out, name, attempts, err := r.resolveFile(ctx, req, f, maxBytes)
if err != nil {
return protocol.Response{}, err
}
if name == "" {
- failures = append(failures, Failure{Path: f.Path, Reasons: reasons})
+ failures = append(failures, Failure{Path: f.Path, Reasons: attemptReasons(attempts), Attempts: attempts})
continue
}
if !seen[name] {
@@ -294,11 +323,31 @@ func resolverName(used []string) string {
return fmt.Sprintf("%s+%d", resolverNamePrefix, len(used))
}
+// attemptReasons returns the text form of the attempts, the reasons of a
+// Failure.
+func attemptReasons(attempts []Attempt) []string {
+ if len(attempts) == 0 {
+ return []string{"no strategy resolved this conflict"}
+ }
+ reasons := make([]string, 0, len(attempts))
+ for _, a := range attempts {
+ switch a.Kind {
+ case AttemptRejected:
+ reasons = append(reasons, fmt.Sprintf("%s: rejected: %v", a.Strategy, a.Err))
+ case AttemptCheckFailed:
+ reasons = append(reasons, fmt.Sprintf("%s: check failed: %v", a.Strategy, a.Err))
+ default:
+ reasons = append(reasons, fmt.Sprintf("%s: %v", a.Strategy, a.Err))
+ }
+ }
+ return reasons
+}
+
// resolveFile runs the chain for one file. It returns the strategy name that
-// accepted the answer, or an empty name and the collected reasons. err is set
-// only when ctx ended.
-func (r *Resolver) resolveFile(ctx context.Context, req protocol.Request, f File, maxBytes int64) (protocol.ResolvedFile, string, []string, error) {
- var reasons []string
+// accepted the answer, or an empty name and the attempts of the strategies
+// that applied. err is set only when ctx ended.
+func (r *Resolver) resolveFile(ctx context.Context, req protocol.Request, f File, maxBytes int64) (protocol.ResolvedFile, string, []Attempt, error) {
+ var attempts []Attempt
for _, b := range r.bindings {
if err := ctx.Err(); err != nil {
return protocol.ResolvedFile{}, "", nil, err
@@ -311,21 +360,21 @@ func (r *Resolver) resolveFile(ctx context.Context, req protocol.Request, f File
if ctxErr := ctx.Err(); ctxErr != nil {
return protocol.ResolvedFile{}, "", nil, ctxErr
}
- reasons = append(reasons, fmt.Sprintf("%s: %v", b.Name, err))
+ attempts = append(attempts, Attempt{Strategy: b.Name, Kind: AttemptFailed, Err: err})
continue
}
if !ok {
continue
}
if err := checkResolution(res, maxBytes); err != nil {
- reasons = append(reasons, fmt.Sprintf("%s: rejected: %v", b.Name, err))
+ attempts = append(attempts, Attempt{Strategy: b.Name, Kind: AttemptRejected, Err: err})
continue
}
if err := r.runChecks(ctx, f.Path, res); err != nil {
if ctxErr := ctx.Err(); ctxErr != nil {
return protocol.ResolvedFile{}, "", nil, ctxErr
}
- reasons = append(reasons, fmt.Sprintf("%s: check failed: %v", b.Name, err))
+ attempts = append(attempts, Attempt{Strategy: b.Name, Kind: AttemptCheckFailed, Err: err})
continue
}
r.record(ctx, f, res)
@@ -336,10 +385,7 @@ func (r *Resolver) resolveFile(ctx context.Context, req protocol.Request, f File
Content: contentOf(res),
}, b.Name, nil, nil
}
- if len(reasons) == 0 {
- reasons = []string{"no strategy resolved this conflict"}
- }
- return protocol.ResolvedFile{}, "", reasons, nil
+ return protocol.ResolvedFile{}, "", attempts, nil
}
func modeOf(res Resolution) string {