Git designed for AI agents.
graft-resolve: the benchmark miner, runner and report (WP1, part 1)
Federico Elia2893dfcba0cd
diff --git a/tools/graft-agents/resolve/bench/bench_test.go b/tools/graft-agents/resolve/bench/bench_test.gonew file mode 100644index 0000000..52351fd--- /dev/null+++ b/tools/graft-agents/resolve/bench/bench_test.go@@ -0,0 +1,281 @@+package bench++import (+ "context"+ "errors"+ "os"+ "os/exec"+ "path/filepath"+ "strings"+ "testing"+ "time"++ "github.com/feder-cr/git-ai/tools/graft-agents/protocol"+ "github.com/feder-cr/git-ai/tools/graft-agents/resolve"+)++// fixture is a repository with crafted merges.+type fixture struct {+ t *testing.T+ dir string+ now int64+}++func newFixture(t *testing.T) *fixture {+ t.Helper()+ if _, err := exec.LookPath("git"); err != nil {+ t.Skip("git is not installed")+ }+ f := &fixture{t: t, dir: t.TempDir(), now: 1767225600}+ f.git("init", "-q", "-b", "main")+ return f+}++// git runs git in the fixture with a fixed identity and date, given on the command line so that no+// configuration file is touched.+func (f *fixture) git(args ...string) string {+ f.t.Helper()+ f.now += 60+ cmd := exec.Command("git", append([]string{"-C", f.dir, "-c", "user.name=Bench", "-c", "[email protected]",+ "-c", "commit.gpgsign=false", "-c", "init.defaultBranch=main"}, args...)...)+ date := time.Unix(f.now, 0).UTC().Format(time.RFC3339)+ cmd.Env = append(os.Environ(), "GIT_AUTHOR_DATE="+date, "GIT_COMMITTER_DATE="+date, "GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_SYSTEM=/dev/null")+ out, err := cmd.CombinedOutput()+ if err != nil {+ f.t.Fatalf("git %s: %v: %s", strings.Join(args, " "), err, out)+ }+ return strings.TrimSpace(string(out))+}++func (f *fixture) write(name, content string) {+ f.t.Helper()+ p := filepath.Join(f.dir, name)+ if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {+ f.t.Fatal(err)+ }+ if err := os.WriteFile(p, []byte(content), 0o644); err != nil {+ f.t.Fatal(err)+ }+}++func (f *fixture) commit(msg string, files map[string]string) string {+ f.t.Helper()+ for name, content := range files {+ f.write(name, content)+ }+ f.git("add", "-A")+ f.git("commit", "-q", "-m", msg)+ return f.git("rev-parse", "HEAD")+}++// conflictMerge merges branch into the current branch, which conflicts, and commits the human+// resolution.+func (f *fixture) conflictMerge(branch string, resolved map[string]string) string {+ f.t.Helper()+ cmd := exec.Command("git", "-C", f.dir, "-c", "user.name=Bench", "-c", "[email protected]", "merge", "--no-ff", "-q", branch)+ cmd.Env = append(os.Environ(), "GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_SYSTEM=/dev/null")+ if out, err := cmd.CombinedOutput(); err == nil {+ f.t.Fatalf("merge of %s did not conflict: %s", branch, out)+ }+ return f.commit("Merge "+branch, resolved)+}++const (+ goBase = "package a\n\nfunc A() int {\n\treturn 1\n}\n"+ goOurs = "package a\n\nfunc A() int {\n\treturn 2\n}\n"+ goTheirs = "package a\n\nfunc A() int {\n\treturn 3\n}\n"+ goHuman = "package a\n\nfunc A() int {\n\treturn 2 + 3\n}\n"+)++// buildHistory creates: a conflicting merge of a.go resolved by hand (one case), a conflicting+// merge of b.py where the human took one side (not a case), a conflict in notes.txt (no family),+// and a clean merge.+func buildHistory(f *fixture) (mergeA string) {+ f.commit("base", map[string]string{"a.go": goBase, "b.py": "x = 1\n", "notes.txt": "one\n"})++ f.git("checkout", "-q", "-b", "feature")+ f.commit("feature: A returns 3", map[string]string{"a.go": goTheirs})+ f.git("checkout", "-q", "main")+ f.commit("main: A returns 2", map[string]string{"a.go": goOurs})+ mergeA = f.conflictMerge("feature", map[string]string{"a.go": goHuman})++ f.git("checkout", "-q", "-b", "feature2")+ f.commit("feature2", map[string]string{"b.py": "x = 3\n", "notes.txt": "three\n"})+ f.git("checkout", "-q", "main")+ f.commit("main2", map[string]string{"b.py": "x = 2\n", "notes.txt": "two\n"})+ f.conflictMerge("feature2", map[string]string{"b.py": "x = 2\n", "notes.txt": "two and three\n"})++ f.git("checkout", "-q", "-b", "feature3")+ f.commit("feature3", map[string]string{"c.go": "package c\n"})+ f.git("checkout", "-q", "main")+ f.commit("main3", map[string]string{"d.go": "package d\n"})+ f.git("merge", "--no-ff", "-q", "-m", "clean merge", "feature3")+ return mergeA+}++func TestMineKeepsRealResolutions(t *testing.T) {+ f := newFixture(t)+ mergeA := buildHistory(f)+ head := f.git("rev-parse", "HEAD")++ cases, stats, err := Mine(context.Background(), Repo{Name: "fixture", Commit: head, License: "MIT"}, f.dir, MineOptions{})+ if err != nil {+ t.Fatal(err)+ }+ if len(cases) != 1 {+ t.Fatalf("got %d cases, want 1: %+v (stats %+v)", len(cases), cases, stats)+ }+ c := cases[0]+ if c.Merge != mergeA || c.Path != "a.go" || c.Family != FamilyGo || c.Kind != protocol.KindContent || c.License != "MIT" {+ t.Fatalf("unexpected case %+v", c)+ }+ if c.BaseBlob == nil || c.OursBlob == nil || c.TheirsBlob == nil || c.HumanBlob == nil {+ t.Fatalf("missing sides in %+v", c)+ }+ want := MineStats{Merges: 3, ConflictingMerges: 2, ConflictingFiles: 3, OtherFamily: 1, OneSide: 1, Kept: 1}+ if stats != want {+ t.Fatalf("stats %+v, want %+v", stats, want)+ }++ // The manifest replays identically.+ file := filepath.Join(t.TempDir(), "manifest.json")+ if err := SaveManifest(file, Manifest{Version: ManifestVersion, Cases: cases}); err != nil {+ t.Fatal(err)+ }+ m, err := LoadManifest(file)+ if err != nil {+ t.Fatal(err)+ }+ if len(m.Cases) != 1 || m.Cases[0].ID != c.ID || *m.Cases[0].HumanBlob != *c.HumanBlob {+ t.Fatalf("manifest round trip changed the case: %+v", m.Cases)+ }+}++// fixedStrategy answers every file with content, or fails with err.+type fixedStrategy struct {+ content []byte+ err error+}++func (s fixedStrategy) Resolve(_ context.Context, _ resolve.File, _ protocol.Request) (resolve.Resolution, bool, error) {+ if s.err != nil {+ return resolve.Resolution{}, false, s.err+ }+ return resolve.Resolution{Action: protocol.ActionWrite, Mode: protocol.ModeFile, Content: s.content}, true, nil+}++func TestRunnerOutcomes(t *testing.T) {+ f := newFixture(t)+ buildHistory(f)+ head := f.git("rev-parse", "HEAD")+ cases, _, err := Mine(context.Background(), Repo{Name: "fixture", Commit: head, License: "MIT"}, f.dir, MineOptions{})+ if err != nil {+ t.Fatal(err)+ }++ for _, tc := range []struct {+ name string+ strategy resolve.Strategy+ want string+ exact bool+ }{+ {"correct", fixedStrategy{content: []byte(goHuman)}, OutcomeCorrect, true},+ {"correct after normalizing", fixedStrategy{content: []byte(strings.ReplaceAll(goHuman, "\n", " \r\n") + "\n\n")}, OutcomeCorrect, false},+ {"wrong", fixedStrategy{content: []byte(goOurs)}, OutcomeWrong, false},+ {"rejected by the marker rule", fixedStrategy{content: []byte("<<<<<<< ours\nx\n=======\ny\n>>>>>>> theirs\n")}, OutcomeRejected, false},+ {"model error", fixedStrategy{err: errors.New("provider: 500")}, OutcomeModelError, false},+ {"budget", fixedStrategy{err: resolve.ErrBudgetExhausted}, OutcomeNotRun, false},+ } {+ t.Run(tc.name, func(t *testing.T) {+ res, err := resolve.NewResolver([]resolve.Binding{{Name: "fake", Strategy: tc.strategy}}, nil, time.Minute)+ if err != nil {+ t.Fatal(err)+ }+ results, err := Runner{Resolver: res, Clone: func(string) string { return f.dir }}.Run(context.Background(), cases)+ if err != nil {+ t.Fatal(err)+ }+ if len(results) != 1 || results[0].Outcome != tc.want || results[0].Exact != tc.exact {+ t.Fatalf("results %+v, want outcome %s exact %v", results, tc.want, tc.exact)+ }+ })+ }+}++func TestNotSentReason(t *testing.T) {+ big := strings.Repeat("x", 3000)+ for _, tc := range []struct {+ path string+ side string+ generated bool+ want string+ }{+ {"a.go", "package a\n", false, ""},+ {"a.bin", "a\x00b", false, NotSentBinary},+ {"big.go", strings.Repeat("a\n", 600), false, NotSentLarge},+ {"web/package-lock.json", "{}", false, NotSentLockfile},+ {"go.sum", "x\n", false, NotSentLockfile},+ {"gen.go", "// Code generated by protoc-gen-go. DO NOT EDIT.\npackage gen\n", false, NotSentGenerated},+ {"x.ts", "// @generated\n", false, NotSentGenerated},+ {"y.py", "x = 1\n", true, NotSentGenerated},+ {"app.min.js", "a\n", false, NotSentMinified},+ {"bundle.js", big + "\n", false, NotSentMinified},+ {"code.js", strings.Repeat("let a = 1;\n", 400), false, ""},+ } {+ maxBytes := int64(protocol.DefaultMaxFileBytes)+ if tc.path == "big.go" {+ maxBytes = 100+ }+ got := notSentReason(tc.path, [][]byte{[]byte(tc.side)}, maxBytes, tc.generated)+ if got != tc.want {+ t.Errorf("%s: got %q, want %q", tc.path, got, tc.want)+ }+ }+}++func TestSampleIsStratifiedAndStable(t *testing.T) {+ var cases []Case+ for i := 0; i < 30; i++ {+ family := []string{FamilyGo, FamilyPython, FamilyTSJS}[i%3]+ cases = append(cases, Case{ID: string(rune('a'+i)) + family, Family: family})+ }+ a, b := Sample(cases, 9, 7), Sample(cases, 9, 7)+ counts := map[string]int{}+ for i := range a {+ if a[i].ID != b[i].ID {+ t.Fatalf("the sample is not stable: %v vs %v", a, b)+ }+ counts[a[i].Family]+++ }+ if counts[FamilyGo] != 3 || counts[FamilyPython] != 3 || counts[FamilyTSJS] != 3 {+ t.Fatalf("the sample is not stratified: %v", counts)+ }+ if c := Sample(cases, 9, 8); c[0].ID == a[0].ID && c[1].ID == a[1].ID && c[2].ID == a[2].ID {+ t.Fatalf("another seed gave the same sample")+ }+ if len(Sample(cases, 0, 1)) != 30 {+ t.Fatal("n <= 0 must take every case")+ }+}++func TestReportCounts(t *testing.T) {+ rep := NewReport("test", []Result{+ {Family: FamilyGo, Outcome: OutcomeCorrect, Exact: true},+ {Family: FamilyGo, Outcome: OutcomeWrong},+ {Family: FamilyPython, Outcome: OutcomeRejected},+ {Family: FamilyPython, Outcome: OutcomeNotSent, Reason: NotSentGenerated},+ })+ if rep.Total.Cases != 4 || rep.Total.Accepted() != 2 || rep.Total.ErrorRate() != 0.5 {+ t.Fatalf("totals %+v", rep.Total)+ }+ if rep.Families[FamilyGo].Correct != 1 || rep.NotSent[NotSentGenerated] != 1 {+ t.Fatalf("families %+v, not sent %+v", rep.Families, rep.NotSent)+ }+ var b strings.Builder+ if err := rep.WriteMarkdown(&b); err != nil {+ t.Fatal(err)+ }+ if !strings.Contains(b.String(), "| total | 4 | 1 (1) | 1 | 1 | 1 |") {+ t.Fatalf("markdown:\n%s", b.String())+ }+}diff --git a/tools/graft-agents/resolve/bench/corpus.go b/tools/graft-agents/resolve/bench/corpus.gonew file mode 100644index 0000000..8e4e45b--- /dev/null+++ b/tools/graft-agents/resolve/bench/corpus.go@@ -0,0 +1,158 @@+// Package bench measures the strategies of graft-resolve on real merge conflicts: it mines the+// conflicts of the merge commits of pinned repositories (with the human resolution of each), and+// replays them through a configured chain, judging every accepted result against the human one.+//+// Only a manifest of the mined cases is kept (repositories, commit and object IDs, paths,+// licenses); the contents are read from the clones at run time, so no third-party code is ever+// stored with the benchmark.+package bench++import (+ "encoding/json"+ "errors"+ "fmt"+ "os"+ "path"+ "strings"+)++// Families of the benchmark, by the extension of the conflicted path.+const (+ FamilyTSJS = "ts-js"+ FamilyPython = "python"+ FamilyGo = "go"+ FamilyRustJava = "rust-java"+)++// familyOf returns the family of a path by its extension, or "" for other files.+func familyOf(p string) string {+ switch strings.ToLower(path.Ext(p)) {+ case ".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs":+ return FamilyTSJS+ case ".py", ".pyi":+ return FamilyPython+ case ".go":+ return FamilyGo+ case ".rs", ".java":+ return FamilyRustJava+ }+ return ""+}++// Repo is one repository of the corpus.+type Repo struct {+ // Name identifies the repository in the manifest and the report, for example+ // "golang/go".+ Name string `json:"name"`+ // URL is the clone URL.+ URL string `json:"url"`+ // Commit pins the history: the merges reachable from it are mined.+ Commit string `json:"commit"`+ // License is the SPDX identifier of the repository's license. The corpus only lists+ // permissively licensed repositories.+ License string `json:"license"`+ // MaxMerges bounds the merges that are replayed, newest first (0: the miner's default).+ MaxMerges int `json:"max_merges,omitempty"`+}++// Corpus is the committed list of repositories.+type Corpus struct {+ Repos []Repo `json:"repos"`+}++// permissiveLicenses are the licenses a corpus repository may have.+var permissiveLicenses = map[string]bool{+ "MIT": true, "Apache-2.0": true, "BSD-2-Clause": true, "BSD-3-Clause": true,+ "ISC": true, "PSF-2.0": true, "MIT OR Apache-2.0": true, "Apache-2.0 OR MIT": true,+}++// LoadCorpus reads and validates a corpus file.+func LoadCorpus(file string) (Corpus, error) {+ data, err := os.ReadFile(file)+ if err != nil {+ return Corpus{}, err+ }+ var c Corpus+ dec := json.NewDecoder(strings.NewReader(string(data)))+ dec.DisallowUnknownFields()+ if err := dec.Decode(&c); err != nil {+ return Corpus{}, fmt.Errorf("corpus %s: %w", file, err)+ }+ if len(c.Repos) == 0 {+ return Corpus{}, errors.New("corpus lists no repositories")+ }+ names := map[string]bool{}+ for _, r := range c.Repos {+ switch {+ case r.Name == "" || r.URL == "" || r.Commit == "":+ return Corpus{}, fmt.Errorf("corpus repository %q needs a name, a URL and a commit", r.Name)+ case names[r.Name]:+ return Corpus{}, fmt.Errorf("corpus lists %q twice", r.Name)+ case !permissiveLicenses[r.License]:+ return Corpus{}, fmt.Errorf("corpus repository %q has license %q, not a permissive one", r.Name, r.License)+ case strings.ContainsAny(r.Name, " \t\n") || strings.Contains(r.Name, ".."):+ return Corpus{}, fmt.Errorf("corpus repository name %q is not a plain name", r.Name)+ }+ names[r.Name] = true+ }+ return c, nil+}++// Case is one conflicting file of one replayed merge, with the human resolution.+type Case struct {+ // ID is stable across runs: the repository, the merge commit and the path.+ ID string `json:"id"`+ Repo string `json:"repo"`+ License string `json:"license"`+ Family string `json:"family"`+ // Merge is the merge commit; Ours and Theirs its first and second parent; Base their merge+ // base.+ Merge string `json:"merge"`+ Ours string `json:"ours"`+ Theirs string `json:"theirs"`+ Base string `json:"base"`+ Path string `json:"path"`+ Kind string `json:"kind"`+ // The blobs of each side, nil when the side is absent; Human is the merge commit's version,+ // nil when the human resolution deleted the file.+ BaseBlob *entry `json:"base_blob"`+ OursBlob *entry `json:"ours_blob"`+ TheirsBlob *entry `json:"theirs_blob"`+ HumanBlob *entry `json:"human_blob"`+}++// Manifest is the result of mining: the cases and what was skipped.+type Manifest struct {+ Version int `json:"version"`+ Cases []Case `json:"cases"`+ // Stats holds the counts of the miner per repository.+ Stats map[string]MineStats `json:"stats"`+}++// ManifestVersion is the format of the manifest.+const ManifestVersion = 1++// SaveManifest writes the manifest as indented JSON.+func SaveManifest(file string, m Manifest) error {+ data, err := json.MarshalIndent(m, "", " ")+ if err != nil {+ return err+ }+ return os.WriteFile(file, append(data, '\n'), 0o644)+}++// LoadManifest reads a manifest.+func LoadManifest(file string) (Manifest, error) {+ data, err := os.ReadFile(file)+ if err != nil {+ return Manifest{}, err+ }+ var m Manifest+ if err := json.Unmarshal(data, &m); err != nil {+ return Manifest{}, fmt.Errorf("manifest %s: %w", file, err)+ }+ if m.Version != ManifestVersion {+ return Manifest{}, fmt.Errorf("manifest %s has version %d, want %d", file, m.Version, ManifestVersion)+ }+ return m, nil+}diff --git a/tools/graft-agents/resolve/bench/git.go b/tools/graft-agents/resolve/bench/git.gonew file mode 100644index 0000000..e1703ee--- /dev/null+++ b/tools/graft-agents/resolve/bench/git.go@@ -0,0 +1,150 @@+package bench++import (+ "bytes"+ "context"+ "errors"+ "fmt"+ "os/exec"+ "strings"+)++// gitRepo runs git in one clone of the corpus.+type gitRepo struct {+ dir string+ // bin is the git program, "git" by default.+ bin string+}++// run runs git with args and returns its stdout. A non-zero exit is an error that carries stderr,+// unless the exit code is in okCodes, in which case it is returned as code.+func (g gitRepo) run(ctx context.Context, stdin []byte, okCodes []int, args ...string) ([]byte, int, error) {+ bin := g.bin+ if bin == "" {+ bin = "git"+ }+ cmd := exec.CommandContext(ctx, bin, append([]string{"-C", g.dir}, args...)...)+ var stdout, stderr bytes.Buffer+ cmd.Stdout, cmd.Stderr = &stdout, &stderr+ if stdin != nil {+ cmd.Stdin = bytes.NewReader(stdin)+ }+ err := cmd.Run()+ if err == nil {+ return stdout.Bytes(), 0, nil+ }+ var exit *exec.ExitError+ if errors.As(err, &exit) {+ for _, ok := range okCodes {+ if exit.ExitCode() == ok {+ return stdout.Bytes(), ok, nil+ }+ }+ }+ return nil, 0, fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String()))+}++// merges returns the merge commits reachable from rev with exactly two parents, newest first,+// at most max of them (0 means no limit), each with its parents.+func (g gitRepo) merges(ctx context.Context, rev string, max int) ([][3]string, error) {+ args := []string{"rev-list", "--merges", "--parents"}+ if max > 0 {+ args = append(args, fmt.Sprintf("--max-count=%d", max))+ }+ out, _, err := g.run(ctx, nil, nil, append(args, rev)...)+ if err != nil {+ return nil, err+ }++ var merges [][3]string+ for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {+ fields := strings.Fields(line)+ if len(fields) != 3 {+ // Octopus merges are not replayed: the resolver merges two sides.+ continue+ }+ merges = append(merges, [3]string{fields[0], fields[1], fields[2]})+ }+ return merges, nil+}++// mergeBases returns the merge bases of a and b.+func (g gitRepo) mergeBases(ctx context.Context, a, b string) ([]string, error) {+ out, code, err := g.run(ctx, nil, []int{1}, "merge-base", "--all", a, b)+ if err != nil {+ return nil, err+ }+ if code == 1 {+ return nil, nil+ }+ return strings.Fields(string(out)), nil+}++// conflicts replays the merge of ours and theirs and returns the paths that conflict, as git+// merge-tree --write-tree reports them.+func (g gitRepo) conflicts(ctx context.Context, ours, theirs string) ([]string, error) {+ out, code, err := g.run(ctx, nil, []int{1}, "merge-tree", "--write-tree", "--name-only", "--no-messages", "-z", ours, theirs)+ if err != nil {+ return nil, err+ }+ if code == 0 {+ return nil, nil+ }++ // The output is the tree OID, then the conflicted paths, each terminated by a NUL; an empty+ // entry ends the list.+ parts := strings.Split(string(out), "\x00")+ var paths []string+ seen := map[string]bool{}+ for _, p := range parts[1:] {+ if p == "" {+ break+ }+ if !seen[p] {+ seen[p] = true+ paths = append(paths, p)+ }+ }+ return paths, nil+}++// entry is a blob of a tree: its object ID and mode.+type entry struct {+ OID string `json:"oid"`+ Mode string `json:"mode"`+}++// entryAt returns the blob at path in the tree of rev, or nil when the path is absent or not a+// blob.+func (g gitRepo) entryAt(ctx context.Context, rev, path string) (*entry, error) {+ out, _, err := g.run(ctx, nil, nil, "ls-tree", "-z", rev, "--", path)+ if err != nil {+ return nil, err+ }+ line := strings.TrimSuffix(string(out), "\x00")+ if line == "" {+ return nil, nil+ }+ // "<mode> SP <type> SP <oid> TAB <path>"+ meta, name, ok := strings.Cut(line, "\t")+ if !ok || name != path {+ return nil, nil+ }+ fields := strings.Fields(meta)+ if len(fields) != 3 || fields[1] != "blob" {+ return nil, nil+ }+ return &entry{OID: fields[2], Mode: fields[0]}, nil+}++// blob returns the content of a blob.+func (g gitRepo) blob(ctx context.Context, oid string) ([]byte, error) {+ out, _, err := g.run(ctx, nil, nil, "cat-file", "blob", oid)+ return out, err+}++// message returns the full commit message of a commit.+func (g gitRepo) message(ctx context.Context, oid string) (string, error) {+ out, _, err := g.run(ctx, nil, nil, "log", "-1", "--format=%B", oid)+ return strings.TrimRight(string(out), "\n"), err+}diff --git a/tools/graft-agents/resolve/bench/judge.go b/tools/graft-agents/resolve/bench/judge.gonew file mode 100644index 0000000..bd37b8c--- /dev/null+++ b/tools/graft-agents/resolve/bench/judge.go@@ -0,0 +1,135 @@+package bench++import (+ "bytes"+ "path"+ "regexp"+ "strings"+)++// normalize returns content with line endings as LF, trailing whitespace of every line removed+// and trailing blank lines dropped: the equivalence by which a result is judged correct.+func normalize(content []byte) []byte {+ text := strings.ReplaceAll(string(content), "\r\n", "\n")+ lines := strings.Split(text, "\n")+ for i, line := range lines {+ lines[i] = strings.TrimRight(line, " \t\r")+ }+ for len(lines) > 0 && lines[len(lines)-1] == "" {+ lines = lines[:len(lines)-1]+ }+ return []byte(strings.Join(lines, "\n"))+}++// judge reports whether a result equals the human resolution after normalization, and whether it+// is byte-identical. A nil content is a deletion.+func judge(result, human []byte, resultDeleted, humanDeleted bool) (correct, exact bool) {+ if resultDeleted || humanDeleted {+ return resultDeleted == humanDeleted, resultDeleted == humanDeleted+ }+ return bytes.Equal(normalize(result), normalize(human)), bytes.Equal(result, human)+}++// Reasons why a file is never sent to the resolver, as the server decides (PLAN10, 3.4).+const (+ NotSentBinary = "binary"+ NotSentLarge = "large"+ NotSentLockfile = "lockfile"+ NotSentGenerated = "generated"+ NotSentMinified = "minified"+)++// lockfiles are the base names of the lockfiles that always go to the agent.+var lockfiles = map[string]bool{+ "package-lock.json": true, "yarn.lock": true, "pnpm-lock.yaml": true,+ "poetry.lock": true, "Cargo.lock": true, "go.sum": true,+}++// generatedMarkers are looked for in the first lines of every side.+var generatedMarkers = []*regexp.Regexp{+ regexp.MustCompile(`Code generated .* DO NOT EDIT\.`),+ regexp.MustCompile(`@generated`),+ regexp.MustCompile(`DO NOT EDIT`),+ regexp.MustCompile(`(?i)auto-generated`),+}++// generatedMarkerLines is the number of first lines searched for a marker.+const generatedMarkerLines = 10++// minifiedGlobs name minified files.+var minifiedGlobs = []string{"*.min.js", "*.min.mjs", "*.min.css", "*.bundle.js"}++// Minified thresholds: a side of more than minifiedMinBytes whose average line length is above+// minifiedAverageLine, or with a line longer than minifiedLongestLine.+const (+ minifiedMinBytes = 2 << 10+ minifiedAverageLine = 300+ minifiedLongestLine = 5000+)++// notSentReason returns why the file at p with the given sides would not be sent to the resolver,+// or "" when it would be. generated reports linguist-generated for the path.+func notSentReason(p string, sides [][]byte, maxBytes int64, generated bool) string {+ for _, s := range sides {+ if bytes.IndexByte(s, 0) >= 0 {+ return NotSentBinary+ }+ }+ for _, s := range sides {+ if int64(len(s)) > maxBytes {+ return NotSentLarge+ }+ }+ if lockfiles[path.Base(p)] {+ return NotSentLockfile+ }+ if generated {+ return NotSentGenerated+ }+ for _, s := range sides {+ head := firstLines(s, generatedMarkerLines)+ for _, m := range generatedMarkers {+ if m.Match(head) {+ return NotSentGenerated+ }+ }+ }+ base := path.Base(p)+ for _, g := range minifiedGlobs {+ if ok, _ := path.Match(g, base); ok {+ return NotSentMinified+ }+ }+ for _, s := range sides {+ if looksMinified(s) {+ return NotSentMinified+ }+ }+ return ""+}++func firstLines(s []byte, n int) []byte {+ end := 0+ for i := 0; i < n; i++ {+ j := bytes.IndexByte(s[end:], '\n')+ if j < 0 {+ return s+ }+ end += j + 1+ }+ return s[:end]+}++func looksMinified(s []byte) bool {+ if len(s) <= minifiedMinBytes {+ return false+ }+ lines := bytes.Split(s, []byte("\n"))+ longest := 0+ for _, l := range lines {+ if len(l) > longest {+ longest = len(l)+ }+ }+ return longest > minifiedLongestLine || len(s)/len(lines) > minifiedAverageLine+}diff --git a/tools/graft-agents/resolve/bench/mine.go b/tools/graft-agents/resolve/bench/mine.gonew file mode 100644index 0000000..75dca47--- /dev/null+++ b/tools/graft-agents/resolve/bench/mine.go@@ -0,0 +1,145 @@+package bench++import (+ "context"+ "crypto/sha256"+ "encoding/hex"+ "fmt"++ "github.com/feder-cr/git-ai/tools/graft-agents/protocol"+)++// DefaultMaxMerges bounds the merges replayed per repository when the corpus sets no bound.+const DefaultMaxMerges = 3000++// MineOptions bound the mining of one repository.+type MineOptions struct {+ // MaxCases bounds the cases kept per repository, newest merges first (0: no bound).+ MaxCases int+}++// MineStats counts what the miner saw in one repository.+type MineStats struct {+ Merges int `json:"merges"`+ ConflictingMerges int `json:"conflicting_merges"`+ // SkippedMergeBases counts merges whose parents have no single merge base (criss-cross or+ // unrelated histories): the resolver is given one base.+ SkippedMergeBases int `json:"skipped_merge_bases"`+ ConflictingFiles int `json:"conflicting_files"`+ // OtherFamily counts conflicting files of no benchmark family.+ OtherFamily int `json:"other_family"`+ // OneSide counts conflicting files whose human resolution is one of the two sides, which+ // measures nothing about merging.+ OneSide int `json:"one_side"`+ Kept int `json:"kept"`+}++// caseID returns the stable ID of a case.+func caseID(repo, merge, p string) string {+ sum := sha256.Sum256([]byte(repo + "\x00" + merge + "\x00" + p))+ return hex.EncodeToString(sum[:8])+}++// Mine replays the merges of repo, cloned at dir, and returns the cases: the conflicting files of+// a benchmark family whose human resolution differs from both sides.+func Mine(ctx context.Context, repo Repo, dir string, opts MineOptions) ([]Case, MineStats, error) {+ g := gitRepo{dir: dir}+ maxMerges := repo.MaxMerges+ if maxMerges <= 0 {+ maxMerges = DefaultMaxMerges+ }++ merges, err := g.merges(ctx, repo.Commit, maxMerges)+ if err != nil {+ return nil, MineStats{}, err+ }++ var stats MineStats+ var cases []Case+ for _, m := range merges {+ if opts.MaxCases > 0 && len(cases) >= opts.MaxCases {+ break+ }+ stats.Merges+++ merge, ours, theirs := m[0], m[1], m[2]++ paths, err := g.conflicts(ctx, ours, theirs)+ if err != nil {+ return nil, stats, fmt.Errorf("merge %s: %w", merge, err)+ }+ if len(paths) == 0 {+ continue+ }+ stats.ConflictingMerges++++ bases, err := g.mergeBases(ctx, ours, theirs)+ if err != nil {+ return nil, stats, fmt.Errorf("merge %s: %w", merge, err)+ }+ if len(bases) != 1 {+ stats.SkippedMergeBases+++ continue+ }+ base := bases[0]++ for _, p := range paths {+ if opts.MaxCases > 0 && len(cases) >= opts.MaxCases {+ break+ }+ stats.ConflictingFiles+++ family := familyOf(p)+ if family == "" {+ stats.OtherFamily+++ continue+ }++ c := Case{+ ID: caseID(repo.Name, merge, p), Repo: repo.Name, License: repo.License, Family: family,+ Merge: merge, Ours: ours, Theirs: theirs, Base: base, Path: p,+ }+ for _, side := range []struct {+ rev string+ dst **entry+ }{{base, &c.BaseBlob}, {ours, &c.OursBlob}, {theirs, &c.TheirsBlob}, {merge, &c.HumanBlob}} {+ e, err := g.entryAt(ctx, side.rev, p)+ if err != nil {+ return nil, stats, fmt.Errorf("merge %s, %s: %w", merge, p, err)+ }+ *side.dst = e+ }+ if c.OursBlob == nil && c.TheirsBlob == nil {+ continue+ }+ if sameEntry(c.HumanBlob, c.OursBlob) || sameEntry(c.HumanBlob, c.TheirsBlob) {+ stats.OneSide+++ continue+ }+ c.Kind = kindOf(c)++ stats.Kept+++ cases = append(cases, c)+ }+ }+ return cases, stats, nil+}++func sameEntry(a, b *entry) bool {+ if a == nil || b == nil {+ return a == nil && b == nil+ }+ return a.OID == b.OID && a.Mode == b.Mode+}++// kindOf returns the protocol kind of the conflict of a case, as the server computes it+// (landConflictKind in internal/gitaly/service/operations/land_resolve.go).+func kindOf(c Case) string {+ switch {+ case c.OursBlob == nil || c.TheirsBlob == nil:+ return protocol.KindModifyDelete+ case c.BaseBlob == nil:+ return protocol.KindAddAdd+ case c.OursBlob.OID == c.TheirsBlob.OID:+ return protocol.KindMode+ }+ return protocol.KindContent+}diff --git a/tools/graft-agents/resolve/bench/report.go b/tools/graft-agents/resolve/bench/report.gonew file mode 100644index 0000000..a0ed608--- /dev/null+++ b/tools/graft-agents/resolve/bench/report.go@@ -0,0 +1,125 @@+package bench++import (+ "fmt"+ "io"+ "sort"+ "strings"+)++// Counts aggregates the outcomes of a set of cases.+type Counts struct {+ Cases int `json:"cases"`+ Correct int `json:"correct"`+ Exact int `json:"exact"`+ Wrong int `json:"wrong"`+ Rejected int `json:"rejected_by_checks"`+ NotSent int `json:"not_sent"`+ Model int `json:"model_error"`+ NotRun int `json:"not_run_cap"`+ Abstained int `json:"abstained"`+}++// Accepted is the number of results that would land: correct and wrong.+func (c Counts) Accepted() int { return c.Correct + c.Wrong }++// ErrorRate is wrong / accepted, the share of the landed AI resolutions that differ from the+// human one; 0 when nothing was accepted.+func (c Counts) ErrorRate() float64 {+ if c.Accepted() == 0 {+ return 0+ }+ return float64(c.Wrong) / float64(c.Accepted())+}++func (c *Counts) add(r Result) {+ c.Cases+++ switch r.Outcome {+ case OutcomeCorrect:+ c.Correct+++ if r.Exact {+ c.Exact+++ }+ case OutcomeWrong:+ c.Wrong+++ case OutcomeRejected:+ c.Rejected+++ case OutcomeNotSent:+ c.NotSent+++ case OutcomeModelError:+ c.Model+++ case OutcomeNotRun:+ c.NotRun+++ case OutcomeAbstained:+ c.Abstained+++ }+}++// Report is the outcome of a run: the totals, the counts per family and every result.+type Report struct {+ // Label names the run, for example the model.+ Label string `json:"label"`+ Total Counts `json:"total"`+ Families map[string]Counts `json:"families"`+ // NotSent counts the cases that were not sent, by reason.+ NotSent map[string]int `json:"not_sent"`+ // Spend is the AI spend of the run in USD, when a budget was used.+ SpendUSD float64 `json:"spend_usd,omitempty"`+ Results []Result `json:"results"`+}++// NewReport aggregates results.+func NewReport(label string, results []Result) Report {+ rep := Report{Label: label, Families: map[string]Counts{}, NotSent: map[string]int{}, Results: results}+ for _, r := range results {+ rep.Total.add(r)+ fc := rep.Families[r.Family]+ fc.add(r)+ rep.Families[r.Family] = fc+ if r.Outcome == OutcomeNotSent {+ rep.NotSent[r.Reason]+++ }+ }+ return rep+}++// WriteMarkdown writes the report as Markdown tables.+func (rep Report) WriteMarkdown(w io.Writer) error {+ var b strings.Builder+ fmt.Fprintf(&b, "## graft-resolve bench: %s\n\n", rep.Label)+ fmt.Fprintf(&b, "Cases run: %d", rep.Total.Cases)+ if rep.SpendUSD > 0 {+ fmt.Fprintf(&b, ", AI spend $%.2f", rep.SpendUSD)+ }+ b.WriteString(".\n\n")+ b.WriteString("| family | cases | correct (exact) | wrong | rejected by checks | not sent | model error | not run (cap) | abstained | error rate |\n")+ b.WriteString("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n")+ families := make([]string, 0, len(rep.Families))+ for f := range rep.Families {+ families = append(families, f)+ }+ sort.Strings(families)+ row := func(name string, c Counts) {+ fmt.Fprintf(&b, "| %s | %d | %d (%d) | %d | %d | %d | %d | %d | %d | %.1f%% |\n",+ name, c.Cases, c.Correct, c.Exact, c.Wrong, c.Rejected, c.NotSent, c.Model, c.NotRun, c.Abstained, 100*c.ErrorRate())+ }+ for _, f := range families {+ row(f, rep.Families[f])+ }+ row("total", rep.Total)+ b.WriteString("\nThe error rate is wrong / (correct + wrong): the share of the results that would land and differ from the human resolution. Correct means equal after normalizing line endings and trailing whitespace; exact means byte-identical.\n")+ if len(rep.NotSent) > 0 {+ reasons := make([]string, 0, len(rep.NotSent))+ for r := range rep.NotSent {+ reasons = append(reasons, r)+ }+ sort.Strings(reasons)+ parts := make([]string, 0, len(reasons))+ for _, r := range reasons {+ parts = append(parts, fmt.Sprintf("%s %d", r, rep.NotSent[r]))+ }+ fmt.Fprintf(&b, "\nNot sent, by reason: %s.\n", strings.Join(parts, ", "))+ }+ _, err := io.WriteString(w, b.String())+ return err+}diff --git a/tools/graft-agents/resolve/bench/run.go b/tools/graft-agents/resolve/bench/run.gonew file mode 100644index 0000000..511f7f5--- /dev/null+++ b/tools/graft-agents/resolve/bench/run.go@@ -0,0 +1,243 @@+package bench++import (+ "context"+ "errors"+ "fmt"+ "math/rand"+ "sort"+ "strings"+ "time"++ "github.com/feder-cr/git-ai/tools/graft-agents/protocol"+ "github.com/feder-cr/git-ai/tools/graft-agents/resolve"+)++// Outcomes of a case.+const (+ // OutcomeCorrect: the chain resolved the file and the result equals the human resolution.+ OutcomeCorrect = "correct"+ // OutcomeWrong: the chain resolved the file, the checks accepted it, and it differs from the+ // human resolution. A wrong answer lands without review.+ OutcomeWrong = "wrong"+ // OutcomeRejected: a result was rejected by the rules (markers, mode, size, syntax) or a+ // configured check; the land would fail and the agent would get the package.+ OutcomeRejected = "rejected_by_checks"+ // OutcomeNotSent: the file is never sent to the resolver (binary, large, lockfile, generated,+ // minified).+ OutcomeNotSent = "not_sent"+ // OutcomeModelError: a strategy failed (for a model, the call or its answer).+ OutcomeModelError = "model_error"+ // OutcomeNotRun: the AI budget of the run was spent before the case.+ OutcomeNotRun = "not_run_cap"+ // OutcomeAbstained: no strategy applied to the file.+ OutcomeAbstained = "abstained"+)++// Result is the outcome of one case.+type Result struct {+ ID string `json:"id"`+ Repo string `json:"repo"`+ Family string `json:"family"`+ Path string `json:"path"`+ Outcome string `json:"outcome"`+ Exact bool `json:"exact,omitempty"`+ Strategy string `json:"strategy,omitempty"`+ Reason string `json:"reason,omitempty"`+ Seconds float64 `json:"seconds"`+}++// Sample returns n cases of cases, stratified by family: each family's cases are shuffled with the+// seed and the families are taken in turn, so that the sample is the same for the same manifest and+// seed. n <= 0 takes every case.+func Sample(cases []Case, n int, seed int64) []Case {+ byFamily := map[string][]Case{}+ for _, c := range cases {+ byFamily[c.Family] = append(byFamily[c.Family], c)+ }+ families := make([]string, 0, len(byFamily))+ for f := range byFamily {+ families = append(families, f)+ }+ sort.Strings(families)++ rnd := rand.New(rand.NewSource(seed))+ for _, f := range families {+ group := byFamily[f]+ sort.Slice(group, func(i, j int) bool { return group[i].ID < group[j].ID })+ rnd.Shuffle(len(group), func(i, j int) { group[i], group[j] = group[j], group[i] })+ }++ if n <= 0 {+ n = len(cases)+ }+ var sample []Case+ for len(sample) < n {+ took := false+ for _, f := range families {+ if len(byFamily[f]) == 0 || len(sample) >= n {+ continue+ }+ sample = append(sample, byFamily[f][0])+ byFamily[f] = byFamily[f][1:]+ took = true+ }+ if !took {+ break+ }+ }+ return sample+}++// Runner replays cases through a resolver.+type Runner struct {+ Resolver *resolve.Resolver+ // Clone returns the directory of the clone of a corpus repository.+ Clone func(repo string) string+ // MaxFileBytes is the server's max_file_bytes (protocol.DefaultMaxFileBytes when 0).+ MaxFileBytes int64+ // Progress, when set, is called after each case.+ Progress func(done, total int, r Result)+}++// Run replays every case and returns the results in the order of cases.+func (r Runner) Run(ctx context.Context, cases []Case) ([]Result, error) {+ maxBytes := r.MaxFileBytes+ if maxBytes <= 0 {+ maxBytes = protocol.DefaultMaxFileBytes+ }++ results := make([]Result, 0, len(cases))+ for i, c := range cases {+ if err := ctx.Err(); err != nil {+ return results, err+ }+ res, err := r.runCase(ctx, c, maxBytes)+ if err != nil {+ return results, fmt.Errorf("case %s (%s %s): %w", c.ID, c.Repo, c.Path, err)+ }+ results = append(results, res)+ if r.Progress != nil {+ r.Progress(i+1, len(cases), res)+ }+ }+ return results, nil+}++func (r Runner) runCase(ctx context.Context, c Case, maxBytes int64) (Result, error) {+ res := Result{ID: c.ID, Repo: c.Repo, Family: c.Family, Path: c.Path}+ g := gitRepo{dir: r.Clone(c.Repo)}++ sides := map[string]*protocol.Side{}+ var contents [][]byte+ for name, e := range map[string]*entry{"base": c.BaseBlob, "ours": c.OursBlob, "theirs": c.TheirsBlob} {+ if e == nil {+ continue+ }+ content, err := g.blob(ctx, e.OID)+ if err != nil {+ return res, err+ }+ sides[name] = &protocol.Side{OID: e.OID, Mode: e.Mode, Content: content}+ contents = append(contents, content)+ }+ var human []byte+ if c.HumanBlob != nil {+ var err error+ if human, err = g.blob(ctx, c.HumanBlob.OID); err != nil {+ return res, err+ }+ }++ generated, err := g.linguistGenerated(ctx, c.Ours, c.Path)+ if err != nil {+ return res, err+ }+ if reason := notSentReason(c.Path, contents, maxBytes, generated); reason != "" {+ res.Outcome, res.Reason = OutcomeNotSent, reason+ return res, nil+ }++ oursMessage, err := g.message(ctx, c.Ours)+ if err != nil {+ return res, err+ }+ theirsMessage, err := g.message(ctx, c.Theirs)+ if err != nil {+ return res, err+ }++ req := protocol.Request{+ Version: protocol.Version,+ Repository: protocol.Repository{Storage: "bench", RelativePath: c.Repo},+ Branch: "bench",+ Tip: c.Ours,+ Commit: c.Theirs,+ MergeBase: c.Base,+ Messages: protocol.Messages{Tip: oursMessage, Commit: theirsMessage},+ Actor: "bench",+ Files: []protocol.FileConflict{{+ Path: c.Path, Kind: c.Kind, Base: sides["base"], Ours: sides["ours"], Theirs: sides["theirs"],+ }},+ Limits: protocol.Limits{MaxFileBytes: maxBytes},+ }++ start := time.Now()+ resp, err := r.Resolver.Resolve(ctx, req)+ res.Seconds = time.Since(start).Seconds()+ if err != nil {+ var unresolved *resolve.UnresolvedError+ if !errors.As(err, &unresolved) || len(unresolved.Failures) != 1 {+ return res, err+ }+ res.Outcome, res.Strategy, res.Reason = classifyFailure(unresolved.Failures[0])+ return res, nil+ }++ f := resp.Files[0]+ res.Strategy = strings.TrimPrefix(resp.Resolver, "graft-resolve/")+ correct, exact := judge(f.Content, human, f.Action == protocol.ActionDelete, c.HumanBlob == nil)+ res.Exact = exact+ if correct {+ res.Outcome = OutcomeCorrect+ } else {+ res.Outcome = OutcomeWrong+ }+ return res, nil+}++// classifyFailure returns the outcome of a file that the chain did not resolve: the last attempt+// decides, since the chain went on past the earlier ones.+func classifyFailure(f resolve.Failure) (outcome, strategy, reason string) {+ if len(f.Attempts) == 0 {+ return OutcomeAbstained, "", strings.Join(f.Reasons, "; ")+ }+ last := f.Attempts[len(f.Attempts)-1]+ reason = truncate(fmt.Sprint(last.Err), 512)+ switch {+ case errors.Is(last.Err, resolve.ErrBudgetExhausted):+ return OutcomeNotRun, last.Strategy, reason+ case last.Kind == resolve.AttemptRejected || last.Kind == resolve.AttemptCheckFailed:+ return OutcomeRejected, last.Strategy, reason+ }+ return OutcomeModelError, last.Strategy, reason+}++func truncate(s string, n int) string {+ if len(s) <= n {+ return s+ }+ return s[:n]+}++// linguistGenerated reports whether the .gitattributes of rev's tree mark path as+// linguist-generated.+func (g gitRepo) linguistGenerated(ctx context.Context, rev, p string) (bool, error) {+ out, _, err := g.run(ctx, nil, nil, "check-attr", "--source", rev, "linguist-generated", "--", p)+ if err != nil {+ return false, err+ }+ // "<path>: linguist-generated: <value>"+ value := strings.TrimSpace(string(out[strings.LastIndex(string(out), ":")+1:]))+ return value == "set" || value == "true", nil+}diff --git a/tools/graft-agents/resolve/resolve.go b/tools/graft-agents/resolve/resolve.goindex 7982c46..e54484c 100644--- a/tools/graft-agents/resolve/resolve.go+++ b/tools/graft-agents/resolve/resolve.go@@ -203,8 +203,37 @@ func NewResolver(bindings []Binding, checks []Check, timeout time.Duration) (*Retype Failure struct {Path stringReasons []string+ // Attempts are the strategies that applied to the file and did not resolve+ // it, in chain order, with what happened. Reasons holds the same in text.+ Attempts []Attempt+}++// AttemptKind says why a strategy that applied to a file did not resolve it.+type AttemptKind string++const (+ // AttemptFailed: the strategy returned an error (for a model, the call or+ // its answer failed; errors.Is(Err, ErrBudgetExhausted) when the AI budget+ // is spent).+ AttemptFailed AttemptKind = "failed"+ // AttemptRejected: the result broke a rule that every result must keep+ // (conflict markers, mode, size, or its syntax).+ AttemptRejected AttemptKind = "rejected"+ // AttemptCheckFailed: a configured check rejected the result.+ AttemptCheckFailed AttemptKind = "check_failed"+)++// Attempt is one strategy that applied to a file without resolving it.+type Attempt struct {+ Strategy string+ Kind AttemptKind+ Err error}+// ErrBudgetExhausted is returned by a strategy that does not call its model+// because the AI budget is spent.+var ErrBudgetExhausted = errors.New("AI budget exhausted")+// UnresolvedError reports files that stayed unresolved.type UnresolvedError struct {Failures []Failure@@ -240,12 +269,12 @@ func (r *Resolver) Resolve(ctx context.Context, req protocol.Request) (protocol.var failures []Failurefor _, f := range req.Files {- out, name, reasons, err := r.resolveFile(ctx, req, f, maxBytes)+ out, name, attempts, err := r.resolveFile(ctx, req, f, maxBytes)if err != nil {return protocol.Response{}, err}if name == "" {- failures = append(failures, Failure{Path: f.Path, Reasons: reasons})+ failures = append(failures, Failure{Path: f.Path, Reasons: attemptReasons(attempts), Attempts: attempts})continue}if !seen[name] {@@ -294,11 +323,31 @@ func resolverName(used []string) string {return fmt.Sprintf("%s+%d", resolverNamePrefix, len(used))}+// attemptReasons returns the text form of the attempts, the reasons of a+// Failure.+func attemptReasons(attempts []Attempt) []string {+ if len(attempts) == 0 {+ return []string{"no strategy resolved this conflict"}+ }+ reasons := make([]string, 0, len(attempts))+ for _, a := range attempts {+ switch a.Kind {+ case AttemptRejected:+ reasons = append(reasons, fmt.Sprintf("%s: rejected: %v", a.Strategy, a.Err))+ case AttemptCheckFailed:+ reasons = append(reasons, fmt.Sprintf("%s: check failed: %v", a.Strategy, a.Err))+ default:+ reasons = append(reasons, fmt.Sprintf("%s: %v", a.Strategy, a.Err))+ }+ }+ return reasons+}+// resolveFile runs the chain for one file. It returns the strategy name that-// accepted the answer, or an empty name and the collected reasons. err is set-// only when ctx ended.-func (r *Resolver) resolveFile(ctx context.Context, req protocol.Request, f File, maxBytes int64) (protocol.ResolvedFile, string, []string, error) {- var reasons []string+// accepted the answer, or an empty name and the attempts of the strategies+// that applied. err is set only when ctx ended.+func (r *Resolver) resolveFile(ctx context.Context, req protocol.Request, f File, maxBytes int64) (protocol.ResolvedFile, string, []Attempt, error) {+ var attempts []Attemptfor _, b := range r.bindings {if err := ctx.Err(); err != nil {return protocol.ResolvedFile{}, "", nil, err@@ -311,21 +360,21 @@ func (r *Resolver) resolveFile(ctx context.Context, req protocol.Request, f Fileif ctxErr := ctx.Err(); ctxErr != nil {return protocol.ResolvedFile{}, "", nil, ctxErr}- reasons = append(reasons, fmt.Sprintf("%s: %v", b.Name, err))+ attempts = append(attempts, Attempt{Strategy: b.Name, Kind: AttemptFailed, Err: err})continue}if !ok {continue}if err := checkResolution(res, maxBytes); err != nil {- reasons = append(reasons, fmt.Sprintf("%s: rejected: %v", b.Name, err))+ attempts = append(attempts, Attempt{Strategy: b.Name, Kind: AttemptRejected, Err: err})continue}if err := r.runChecks(ctx, f.Path, res); err != nil {if ctxErr := ctx.Err(); ctxErr != nil {return protocol.ResolvedFile{}, "", nil, ctxErr}- reasons = append(reasons, fmt.Sprintf("%s: check failed: %v", b.Name, err))+ attempts = append(attempts, Attempt{Strategy: b.Name, Kind: AttemptCheckFailed, Err: err})continue}r.record(ctx, f, res)@@ -336,10 +385,7 @@ func (r *Resolver) resolveFile(ctx context.Context, req protocol.Request, f FileContent: contentOf(res),}, b.Name, nil, nil}- if len(reasons) == 0 {- reasons = []string{"no strategy resolved this conflict"}- }- return protocol.ResolvedFile{}, "", reasons, nil+ return protocol.ResolvedFile{}, "", attempts, nil}func modeOf(res Resolution) string {