Skip to content

Git designed for AI agents.

graft-resolve: bench mine and bench run, and the corpus (WP1, part 2)

Federico Elia021ee4b9c5e5
diff --git a/tools/graft-agents/cmd/graft-resolve/bench.go b/tools/graft-agents/cmd/graft-resolve/bench.go
new file mode 100644
index 0000000..10a1926
--- /dev/null
+++ b/tools/graft-agents/cmd/graft-resolve/bench.go
@@ -0,0 +1,156 @@
+package main
+
+import (
+ "context"
+ "encoding/json"
+ "flag"
+ "fmt"
+ "io"
+ "os"
+ "os/signal"
+ "syscall"
+
+ "github.com/feder-cr/git-ai/tools/graft-agents/resolve/bench"
+)
+
+// runBench runs the benchmark subcommands:
+//
+// graft-resolve bench mine --corpus FILE --clones DIR --manifest OUT [--max-cases-per-repo N]
+// graft-resolve bench run --config FILE --manifest FILE --clones DIR [--sample N] [--seed S]
+// [--label L] [--json-out FILE] [--md-out FILE]
+func runBench(args []string, stdout, stderr io.Writer) int {
+ if len(args) == 0 {
+ fmt.Fprintln(stderr, "usage: graft-resolve bench mine|run [flags]")
+ return 2
+ }
+ ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+ defer stop()
+
+ switch args[0] {
+ case "mine":
+ return benchMine(ctx, args[1:], stdout, stderr)
+ case "run":
+ return benchRun(ctx, args[1:], stdout, stderr)
+ }
+ fmt.Fprintf(stderr, "graft-resolve bench: unknown subcommand %q, want mine or run\n", args[0])
+ return 2
+}
+
+func benchMine(ctx context.Context, args []string, stdout, stderr io.Writer) int {
+ fs := flag.NewFlagSet("graft-resolve bench mine", flag.ContinueOnError)
+ fs.SetOutput(stderr)
+ corpusPath := fs.String("corpus", "", "corpus file (JSON)")
+ clones := fs.String("clones", "", "directory of the clones of the corpus repositories")
+ manifestPath := fs.String("manifest", "", "manifest to write")
+ maxCases := fs.Int("max-cases-per-repo", 0, "bound on the cases kept per repository (0: none)")
+ if err := fs.Parse(args); err != nil {
+ return 2
+ }
+ if *corpusPath == "" || *clones == "" || *manifestPath == "" || fs.NArg() != 0 {
+ fmt.Fprintln(stderr, "graft-resolve bench mine: --corpus, --clones and --manifest are required")
+ return 2
+ }
+
+ corpus, err := bench.LoadCorpus(*corpusPath)
+ if err != nil {
+ fmt.Fprintf(stderr, "graft-resolve bench mine: %v\n", err)
+ return 1
+ }
+
+ manifest := bench.Manifest{Version: bench.ManifestVersion, Stats: map[string]bench.MineStats{}}
+ for _, repo := range corpus.Repos {
+ dir, err := bench.EnsureClone(ctx, *clones, repo)
+ if err != nil {
+ fmt.Fprintf(stderr, "graft-resolve bench mine: %s: %v\n", repo.Name, err)
+ return 1
+ }
+ cases, stats, err := bench.Mine(ctx, repo, dir, bench.MineOptions{MaxCases: *maxCases})
+ if err != nil {
+ fmt.Fprintf(stderr, "graft-resolve bench mine: %s: %v\n", repo.Name, err)
+ return 1
+ }
+ manifest.Cases = append(manifest.Cases, cases...)
+ manifest.Stats[repo.Name] = stats
+ fmt.Fprintf(stdout, "%s: %d merges, %d conflicting, %d conflicting files, %d kept\n",
+ repo.Name, stats.Merges, stats.ConflictingMerges, stats.ConflictingFiles, stats.Kept)
+ }
+
+ if err := bench.SaveManifest(*manifestPath, manifest); err != nil {
+ fmt.Fprintf(stderr, "graft-resolve bench mine: %v\n", err)
+ return 1
+ }
+ fmt.Fprintf(stdout, "%d cases in %s\n", len(manifest.Cases), *manifestPath)
+ return 0
+}
+
+func benchRun(ctx context.Context, args []string, stdout, stderr io.Writer) int {
+ fs := flag.NewFlagSet("graft-resolve bench run", flag.ContinueOnError)
+ fs.SetOutput(stderr)
+ configPath := fs.String("config", "", "resolver configuration (TOML), as in production")
+ manifestPath := fs.String("manifest", "", "manifest written by bench mine")
+ clones := fs.String("clones", "", "directory of the clones of the corpus repositories")
+ sample := fs.Int("sample", 0, "number of cases, stratified by family (0: every case)")
+ seed := fs.Int64("seed", 1, "seed of the sample")
+ label := fs.String("label", "", "name of the run in the report")
+ jsonOut := fs.String("json-out", "", "report as JSON")
+ mdOut := fs.String("md-out", "", "report as Markdown")
+ if err := fs.Parse(args); err != nil {
+ return 2
+ }
+ if *manifestPath == "" || *clones == "" || fs.NArg() != 0 {
+ fmt.Fprintln(stderr, "graft-resolve bench run: --manifest and --clones are required")
+ return 2
+ }
+
+ res, err := loadResolver(*configPath)
+ if err != nil {
+ fmt.Fprintf(stderr, "graft-resolve bench run: %v\n", err)
+ return 1
+ }
+ manifest, err := bench.LoadManifest(*manifestPath)
+ if err != nil {
+ fmt.Fprintf(stderr, "graft-resolve bench run: %v\n", err)
+ return 1
+ }
+
+ cases := bench.Sample(manifest.Cases, *sample, *seed)
+ runner := bench.Runner{
+ Resolver: res,
+ Clone: func(repo string) string { return bench.CloneDir(*clones, repo) },
+ Progress: func(done, total int, r bench.Result) {
+ fmt.Fprintf(stderr, "[%d/%d] %s %s %s: %s %s\n", done, total, r.Family, r.Repo, r.Path, r.Outcome, r.Reason)
+ },
+ }
+ results, runErr := runner.Run(ctx, cases)
+ report := bench.NewReport(*label, results)
+
+ if *jsonOut != "" {
+ data, err := json.MarshalIndent(report, "", " ")
+ if err == nil {
+ err = os.WriteFile(*jsonOut, append(data, '\n'), 0o644)
+ }
+ if err != nil {
+ fmt.Fprintf(stderr, "graft-resolve bench run: %v\n", err)
+ return 1
+ }
+ }
+ out := stdout
+ if *mdOut != "" {
+ f, err := os.Create(*mdOut)
+ if err != nil {
+ fmt.Fprintf(stderr, "graft-resolve bench run: %v\n", err)
+ return 1
+ }
+ defer f.Close()
+ out = io.MultiWriter(stdout, f)
+ }
+ if err := report.WriteMarkdown(out); err != nil {
+ fmt.Fprintf(stderr, "graft-resolve bench run: %v\n", err)
+ return 1
+ }
+ if runErr != nil {
+ fmt.Fprintf(stderr, "graft-resolve bench run: stopped after %d of %d cases: %v\n", len(results), len(cases), runErr)
+ return 1
+ }
+ return 0
+}
diff --git a/tools/graft-agents/cmd/graft-resolve/bench_test.go b/tools/graft-agents/cmd/graft-resolve/bench_test.go
new file mode 100644
index 0000000..89594bd
--- /dev/null
+++ b/tools/graft-agents/cmd/graft-resolve/bench_test.go
@@ -0,0 +1,91 @@
+package main
+
+import (
+ "bytes"
+ "encoding/json"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/feder-cr/git-ai/tools/graft-agents/resolve/bench"
+)
+
+// gitIn runs git in dir with a fixed identity, given on the command line.
+func gitIn(t *testing.T, dir string, args ...string) string {
+ t.Helper()
+ cmd := exec.Command("git", append([]string{"-C", dir, "-c", "user.name=Bench", "-c", "[email protected]",
+ "-c", "commit.gpgsign=false"}, args...)...)
+ cmd.Env = append(os.Environ(), "GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_SYSTEM=/dev/null")
+ out, err := cmd.CombinedOutput()
+ if err != nil && !(len(args) > 0 && args[0] == "merge") {
+ t.Fatalf("git %s: %v: %s", strings.Join(args, " "), err, out)
+ }
+ return strings.TrimSpace(string(out))
+}
+
+// TestBenchMineAndRun mines a repository with one hand-resolved conflict through a file:// clone
+// and runs the default chain (trivial only) on it, which abstains.
+func TestBenchMineAndRun(t *testing.T) {
+ if _, err := exec.LookPath("git"); err != nil {
+ t.Skip("git is not installed")
+ }
+ src := t.TempDir()
+ gitIn(t, src, "init", "-q", "-b", "main")
+ write := func(content string) {
+ if err := os.WriteFile(filepath.Join(src, "a.py"), []byte(content), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ gitIn(t, src, "add", "-A")
+ }
+ write("x = 1\n")
+ gitIn(t, src, "commit", "-q", "-m", "base")
+ gitIn(t, src, "checkout", "-q", "-b", "feature")
+ write("x = 3\n")
+ gitIn(t, src, "commit", "-q", "-m", "three")
+ gitIn(t, src, "checkout", "-q", "main")
+ write("x = 2\n")
+ gitIn(t, src, "commit", "-q", "-m", "two")
+ gitIn(t, src, "merge", "--no-ff", "-q", "feature")
+ write("x = 2 + 3\n")
+ gitIn(t, src, "commit", "-q", "-m", "merge feature")
+ head := gitIn(t, src, "rev-parse", "HEAD")
+
+ work := t.TempDir()
+ corpus := filepath.Join(work, "corpus.json")
+ data, _ := json.Marshal(bench.Corpus{Repos: []bench.Repo{{Name: "local/fixture", URL: "file://" + src, Commit: head, License: "MIT"}}})
+ if err := os.WriteFile(corpus, data, 0o644); err != nil {
+ t.Fatal(err)
+ }
+ clones, manifest := filepath.Join(work, "clones"), filepath.Join(work, "manifest.json")
+
+ var stdout, stderr bytes.Buffer
+ if code := run([]string{"bench", "mine", "--corpus", corpus, "--clones", clones, "--manifest", manifest}, nil, &stdout, &stderr); code != 0 {
+ t.Fatalf("mine exited %d: %s", code, stderr.String())
+ }
+ if !strings.Contains(stdout.String(), "1 cases in") {
+ t.Fatalf("mine output: %s", stdout.String())
+ }
+
+ stdout.Reset()
+ stderr.Reset()
+ jsonOut := filepath.Join(work, "report.json")
+ if code := run([]string{"bench", "run", "--manifest", manifest, "--clones", clones, "--label", "trivial", "--json-out", jsonOut}, nil, &stdout, &stderr); code != 0 {
+ t.Fatalf("run exited %d: %s", code, stderr.String())
+ }
+ var report bench.Report
+ raw, err := os.ReadFile(jsonOut)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := json.Unmarshal(raw, &report); err != nil {
+ t.Fatal(err)
+ }
+ if report.Total.Cases != 1 || report.Total.Abstained != 1 || report.Results[0].Family != bench.FamilyPython {
+ t.Fatalf("report %+v", report)
+ }
+ if !strings.Contains(stdout.String(), "| total | 1 |") {
+ t.Fatalf("markdown: %s", stdout.String())
+ }
+}
diff --git a/tools/graft-agents/cmd/graft-resolve/main.go b/tools/graft-agents/cmd/graft-resolve/main.go
index 168009b..785254c 100644
--- a/tools/graft-agents/cmd/graft-resolve/main.go
+++ b/tools/graft-agents/cmd/graft-resolve/main.go
@@ -5,6 +5,7 @@
//
// graft-resolve [--config FILE] [--max-request-bytes N] < request.json > response.json
// graft-resolve serve --listen ADDR [--config FILE] [--token-file FILE] [--max-request-bytes N]
+// graft-resolve bench mine|run [flags] (see bench.go and doc/graft/PLAN10.md)
//
// In command mode the request is read from stdin and the response written to
// stdout. An unresolvable request exits 1 with the reason on stderr. In serve
@@ -45,6 +46,9 @@ func run(args []string, stdin io.Reader, stdout, stderr io.Writer) int {
if len(args) > 0 && args[0] == "serve" {
return runServe(args[1:], stderr)
}
+ if len(args) > 0 && args[0] == "bench" {
+ return runBench(args[1:], stdout, stderr)
+ }
return runCommand(args, stdin, stdout, stderr)
}
diff --git a/tools/graft-agents/resolve/bench/bench_test.go b/tools/graft-agents/resolve/bench/bench_test.go
index 52351fd..b3e1096 100644
--- a/tools/graft-agents/resolve/bench/bench_test.go
+++ b/tools/graft-agents/resolve/bench/bench_test.go
@@ -279,3 +279,17 @@ func TestReportCounts(t *testing.T) {
t.Fatalf("markdown:\n%s", b.String())
}
}
+
+// TestCommittedCorpusLoads checks the corpus file of the benchmark: permissive licenses, no
+// duplicate, a pinned commit for every repository.
+func TestCommittedCorpusLoads(t *testing.T) {
+ c, err := LoadCorpus("corpus.json")
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, r := range c.Repos {
+ if len(r.Commit) != 40 {
+ t.Errorf("%s: commit %q is not a full object ID", r.Name, r.Commit)
+ }
+ }
+}
diff --git a/tools/graft-agents/resolve/bench/clone.go b/tools/graft-agents/resolve/bench/clone.go
new file mode 100644
index 0000000..1c433b7
--- /dev/null
+++ b/tools/graft-agents/resolve/bench/clone.go
@@ -0,0 +1,44 @@
+package bench
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "strings"
+)
+
+// CloneDir returns the directory of the clone of a corpus repository under root.
+func CloneDir(root, name string) string {
+ return filepath.Join(root, strings.ReplaceAll(name, "/", "__")+".git")
+}
+
+// EnsureClone makes a bare clone of repo under root unless one exists, and makes sure that it has
+// the pinned commit. The clone is complete, not partial: replaying merges reads the blobs of
+// every changed file, which a partial clone would fetch one at a time.
+func EnsureClone(ctx context.Context, root string, repo Repo) (string, error) {
+ dir := CloneDir(root, repo.Name)
+ if _, err := os.Stat(dir); errors.Is(err, fs.ErrNotExist) {
+ if err := os.MkdirAll(root, 0o755); err != nil {
+ return "", err
+ }
+ g := gitRepo{dir: root}
+ if _, _, err := g.run(ctx, nil, nil, "clone", "--bare", "--quiet", repo.URL, dir); err != nil {
+ return "", err
+ }
+ } else if err != nil {
+ return "", err
+ }
+
+ g := gitRepo{dir: dir}
+ if _, code, err := g.run(ctx, nil, []int{1, 128}, "cat-file", "-e", repo.Commit+"^{commit}"); err != nil {
+ return "", err
+ } else if code != 0 {
+ if _, _, err := g.run(ctx, nil, nil, "fetch", "--quiet", "origin", repo.Commit); err != nil {
+ return "", fmt.Errorf("fetching the pinned commit of %s: %w", repo.Name, err)
+ }
+ }
+ return dir, nil
+}
diff --git a/tools/graft-agents/resolve/bench/corpus.json b/tools/graft-agents/resolve/bench/corpus.json
new file mode 100644
index 0000000..3b46281
--- /dev/null
+++ b/tools/graft-agents/resolve/bench/corpus.json
@@ -0,0 +1,17 @@
+{
+ "repos": [
+ {"name": "microsoft/TypeScript", "url": "https://github.com/microsoft/TypeScript.git", "commit": "5237fd46890956e4bb8177a1a05dcd69e4166560", "license": "Apache-2.0"},
+ {"name": "microsoft/vscode", "url": "https://github.com/microsoft/vscode.git", "commit": "80aa091c84c6201a3b3cc648a6b12e388e1a2c39", "license": "MIT"},
+ {"name": "facebook/react", "url": "https://github.com/facebook/react.git", "commit": "098d2e87a1f7142947418d15f63f975b58c4685b", "license": "MIT"},
+ {"name": "django/django", "url": "https://github.com/django/django.git", "commit": "d430084791a61b31cecec132bce3dca19194f7f2", "license": "BSD-3-Clause"},
+ {"name": "pandas-dev/pandas", "url": "https://github.com/pandas-dev/pandas.git", "commit": "4f72cac84d11ea782685bfced9de9db9b492686c", "license": "BSD-3-Clause"},
+ {"name": "pallets/flask", "url": "https://github.com/pallets/flask.git", "commit": "d086db856be187255b8ec61ef409357393020f32", "license": "BSD-3-Clause"},
+ {"name": "kubernetes/kubernetes", "url": "https://github.com/kubernetes/kubernetes.git", "commit": "7f9d9d6a3d6e48be4d8d261c9474618d7f8d303d", "license": "Apache-2.0"},
+ {"name": "prometheus/prometheus", "url": "https://github.com/prometheus/prometheus.git", "commit": "c052a79dc7fd0f2df2219ffe336f510b0d149b72", "license": "Apache-2.0"},
+ {"name": "golang/go", "url": "https://github.com/golang/go.git", "commit": "dfc6f7db9bd75868ffb5cbc9501a2f4899122d2f", "license": "BSD-3-Clause"},
+ {"name": "rust-lang/rust-analyzer", "url": "https://github.com/rust-lang/rust-analyzer.git", "commit": "5ec20d044dbf04895877a55921ddc0ae6e73e127", "license": "MIT OR Apache-2.0"},
+ {"name": "tokio-rs/tokio", "url": "https://github.com/tokio-rs/tokio.git", "commit": "cdd9dae3aa5056e23dc4d5bc4f852ab8fbf27aff", "license": "MIT"},
+ {"name": "apache/kafka", "url": "https://github.com/apache/kafka.git", "commit": "f90822763913e10a02478c235b34a5ea9f36856e", "license": "Apache-2.0"},
+ {"name": "spring-projects/spring-boot", "url": "https://github.com/spring-projects/spring-boot.git", "commit": "57a1ca37c36f3cf355211aa8dea8c500b2dd3a09", "license": "Apache-2.0"}
+ ]
+}