Git designed for AI agents.
graft-resolve: bench mine and bench run, and the corpus (WP1, part 2)
Federico Elia021ee4b9c5e5
diff --git a/tools/graft-agents/cmd/graft-resolve/bench.go b/tools/graft-agents/cmd/graft-resolve/bench.gonew file mode 100644index 0000000..10a1926--- /dev/null+++ b/tools/graft-agents/cmd/graft-resolve/bench.go@@ -0,0 +1,156 @@+package main++import (+ "context"+ "encoding/json"+ "flag"+ "fmt"+ "io"+ "os"+ "os/signal"+ "syscall"++ "github.com/feder-cr/git-ai/tools/graft-agents/resolve/bench"+)++// runBench runs the benchmark subcommands:+//+// graft-resolve bench mine --corpus FILE --clones DIR --manifest OUT [--max-cases-per-repo N]+// graft-resolve bench run --config FILE --manifest FILE --clones DIR [--sample N] [--seed S]+// [--label L] [--json-out FILE] [--md-out FILE]+func runBench(args []string, stdout, stderr io.Writer) int {+ if len(args) == 0 {+ fmt.Fprintln(stderr, "usage: graft-resolve bench mine|run [flags]")+ return 2+ }+ ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)+ defer stop()++ switch args[0] {+ case "mine":+ return benchMine(ctx, args[1:], stdout, stderr)+ case "run":+ return benchRun(ctx, args[1:], stdout, stderr)+ }+ fmt.Fprintf(stderr, "graft-resolve bench: unknown subcommand %q, want mine or run\n", args[0])+ return 2+}++func benchMine(ctx context.Context, args []string, stdout, stderr io.Writer) int {+ fs := flag.NewFlagSet("graft-resolve bench mine", flag.ContinueOnError)+ fs.SetOutput(stderr)+ corpusPath := fs.String("corpus", "", "corpus file (JSON)")+ clones := fs.String("clones", "", "directory of the clones of the corpus repositories")+ manifestPath := fs.String("manifest", "", "manifest to write")+ maxCases := fs.Int("max-cases-per-repo", 0, "bound on the cases kept per repository (0: none)")+ if err := fs.Parse(args); err != nil {+ return 2+ }+ if *corpusPath == "" || *clones == "" || *manifestPath == "" || fs.NArg() != 0 {+ fmt.Fprintln(stderr, "graft-resolve bench mine: --corpus, --clones and --manifest are required")+ return 2+ }++ corpus, err := bench.LoadCorpus(*corpusPath)+ if err != nil {+ fmt.Fprintf(stderr, "graft-resolve bench mine: %v\n", err)+ return 1+ }++ manifest := bench.Manifest{Version: bench.ManifestVersion, Stats: map[string]bench.MineStats{}}+ for _, repo := range corpus.Repos {+ dir, err := bench.EnsureClone(ctx, *clones, repo)+ if err != nil {+ fmt.Fprintf(stderr, "graft-resolve bench mine: %s: %v\n", repo.Name, err)+ return 1+ }+ cases, stats, err := bench.Mine(ctx, repo, dir, bench.MineOptions{MaxCases: *maxCases})+ if err != nil {+ fmt.Fprintf(stderr, "graft-resolve bench mine: %s: %v\n", repo.Name, err)+ return 1+ }+ manifest.Cases = append(manifest.Cases, cases...)+ manifest.Stats[repo.Name] = stats+ fmt.Fprintf(stdout, "%s: %d merges, %d conflicting, %d conflicting files, %d kept\n",+ repo.Name, stats.Merges, stats.ConflictingMerges, stats.ConflictingFiles, stats.Kept)+ }++ if err := bench.SaveManifest(*manifestPath, manifest); err != nil {+ fmt.Fprintf(stderr, "graft-resolve bench mine: %v\n", err)+ return 1+ }+ fmt.Fprintf(stdout, "%d cases in %s\n", len(manifest.Cases), *manifestPath)+ return 0+}++func benchRun(ctx context.Context, args []string, stdout, stderr io.Writer) int {+ fs := flag.NewFlagSet("graft-resolve bench run", flag.ContinueOnError)+ fs.SetOutput(stderr)+ configPath := fs.String("config", "", "resolver configuration (TOML), as in production")+ manifestPath := fs.String("manifest", "", "manifest written by bench mine")+ clones := fs.String("clones", "", "directory of the clones of the corpus repositories")+ sample := fs.Int("sample", 0, "number of cases, stratified by family (0: every case)")+ seed := fs.Int64("seed", 1, "seed of the sample")+ label := fs.String("label", "", "name of the run in the report")+ jsonOut := fs.String("json-out", "", "report as JSON")+ mdOut := fs.String("md-out", "", "report as Markdown")+ if err := fs.Parse(args); err != nil {+ return 2+ }+ if *manifestPath == "" || *clones == "" || fs.NArg() != 0 {+ fmt.Fprintln(stderr, "graft-resolve bench run: --manifest and --clones are required")+ return 2+ }++ res, err := loadResolver(*configPath)+ if err != nil {+ fmt.Fprintf(stderr, "graft-resolve bench run: %v\n", err)+ return 1+ }+ manifest, err := bench.LoadManifest(*manifestPath)+ if err != nil {+ fmt.Fprintf(stderr, "graft-resolve bench run: %v\n", err)+ return 1+ }++ cases := bench.Sample(manifest.Cases, *sample, *seed)+ runner := bench.Runner{+ Resolver: res,+ Clone: func(repo string) string { return bench.CloneDir(*clones, repo) },+ Progress: func(done, total int, r bench.Result) {+ fmt.Fprintf(stderr, "[%d/%d] %s %s %s: %s %s\n", done, total, r.Family, r.Repo, r.Path, r.Outcome, r.Reason)+ },+ }+ results, runErr := runner.Run(ctx, cases)+ report := bench.NewReport(*label, results)++ if *jsonOut != "" {+ data, err := json.MarshalIndent(report, "", " ")+ if err == nil {+ err = os.WriteFile(*jsonOut, append(data, '\n'), 0o644)+ }+ if err != nil {+ fmt.Fprintf(stderr, "graft-resolve bench run: %v\n", err)+ return 1+ }+ }+ out := stdout+ if *mdOut != "" {+ f, err := os.Create(*mdOut)+ if err != nil {+ fmt.Fprintf(stderr, "graft-resolve bench run: %v\n", err)+ return 1+ }+ defer f.Close()+ out = io.MultiWriter(stdout, f)+ }+ if err := report.WriteMarkdown(out); err != nil {+ fmt.Fprintf(stderr, "graft-resolve bench run: %v\n", err)+ return 1+ }+ if runErr != nil {+ fmt.Fprintf(stderr, "graft-resolve bench run: stopped after %d of %d cases: %v\n", len(results), len(cases), runErr)+ return 1+ }+ return 0+}diff --git a/tools/graft-agents/cmd/graft-resolve/bench_test.go b/tools/graft-agents/cmd/graft-resolve/bench_test.gonew file mode 100644index 0000000..89594bd--- /dev/null+++ b/tools/graft-agents/cmd/graft-resolve/bench_test.go@@ -0,0 +1,91 @@+package main++import (+ "bytes"+ "encoding/json"+ "os"+ "os/exec"+ "path/filepath"+ "strings"+ "testing"++ "github.com/feder-cr/git-ai/tools/graft-agents/resolve/bench"+)++// gitIn runs git in dir with a fixed identity, given on the command line.+func gitIn(t *testing.T, dir string, args ...string) string {+ t.Helper()+ cmd := exec.Command("git", append([]string{"-C", dir, "-c", "user.name=Bench", "-c", "[email protected]",+ "-c", "commit.gpgsign=false"}, args...)...)+ cmd.Env = append(os.Environ(), "GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_SYSTEM=/dev/null")+ out, err := cmd.CombinedOutput()+ if err != nil && !(len(args) > 0 && args[0] == "merge") {+ t.Fatalf("git %s: %v: %s", strings.Join(args, " "), err, out)+ }+ return strings.TrimSpace(string(out))+}++// TestBenchMineAndRun mines a repository with one hand-resolved conflict through a file:// clone+// and runs the default chain (trivial only) on it, which abstains.+func TestBenchMineAndRun(t *testing.T) {+ if _, err := exec.LookPath("git"); err != nil {+ t.Skip("git is not installed")+ }+ src := t.TempDir()+ gitIn(t, src, "init", "-q", "-b", "main")+ write := func(content string) {+ if err := os.WriteFile(filepath.Join(src, "a.py"), []byte(content), 0o644); err != nil {+ t.Fatal(err)+ }+ gitIn(t, src, "add", "-A")+ }+ write("x = 1\n")+ gitIn(t, src, "commit", "-q", "-m", "base")+ gitIn(t, src, "checkout", "-q", "-b", "feature")+ write("x = 3\n")+ gitIn(t, src, "commit", "-q", "-m", "three")+ gitIn(t, src, "checkout", "-q", "main")+ write("x = 2\n")+ gitIn(t, src, "commit", "-q", "-m", "two")+ gitIn(t, src, "merge", "--no-ff", "-q", "feature")+ write("x = 2 + 3\n")+ gitIn(t, src, "commit", "-q", "-m", "merge feature")+ head := gitIn(t, src, "rev-parse", "HEAD")++ work := t.TempDir()+ corpus := filepath.Join(work, "corpus.json")+ data, _ := json.Marshal(bench.Corpus{Repos: []bench.Repo{{Name: "local/fixture", URL: "file://" + src, Commit: head, License: "MIT"}}})+ if err := os.WriteFile(corpus, data, 0o644); err != nil {+ t.Fatal(err)+ }+ clones, manifest := filepath.Join(work, "clones"), filepath.Join(work, "manifest.json")++ var stdout, stderr bytes.Buffer+ if code := run([]string{"bench", "mine", "--corpus", corpus, "--clones", clones, "--manifest", manifest}, nil, &stdout, &stderr); code != 0 {+ t.Fatalf("mine exited %d: %s", code, stderr.String())+ }+ if !strings.Contains(stdout.String(), "1 cases in") {+ t.Fatalf("mine output: %s", stdout.String())+ }++ stdout.Reset()+ stderr.Reset()+ jsonOut := filepath.Join(work, "report.json")+ if code := run([]string{"bench", "run", "--manifest", manifest, "--clones", clones, "--label", "trivial", "--json-out", jsonOut}, nil, &stdout, &stderr); code != 0 {+ t.Fatalf("run exited %d: %s", code, stderr.String())+ }+ var report bench.Report+ raw, err := os.ReadFile(jsonOut)+ if err != nil {+ t.Fatal(err)+ }+ if err := json.Unmarshal(raw, &report); err != nil {+ t.Fatal(err)+ }+ if report.Total.Cases != 1 || report.Total.Abstained != 1 || report.Results[0].Family != bench.FamilyPython {+ t.Fatalf("report %+v", report)+ }+ if !strings.Contains(stdout.String(), "| total | 1 |") {+ t.Fatalf("markdown: %s", stdout.String())+ }+}diff --git a/tools/graft-agents/cmd/graft-resolve/main.go b/tools/graft-agents/cmd/graft-resolve/main.goindex 168009b..785254c 100644--- a/tools/graft-agents/cmd/graft-resolve/main.go+++ b/tools/graft-agents/cmd/graft-resolve/main.go@@ -5,6 +5,7 @@//// graft-resolve [--config FILE] [--max-request-bytes N] < request.json > response.json// graft-resolve serve --listen ADDR [--config FILE] [--token-file FILE] [--max-request-bytes N]+// graft-resolve bench mine|run [flags] (see bench.go and doc/graft/PLAN10.md)//// In command mode the request is read from stdin and the response written to// stdout. An unresolvable request exits 1 with the reason on stderr. In serve@@ -45,6 +46,9 @@ func run(args []string, stdin io.Reader, stdout, stderr io.Writer) int {if len(args) > 0 && args[0] == "serve" {return runServe(args[1:], stderr)}+ if len(args) > 0 && args[0] == "bench" {+ return runBench(args[1:], stdout, stderr)+ }return runCommand(args, stdin, stdout, stderr)}diff --git a/tools/graft-agents/resolve/bench/bench_test.go b/tools/graft-agents/resolve/bench/bench_test.goindex 52351fd..b3e1096 100644--- a/tools/graft-agents/resolve/bench/bench_test.go+++ b/tools/graft-agents/resolve/bench/bench_test.go@@ -279,3 +279,17 @@ func TestReportCounts(t *testing.T) {t.Fatalf("markdown:\n%s", b.String())}}++// TestCommittedCorpusLoads checks the corpus file of the benchmark: permissive licenses, no+// duplicate, a pinned commit for every repository.+func TestCommittedCorpusLoads(t *testing.T) {+ c, err := LoadCorpus("corpus.json")+ if err != nil {+ t.Fatal(err)+ }+ for _, r := range c.Repos {+ if len(r.Commit) != 40 {+ t.Errorf("%s: commit %q is not a full object ID", r.Name, r.Commit)+ }+ }+}diff --git a/tools/graft-agents/resolve/bench/clone.go b/tools/graft-agents/resolve/bench/clone.gonew file mode 100644index 0000000..1c433b7--- /dev/null+++ b/tools/graft-agents/resolve/bench/clone.go@@ -0,0 +1,44 @@+package bench++import (+ "context"+ "errors"+ "fmt"+ "io/fs"+ "os"+ "path/filepath"+ "strings"+)++// CloneDir returns the directory of the clone of a corpus repository under root.+func CloneDir(root, name string) string {+ return filepath.Join(root, strings.ReplaceAll(name, "/", "__")+".git")+}++// EnsureClone makes a bare clone of repo under root unless one exists, and makes sure that it has+// the pinned commit. The clone is complete, not partial: replaying merges reads the blobs of+// every changed file, which a partial clone would fetch one at a time.+func EnsureClone(ctx context.Context, root string, repo Repo) (string, error) {+ dir := CloneDir(root, repo.Name)+ if _, err := os.Stat(dir); errors.Is(err, fs.ErrNotExist) {+ if err := os.MkdirAll(root, 0o755); err != nil {+ return "", err+ }+ g := gitRepo{dir: root}+ if _, _, err := g.run(ctx, nil, nil, "clone", "--bare", "--quiet", repo.URL, dir); err != nil {+ return "", err+ }+ } else if err != nil {+ return "", err+ }++ g := gitRepo{dir: dir}+ if _, code, err := g.run(ctx, nil, []int{1, 128}, "cat-file", "-e", repo.Commit+"^{commit}"); err != nil {+ return "", err+ } else if code != 0 {+ if _, _, err := g.run(ctx, nil, nil, "fetch", "--quiet", "origin", repo.Commit); err != nil {+ return "", fmt.Errorf("fetching the pinned commit of %s: %w", repo.Name, err)+ }+ }+ return dir, nil+}diff --git a/tools/graft-agents/resolve/bench/corpus.json b/tools/graft-agents/resolve/bench/corpus.jsonnew file mode 100644index 0000000..3b46281--- /dev/null+++ b/tools/graft-agents/resolve/bench/corpus.json@@ -0,0 +1,17 @@+{+ "repos": [+ {"name": "microsoft/TypeScript", "url": "https://github.com/microsoft/TypeScript.git", "commit": "5237fd46890956e4bb8177a1a05dcd69e4166560", "license": "Apache-2.0"},+ {"name": "microsoft/vscode", "url": "https://github.com/microsoft/vscode.git", "commit": "80aa091c84c6201a3b3cc648a6b12e388e1a2c39", "license": "MIT"},+ {"name": "facebook/react", "url": "https://github.com/facebook/react.git", "commit": "098d2e87a1f7142947418d15f63f975b58c4685b", "license": "MIT"},+ {"name": "django/django", "url": "https://github.com/django/django.git", "commit": "d430084791a61b31cecec132bce3dca19194f7f2", "license": "BSD-3-Clause"},+ {"name": "pandas-dev/pandas", "url": "https://github.com/pandas-dev/pandas.git", "commit": "4f72cac84d11ea782685bfced9de9db9b492686c", "license": "BSD-3-Clause"},+ {"name": "pallets/flask", "url": "https://github.com/pallets/flask.git", "commit": "d086db856be187255b8ec61ef409357393020f32", "license": "BSD-3-Clause"},+ {"name": "kubernetes/kubernetes", "url": "https://github.com/kubernetes/kubernetes.git", "commit": "7f9d9d6a3d6e48be4d8d261c9474618d7f8d303d", "license": "Apache-2.0"},+ {"name": "prometheus/prometheus", "url": "https://github.com/prometheus/prometheus.git", "commit": "c052a79dc7fd0f2df2219ffe336f510b0d149b72", "license": "Apache-2.0"},+ {"name": "golang/go", "url": "https://github.com/golang/go.git", "commit": "dfc6f7db9bd75868ffb5cbc9501a2f4899122d2f", "license": "BSD-3-Clause"},+ {"name": "rust-lang/rust-analyzer", "url": "https://github.com/rust-lang/rust-analyzer.git", "commit": "5ec20d044dbf04895877a55921ddc0ae6e73e127", "license": "MIT OR Apache-2.0"},+ {"name": "tokio-rs/tokio", "url": "https://github.com/tokio-rs/tokio.git", "commit": "cdd9dae3aa5056e23dc4d5bc4f852ab8fbf27aff", "license": "MIT"},+ {"name": "apache/kafka", "url": "https://github.com/apache/kafka.git", "commit": "f90822763913e10a02478c235b34a5ea9f36856e", "license": "Apache-2.0"},+ {"name": "spring-projects/spring-boot", "url": "https://github.com/spring-projects/spring-boot.git", "commit": "57a1ca37c36f3cf355211aa8dea8c500b2dd3a09", "license": "Apache-2.0"}+ ]+}