Git designed for AI agents.
gitai land: optional verification of a resolution (c1 WP7)
Federico Elia38fa5cb81591
diff --git a/doc/graft/CONFIGURATION.md b/doc/graft/CONFIGURATION.mdindex 4f7fddd..7609bc9 100644--- a/doc/graft/CONFIGURATION.md+++ b/doc/graft/CONFIGURATION.md@@ -196,6 +196,9 @@ here. The protocol, the validation rules and `graft-resolve` are described in| `minified.globs` | array of strings | empty | Base-name globs of minified files, besides the built-in `*.min.js`, `*.min.mjs`, `*.min.css` and `*.bundle.js`. Such files are never sent. || `minified.average_line_bytes` | int | `300` | A side larger than 2 KiB whose average line is longer is minified. || `minified.longest_line_bytes` | int | `5000` | A side larger than 2 KiB with a longer line is minified. |+| `verify.enabled` | bool | `false` | Run `verify.command` on the merged tree of every resolution (a resolver's or an agent's) before it lands; it applies even when `enabled` is false. |+| `verify.command` | array of strings | empty | Program and arguments, run in a temporary checkout of the merged tree with `PATH`, `HOME` and `TMPDIR` only. A non-zero exit rejects the resolution. |+| `verify.timeout` | string | `"5m"` | Bound on one run (Go duration); a run that does not finish rejects the resolution. |A file that is never sent goes to the agent with its reason in the conflictpackage: large (a side over `max_file_bytes`), binary (a NUL byte), lockfile,diff --git a/doc/graft/PLAN10.md b/doc/graft/PLAN10.mdindex 07c41b1..765b873 100644--- a/doc/graft/PLAN10.md+++ b/doc/graft/PLAN10.md@@ -192,9 +192,9 @@ need the agent. Nothing partial lands.`[graft.land.resolver.verify]`: `enabled = false` (default), `command` (argv run in atemporary checkout of the merged tree, minimal environment), `timeout` (default 5min, hard). A failure or timeout rejects the resolution, and the package says so.-RESOLVE.md gets "finding and reverting AI resolutions": the trailers-(`graft log --grep Graft-Resolved-By`), the info log, graft-verify's bisect and-revert.+RESOLVE.md gets "finding and reverting AI resolutions": the trailers (`git log+--grep='^Graft-Resolved-By:'` in a clone, `graft show --json`; `graft log` lists+changesets and has no grep), the info log, graft-verify's bisect and revert.### 3.7 The benchmark (G8)diff --git a/doc/graft/RESOLVE.md b/doc/graft/RESOLVE.mdindex 182ccf8..db71280 100644--- a/doc/graft/RESOLVE.md+++ b/doc/graft/RESOLVE.md@@ -256,6 +256,44 @@ resolver when the rules above let it. Use `paths_deny` for paths that mustnever leave the server. `graft-resolve bench` applies the same rules (3 to 7);both are tested against `internal/graft/resolver/testdata/never-send.json`.+## Optional verification of a resolution++`[graft.land.resolver.verify]`, off by default, runs a command on the merged+tree of every resolution before it lands, a resolver's answer or an agent's+resolution alike:++```toml+[graft.land.resolver.verify]+enabled = true+command = ["/usr/local/bin/graft-check-build"]+timeout = "5m"+```++The server archives the merged tree (the resolution included) into a+temporary directory and runs `command` there, with a minimal environment+(`PATH`, and `HOME` and `TMPDIR` in a directory of their own). A non-zero exit or a+timeout rejects the resolution: the land fails, every file of the package is+`REJECTED`, and the reason is `verification failed:` with the exit status and+the first 4 KiB of the output. The command runs on the server, with the+server's rights, on code that agents wrote: treat it as a custom hook. It+also costs time on every resolution, which is why it is off by default; the+trailers and `graft-verify` remain the safety net.++## Finding and reverting AI resolutions++Every resolution carries its trailers, so it can be found and undone:++- `Graft-Resolved-By` names the resolver (`graft-resolve/llm`, `agent/<actor>`)+ and `Graft-Resolved-Path` each resolved path; `Graft-Checked-By` says that a+ resolver checked an agent's resolution. In a clone, `git log+ --grep='^Graft-Resolved-By:'` lists them; `graft show --json REV` prints the+ trailers of one commit.+- The server logs every resolution at info level (`graft: conflicts of land+ resolved`, with the actor, the resolver, the paths and the explanation).+- `graft-verify` ([VERIFY.md](VERIFY.md)) tests the tip after landing,+ bisects a failure to the commit that broke it and reverts it with+ `UserRevert`, whether it was an AI resolution or not.+## The conflict package and the re-landWhen a land with `resolve_conflicts` (or with a `resolution`) fails on itsdiff --git a/internal/gitaly/config/config_graft.go b/internal/gitaly/config/config_graft.goindex 468558f..ab5740b 100644--- a/internal/gitaly/config/config_graft.go+++ b/internal/gitaly/config/config_graft.go@@ -314,6 +314,50 @@ type GraftResolver struct {GeneratedMarkers []string `json:"generated_markers" toml:"generated_markers,omitempty"`// Minified tells minified files apart; they are never sent to the resolver.Minified GraftResolverMinified `json:"minified" toml:"minified,omitempty"`+ // Verify runs a command on the merged tree of every resolution before it+ // lands (a resolver's answer or an agent's resolution). Off by default.+ Verify GraftResolverVerify `json:"verify" toml:"verify,omitempty"`+}++// GraftResolverVerify is the optional verification of a resolution: Command+// runs in a temporary checkout of the merged tree, and a failure or a timeout+// rejects the resolution.+type GraftResolverVerify struct {+ // Enabled turns the verification on. Off by default.+ Enabled bool `json:"enabled" toml:"enabled,omitempty"`+ // Command is the program and arguments, run in the checkout with a+ // minimal environment (PATH, HOME and TMPDIR inside the checkout).+ Command []string `json:"command" toml:"command,omitempty"`+ // Timeout bounds one run, as a Go duration. Empty means+ // DefaultGraftResolverVerifyTimeout.+ Timeout string `json:"timeout" toml:"timeout,omitempty"`+}++// DefaultGraftResolverVerifyTimeout is the default GraftResolverVerify.Timeout.+const DefaultGraftResolverVerifyTimeout = "5m"++// TimeoutOrDefault parses Timeout, applying the default.+func (v GraftResolverVerify) TimeoutOrDefault() (time.Duration, error) {+ s := v.Timeout+ if s == "" {+ s = DefaultGraftResolverVerifyTimeout+ }+ d, err := time.ParseDuration(s)+ if err != nil || d <= 0 {+ return 0, fmt.Errorf("graft land resolver verify: invalid timeout %q", v.Timeout)+ }+ return d, nil+}++func (v GraftResolverVerify) validate() error {+ if !v.Enabled {+ return nil+ }+ if len(v.Command) == 0 || v.Command[0] == "" {+ return fmt.Errorf("graft land resolver verify: command is required when enabled")+ }+ _, err := v.TimeoutOrDefault()+ return err}// GraftResolverMinified are the rules that mark a conflicting file minified.@@ -407,6 +451,10 @@ func (g Graft) Validate() error {}func (r GraftResolver) validate() error {+ // The verification also applies to agents' resolutions, which land without a resolver.+ if err := r.Verify.validate(); err != nil {+ return err+ }if !r.Enabled {return nil}diff --git a/internal/gitaly/config/config_graft_resolver_test.go b/internal/gitaly/config/config_graft_resolver_test.goindex 10791f3..1e4b780 100644--- a/internal/gitaly/config/config_graft_resolver_test.go+++ b/internal/gitaly/config/config_graft_resolver_test.go@@ -41,6 +41,27 @@ func TestGraftResolverValidateSendRules(t *testing.T) {}}+func TestGraftResolverValidateVerify(t *testing.T) {+ t.Parallel()++ // Off by default, and validated even without a resolver (agents' resolutions get it too).+ require.NoError(t, GraftResolver{}.validate())+ require.NoError(t, GraftResolver{Verify: GraftResolverVerify{Enabled: true, Command: []string{"make", "test"}}}.validate())++ d, err := GraftResolverVerify{}.TimeoutOrDefault()+ require.NoError(t, err)+ require.Equal(t, "5m0s", d.String())++ for name, v := range map[string]GraftResolverVerify{+ "no command": {Enabled: true},+ "empty program": {Enabled: true, Command: []string{""}},+ "bad timeout": {Enabled: true, Command: []string{"x"}, Timeout: "soon"},+ "zero timeout": {Enabled: true, Command: []string{"x"}, Timeout: "0s"},+ } {+ require.Error(t, GraftResolver{Verify: v}.validate(), name)+ }+}+func TestGraftResolverMinifiedDefaults(t *testing.T) {t.Parallel()diff --git a/internal/gitaly/service/operations/land_reland.go b/internal/gitaly/service/operations/land_reland.goindex a281769..0e8723c 100644--- a/internal/gitaly/service/operations/land_reland.go+++ b/internal/gitaly/service/operations/land_reland.go@@ -110,7 +110,8 @@ func (s *Server) landWithResolution(for _, f := range l.files {resp.Files = append(resp.Files, resolved[f.conflict.path])}- return s.landWriteResolution(ctx, in, repo, tip, commitID, l.mergedTree, committer, actor, resp, checkedBy)+ res, err := s.landWriteResolution(ctx, in, repo, tip, commitID, l.mergedTree, committer, actor, resp, checkedBy)+ return res, l.verifyRejected(ctx, maxFiles, err)}// landResolutionFiles maps the files of the resolution to the conflicting paths. It returns adiff --git a/internal/gitaly/service/operations/land_resolve.go b/internal/gitaly/service/operations/land_resolve.goindex 975afc8..c958f41 100644--- a/internal/gitaly/service/operations/land_resolve.go+++ b/internal/gitaly/service/operations/land_resolve.go@@ -158,7 +158,22 @@ func (s *Server) resolveLandConflicts(}}- return s.landWriteResolution(ctx, in, repo, tip, commitID, l.mergedTree, committer, actor, resp, "")+ res, err := s.landWriteResolution(ctx, in, repo, tip, commitID, l.mergedTree, committer, actor, resp, "")+ return res, l.verifyRejected(ctx, maxFiles, err)+}++// verifyRejected turns a resolution that the verification rejected into the unresolved land with+// its package, every file rejected with the reason of the verification. Any other error is+// returned as is.+func (l landListing) verifyRejected(ctx context.Context, maxFiles int, err error) error {+ var verifyErr *landVerifyError+ if !errors.As(err, &verifyErr) {+ return err+ }+ for _, f := range l.files {+ f.outcome, f.reason = gitalypb.LandConflictFile_OUTCOME_REJECTED, verifyErr.Error()+ }+ return l.unresolved(ctx, maxFiles, "", verifyErr.Error(), err)}// landListing is the conflicting files of the merge of a tip and a change, with what the conflict@@ -372,6 +387,12 @@ func (s *Server) landWriteResolution(if err := tree.Write(ctx, repo); err != nil {return landResolution{}, fmt.Errorf("writing merged tree: %w", err)}+ // The optional verification runs on the merged tree, the resolution included, before it lands.+ if s.resolveCfg.Verify.Enabled {+ if err := s.landVerify(ctx, repo, tree.OID); err != nil {+ return landResolution{}, err+ }+ }newTip, err := repo.WriteCommit(ctx, localrepo.WriteCommitConfig{TreeID: tree.OID,diff --git a/internal/gitaly/service/operations/land_verify.go b/internal/gitaly/service/operations/land_verify.gonew file mode 100644index 0000000..00c4845--- /dev/null+++ b/internal/gitaly/service/operations/land_verify.go@@ -0,0 +1,189 @@+package operations++import (+ "archive/tar"+ "context"+ "errors"+ "fmt"+ "io"+ "os"+ "path"+ "path/filepath"+ "strings"++ "github.com/feder-cr/git-ai/internal/command"+ "github.com/feder-cr/git-ai/internal/git"+ "github.com/feder-cr/git-ai/internal/git/gitcmd"+ "github.com/feder-cr/git-ai/internal/git/localrepo"+)++// This file is the optional verification of a resolution ([graft.land.resolver.verify], PLAN10+// 3.6): before a resolution lands, the configured command runs in a temporary checkout of the+// merged tree, the resolution included. A failure or a timeout rejects the resolution, and the+// conflict package says why. It applies to resolver answers and agents' resolutions alike.++// landVerifyOutputBytes bounds the output of the command kept as the reason.+const landVerifyOutputBytes = 4 << 10++// landVerifyError is a resolution the verification rejected.+type landVerifyError struct {+ reason string+}++func (e *landVerifyError) Error() string {+ return "verification failed: " + e.reason+}++// landVerify runs the verification command on tree. It returns a *landVerifyError when the+// command fails or times out, and another error when the verification could not be run.+func (s *Server) landVerify(ctx context.Context, repo *localrepo.Repo, tree git.ObjectID) error {+ v := s.resolveCfg.Verify+ timeout, err := v.TimeoutOrDefault()+ if err != nil {+ return err+ }++ dir, err := os.MkdirTemp("", "graft-verify-")+ if err != nil {+ return fmt.Errorf("verification directory: %w", err)+ }+ defer func() { _ = os.RemoveAll(dir) }()+ checkout, home := filepath.Join(dir, "tree"), filepath.Join(dir, "home")+ for _, d := range []string{checkout, home} {+ if err := os.Mkdir(d, 0o700); err != nil {+ return fmt.Errorf("verification directory: %w", err)+ }+ }++ archive, err := os.Create(filepath.Join(dir, "tree.tar"))+ if err != nil {+ return fmt.Errorf("verification archive: %w", err)+ }+ defer archive.Close()+ var stderr landBoundedBuffer+ stderr.limit = landVerifyOutputBytes+ if err := repo.ExecAndWait(ctx,+ gitcmd.Command{Name: "archive", Flags: []gitcmd.Option{gitcmd.ValueFlag{Name: "--format", Value: "tar"}}, Args: []string{tree.String()}},+ gitcmd.WithStdout(archive),+ gitcmd.WithStderr(&stderr),+ ); err != nil {+ return fmt.Errorf("archiving the merged tree: %w, stderr: %q", err, stderr.String())+ }+ if _, err := archive.Seek(0, io.SeekStart); err != nil {+ return fmt.Errorf("verification archive: %w", err)+ }+ if err := landExtractTar(archive, checkout); err != nil {+ return fmt.Errorf("checking out the merged tree: %w", err)+ }++ runCtx, cancel := context.WithTimeout(ctx, timeout)+ defer cancel()+ var output landBoundedBuffer+ output.limit = landVerifyOutputBytes+ cmd, err := command.New(runCtx, s.logger, v.Command,+ command.WithDir(checkout),+ command.WithEnvironment([]string{"PATH=" + os.Getenv("PATH"), "HOME=" + home, "TMPDIR=" + home}),+ command.WithStdout(&output),+ command.WithStderr(&output),+ )+ if err != nil {+ return fmt.Errorf("starting the verification: %w", err)+ }+ runErr := cmd.Wait()+ switch {+ case errors.Is(runCtx.Err(), context.DeadlineExceeded):+ return &landVerifyError{reason: fmt.Sprintf("the command did not finish within %s", timeout)}+ case ctx.Err() != nil:+ return ctx.Err()+ case runErr != nil:+ reason := strings.TrimSpace(output.String())+ if code, ok := command.ExitStatus(runErr); ok {+ reason = fmt.Sprintf("exit status %d: %s", code, reason)+ }+ return &landVerifyError{reason: reason}+ }+ return nil+}++// landExtractTar writes the files of the archive of a tree under dir. It refuses an entry that+// would leave dir, or that lies under a symbolic link it wrote: a link is written as a link and+// never followed.+func landExtractTar(r io.Reader, dir string) error {+ links := map[string]bool{}+ tr := tar.NewReader(r)+ for {+ h, err := tr.Next()+ if errors.Is(err, io.EOF) {+ return nil+ }+ if err != nil {+ return err+ }+ if h.Typeflag == tar.TypeXGlobalHeader {+ continue // git archive's commit header+ }+ name := strings.TrimSuffix(h.Name, "/")+ if name == "" || path.IsAbs(name) || path.Clean(name) != name || name == ".." || strings.HasPrefix(name, "../") {+ return fmt.Errorf("unsafe archive entry %q", h.Name)+ }+ for p := path.Dir(name); p != "."; p = path.Dir(p) {+ if links[p] {+ return fmt.Errorf("archive entry %q lies under the symbolic link %q", h.Name, p)+ }+ }+ target := filepath.Join(dir, filepath.FromSlash(name))+ switch h.Typeflag {+ case tar.TypeDir:+ if err := os.MkdirAll(target, 0o755); err != nil {+ return err+ }+ case tar.TypeReg:+ if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {+ return err+ }+ mode := os.FileMode(0o644)+ if h.Mode&0o111 != 0 {+ mode = 0o755+ }+ f, err := os.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_EXCL, mode)+ if err != nil {+ return err+ }+ if _, err := io.Copy(f, tr); err != nil {+ _ = f.Close()+ return err+ }+ if err := f.Close(); err != nil {+ return err+ }+ case tar.TypeSymlink:+ if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {+ return err+ }+ if err := os.Symlink(h.Linkname, target); err != nil {+ return err+ }+ links[name] = true+ default:+ return fmt.Errorf("archive entry %q has the unexpected type %q", h.Name, h.Typeflag)+ }+ }+}++// landBoundedBuffer keeps the first limit bytes written to it and drops the rest, so that a+// command cannot fill the server's memory with its output.+type landBoundedBuffer struct {+ buf []byte+ limit int+}++func (b *landBoundedBuffer) Write(p []byte) (int, error) {+ if room := b.limit - len(b.buf); room > 0 {+ b.buf = append(b.buf, p[:min(room, len(p))]...)+ }+ return len(p), nil+}++func (b *landBoundedBuffer) String() string {+ return strings.ToValidUTF8(string(b.buf), "")+}diff --git a/internal/gitaly/service/operations/land_verify_test.go b/internal/gitaly/service/operations/land_verify_test.gonew file mode 100644index 0000000..360558c--- /dev/null+++ b/internal/gitaly/service/operations/land_verify_test.go@@ -0,0 +1,136 @@+package operations++import (+ "archive/tar"+ "bytes"+ "os"+ "path/filepath"+ "testing"++ "github.com/stretchr/testify/require"++ "github.com/feder-cr/git-ai/internal/git"+ "github.com/feder-cr/git-ai/internal/gitaly/config"+ "github.com/feder-cr/git-ai/proto/go/gitalypb"+)++// verifyScript writes a verification command that runs body in the checkout.+func verifyScript(t *testing.T, body string) []string {+ t.Helper()+ p := filepath.Join(t.TempDir(), "verify.sh")+ require.NoError(t, os.WriteFile(p, []byte("#!/bin/sh\n"+body+"\n"), 0o755))+ return []string{p}+}++func withVerify(rcfg config.GraftResolver, argv []string, timeout string) config.GraftResolver {+ rcfg.Verify = config.GraftResolverVerify{Enabled: true, Command: argv, Timeout: timeout}+ return rcfg+}++func TestUserLand_verify(t *testing.T) {+ t.Parallel()++ t.Run("a resolution that passes lands", func(t *testing.T) {+ t.Parallel()++ dir, argv := resolverScript(t, "")+ writeAnswer(t, dir, validAnswer())+ // The checkout holds the merged tree with the resolution: a and b resolved, d from the change.+ fx := newResolveFixture(t, withVerify(enabledResolver(argv), verifyScript(t, `grep -qx merged-a a && grep -qx merged-b b && grep -qx d d`), ""))++ resp, err := fx.client.UserLand(fx.ctx, fx.request("Land"))+ require.NoError(t, err)+ requireBranchAt(t, fx.cfg, fx.repoPath, git.DefaultBranch, git.ObjectID(resp.GetLandedCommitId()))+ })++ t.Run("a failing command rejects every file", func(t *testing.T) {+ t.Parallel()++ dir, argv := resolverScript(t, "")+ writeAnswer(t, dir, validAnswer())+ fx := newResolveFixture(t, withVerify(enabledResolver(argv), verifyScript(t, `echo "2 tests failed"; exit 3`), ""))++ _, err := fx.client.UserLand(fx.ctx, fx.request("Land"))+ requireMergeConflict(t, err)+ pkg := landConflictDetail(err)+ require.NotNil(t, pkg)+ for _, f := range pkg.GetFiles() {+ require.Equal(t, gitalypb.LandConflictFile_OUTCOME_REJECTED, f.GetOutcome())+ require.Equal(t, "verification failed: exit status 3: 2 tests failed", f.GetReason())+ }+ requireBranchAt(t, fx.cfg, fx.repoPath, git.DefaultBranch, fx.tip)+ })++ t.Run("a command that does not finish is rejected", func(t *testing.T) {+ t.Parallel()++ dir, argv := resolverScript(t, "")+ writeAnswer(t, dir, validAnswer())+ fx := newResolveFixture(t, withVerify(enabledResolver(argv), verifyScript(t, `sleep 30`), "300ms"))++ _, err := fx.client.UserLand(fx.ctx, fx.request("Land"))+ pkg := landConflictDetail(err)+ require.NotNil(t, pkg)+ require.Contains(t, packageFile(t, pkg, "a").GetReason(), "did not finish within 300ms")+ requireBranchAt(t, fx.cfg, fx.repoPath, git.DefaultBranch, fx.tip)+ })++ t.Run("an agent's resolution is verified too", func(t *testing.T) {+ t.Parallel()++ fx := newResolveFixture(t, withVerify(config.GraftResolver{}, verifyScript(t, `grep -qx agent-a a`), ""))+ req := fx.request("Land")+ req.Resolution = resolutionOf(fx, "wrong\n", "agent-b\n")+ _, err := fx.client.UserLand(fx.ctx, req)+ pkg := landConflictDetail(err)+ require.NotNil(t, pkg)+ require.Equal(t, gitalypb.LandConflictFile_OUTCOME_REJECTED, packageFile(t, pkg, "a").GetOutcome())++ req.Resolution = resolutionOf(fx, "agent-a\n", "agent-b\n")+ _, err = fx.client.UserLand(fx.ctx, req)+ require.NoError(t, err)+ })+}++func TestLandExtractTar(t *testing.T) {+ t.Parallel()++ archive := func(entries ...*tar.Header) *bytes.Buffer {+ var b bytes.Buffer+ w := tar.NewWriter(&b)+ for _, h := range entries {+ require.NoError(t, w.WriteHeader(h))+ if h.Typeflag == tar.TypeReg {+ _, err := w.Write(make([]byte, h.Size))+ require.NoError(t, err)+ }+ }+ require.NoError(t, w.Close())+ return &b+ }+ file := func(name string) *tar.Header {+ return &tar.Header{Name: name, Typeflag: tar.TypeReg, Mode: 0o644, Size: 1}+ }++ dir := t.TempDir()+ require.NoError(t, landExtractTar(archive(+ &tar.Header{Name: "d/", Typeflag: tar.TypeDir, Mode: 0o755},+ file("d/f"),+ &tar.Header{Name: "x", Typeflag: tar.TypeReg, Mode: 0o755, Size: 1},+ &tar.Header{Name: "l", Typeflag: tar.TypeSymlink, Linkname: "/etc"},+ ), dir))+ info, err := os.Stat(filepath.Join(dir, "x"))+ require.NoError(t, err)+ require.Equal(t, os.FileMode(0o755), info.Mode().Perm())+ target, err := os.Readlink(filepath.Join(dir, "l"))+ require.NoError(t, err)+ require.Equal(t, "/etc", target)++ for name, a := range map[string]*bytes.Buffer{+ "under a link": archive(&tar.Header{Name: "l", Typeflag: tar.TypeSymlink, Linkname: "/tmp"}, file("l/x")),+ "leaves the dir": archive(file("../x")),+ "absolute": archive(file("/x")),+ } {+ require.Error(t, landExtractTar(a, t.TempDir()), name)+ }+}