Git designed for AI agents.
graft: every writer of revisions to a git stdin is listed and guarded
Federico Elia59a9800a5700
diff --git a/doc/graft/AGENT-SECURITY.md b/doc/graft/AGENT-SECURITY.mdindex 5df6cd7..39151f0 100644--- a/doc/graft/AGENT-SECURITY.md+++ b/doc/graft/AGENT-SECURITY.md@@ -101,9 +101,15 @@ the hiding:accepting a line break in a path (`AllowLineBreaksInPath`), as their testsrequire. A path with a tab, a space or other special characters is allowedeverywhere.-2. **The sink.** `gitpipe.Revlist`, the one place that joins revisions into- `git rev-list --stdin`, refuses a revision with a line feed, a carriage return- or a NUL, whatever its caller validated.+2. **The sinks.** Every writer of revisions into a newline-delimited git stdin+ (`git rev-list --stdin`) refuses a revision with a line feed, a carriage+ return or a NUL, whatever its caller validated, through+ `git.ValidateStdinRevision`. Today there are two: `gitpipe.Revlist` (ListCommits,+ ListBlobs, ListLFSPointers, GetTagSignatures and the other revision walks) and+ `ObjectsSize`. A test (`internal/git/stdin_writers_test.go`) lists every+ `--stdin` writer by file and the way it is safe (a request's revisions+ through the helper; NUL-delimited `-z`; input the server builds), and fails+ on a writer it does not know, so a new one is classified when it is added.3. **The filter reads lines as git does.** The `--exclude-hidden` flag is putbefore a line that git reads as `--all` or `--glob=...`: without its linefeed and the carriage return before it, and cut at the first NUL.diff --git a/internal/git/gitpipe/revision.go b/internal/git/gitpipe/revision.goindex 1a101ce..765aabc 100644--- a/internal/git/gitpipe/revision.go+++ b/internal/git/gitpipe/revision.go@@ -296,14 +296,11 @@ func Revlist(go func() {defer close(resultChan)- // The revisions go to git-rev-list(1) one per line on its stdin: whatever a caller validated,- // a revision with a line break is more than one revision, and one with a NUL is read only up- // to it. The guard is here, at the sink, so that it holds for every caller.+ // The revisions go to git-rev-list(1) one per line on its stdin: the guard is here, at the+ // sink, so that it holds for every caller.for _, revision := range revisions {- if strings.ContainsAny(revision, "\n\r\x00") {- sendRevisionResult(ctx, resultChan, RevisionResult{- err: fmt.Errorf("revision %q: a revision can't contain a line break or NUL", revision),- })+ if err := git.ValidateStdinRevision(revision); err != nil {+ sendRevisionResult(ctx, resultChan, RevisionResult{err: err})return}}diff --git a/internal/git/revision.go b/internal/git/revision.goindex d043371..b5dfb21 100644--- a/internal/git/revision.go+++ b/internal/git/revision.go@@ -3,6 +3,7 @@ package gitimport ("bytes""fmt"+ "strings")type validateRevisionConfig struct {@@ -131,3 +132,15 @@ func hasControlCharacter(b []byte) bool {}return false}++// ValidateStdinRevision checks a revision that is about to be written to the standard input of a+// git command that reads revisions one per line (git-rev-list(1) --stdin): whatever its caller+// validated, a revision with a line feed or a carriage return is more than one revision, and one+// with a NUL is read only up to it. Every writer of such a stdin calls it for each revision, and a+// test lists the writers (stdin_writers_test.go).+func ValidateStdinRevision(revision string) error {+ if strings.ContainsAny(revision, "\n\r\x00") {+ return fmt.Errorf("revision %q: a revision can't contain a line break or NUL", revision)+ }+ return nil+}diff --git a/internal/git/stdin_writers_test.go b/internal/git/stdin_writers_test.gonew file mode 100755index 0000000..91b2317--- /dev/null+++ b/internal/git/stdin_writers_test.go@@ -0,0 +1,103 @@+package git++import (+ "io/fs"+ "os"+ "path/filepath"+ "runtime"+ "strings"+ "testing"++ "github.com/stretchr/testify/require"+)++// Where a git command reads revisions from its stdin one per line, a line break in a revision is+// another revision. This test lists every non-test file that gives a git command the flag --stdin,+// and how each is safe, so that a new writer is classified when it is added.+const (+ // stdinRevisionsOfRequests: revisions that callers send, written one per line; the file calls+ // ValidateStdinRevision for each.+ stdinRevisionsOfRequests = "revisions of requests, through ValidateStdinRevision"+ // stdinNULDelimited: the input is NUL-delimited (-z), so a line break is data.+ stdinNULDelimited = "NUL-delimited (-z)"+ // stdinServerInput: the input is built by the server from object IDs, packs or content, never+ // from a caller's revision or pattern.+ stdinServerInput = "built by the server from object IDs, packs or content"+ // stdinNamesTheFlag: not a writer; the file only compares the flag.+ stdinNamesTheFlag = "not a writer"+)++var stdinWriters = map[string]string{+ "internal/git/gitpipe/revision.go": stdinRevisionsOfRequests,+ "internal/gitaly/service/repository/objects_size.go": stdinRevisionsOfRequests,+ "internal/gitaly/service/diff/find_changed_paths.go": stdinNULDelimited,+ "internal/gitaly/service/operations/land_package.go": stdinNULDelimited,+ "internal/git/localrepo/merge_tree_stream.go": stdinNULDelimited,+ "internal/graft/corestore/syncer/syncer.go": stdinNULDelimited,+ "internal/git/gitattributes/check_attr.go": stdinNULDelimited,+ "internal/git/updateref/updateref.go": stdinNULDelimited,+ "internal/git/localrepo/bundle.go": stdinServerInput,+ "internal/git/localrepo/objects.go": stdinServerInput,+ "internal/git/localrepo/tree.go": stdinServerInput,+ "internal/git/localrepo/rebase.go": stdinServerInput,+ "internal/git/gitcmd/blob.go": stdinServerInput,+ "internal/graft/corestore/gitops/gitops.go": stdinServerInput,+ "internal/gitaly/service/storagecore/verify_objects.go": stdinServerInput,+ "internal/gitaly/storage/storagemgr/partition/transaction_manager.go": stdinServerInput,+ "internal/git/gitcmd/agent_walk.go": stdinNamesTheFlag,+}++func TestStdinWriters_areAllClassified(t *testing.T) {+ t.Parallel()++ _, self, _, ok := runtime.Caller(0)+ require.True(t, ok)+ moduleRoot := filepath.Join(filepath.Dir(self), "..", "..")++ found := map[string]bool{}+ require.NoError(t, filepath.WalkDir(moduleRoot, func(path string, d fs.DirEntry, err error) error {+ if err != nil {+ return err+ }+ if d.IsDir() {+ switch d.Name() {+ case "_build", ".git", "node_modules", "testdata":+ return filepath.SkipDir+ }+ return nil+ }+ if !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {+ return nil+ }+ raw, err := os.ReadFile(path)+ require.NoError(t, err)+ if strings.Contains(string(raw), `"--stdin"`) {+ rel, err := filepath.Rel(moduleRoot, path)+ require.NoError(t, err)+ found[filepath.ToSlash(rel)] = true++ if stdinWriters[filepath.ToSlash(rel)] == stdinRevisionsOfRequests {+ require.Contains(t, string(raw), "ValidateStdinRevision", "%s writes revisions of requests to a git stdin without the guard", rel)+ }+ }+ return nil+ }))++ for file := range found {+ require.Contains(t, stdinWriters, file,+ "%s gives a git command --stdin: classify it in stdinWriters (are its revisions written one per line? then call ValidateStdinRevision for each)", file)+ }+ for file := range stdinWriters {+ require.Contains(t, found, file, "%s is classified but no longer uses --stdin: remove it from stdinWriters", file)+ }+}++func TestValidateStdinRevision(t *testing.T) {+ t.Parallel()++ require.NoError(t, ValidateStdinRevision("refs/heads/main"))+ require.NoError(t, ValidateStdinRevision("--glob=refs/heads/*"))+ for _, revision := range []string{"main\nrefs/graft/green/main", "main\r", "--all\x00", "\n"} {+ require.ErrorContains(t, ValidateStdinRevision(revision), "a revision can't contain a line break or NUL", "%q", revision)+ }+}diff --git a/internal/gitaly/service/commit/agent_injection_service_test.go b/internal/gitaly/service/commit/agent_injection_service_test.goindex 6928d83..0d36281 100755--- a/internal/gitaly/service/commit/agent_injection_service_test.go+++ b/internal/gitaly/service/commit/agent_injection_service_test.go@@ -1,6 +1,7 @@package commitimport (+ "bytes""context""errors""io"@@ -12,7 +13,9 @@ import ("google.golang.org/grpc/metadata""google.golang.org/grpc/status"+ "github.com/feder-cr/git-ai/internal/git""github.com/feder-cr/git-ai/internal/git/gittest"+ "github.com/feder-cr/git-ai/internal/gitaly/config""github.com/feder-cr/git-ai/internal/gitaly/service""github.com/feder-cr/git-ai/internal/gitaly/service/blob""github.com/feder-cr/git-ai/internal/gitaly/service/ref"@@ -23,15 +26,24 @@ import ("github.com/feder-cr/git-ai/proto/go/gitalypb")-// TestRevisionInjection_rpcsRefuseLineBreaks: revisions travel to git one per line on its stdin, so-// a revision with a line feed, a carriage return or a NUL is more than one revision to git, or one-// that the filter that leaves refs/graft/ out of an agent's walks reads differently from git. The-// pseudo-revision patterns after "=" took any bytes, and the request check read the whole as one-// name: "--glob=x\nrefs/graft/green/main" named a hidden reference without the check seeing it.-// Every RPC that sends revisions to git over stdin refuses them now, for every caller, with-// InvalidArgument, and no proposal is returned.-func TestRevisionInjection_rpcsRefuseLineBreaks(t *testing.T) {- t.Parallel()+// injectionFixture is a service with the commit, blob and ref services, and a repository whose HEAD+// is a normal branch and whose only commit that no branch reaches is a proposal under refs/graft/.+type injectionFixture struct {+ ctx context.Context+ agent context.Context+ cfg config.Cfg+ repo *gitalypb.Repository+ repoPath string+ commits gitalypb.CommitServiceClient+ blobs gitalypb.BlobServiceClient+ refs gitalypb.RefServiceClient+ main git.ObjectID+ proposal git.ObjectID+ proposedBlob git.ObjectID+}++func newInjectionFixture(t *testing.T, mainEntries ...gittest.TreeEntry) injectionFixture {+ t.Helper()ctx := testhelper.Context(t)cfg := testcfg.Build(t)@@ -42,48 +54,74 @@ func TestRevisionInjection_rpcsRefuseLineBreaks(t *testing.T) {gitalypb.RegisterRepositoryServiceServer(srv, repository.NewServer(deps))}, testserver.WithInterceptors(agentInterceptors))conn := dial(t, cfg.SocketPath)- commits := gitalypb.NewCommitServiceClient(conn)- blobs := gitalypb.NewBlobServiceClient(conn)- refs := gitalypb.NewRefServiceClient(conn)repo, repoPath := gittest.CreateRepository(t, ctx, cfg)- main := gittest.WriteCommit(t, cfg, repoPath, gittest.WithBranch("main"),- gittest.WithTreeEntries(gittest.TreeEntry{Mode: "100644", Path: "a", Content: "a\n"}))- gittest.WriteCommit(t, cfg, repoPath, gittest.WithParents(main),+ if len(mainEntries) == 0 {+ mainEntries = []gittest.TreeEntry{{Mode: "100644", Path: "a", Content: "a\n"}}+ }+ main := gittest.WriteCommit(t, cfg, repoPath, gittest.WithBranch("main"), gittest.WithTreeEntries(mainEntries...))+ gittest.Exec(t, cfg, "-C", repoPath, "symbolic-ref", "HEAD", "refs/heads/main")+ proposal := gittest.WriteCommit(t, cfg, repoPath, gittest.WithParents(main),gittest.WithReference("refs/graft/proposals/0123456789abcdef/1"), gittest.WithMessage("graft: proposal\n"),gittest.WithTreeEntries(gittest.TreeEntry{Mode: "100644", Path: "a", Content: "proposed by a model\n"}))- agent := metadata.AppendToOutgoingContext(ctx, testAgentHeader, "agent-a")-- drain := func(recv func() error) error {- for {- if err := recv(); err != nil {- if errors.Is(err, io.EOF) {- return nil- }- return err++ return injectionFixture{+ ctx: ctx,+ agent: metadata.AppendToOutgoingContext(ctx, testAgentHeader, "agent-a"),+ cfg: cfg,+ repo: repo,+ repoPath: repoPath,+ commits: gitalypb.NewCommitServiceClient(conn),+ blobs: gitalypb.NewBlobServiceClient(conn),+ refs: gitalypb.NewRefServiceClient(conn),+ main: main,+ proposal: proposal,+ proposedBlob: gittest.ResolveRevision(t, cfg, repoPath, proposal.String()+":a"),+ }+}++func drainStream(recv func() error) error {+ for {+ if err := recv(); err != nil {+ if errors.Is(err, io.EOF) {+ return nil}+ return err}}+}++// TestRevisionInjection_rpcsRefuseLineBreaks: revisions travel to git one per line on its stdin, so+// a revision with a line feed, a carriage return or a NUL is more than one revision to git, or one+// that the filter that leaves refs/graft/ out of an agent's walks reads differently from git. The+// pseudo-revision patterns after "=" took any bytes, and the request check read the whole as one+// name: "--glob=x\nrefs/graft/green/main" named a hidden reference without the check seeing it.+// Every RPC that sends revisions to git over stdin refuses them now, for every caller, with+// InvalidArgument.+func TestRevisionInjection_rpcsRefuseLineBreaks(t *testing.T) {+ t.Parallel()++ fx := newInjectionFixture(t)rpcs := map[string]func(context.Context, ...string) error{"ListCommits": func(ctx context.Context, revs ...string) error {- stream, err := commits.ListCommits(ctx, &gitalypb.ListCommitsRequest{Repository: repo, Revisions: revs})+ stream, err := fx.commits.ListCommits(ctx, &gitalypb.ListCommitsRequest{Repository: fx.repo, Revisions: revs})require.NoError(t, err)- return drain(func() error { _, err := stream.Recv(); return err })+ return drainStream(func() error { _, err := stream.Recv(); return err })},"ListBlobs": func(ctx context.Context, revs ...string) error {- stream, err := blobs.ListBlobs(ctx, &gitalypb.ListBlobsRequest{Repository: repo, Revisions: revs, Limit: 100})+ stream, err := fx.blobs.ListBlobs(ctx, &gitalypb.ListBlobsRequest{Repository: fx.repo, Revisions: revs, Limit: 100})require.NoError(t, err)- return drain(func() error { _, err := stream.Recv(); return err })+ return drainStream(func() error { _, err := stream.Recv(); return err })},"ListLFSPointers": func(ctx context.Context, revs ...string) error {- stream, err := blobs.ListLFSPointers(ctx, &gitalypb.ListLFSPointersRequest{Repository: repo, Revisions: revs, Limit: 100})+ stream, err := fx.blobs.ListLFSPointers(ctx, &gitalypb.ListLFSPointersRequest{Repository: fx.repo, Revisions: revs, Limit: 100})require.NoError(t, err)- return drain(func() error { _, err := stream.Recv(); return err })+ return drainStream(func() error { _, err := stream.Recv(); return err })},"GetTagSignatures": func(ctx context.Context, revs ...string) error {- stream, err := refs.GetTagSignatures(ctx, &gitalypb.GetTagSignaturesRequest{Repository: repo, TagRevisions: revs})+ stream, err := fx.refs.GetTagSignatures(ctx, &gitalypb.GetTagSignaturesRequest{Repository: fx.repo, TagRevisions: revs})require.NoError(t, err)- return drain(func() error { _, err := stream.Recv(); return err })+ return drainStream(func() error { _, err := stream.Recv(); return err })},}injections := map[string][]string{@@ -94,17 +132,164 @@ func TestRevisionInjection_rpcsRefuseLineBreaks(t *testing.T) {"tags pattern, then a name": {"--tags=x\nrefs/graft/green/main"},"CR alone in the pattern": {"--glob=refs/*\r--all"},"NUL in the pattern": {"--glob=refs/*\x00"},+ "line feed in a path": {"main:a\nrefs/graft/proposals/0123456789abcdef/1"},}for name, call := range rpcs {for desc, revs := range injections {- for who, callCtx := range map[string]context.Context{"agent": agent, "other caller": ctx} {- err := call(callCtx, revs...)- require.Error(t, err, "%s %s as %s", name, desc, who)- require.Equal(t, codes.InvalidArgument, status.Code(err), "%s %s as %s: %v", name, desc, who, err)+ for who, callCtx := range map[string]context.Context{"agent": fx.agent, "other caller": fx.ctx} {+ t.Run(name+"/"+desc+"/"+who, func(t *testing.T) {+ t.Parallel()++ err := call(callCtx, revs...)+ require.Error(t, err)+ require.Equal(t, codes.InvalidArgument, status.Code(err), "%v", err)+ })}}- // Control: an ordinary pattern is not refused.- require.NoError(t, call(agent, "--glob=refs/heads/*"), name)+ t.Run(name+"/control, an ordinary pattern", func(t *testing.T) {+ t.Parallel()+ require.NoError(t, call(fx.agent, "--glob=refs/heads/*"))+ })+ }+}++// TestRevisionInjection_nameAndPathRPCsReachNothingHidden: the RPCs that name an object by revision+// and path (GetBlobs, TreeEntry) hand git a NUL-delimited request, where a line break is data. A+// revision with one names no object, and never the proposal's. (The plain hidden names are refused+// by the agent request check of the auth interceptor, which this server does not install: see+// TestCheckAgentRefs.)+func TestRevisionInjection_nameAndPathRPCsReachNothingHidden(t *testing.T) {+ t.Parallel()++ fx := newInjectionFixture(t)+ for desc, revision := range map[string]string{+ "line feed and a hidden name": "main\nrefs/graft/proposals/0123456789abcdef/1",+ "carriage return": "main\rrefs/graft/proposals/0123456789abcdef/1",+ "NUL": "main\x00refs/graft/proposals/0123456789abcdef/1",+ "a pattern": "--glob=x\nrefs/graft/green/main",+ } {+ for who, callCtx := range map[string]context.Context{"agent": fx.agent, "other caller": fx.ctx} {+ t.Run("GetBlobs/"+desc+"/"+who, func(t *testing.T) {+ t.Parallel()++ stream, err := fx.blobs.GetBlobs(callCtx, &gitalypb.GetBlobsRequest{+ Repository: fx.repo,+ RevisionPaths: []*gitalypb.GetBlobsRequest_RevisionPath{{Revision: revision, Path: []byte("a")}},+ Limit: -1,+ })+ require.NoError(t, err)+ var data []byte+ err = drainStream(func() error {+ resp, err := stream.Recv()+ data = append(data, resp.GetData()...)+ return err+ })+ require.NotContains(t, string(data), "proposed by a model")+ })+ t.Run("TreeEntry/"+desc+"/"+who, func(t *testing.T) {+ t.Parallel()++ stream, err := fx.commits.TreeEntry(callCtx, &gitalypb.TreeEntryRequest{+ Repository: fx.repo, Revision: []byte(revision), Path: []byte("a"),+ })+ require.NoError(t, err)+ var data []byte+ _ = drainStream(func() error {+ resp, err := stream.Recv()+ data = append(data, resp.GetData()...)+ return err+ })+ require.NotContains(t, string(data), "proposed by a model")+ })+ }+ }+}++// TestRevisionInjection_otherSpellingsReachNothingHidden: spellings of a revision that are not+// control characters and that someone might try (@, @{-1}, -1) resolve against the repository's HEAD,+// a normal branch, or fail: none reaches a commit that only refs/graft/ reaches.+func TestRevisionInjection_otherSpellingsReachNothingHidden(t *testing.T) {+ t.Parallel()++ fx := newInjectionFixture(t)+ for _, revision := range []string{"@", "@{-1}", "-1", "HEAD", "HEAD@{1}", "@{upstream}", "main@{0}", "FETCH_HEAD", "ORIG_HEAD"} {+ t.Run(revision, func(t *testing.T) {+ t.Parallel()++ for who, callCtx := range map[string]context.Context{"agent": fx.agent, "other caller": fx.ctx} {+ stream, err := fx.commits.ListCommits(callCtx, &gitalypb.ListCommitsRequest{Repository: fx.repo, Revisions: []string{revision}})+ require.NoError(t, err)+ var ids []string+ _ = drainStream(func() error {+ resp, err := stream.Recv()+ for _, c := range resp.GetCommits() {+ ids = append(ids, c.GetId())+ }+ return err+ })+ require.NotContains(t, ids, fx.proposal.String(), "%s: %s", who, revision)+ }+ })+ }+}++// TestRevisionPaths_stillAcceptWhitespaceAndNonASCII: the refusal of control characters leaves the+// paths that people have: a tab, a space, a non-ASCII name, in ListBlobs and GetBlobs.+func TestRevisionPaths_stillAcceptWhitespaceAndNonASCII(t *testing.T) {+ t.Parallel()++ paths := map[string]string{+ "a b.txt": "space\n",+ "tab\there": "tab\n",+ "é à ü.txt": "latin\n",+ "日本語.txt": "cjk\n",+ "emoji 🙂.txt": "emoji\n",+ "quote'\"s.go": "quote\n",+ }+ var entries []gittest.TreeEntry+ for path, content := range paths {+ entries = append(entries, gittest.TreeEntry{Mode: "100644", Path: path, Content: content})+ }+ fx := newInjectionFixture(t, entries...)++ for path, content := range paths {+ for who, callCtx := range map[string]context.Context{"agent": fx.agent, "other caller": fx.ctx} {+ t.Run("ListBlobs/"+path+"/"+who, func(t *testing.T) {+ t.Parallel()++ want := gittest.ResolveRevision(t, fx.cfg, fx.repoPath, "main:"+path)+ stream, err := fx.blobs.ListBlobs(callCtx, &gitalypb.ListBlobsRequest{+ Repository: fx.repo, Revisions: []string{"main:" + path}, Limit: 10,+ })+ require.NoError(t, err)+ var oids []string+ require.NoError(t, drainStream(func() error {+ resp, err := stream.Recv()+ for _, b := range resp.GetBlobs() {+ oids = append(oids, b.GetOid())+ }+ return err+ }))+ require.Equal(t, []string{want.String()}, oids)+ })+ t.Run("GetBlobs/"+path+"/"+who, func(t *testing.T) {+ t.Parallel()++ stream, err := fx.blobs.GetBlobs(callCtx, &gitalypb.GetBlobsRequest{+ Repository: fx.repo,+ RevisionPaths: []*gitalypb.GetBlobsRequest_RevisionPath{{Revision: "main", Path: []byte(path)}},+ Limit: -1,+ })+ require.NoError(t, err)+ var data bytes.Buffer+ require.NoError(t, drainStream(func() error {+ resp, err := stream.Recv()+ data.Write(resp.GetData())+ return err+ }))+ require.Equal(t, content, data.String())+ })+ }}}diff --git a/internal/gitaly/service/repository/objects_size.go b/internal/gitaly/service/repository/objects_size.goindex 9cb92b1..6da078a 100644--- a/internal/gitaly/service/repository/objects_size.go+++ b/internal/gitaly/service/repository/objects_size.go@@ -66,6 +66,10 @@ func (s *server) ObjectsSize(server gitalypb.RepositoryService_ObjectsSizeServerreturn structerr.NewInvalidArgument("validating revision: %w", err).WithMetadata("revision", revision)}+ if err := git.ValidateStdinRevision(string(revision)); err != nil {+ return structerr.NewInvalidArgument("validating revision: %w", err).WithMetadata("revision", revision)+ }+// Each revision must be separated by a newline when the `--stdin` option is used, as Git// parses these differently to command-line arguments.if _, err := cmd.Write([]byte(fmt.Sprintf("%s\n", revision))); err != nil {