Git designed for AI agents.
quarantine: links first, one fsync per directory, vanishing fan-out dirs
Federico Eliafdb40f0648d8
diff --git a/doc/graft/LIMITATIONS.md b/doc/graft/LIMITATIONS.mdindex e1ac8b1..cb689b0 100644--- a/doc/graft/LIMITATIONS.md+++ b/doc/graft/LIMITATIONS.md@@ -140,6 +140,18 @@ Git hooks use private.does this. The quickstart creates a bare repository on disk and seeds itwith a push.- `graft clone` needs the branch to exist. An empty repository cannot be cloned.+- Durability of written objects assumes a local journaling filesystem, ext4 (in+ any `data=` mode) or xfs, not NFS. The objects of a write are made durable by+ their writer, and the migration of the write's quarantine into the repository+ makes the new names durable with one fsync per directory: that relies on a+ hard link being one journaled operation and on the fsync of a directory making+ every earlier change of the journal durable. On other filesystems an object+ written just before a crash may be missing although the write succeeded.+- Custom hooks get the server's `core.fsync` configuration, so the objects that+ their git commands write into a quarantine are durable. When a custom hook+ runs with a quarantine, the migration of that quarantine also fsyncs its files,+ because a hook may write objects without git; that costs about one fsync per+ file of the write.### Landing (`UserLand`)diff --git a/internal/git/quarantine/migrate_crash_test.go b/internal/git/quarantine/migrate_crash_test.gonew file mode 100644index 0000000..c4c0956--- /dev/null+++ b/internal/git/quarantine/migrate_crash_test.go@@ -0,0 +1,145 @@+package quarantine++import (+ "context"+ "io/fs"+ "os"+ "os/exec"+ "path/filepath"+ "strconv"+ "strings"+ "syscall"+ "testing"++ "github.com/feder-cr/git-ai/internal/git/gittest"+ "github.com/feder-cr/git-ai/internal/testhelper"+ "github.com/feder-cr/git-ai/internal/testhelper/testcfg"+ "github.com/stretchr/testify/require"+)++// TestMigrateCrashChild is the child process of TestMigrateCrashLeavesAPrefixOfTheLinks. It migrates+// the quarantine it is given and kills itself right before the link that it is told.+func TestMigrateCrashChild(t *testing.T) {+ source := os.Getenv("QUARANTINE_CRASH_SOURCE")+ if source == "" {+ t.Skip("only runs as the child of TestMigrateCrashLeavesAPrefixOfTheLinks")+ }+ target := os.Getenv("QUARANTINE_CRASH_TARGET")+ crashAt, err := strconv.Atoi(os.Getenv("QUARANTINE_CRASH_AT"))+ require.NoError(t, err)++ links := 0+ beforeLink = func(string) {+ if links == crashAt {+ _ = syscall.Kill(os.Getpid(), syscall.SIGKILL)+ select {}+ }+ links+++ }++ require.NoError(t, migrate(context.Background(), source, target, false))+}++// copyTree copies the directory tree at from to to.+func copyTree(t *testing.T, from, to string) {+ t.Helper()++ require.NoError(t, filepath.WalkDir(from, func(path string, entry fs.DirEntry, err error) error {+ if err != nil {+ return err+ }+ rel, err := filepath.Rel(from, path)+ if err != nil {+ return err+ }+ if entry.IsDir() {+ return os.MkdirAll(filepath.Join(to, rel), 0o755)+ }+ data, err := os.ReadFile(path)+ if err != nil {+ return err+ }+ return os.WriteFile(filepath.Join(to, rel), data, 0o444)+ }))+}++// TestMigrateCrashLeavesAPrefixOfTheLinks kills a migration before each of its links in turn, in a+// child process, and checks what the repository holds afterwards: the names that the migration+// linked before the crash, in link order, so a pack index never without its pack, and git fsck+// finds every object it holds intact. A crash of the process leaves the page cache; a power cut,+// which the filesystem assumption of migrate covers, cannot be simulated here.+func TestMigrateCrashLeavesAPrefixOfTheLinks(t *testing.T) {+ ctx := testhelper.Context(t)+ cfg := testcfg.Build(t)++ // A quarantine with loose objects and a pack, written by git.+ _, scratchPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{SkipCreationViaService: true})+ packed := gittest.WriteCommit(t, cfg, scratchPath, gittest.WithMessage("packed"))+ gittest.ExecOpts(t, cfg, gittest.ExecConfig{Stdin: strings.NewReader(packed.String() + "\n")},+ "-C", scratchPath, "pack-objects", "-q", "--revs", filepath.Join(scratchPath, "objects", "pack", "pack"))+ gittest.Exec(t, cfg, "-C", scratchPath, "prune-packed")+ gittest.WriteCommit(t, cfg, scratchPath, gittest.WithMessage("loose"),+ gittest.WithTreeEntries(gittest.TreeEntry{Mode: "100644", Path: "loose", Content: "loose\n"}))+ quarantine := filepath.Join(t.TempDir(), "quarantine")+ copyTree(t, filepath.Join(scratchPath, "objects"), quarantine)++ // The order of the links, from a migration that completes.+ var order []string+ originalBefore := beforeLink+ t.Cleanup(func() { beforeLink = originalBefore })+ beforeLink = func(targetPath string) { order = append(order, targetPath) }+ fullTarget := filepath.Join(t.TempDir(), "objects")+ require.NoError(t, os.MkdirAll(fullTarget, 0o755))+ source := filepath.Join(t.TempDir(), "source")+ copyTree(t, quarantine, source)+ require.NoError(t, migrate(ctx, source, fullTarget, false))+ beforeLink = originalBefore++ var relOrder []string+ for _, path := range order {+ rel, err := filepath.Rel(fullTarget, path)+ require.NoError(t, err)+ relOrder = append(relOrder, rel)+ }+ var idx, pack int = -1, -1+ for i, rel := range relOrder {+ switch {+ case strings.HasSuffix(rel, ".pack"):+ pack = i+ case strings.HasSuffix(rel, ".idx"):+ idx = i+ }+ }+ require.True(t, pack >= 0 && idx > pack, "the pack is linked before its index: %v", relOrder)++ for crashAt := 0; crashAt <= len(relOrder); crashAt++ {+ _, repoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{SkipCreationViaService: true})+ target := filepath.Join(repoPath, "objects")+ source := filepath.Join(t.TempDir(), "source")+ copyTree(t, quarantine, source)++ cmd := exec.Command(os.Args[0], "-test.run", "^TestMigrateCrashChild$")+ cmd.Env = append(os.Environ(),+ "QUARANTINE_CRASH_SOURCE="+source,+ "QUARANTINE_CRASH_TARGET="+target,+ "QUARANTINE_CRASH_AT="+strconv.Itoa(crashAt),+ )+ output, err := cmd.CombinedOutput()+ if crashAt < len(relOrder) {+ require.Error(t, err, "the child is killed before link %d", crashAt)+ } else {+ require.NoError(t, err, "%s", output)+ }++ for i, rel := range relOrder {+ _, statErr := os.Stat(filepath.Join(target, rel))+ if i < crashAt {+ require.NoError(t, statErr, "crash at %d: %s linked before the crash", crashAt, rel)+ } else {+ require.ErrorIs(t, statErr, fs.ErrNotExist, "crash at %d: %s linked after the crash", crashAt, rel)+ }+ }++ gittest.Exec(t, cfg, "-C", repoPath, "fsck", "--no-dangling")+ }+}diff --git a/internal/git/quarantine/migrate_test.go b/internal/git/quarantine/migrate_test.gonew file mode 100644index 0000000..77d7e00--- /dev/null+++ b/internal/git/quarantine/migrate_test.go@@ -0,0 +1,281 @@+package quarantine++import (+ "bytes"+ "context"+ "fmt"+ "os"+ "path/filepath"+ "sort"+ "sync"+ "sync/atomic"+ "testing"++ "github.com/feder-cr/git-ai/internal/git/gittest"+ "github.com/feder-cr/git-ai/internal/gitaly/config"+ "github.com/feder-cr/git-ai/internal/gitaly/storage"+ "github.com/feder-cr/git-ai/internal/testhelper"+ "github.com/feder-cr/git-ai/internal/testhelper/testcfg"+ "github.com/stretchr/testify/require"+)++// The tests of this file replace beforeLink, syncDirFile and syncFile, so they are not parallel.++// writeQuarantine writes a quarantine at dir with one object in each of the fan-out directories+// fanouts and a pack with its index, and returns the relative paths of its files.+func writeQuarantine(t *testing.T, dir string, fanouts []string) []string {+ t.Helper()++ var files []string+ for _, fanout := range fanouts {+ files = append(files, filepath.Join(fanout, "object"+fanout))+ }+ files = append(files, filepath.Join("pack", "pack-1.pack"), filepath.Join("pack", "pack-1.idx"))++ for _, file := range files {+ require.NoError(t, os.MkdirAll(filepath.Join(dir, filepath.Dir(file)), 0o755))+ require.NoError(t, os.WriteFile(filepath.Join(dir, file), []byte(file), 0o444))+ }+ return files+}++// replaceHooks replaces the test hooks of migrate for the test and counts the fsyncs.+func replaceHooks(t *testing.T, before func(targetPath string)) (dirSyncs, fileSyncs *atomic.Int32) {+ t.Helper()++ dirSyncs, fileSyncs = &atomic.Int32{}, &atomic.Int32{}+ originalBefore, originalSyncDir, originalSyncFile := beforeLink, syncDirFile, syncFile+ t.Cleanup(func() { beforeLink, syncDirFile, syncFile = originalBefore, originalSyncDir, originalSyncFile })++ if before != nil {+ beforeLink = before+ }+ syncDirFile = func(f *os.File) error {+ dirSyncs.Add(1)+ return originalSyncDir(f)+ }+ syncFile = func(path string) error {+ fileSyncs.Add(1)+ return originalSyncFile(path)+ }+ return dirSyncs, fileSyncs+}++// TestMigrateRecreatesTargetDirectoriesThatVanish removes each empty target fan-out directory+// right before the first link into it, as housekeeping (git prune-packed) may. Before, the link+// and its rename fallback then failed with ENOENT. The migration creates the directory again and+// links every object.+func TestMigrateRecreatesTargetDirectoriesThatVanish(t *testing.T) {+ ctx := testhelper.Context(t)+ root := t.TempDir()+ source, target := filepath.Join(root, "quarantine"), filepath.Join(root, "objects")+ fanouts := []string{"0a", "1b", "2c"}+ files := writeQuarantine(t, source, fanouts)++ // The target fan-out directories exist and are empty, as housekeeping leaves them.+ require.NoError(t, os.MkdirAll(filepath.Join(target, "pack"), 0o755))+ for _, fanout := range fanouts {+ require.NoError(t, os.Mkdir(filepath.Join(target, fanout), 0o755))+ }++ var removed sync.Map+ replaceHooks(t, func(targetPath string) {+ dir := filepath.Dir(targetPath)+ if filepath.Base(dir) == "pack" {+ return+ }+ if _, done := removed.LoadOrStore(dir, true); !done {+ require.NoError(t, os.Remove(dir))+ }+ })++ require.NoError(t, migrate(ctx, source, target, false))++ for _, file := range files {+ require.FileExists(t, filepath.Join(target, file))+ }+ for _, fanout := range fanouts {+ _, ok := removed.Load(filepath.Join(target, fanout))+ require.True(t, ok, "the directory %s was removed during the migration", fanout)+ }+ require.NoDirExists(t, source)+}++// TestMigrateSyncsEachDirectoryOnce counts the fsyncs: one per target directory that received a+// link, one for the object directory, and none per file, whose writers made them durable.+func TestMigrateSyncsEachDirectoryOnce(t *testing.T) {+ ctx := testhelper.Context(t)+ root := t.TempDir()+ source, target := filepath.Join(root, "quarantine"), filepath.Join(root, "objects")+ fanouts := []string{"0a", "1b", "2c", "3d"}+ writeQuarantine(t, source, fanouts)+ require.NoError(t, os.MkdirAll(target, 0o755))++ dirSyncs, fileSyncs := replaceHooks(t, nil)+ require.NoError(t, migrate(ctx, source, target, false))++ require.EqualValues(t, len(fanouts)+1+1, dirSyncs.Load(), "the fan-out directories, pack and the object directory")+ require.Zero(t, fileSyncs.Load())+}++// TestMigrateSyncsTheFilesOfAnUntrustedWriter fsyncs every file of a quarantine that a writer+// Gitaly does not control wrote into, and still each directory once.+func TestMigrateSyncsTheFilesOfAnUntrustedWriter(t *testing.T) {+ ctx := testhelper.Context(t)+ root := t.TempDir()+ source, target := filepath.Join(root, "quarantine"), filepath.Join(root, "objects")+ fanouts := []string{"0a", "1b"}+ files := writeQuarantine(t, source, fanouts)+ require.NoError(t, os.MkdirAll(target, 0o755))++ dirSyncs, fileSyncs := replaceHooks(t, nil)+ require.NoError(t, migrate(ctx, source, target, true))++ require.EqualValues(t, len(files), fileSyncs.Load())+ require.EqualValues(t, len(fanouts)+1+1, dirSyncs.Load())+}++// fakeTransaction makes storage.NeedsSync false, as a transaction of the transaction manager does.+type fakeTransaction struct {+ storage.Transaction+}++// TestMigrateDoesNotSyncUnderATransaction leaves durability to the transaction manager.+func TestMigrateDoesNotSyncUnderATransaction(t *testing.T) {+ ctx := storage.ContextWithTransaction(testhelper.Context(t), fakeTransaction{})+ require.False(t, storage.NeedsSync(ctx))++ root := t.TempDir()+ source, target := filepath.Join(root, "quarantine"), filepath.Join(root, "objects")+ files := writeQuarantine(t, source, []string{"0a"})+ require.NoError(t, os.MkdirAll(target, 0o755))++ dirSyncs, fileSyncs := replaceHooks(t, nil)+ require.NoError(t, migrate(ctx, source, target, true))++ require.Zero(t, dirSyncs.Load())+ require.Zero(t, fileSyncs.Load())+ for _, file := range files {+ require.FileExists(t, filepath.Join(target, file))+ }+}++// TestMigrateKeepsThePackOrder links the data of a pack before its index.+func TestMigrateKeepsThePackOrder(t *testing.T) {+ ctx := testhelper.Context(t)+ root := t.TempDir()+ source, target := filepath.Join(root, "quarantine"), filepath.Join(root, "objects")+ writeQuarantine(t, source, nil)+ require.NoError(t, os.MkdirAll(target, 0o755))++ var linked []string+ var mu sync.Mutex+ replaceHooks(t, func(targetPath string) {+ mu.Lock()+ defer mu.Unlock()+ linked = append(linked, filepath.Base(targetPath))+ })++ require.NoError(t, migrate(ctx, source, target, false))+ require.Equal(t, []string{"pack-1.pack", "pack-1.idx"}, linked)+}++// TestMarkUntrustedWriter marks the quarantine whose object directory a hook was given, and only+// while the quarantine exists.+func TestMarkUntrustedWriter(t *testing.T) {+ ctx := testhelper.Context(t)+ cfg := testcfg.Build(t)+ repo, _ := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{SkipCreationViaService: true})+ locator := config.NewLocator(cfg)++ marked, cleanupMarked, err := New(ctx, repo, testhelper.NewLogger(t), locator)+ require.NoError(t, err)+ defer cleanupMarked()+ other, cleanupOther, err := New(ctx, repo, testhelper.NewLogger(t), locator)+ require.NoError(t, err)+ defer cleanupOther()++ MarkUntrustedWriter(marked.dir.Path() + "/")+ require.True(t, marked.untrustedWriter.Load())+ require.False(t, other.untrustedWriter.Load())++ // A path that is no quarantine marks nothing.+ MarkUntrustedWriter(t.TempDir())+ require.False(t, other.untrustedWriter.Load())++ path := other.dir.Path()+ cleanupOther()+ _, registered := quarantines.Load(filepath.Clean(path))+ require.False(t, registered, "a quarantine leaves the registry when it is cleaned up")+}++// TestMigrateWhileGitPrunesPacked runs migrations into a repository while git prune-packed, which+// housekeeping runs after a repack, removes its empty fan-out directories in a loop.+func TestMigrateWhileGitPrunesPacked(t *testing.T) {+ ctx := testhelper.Context(t)+ cfg := testcfg.Build(t)+ _, repoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{SkipCreationViaService: true})+ objects := filepath.Join(repoPath, "objects")++ // Every fan-out directory exists and is empty, the state that prune-packed cleans up.+ for i := 0; i < 256; i++ {+ require.NoError(t, os.MkdirAll(filepath.Join(objects, fmt.Sprintf("%02x", i)), 0o755))+ }++ stop := make(chan struct{})+ var pruned sync.WaitGroup+ pruned.Add(1)+ runs := 0+ go func() {+ defer pruned.Done()+ for {+ select {+ case <-stop:+ return+ default:+ }+ var stderr bytes.Buffer+ cmd := gittest.NewCommand(t, cfg, "-C", repoPath, "prune-packed", "--quiet")+ cmd.Stderr = &stderr+ if err := cmd.Run(); err != nil {+ t.Errorf("prune-packed: %v: %s", err, stderr.String())+ return+ }+ runs+++ // Recreate empty directories, so that the race stays possible.+ for i := 0; i < 256; i += 7 {+ _ = os.Mkdir(filepath.Join(objects, fmt.Sprintf("%02x", i)), 0o755)+ }+ }+ }()++ for i := 0; i < 200; i++ {+ source := filepath.Join(t.TempDir(), "quarantine")+ fanouts := []string{fmt.Sprintf("%02x", (i*7)%256), fmt.Sprintf("%02x", (i*13+1)%256)}+ var files []string+ for _, fanout := range fanouts {+ file := filepath.Join(fanout, fmt.Sprintf("object-%03d", i))+ require.NoError(t, os.MkdirAll(filepath.Join(source, fanout), 0o755))+ require.NoError(t, os.WriteFile(filepath.Join(source, file), []byte(file), 0o444))+ files = append(files, file)+ }++ require.NoError(t, migrate(context.Background(), source, objects, false), "migration %d", i)+ for _, file := range files {+ require.FileExists(t, filepath.Join(objects, file))+ }+ }++ close(stop)+ pruned.Wait()+ require.Positive(t, runs)++ var names []string+ entries, err := os.ReadDir(objects)+ require.NoError(t, err)+ for _, entry := range entries {+ names = append(names, entry.Name())+ }+ sort.Strings(names)+ t.Logf("%d prune-packed runs during 200 migrations", runs)+}diff --git a/internal/git/quarantine/quarantine.go b/internal/git/quarantine/quarantine.goindex 6c5771e..a04f67d 100644--- a/internal/git/quarantine/quarantine.go+++ b/internal/git/quarantine/quarantine.go@@ -4,32 +4,67 @@ import ("context""errors""fmt"+ "io/fs""os""path/filepath""sort""strings"+ "sync"+ "sync/atomic""github.com/feder-cr/git-ai/internal/gitaly/storage""github.com/feder-cr/git-ai/internal/gitaly/storage/mode""github.com/feder-cr/git-ai/internal/log"- "github.com/feder-cr/git-ai/internal/safe""github.com/feder-cr/git-ai/internal/structerr""github.com/feder-cr/git-ai/internal/tempdir""github.com/feder-cr/git-ai/proto/go/gitalypb"+ "github.com/prometheus/client_golang/prometheus"+ "github.com/prometheus/client_golang/prometheus/promauto""google.golang.org/protobuf/proto")+// untrustedMigrationsTotal counts the migrations that fsynced the files of their quarantine+// because a writer that Gitaly does not control ran with it.+var untrustedMigrationsTotal = promauto.NewCounter(prometheus.CounterOpts{+ Name: "graft_quarantine_untrusted_migrations_total",+ Help: "Quarantine migrations that fsynced the files of the quarantine because a custom hook ran with it.",+})+// Dir is a quarantine directory for Git objects. Instead of writing new commits into the main// repository, they're instead written into a temporary quarantine directory. This staging area can// either be migrated into the main repository or, alternatively, will automatically be discarded// when the context gets cancelled. If the quarantine environment is discarded without being staged,// then none of the objects which have been created in the quarantine directory will end up in the// main repository.+//+// Durability contract: every file in a quarantine is durable when Migrate starts, its data and its+// inode. The Gitaly writers make it so: they are git processes that the command factory starts with+// core.fsync covering objects, or helpers that write without fsync and pack their objects into one+// fsynced pack before the update (see localrepo.WithoutObjectFsync). Migrate then only makes the+// new names durable, with one fsync per directory. A writer that Gitaly does not control, a custom+// hook, is reported with MarkUntrustedWriter, and Migrate then fsyncs the files of that quarantine+// itself.type Dir struct {repo *gitalypb.RepositoryquarantinedRepo *gitalypb.Repositorydir tempdir.Dirlocator storage.Locator+ // untrustedWriter is set when a writer that Gitaly does not control may have written into+ // the quarantine, see MarkUntrustedWriter.+ untrustedWriter atomic.Bool+}++// quarantines holds the quarantines of this process by the absolute path of their object+// directory, so that MarkUntrustedWriter finds the quarantine that a hook was given.+var quarantines sync.Map++// MarkUntrustedWriter records that a writer that Gitaly does not control, such as a custom hook, ran+// with objectDir as its object directory. When objectDir is a quarantine of this process, its+// files are not trusted to be durable, and its migration fsyncs them first.+func MarkUntrustedWriter(objectDir string) {+ if d, ok := quarantines.Load(filepath.Clean(objectDir)); ok {+ d.(*Dir).untrustedWriter.Store(true)+ }}// New creates a new quarantine directory and returns the directory and a cleanup function.@@ -52,12 +87,20 @@ func New(ctx context.Context, repo *gitalypb.Repository, logger log.Logger, locareturn nil, nil, err}- return &Dir{+ d := &Dir{repo: repo,quarantinedRepo: quarantinedRepo,locator: locator,dir: quarantineDir,- }, cleanup, nil+ }++ key := filepath.Clean(quarantineDir.Path())+ quarantines.Store(key, d)++ return d, func() {+ quarantines.CompareAndDelete(key, d)+ cleanup()+ }, nil}// Apply applies the quarantine on the repository. This is done by setting the quarantineDirectory@@ -109,67 +152,236 @@ func (d *Dir) Migrate(ctx context.Context) error {objectDir = "objects"}- return migrate(ctx, d.dir.Path(), filepath.Join(repoPath, objectDir))+ return migrate(ctx, d.dir.Path(), filepath.Join(repoPath, objectDir), d.untrustedWriter.Load())}-func migrate(ctx context.Context, sourcePath, targetPath string) error {+// migrate moves the objects of the quarantine at sourcePath into the object directory at+// targetPath, in two passes.+//+// The first pass links every file into the target directories, in the order of sortEntries, so+// that a pack's index appears after its data. An empty fan-out directory of the target may be+// removed at any time: git's housekeeping (prune-packed, repack -d, prune) removes empty fan-out+// directories, and git's own writers create their temporary file inside the fan-out directory,+// which pins it. A quarantine is a separate directory, so a target directory that migrate created+// or found can vanish before its first link. A link that fails because its target directory is+// gone creates the directory again and links again; once the link succeeds the directory is not+// empty and stays. Each round needs another removal between the creation and the link, so the loop+// ends; it also ends with the context.+//+// The second pass, unless a transaction manager owns durability (storage.NeedsSync), fsyncs each+// target directory that received a link, once, through a descriptor opened in the first pass, and+// the object directory itself. The files are not fsynced: they are durable before the migration+// (see Dir), unless untrustedWriter is set, in which case they are fsynced in parallel before the+// first pass.+//+// Filesystem assumption (local ext4 in any data mode, or xfs; not NFS): a new hard link is+// durable once its directory is fsynced; a link is one journaled operation that changes the+// directory entry and the link count together; the journal commits its transactions in order, and+// the fsync of a directory forces it up to the directory's last change. So after the second pass+// every link is durable, and a crash before leaves a prefix of the links in their order.+func migrate(ctx context.Context, sourcePath, targetPath string, untrustedWriter bool) error {+ sync := storage.NeedsSync(ctx)++ if sync && untrustedWriter {+ untrustedMigrationsTotal.Inc()+ if err := syncFilesInParallel(ctx, sourcePath); err != nil {+ return fmt.Errorf("syncing the files of an untrusted writer: %w", err)+ }+ }++ m := migration{ctx: ctx, dirs: map[string]*os.File{}}+ defer m.closeDirs()++ if err := m.openDir(targetPath); err != nil {+ return err+ }+ if err := m.link(sourcePath, targetPath); err != nil {+ return err+ }++ if !sync {+ return nil+ }++ // The object directory last: it holds the entries of the fan-out directories that the first+ // pass created.+ for i := len(m.order) - 1; i >= 0; i-- {+ if err := syncDirFile(m.dirs[m.order[i]]); err != nil {+ return fmt.Errorf("sync directory %q: %w", m.order[i], err)+ }+ }++ return nil+}++// migration is the state of the first pass of migrate.+type migration struct {+ ctx context.Context+ // dirs holds a descriptor of each target directory, by path.+ dirs map[string]*os.File+ // order holds the target directories in the order they were opened, parents first.+ order []string+}++// openDir creates the target directory dir if it does not exist and opens a descriptor of it,+// replacing the descriptor of a directory of the same path that was removed.+func (m *migration) openDir(dir string) error {+ if err := os.Mkdir(dir, mode.Directory); err != nil && !errors.Is(err, os.ErrExist) {+ return fmt.Errorf("creating target directory %q: %w", dir, err)+ }++ f, err := os.Open(dir)+ if err != nil {+ return fmt.Errorf("opening target directory %q: %w", dir, err)+ }++ if old, ok := m.dirs[dir]; ok {+ _ = old.Close()+ } else {+ m.order = append(m.order, dir)+ }+ m.dirs[dir] = f++ return nil+}++func (m *migration) closeDirs() {+ for _, f := range m.dirs {+ _ = f.Close()+ }+}++// link links the files of sourcePath into targetPath, recursively, and removes sourcePath.+func (m *migration) link(sourcePath, targetPath string) error {entries, err := os.ReadDir(sourcePath)if err != nil {return fmt.Errorf("reading directory: %w", err)}sortEntries(entries)- syncer := safe.NewSyncer()for _, entry := range entries {- if entry.Name() == "." {- continue- }-nestedTargetPath := filepath.Join(targetPath, entry.Name())nestedSourcePath := filepath.Join(sourcePath, entry.Name())if entry.IsDir() {- if err := os.Mkdir(nestedTargetPath, mode.Directory); err != nil {- if !errors.Is(err, os.ErrExist) {- return fmt.Errorf("creating target directory %q: %w", nestedTargetPath, err)- }+ if err := m.openDir(nestedTargetPath); err != nil {+ return err}-- if err := migrate(ctx, nestedSourcePath, nestedTargetPath); err != nil {+ if err := m.link(nestedSourcePath, nestedTargetPath); err != nil {return fmt.Errorf("migrating directory %q: %w", nestedSourcePath, err)}-- if storage.NeedsSync(ctx) {- if err := syncer.Sync(ctx, nestedTargetPath); err != nil {- return fmt.Errorf("sync directory: %w", err)- }- }-continue}- if err := finalizeObjectFile(nestedSourcePath, nestedTargetPath); err != nil {+ if err := m.linkFile(nestedSourcePath, nestedTargetPath); err != nil {return fmt.Errorf("migrating object file %q: %w", nestedSourcePath, err)}+ }- if storage.NeedsSync(ctx) {- if err := syncer.Sync(ctx, nestedTargetPath); err != nil {- return fmt.Errorf("sync object: %w", err)- }+ if err := os.Remove(sourcePath); err != nil {+ return fmt.Errorf("removing source directory: %w", err)+ }++ return nil+}++// linkFile links the file at sourcePath to targetPath, creating the target directory again when+// it was removed (see migrate).+func (m *migration) linkFile(sourcePath, targetPath string) error {+ for {+ beforeLink(targetPath)++ err := finalizeObjectFile(sourcePath, targetPath)+ if err == nil {+ return nil}+ if !errors.Is(err, fs.ErrNotExist) {+ return err+ }++ if _, statErr := os.Lstat(sourcePath); statErr != nil {+ return fmt.Errorf("%w (source: %w)", err, statErr)+ }+ if ctxErr := m.ctx.Err(); ctxErr != nil {+ return fmt.Errorf("%w (%w)", err, ctxErr)+ }++ // The source exists, so the target directory is gone.+ if err := m.openDir(filepath.Dir(targetPath)); err != nil {+ return err+ }+ }+}++// beforeLink runs before each link of the first pass of migrate. Tests replace it to remove a+// target directory at that point, as housekeeping may.+var beforeLink = func(targetPath string) {}++// syncDirFile fsyncs the descriptor of a target directory. Tests replace it to count the fsyncs.+var syncDirFile = func(f *os.File) error {+ return f.Sync()+}++// syncFile fsyncs the file at path. Tests replace it to count the fsyncs.+var syncFile = func(path string) error {+ f, err := os.Open(path)+ if err != nil {+ return err}+ defer f.Close()++ return f.Sync()+}- if storage.NeedsSync(ctx) {- if err := syncer.Sync(ctx, targetPath); err != nil {- return fmt.Errorf("sync object directory: %w", err)+// maxParallelSyncs bounds the fsyncs that syncFilesInParallel runs at once.+const maxParallelSyncs = 16++// syncFilesInParallel fsyncs every file under dir. Concurrent fsyncs of the files of one+// filesystem share journal commits, which one fsync after another does not.+func syncFilesInParallel(ctx context.Context, dir string) error {+ var files []string+ if err := filepath.WalkDir(dir, func(path string, entry fs.DirEntry, err error) error {+ if err != nil {+ return err+ }+ if !entry.IsDir() {+ files = append(files, path)}+ return nil+ }); err != nil {+ return err}- if err := os.Remove(sourcePath); err != nil {- return fmt.Errorf("removing source directory: %w", err)+ var (+ wg sync.WaitGroup+ mu sync.Mutex+ firstErr error+ )+ slots := make(chan struct{}, maxParallelSyncs)+ for _, path := range files {+ if err := ctx.Err(); err != nil {+ wg.Wait()+ return err+ }++ slots <- struct{}{}+ wg.Add(1)+ go func() {+ defer wg.Done()+ defer func() { <-slots }()++ if err := syncFile(path); err != nil {+ mu.Lock()+ if firstErr == nil {+ firstErr = fmt.Errorf("sync %q: %w", path, err)+ }+ mu.Unlock()+ }+ }()}+ wg.Wait()- return nil+ return firstErr}// finalizeObjectFile will move the object file (either a packfile, its metadata or or a loosediff --git a/internal/git/quarantine/quarantine_test.go b/internal/git/quarantine/quarantine_test.goindex c284930..47fd831 100644--- a/internal/git/quarantine/quarantine_test.go+++ b/internal/git/quarantine/quarantine_test.go@@ -369,7 +369,7 @@ func TestMigrate(t *testing.T) {target := filepath.Join(dir, "target")tc.target.create(t, target)- require.NoError(t, migrate(ctx, source, target))+ require.NoError(t, migrate(ctx, source, target, false))require.Equal(t, tc.expected, listEntries(t, target))require.NoDirExists(t, source)})diff --git a/internal/gitaly/hook/custom.go b/internal/gitaly/hook/custom.goindex 31d4b9a..839620c 100644--- a/internal/gitaly/hook/custom.go+++ b/internal/gitaly/hook/custom.go@@ -12,6 +12,7 @@ import ("github.com/feder-cr/git-ai/internal/cgroups""github.com/feder-cr/git-ai/internal/command""github.com/feder-cr/git-ai/internal/git/gitcmd"+ "github.com/feder-cr/git-ai/internal/git/quarantine""github.com/feder-cr/git-ai/internal/gitaly/storage""github.com/feder-cr/git-ai/internal/helper/env""github.com/feder-cr/git-ai/proto/go/gitalypb"@@ -84,6 +85,17 @@ func (m *GitLabHookManager) newCustomHooksExecutor(ctx context.Context, repo *gi}}+ // A custom hook is code that Gitaly does not control. When it runs with a quarantine as its+ // object directory, it may write objects there that are not durable, which the migration+ // of the quarantine then makes durable itself (see quarantine.Dir). Custom hooks run in the+ // Gitaly process: gitaly-hooks forwards them through the HookService, and updateref calls+ // the hook manager directly, so the quarantine is one of this process.+ if len(hookFiles) > 0 {+ if objectDir := environmentValue(env, "GIT_OBJECT_DIRECTORY"); objectDir != "" {+ quarantine.MarkUntrustedWriter(objectDir)+ }+ }+for _, hookFile := range hookFiles {c, err := command.New(ctx, m.logger, append([]string{hookFile}, args...),command.WithDir(repoPath),@@ -212,6 +224,11 @@ func (m *GitLabHookManager) customHooksEnv(ctx context.Context, payload gitcmd.H// case we're running with bundled Git such that Git can locate its binaries.customEnvs = append(customEnvs, gitExecEnv.EnvironmentVariables...)+ // The git commands of a custom hook write objects as durably as the server's own: a+ // quarantine relies on its files being durable before it is migrated (see quarantine.Dir).+ // The environment carries no other GIT_CONFIG_* variable (command.AllowedEnvironment).+ customEnvs = append(customEnvs, gitcmd.ConfigPairsToGitEnvironment(m.fsyncConfiguration())...)+return append(customEnvs,"GIT_DIR="+repoPath,"GL_REPOSITORY="+payload.Repo.GetGlRepository(),@@ -222,6 +239,38 @@ func (m *GitLabHookManager) customHooksEnv(ctx context.Context, payload gitcmd.H), nil}+// fsyncConfiguration returns the fsync configuration of the server's git commands: the defaults+// of the command factory (gitcmd, GlobalConfiguration), overridden by the administrator's [git]+// configuration, which the command factory applies after them.+func (m *GitLabHookManager) fsyncConfiguration() []gitcmd.ConfigPair {+ pairs := []gitcmd.ConfigPair{+ {Key: "core.fsync", Value: "objects,derived-metadata,reference"},+ {Key: "core.fsyncMethod", Value: "fsync"},+ }++ for _, configured := range m.cfg.Git.Config {+ for i := range pairs {+ if strings.EqualFold(configured.Key, pairs[i].Key) {+ pairs[i].Value = configured.Value+ }+ }+ }++ return pairs+}++// environmentValue returns the value of the variable name in the environment variables envs, the+// last one if it is set several times.+func environmentValue(envs []string, name string) string {+ value := ""+ for _, variable := range envs {+ if strings.HasPrefix(variable, name+"=") {+ value = strings.TrimPrefix(variable, name+"=")+ }+ }+ return value+}+// pushOptionsEnv turns a slice of git push option values into a GIT_PUSH_OPTION_COUNT and individual// GIT_PUSH_OPTION_0, GIT_PUSH_OPTION_1 etc.func pushOptionsEnv(options []string) []string {diff --git a/internal/gitaly/hook/custom_fsync_test.go b/internal/gitaly/hook/custom_fsync_test.gonew file mode 100644index 0000000..a470c64--- /dev/null+++ b/internal/gitaly/hook/custom_fsync_test.go@@ -0,0 +1,65 @@+package hook++import (+ "testing"++ "github.com/feder-cr/git-ai/internal/git/gitcmd"+ "github.com/feder-cr/git-ai/internal/git/gittest"+ "github.com/feder-cr/git-ai/internal/gitaly/config"+ "github.com/feder-cr/git-ai/internal/gitaly/transaction"+ "github.com/feder-cr/git-ai/internal/gitlab"+ "github.com/feder-cr/git-ai/internal/testhelper"+ "github.com/feder-cr/git-ai/internal/testhelper/testcfg"+ "github.com/stretchr/testify/require"+)++// TestCustomHooksGetTheFsyncConfiguration checks that the git commands of a custom hook get the+// fsync configuration of the server's own git commands, as the command factory applies it: its+// defaults, overridden by the administrator's [git] configuration.+func TestCustomHooksGetTheFsyncConfiguration(t *testing.T) {+ t.Parallel()++ ctx := testhelper.Context(t)++ for _, tc := range []struct {+ desc string+ admin []config.GitConfig+ }{+ {desc: "defaults"},+ {desc: "administrator override", admin: []config.GitConfig{{Key: "core.fsync", Value: "all"}, {Key: "core.fsyncMethod", Value: "writeout-only"}}},+ } {+ t.Run(tc.desc, func(t *testing.T) {+ t.Parallel()++ cfg := testcfg.Build(t)+ cfg.Git.Config = tc.admin+ factory := gittest.NewCommandFactory(t, cfg)++ // What the command factory gives every git command, the administrator last.+ global, err := factory.GlobalConfiguration(ctx)+ require.NoError(t, err)+ want := map[string]string{}+ for _, pair := range append(global, cfg.Git.Config...) {+ if pair.Key == "core.fsync" || pair.Key == "core.fsyncMethod" {+ want[pair.Key] = pair.Value+ }+ }+ require.Len(t, want, 2)++ manager := NewManager(cfg, config.NewLocator(cfg), testhelper.SharedLogger(t), factory, nil, &transaction.MockManager{},+ gitlab.NewMockClient(t, gitlab.MockAllowed, gitlab.MockPreReceive, gitlab.MockPostReceive), NewPostReceiveRegistry())++ repo, _ := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{SkipCreationViaService: true})+ env, err := manager.customHooksEnv(ctx, gitcmd.HooksPayload{Repo: repo, UserDetails: &gitcmd.UserDetails{}}, nil, nil)+ require.NoError(t, err)++ got := map[string]string{}+ count := environmentValue(env, "GIT_CONFIG_COUNT")+ require.Equal(t, "2", count)+ for _, i := range []string{"0", "1"} {+ got[environmentValue(env, "GIT_CONFIG_KEY_"+i)] = environmentValue(env, "GIT_CONFIG_VALUE_"+i)+ }+ require.Equal(t, want, got)+ })+ }+}diff --git a/internal/gitaly/hook/testhelper_test.go b/internal/gitaly/hook/testhelper_test.goindex 48e890a..24c18b6 100644--- a/internal/gitaly/hook/testhelper_test.go+++ b/internal/gitaly/hook/testhelper_test.go@@ -50,6 +50,16 @@ func getExpectedEnv(tb testing.TB, ctx context.Context, locator storage.Locator,expectedEnv["PATH"] = fmt.Sprintf("%s:%s", filepath.Dir(execEnv.BinaryPath), os.Getenv("PATH"))+ // Custom hooks get the fsync configuration of the server's git commands (see customHooksEnv+ // and TestCustomHooksGetTheFsyncConfiguration).+ for _, variable := range gitcmd.ConfigPairsToGitEnvironment([]gitcmd.ConfigPair{+ {Key: "core.fsync", Value: "objects,derived-metadata,reference"},+ {Key: "core.fsyncMethod", Value: "fsync"},+ }) {+ kv := strings.SplitN(variable, "=", 2)+ expectedEnv[kv[0]] = kv[1]+ }+result := make([]string, 0, len(expectedEnv))for key, value := range expectedEnv {result = append(result, fmt.Sprintf("%s=%s", key, value))diff --git a/internal/gitaly/service/operations/migrate_untrusted_test.go b/internal/gitaly/service/operations/migrate_untrusted_test.gonew file mode 100644index 0000000..b330a9d--- /dev/null+++ b/internal/gitaly/service/operations/migrate_untrusted_test.go@@ -0,0 +1,108 @@+package operations++import (+ "bytes"+ "compress/zlib"+ "context"+ "fmt"+ "os"+ "path/filepath"+ "strings"+ "testing"++ "github.com/feder-cr/git-ai/internal/featureflag"+ "github.com/feder-cr/git-ai/internal/git/gittest"+ "github.com/feder-cr/git-ai/internal/gitaly/config"+ "github.com/feder-cr/git-ai/internal/testhelper"+ "github.com/feder-cr/git-ai/proto/go/gitalypb"+ "github.com/prometheus/client_golang/prometheus"+ "github.com/stretchr/testify/require"+)++// untrustedMigrations returns graft_quarantine_untrusted_migrations_total.+func untrustedMigrations(t *testing.T) float64 {+ t.Helper()++ families, err := prometheus.DefaultGatherer.Gather()+ require.NoError(t, err)+ for _, family := range families {+ if family.GetName() == "graft_quarantine_untrusted_migrations_total" {+ return family.GetMetric()[0].GetCounter().GetValue()+ }+ }+ return 0+}++// commitAFile writes one file on branch with UserCommitFiles.+func commitAFile(t *testing.T, ctx context.Context, client gitalypb.OperationServiceClient, repo *gitalypb.Repository, branch, path, content string) {+ t.Helper()++ stream, err := client.UserCommitFiles(ctx)+ require.NoError(t, err)+ header := headerRequest(repo, gittest.TestUser, branch, []byte("commit message"), "", "")+ setAuthorAndEmail(header, []byte("Author Name"), []byte("[email protected]"))+ require.NoError(t, stream.Send(header))+ require.NoError(t, stream.Send(createFileHeaderRequest(path)))+ require.NoError(t, stream.Send(actionContentRequest(content)))+ _, err = stream.CloseAndRecv()+ require.NoError(t, err)+}++// writeLooseObjectHook writes a custom pre-receive hook that writes a loose object into the+// quarantine without git: it copies a zlib-compressed object, so no git configuration applies to the+// write. It returns the ID of the object.+func writeLooseObjectHook(t *testing.T, cfg config.Cfg, repoPath string) string {+ t.Helper()++ content := "written by a custom hook\n"+ oid := strings.TrimSpace(string(gittest.ExecOpts(t, cfg, gittest.ExecConfig{Stdin: strings.NewReader(content)},+ "-C", repoPath, "hash-object", "--stdin")))++ var compressed bytes.Buffer+ zw := zlib.NewWriter(&compressed)+ _, err := fmt.Fprintf(zw, "blob %d\x00%s", len(content), content)+ require.NoError(t, err)+ require.NoError(t, zw.Close())++ objectFile := filepath.Join(testhelper.TempDir(t), "object")+ require.NoError(t, os.WriteFile(objectFile, compressed.Bytes(), 0o644))++ gittest.WriteCustomHook(t, repoPath, "pre-receive", []byte(fmt.Sprintf(`#!/bin/sh+cat >/dev/null+mkdir -p "$GIT_OBJECT_DIRECTORY/%[1]s" || exit 1+cp %[3]q "$GIT_OBJECT_DIRECTORY/%[1]s/%[2]s" || exit 1+`, oid[:2], oid[2:], objectFile)))++ return oid+}++// TestUserCommitFiles_migrationOfAnUntrustedWriter is not parallel: it reads a global counter. A+// custom pre-receive hook writes a loose object into the quarantine of UserCommitFiles without git,+// so neither the object nor its directory is durable. The migration of that quarantine fsyncs its+// files, and the object ends up in the repository.+func TestUserCommitFiles_migrationOfAnUntrustedWriter(t *testing.T) {+ testhelper.NewFeatureSets(featureflag.GPGSigning).Run(t, func(t *testing.T, ctx context.Context) {+ ctx, cfg, client := setupOperationsService(t, ctx)+ repo, repoPath := gittest.CreateRepository(t, ctx, cfg)+ oid := writeLooseObjectHook(t, cfg, repoPath)++ before := untrustedMigrations(t)+ commitAFile(t, ctx, client, repo, "untrusted", "file", "content\n")+ require.InDelta(t, 1, untrustedMigrations(t)-before, 0, "the migration fsyncs the files of the quarantine")++ gittest.Exec(t, cfg, "-C", repoPath, "cat-file", "-e", oid)+ })+}++// TestUserCommitFiles_migrationWithoutAnUntrustedWriter is not parallel: it reads a global counter.+// Without a custom hook, the migration trusts the writers of the quarantine.+func TestUserCommitFiles_migrationWithoutAnUntrustedWriter(t *testing.T) {+ testhelper.NewFeatureSets(featureflag.GPGSigning).Run(t, func(t *testing.T, ctx context.Context) {+ ctx, cfg, client := setupOperationsService(t, ctx)+ repo, _ := gittest.CreateRepository(t, ctx, cfg)++ before := untrustedMigrations(t)+ commitAFile(t, ctx, client, repo, "trusted", "file", "content\n")+ require.InDelta(t, 0, untrustedMigrations(t)-before, 0, "the migration does not fsync the files of the quarantine")+ })+}