Skip to content

Git designed for AI agents.

quarantine: links first, one fsync per directory, vanishing fan-out dirs

Federico Eliafdb40f0648d8
diff --git a/doc/graft/LIMITATIONS.md b/doc/graft/LIMITATIONS.md
index e1ac8b1..cb689b0 100644
--- a/doc/graft/LIMITATIONS.md
+++ b/doc/graft/LIMITATIONS.md
@@ -140,6 +140,18 @@ Git hooks use private.
does this. The quickstart creates a bare repository on disk and seeds it
with a push.
- `graft clone` needs the branch to exist. An empty repository cannot be cloned.
+- Durability of written objects assumes a local journaling filesystem, ext4 (in
+ any `data=` mode) or xfs, not NFS. The objects of a write are made durable by
+ their writer, and the migration of the write's quarantine into the repository
+ makes the new names durable with one fsync per directory: that relies on a
+ hard link being one journaled operation and on the fsync of a directory making
+ every earlier change of the journal durable. On other filesystems an object
+ written just before a crash may be missing although the write succeeded.
+- Custom hooks get the server's `core.fsync` configuration, so the objects that
+ their git commands write into a quarantine are durable. When a custom hook
+ runs with a quarantine, the migration of that quarantine also fsyncs its files,
+ because a hook may write objects without git; that costs about one fsync per
+ file of the write.
### Landing (`UserLand`)
diff --git a/internal/git/quarantine/migrate_crash_test.go b/internal/git/quarantine/migrate_crash_test.go
new file mode 100644
index 0000000..c4c0956
--- /dev/null
+++ b/internal/git/quarantine/migrate_crash_test.go
@@ -0,0 +1,145 @@
+package quarantine
+
+import (
+ "context"
+ "io/fs"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "syscall"
+ "testing"
+
+ "github.com/feder-cr/git-ai/internal/git/gittest"
+ "github.com/feder-cr/git-ai/internal/testhelper"
+ "github.com/feder-cr/git-ai/internal/testhelper/testcfg"
+ "github.com/stretchr/testify/require"
+)
+
+// TestMigrateCrashChild is the child process of TestMigrateCrashLeavesAPrefixOfTheLinks. It migrates
+// the quarantine it is given and kills itself right before the link that it is told.
+func TestMigrateCrashChild(t *testing.T) {
+ source := os.Getenv("QUARANTINE_CRASH_SOURCE")
+ if source == "" {
+ t.Skip("only runs as the child of TestMigrateCrashLeavesAPrefixOfTheLinks")
+ }
+ target := os.Getenv("QUARANTINE_CRASH_TARGET")
+ crashAt, err := strconv.Atoi(os.Getenv("QUARANTINE_CRASH_AT"))
+ require.NoError(t, err)
+
+ links := 0
+ beforeLink = func(string) {
+ if links == crashAt {
+ _ = syscall.Kill(os.Getpid(), syscall.SIGKILL)
+ select {}
+ }
+ links++
+ }
+
+ require.NoError(t, migrate(context.Background(), source, target, false))
+}
+
+// copyTree copies the directory tree at from to to.
+func copyTree(t *testing.T, from, to string) {
+ t.Helper()
+
+ require.NoError(t, filepath.WalkDir(from, func(path string, entry fs.DirEntry, err error) error {
+ if err != nil {
+ return err
+ }
+ rel, err := filepath.Rel(from, path)
+ if err != nil {
+ return err
+ }
+ if entry.IsDir() {
+ return os.MkdirAll(filepath.Join(to, rel), 0o755)
+ }
+ data, err := os.ReadFile(path)
+ if err != nil {
+ return err
+ }
+ return os.WriteFile(filepath.Join(to, rel), data, 0o444)
+ }))
+}
+
+// TestMigrateCrashLeavesAPrefixOfTheLinks kills a migration before each of its links in turn, in a
+// child process, and checks what the repository holds afterwards: the names that the migration
+// linked before the crash, in link order, so a pack index never without its pack, and git fsck
+// finds every object it holds intact. A crash of the process leaves the page cache; a power cut,
+// which the filesystem assumption of migrate covers, cannot be simulated here.
+func TestMigrateCrashLeavesAPrefixOfTheLinks(t *testing.T) {
+ ctx := testhelper.Context(t)
+ cfg := testcfg.Build(t)
+
+ // A quarantine with loose objects and a pack, written by git.
+ _, scratchPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{SkipCreationViaService: true})
+ packed := gittest.WriteCommit(t, cfg, scratchPath, gittest.WithMessage("packed"))
+ gittest.ExecOpts(t, cfg, gittest.ExecConfig{Stdin: strings.NewReader(packed.String() + "\n")},
+ "-C", scratchPath, "pack-objects", "-q", "--revs", filepath.Join(scratchPath, "objects", "pack", "pack"))
+ gittest.Exec(t, cfg, "-C", scratchPath, "prune-packed")
+ gittest.WriteCommit(t, cfg, scratchPath, gittest.WithMessage("loose"),
+ gittest.WithTreeEntries(gittest.TreeEntry{Mode: "100644", Path: "loose", Content: "loose\n"}))
+ quarantine := filepath.Join(t.TempDir(), "quarantine")
+ copyTree(t, filepath.Join(scratchPath, "objects"), quarantine)
+
+ // The order of the links, from a migration that completes.
+ var order []string
+ originalBefore := beforeLink
+ t.Cleanup(func() { beforeLink = originalBefore })
+ beforeLink = func(targetPath string) { order = append(order, targetPath) }
+ fullTarget := filepath.Join(t.TempDir(), "objects")
+ require.NoError(t, os.MkdirAll(fullTarget, 0o755))
+ source := filepath.Join(t.TempDir(), "source")
+ copyTree(t, quarantine, source)
+ require.NoError(t, migrate(ctx, source, fullTarget, false))
+ beforeLink = originalBefore
+
+ var relOrder []string
+ for _, path := range order {
+ rel, err := filepath.Rel(fullTarget, path)
+ require.NoError(t, err)
+ relOrder = append(relOrder, rel)
+ }
+ var idx, pack int = -1, -1
+ for i, rel := range relOrder {
+ switch {
+ case strings.HasSuffix(rel, ".pack"):
+ pack = i
+ case strings.HasSuffix(rel, ".idx"):
+ idx = i
+ }
+ }
+ require.True(t, pack >= 0 && idx > pack, "the pack is linked before its index: %v", relOrder)
+
+ for crashAt := 0; crashAt <= len(relOrder); crashAt++ {
+ _, repoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{SkipCreationViaService: true})
+ target := filepath.Join(repoPath, "objects")
+ source := filepath.Join(t.TempDir(), "source")
+ copyTree(t, quarantine, source)
+
+ cmd := exec.Command(os.Args[0], "-test.run", "^TestMigrateCrashChild$")
+ cmd.Env = append(os.Environ(),
+ "QUARANTINE_CRASH_SOURCE="+source,
+ "QUARANTINE_CRASH_TARGET="+target,
+ "QUARANTINE_CRASH_AT="+strconv.Itoa(crashAt),
+ )
+ output, err := cmd.CombinedOutput()
+ if crashAt < len(relOrder) {
+ require.Error(t, err, "the child is killed before link %d", crashAt)
+ } else {
+ require.NoError(t, err, "%s", output)
+ }
+
+ for i, rel := range relOrder {
+ _, statErr := os.Stat(filepath.Join(target, rel))
+ if i < crashAt {
+ require.NoError(t, statErr, "crash at %d: %s linked before the crash", crashAt, rel)
+ } else {
+ require.ErrorIs(t, statErr, fs.ErrNotExist, "crash at %d: %s linked after the crash", crashAt, rel)
+ }
+ }
+
+ gittest.Exec(t, cfg, "-C", repoPath, "fsck", "--no-dangling")
+ }
+}
diff --git a/internal/git/quarantine/migrate_test.go b/internal/git/quarantine/migrate_test.go
new file mode 100644
index 0000000..77d7e00
--- /dev/null
+++ b/internal/git/quarantine/migrate_test.go
@@ -0,0 +1,281 @@
+package quarantine
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "os"
+ "path/filepath"
+ "sort"
+ "sync"
+ "sync/atomic"
+ "testing"
+
+ "github.com/feder-cr/git-ai/internal/git/gittest"
+ "github.com/feder-cr/git-ai/internal/gitaly/config"
+ "github.com/feder-cr/git-ai/internal/gitaly/storage"
+ "github.com/feder-cr/git-ai/internal/testhelper"
+ "github.com/feder-cr/git-ai/internal/testhelper/testcfg"
+ "github.com/stretchr/testify/require"
+)
+
+// The tests of this file replace beforeLink, syncDirFile and syncFile, so they are not parallel.
+
+// writeQuarantine writes a quarantine at dir with one object in each of the fan-out directories
+// fanouts and a pack with its index, and returns the relative paths of its files.
+func writeQuarantine(t *testing.T, dir string, fanouts []string) []string {
+ t.Helper()
+
+ var files []string
+ for _, fanout := range fanouts {
+ files = append(files, filepath.Join(fanout, "object"+fanout))
+ }
+ files = append(files, filepath.Join("pack", "pack-1.pack"), filepath.Join("pack", "pack-1.idx"))
+
+ for _, file := range files {
+ require.NoError(t, os.MkdirAll(filepath.Join(dir, filepath.Dir(file)), 0o755))
+ require.NoError(t, os.WriteFile(filepath.Join(dir, file), []byte(file), 0o444))
+ }
+ return files
+}
+
+// replaceHooks replaces the test hooks of migrate for the test and counts the fsyncs.
+func replaceHooks(t *testing.T, before func(targetPath string)) (dirSyncs, fileSyncs *atomic.Int32) {
+ t.Helper()
+
+ dirSyncs, fileSyncs = &atomic.Int32{}, &atomic.Int32{}
+ originalBefore, originalSyncDir, originalSyncFile := beforeLink, syncDirFile, syncFile
+ t.Cleanup(func() { beforeLink, syncDirFile, syncFile = originalBefore, originalSyncDir, originalSyncFile })
+
+ if before != nil {
+ beforeLink = before
+ }
+ syncDirFile = func(f *os.File) error {
+ dirSyncs.Add(1)
+ return originalSyncDir(f)
+ }
+ syncFile = func(path string) error {
+ fileSyncs.Add(1)
+ return originalSyncFile(path)
+ }
+ return dirSyncs, fileSyncs
+}
+
+// TestMigrateRecreatesTargetDirectoriesThatVanish removes each empty target fan-out directory
+// right before the first link into it, as housekeeping (git prune-packed) may. Before, the link
+// and its rename fallback then failed with ENOENT. The migration creates the directory again and
+// links every object.
+func TestMigrateRecreatesTargetDirectoriesThatVanish(t *testing.T) {
+ ctx := testhelper.Context(t)
+ root := t.TempDir()
+ source, target := filepath.Join(root, "quarantine"), filepath.Join(root, "objects")
+ fanouts := []string{"0a", "1b", "2c"}
+ files := writeQuarantine(t, source, fanouts)
+
+ // The target fan-out directories exist and are empty, as housekeeping leaves them.
+ require.NoError(t, os.MkdirAll(filepath.Join(target, "pack"), 0o755))
+ for _, fanout := range fanouts {
+ require.NoError(t, os.Mkdir(filepath.Join(target, fanout), 0o755))
+ }
+
+ var removed sync.Map
+ replaceHooks(t, func(targetPath string) {
+ dir := filepath.Dir(targetPath)
+ if filepath.Base(dir) == "pack" {
+ return
+ }
+ if _, done := removed.LoadOrStore(dir, true); !done {
+ require.NoError(t, os.Remove(dir))
+ }
+ })
+
+ require.NoError(t, migrate(ctx, source, target, false))
+
+ for _, file := range files {
+ require.FileExists(t, filepath.Join(target, file))
+ }
+ for _, fanout := range fanouts {
+ _, ok := removed.Load(filepath.Join(target, fanout))
+ require.True(t, ok, "the directory %s was removed during the migration", fanout)
+ }
+ require.NoDirExists(t, source)
+}
+
+// TestMigrateSyncsEachDirectoryOnce counts the fsyncs: one per target directory that received a
+// link, one for the object directory, and none per file, whose writers made them durable.
+func TestMigrateSyncsEachDirectoryOnce(t *testing.T) {
+ ctx := testhelper.Context(t)
+ root := t.TempDir()
+ source, target := filepath.Join(root, "quarantine"), filepath.Join(root, "objects")
+ fanouts := []string{"0a", "1b", "2c", "3d"}
+ writeQuarantine(t, source, fanouts)
+ require.NoError(t, os.MkdirAll(target, 0o755))
+
+ dirSyncs, fileSyncs := replaceHooks(t, nil)
+ require.NoError(t, migrate(ctx, source, target, false))
+
+ require.EqualValues(t, len(fanouts)+1+1, dirSyncs.Load(), "the fan-out directories, pack and the object directory")
+ require.Zero(t, fileSyncs.Load())
+}
+
+// TestMigrateSyncsTheFilesOfAnUntrustedWriter fsyncs every file of a quarantine that a writer
+// Gitaly does not control wrote into, and still each directory once.
+func TestMigrateSyncsTheFilesOfAnUntrustedWriter(t *testing.T) {
+ ctx := testhelper.Context(t)
+ root := t.TempDir()
+ source, target := filepath.Join(root, "quarantine"), filepath.Join(root, "objects")
+ fanouts := []string{"0a", "1b"}
+ files := writeQuarantine(t, source, fanouts)
+ require.NoError(t, os.MkdirAll(target, 0o755))
+
+ dirSyncs, fileSyncs := replaceHooks(t, nil)
+ require.NoError(t, migrate(ctx, source, target, true))
+
+ require.EqualValues(t, len(files), fileSyncs.Load())
+ require.EqualValues(t, len(fanouts)+1+1, dirSyncs.Load())
+}
+
+// fakeTransaction makes storage.NeedsSync false, as a transaction of the transaction manager does.
+type fakeTransaction struct {
+ storage.Transaction
+}
+
+// TestMigrateDoesNotSyncUnderATransaction leaves durability to the transaction manager.
+func TestMigrateDoesNotSyncUnderATransaction(t *testing.T) {
+ ctx := storage.ContextWithTransaction(testhelper.Context(t), fakeTransaction{})
+ require.False(t, storage.NeedsSync(ctx))
+
+ root := t.TempDir()
+ source, target := filepath.Join(root, "quarantine"), filepath.Join(root, "objects")
+ files := writeQuarantine(t, source, []string{"0a"})
+ require.NoError(t, os.MkdirAll(target, 0o755))
+
+ dirSyncs, fileSyncs := replaceHooks(t, nil)
+ require.NoError(t, migrate(ctx, source, target, true))
+
+ require.Zero(t, dirSyncs.Load())
+ require.Zero(t, fileSyncs.Load())
+ for _, file := range files {
+ require.FileExists(t, filepath.Join(target, file))
+ }
+}
+
+// TestMigrateKeepsThePackOrder links the data of a pack before its index.
+func TestMigrateKeepsThePackOrder(t *testing.T) {
+ ctx := testhelper.Context(t)
+ root := t.TempDir()
+ source, target := filepath.Join(root, "quarantine"), filepath.Join(root, "objects")
+ writeQuarantine(t, source, nil)
+ require.NoError(t, os.MkdirAll(target, 0o755))
+
+ var linked []string
+ var mu sync.Mutex
+ replaceHooks(t, func(targetPath string) {
+ mu.Lock()
+ defer mu.Unlock()
+ linked = append(linked, filepath.Base(targetPath))
+ })
+
+ require.NoError(t, migrate(ctx, source, target, false))
+ require.Equal(t, []string{"pack-1.pack", "pack-1.idx"}, linked)
+}
+
+// TestMarkUntrustedWriter marks the quarantine whose object directory a hook was given, and only
+// while the quarantine exists.
+func TestMarkUntrustedWriter(t *testing.T) {
+ ctx := testhelper.Context(t)
+ cfg := testcfg.Build(t)
+ repo, _ := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{SkipCreationViaService: true})
+ locator := config.NewLocator(cfg)
+
+ marked, cleanupMarked, err := New(ctx, repo, testhelper.NewLogger(t), locator)
+ require.NoError(t, err)
+ defer cleanupMarked()
+ other, cleanupOther, err := New(ctx, repo, testhelper.NewLogger(t), locator)
+ require.NoError(t, err)
+ defer cleanupOther()
+
+ MarkUntrustedWriter(marked.dir.Path() + "/")
+ require.True(t, marked.untrustedWriter.Load())
+ require.False(t, other.untrustedWriter.Load())
+
+ // A path that is no quarantine marks nothing.
+ MarkUntrustedWriter(t.TempDir())
+ require.False(t, other.untrustedWriter.Load())
+
+ path := other.dir.Path()
+ cleanupOther()
+ _, registered := quarantines.Load(filepath.Clean(path))
+ require.False(t, registered, "a quarantine leaves the registry when it is cleaned up")
+}
+
+// TestMigrateWhileGitPrunesPacked runs migrations into a repository while git prune-packed, which
+// housekeeping runs after a repack, removes its empty fan-out directories in a loop.
+func TestMigrateWhileGitPrunesPacked(t *testing.T) {
+ ctx := testhelper.Context(t)
+ cfg := testcfg.Build(t)
+ _, repoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{SkipCreationViaService: true})
+ objects := filepath.Join(repoPath, "objects")
+
+ // Every fan-out directory exists and is empty, the state that prune-packed cleans up.
+ for i := 0; i < 256; i++ {
+ require.NoError(t, os.MkdirAll(filepath.Join(objects, fmt.Sprintf("%02x", i)), 0o755))
+ }
+
+ stop := make(chan struct{})
+ var pruned sync.WaitGroup
+ pruned.Add(1)
+ runs := 0
+ go func() {
+ defer pruned.Done()
+ for {
+ select {
+ case <-stop:
+ return
+ default:
+ }
+ var stderr bytes.Buffer
+ cmd := gittest.NewCommand(t, cfg, "-C", repoPath, "prune-packed", "--quiet")
+ cmd.Stderr = &stderr
+ if err := cmd.Run(); err != nil {
+ t.Errorf("prune-packed: %v: %s", err, stderr.String())
+ return
+ }
+ runs++
+ // Recreate empty directories, so that the race stays possible.
+ for i := 0; i < 256; i += 7 {
+ _ = os.Mkdir(filepath.Join(objects, fmt.Sprintf("%02x", i)), 0o755)
+ }
+ }
+ }()
+
+ for i := 0; i < 200; i++ {
+ source := filepath.Join(t.TempDir(), "quarantine")
+ fanouts := []string{fmt.Sprintf("%02x", (i*7)%256), fmt.Sprintf("%02x", (i*13+1)%256)}
+ var files []string
+ for _, fanout := range fanouts {
+ file := filepath.Join(fanout, fmt.Sprintf("object-%03d", i))
+ require.NoError(t, os.MkdirAll(filepath.Join(source, fanout), 0o755))
+ require.NoError(t, os.WriteFile(filepath.Join(source, file), []byte(file), 0o444))
+ files = append(files, file)
+ }
+
+ require.NoError(t, migrate(context.Background(), source, objects, false), "migration %d", i)
+ for _, file := range files {
+ require.FileExists(t, filepath.Join(objects, file))
+ }
+ }
+
+ close(stop)
+ pruned.Wait()
+ require.Positive(t, runs)
+
+ var names []string
+ entries, err := os.ReadDir(objects)
+ require.NoError(t, err)
+ for _, entry := range entries {
+ names = append(names, entry.Name())
+ }
+ sort.Strings(names)
+ t.Logf("%d prune-packed runs during 200 migrations", runs)
+}
diff --git a/internal/git/quarantine/quarantine.go b/internal/git/quarantine/quarantine.go
index 6c5771e..a04f67d 100644
--- a/internal/git/quarantine/quarantine.go
+++ b/internal/git/quarantine/quarantine.go
@@ -4,32 +4,67 @@ import (
"context"
"errors"
"fmt"
+ "io/fs"
"os"
"path/filepath"
"sort"
"strings"
+ "sync"
+ "sync/atomic"
"github.com/feder-cr/git-ai/internal/gitaly/storage"
"github.com/feder-cr/git-ai/internal/gitaly/storage/mode"
"github.com/feder-cr/git-ai/internal/log"
- "github.com/feder-cr/git-ai/internal/safe"
"github.com/feder-cr/git-ai/internal/structerr"
"github.com/feder-cr/git-ai/internal/tempdir"
"github.com/feder-cr/git-ai/proto/go/gitalypb"
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/promauto"
"google.golang.org/protobuf/proto"
)
+// untrustedMigrationsTotal counts the migrations that fsynced the files of their quarantine
+// because a writer that Gitaly does not control ran with it.
+var untrustedMigrationsTotal = promauto.NewCounter(prometheus.CounterOpts{
+ Name: "graft_quarantine_untrusted_migrations_total",
+ Help: "Quarantine migrations that fsynced the files of the quarantine because a custom hook ran with it.",
+})
+
// Dir is a quarantine directory for Git objects. Instead of writing new commits into the main
// repository, they're instead written into a temporary quarantine directory. This staging area can
// either be migrated into the main repository or, alternatively, will automatically be discarded
// when the context gets cancelled. If the quarantine environment is discarded without being staged,
// then none of the objects which have been created in the quarantine directory will end up in the
// main repository.
+//
+// Durability contract: every file in a quarantine is durable when Migrate starts, its data and its
+// inode. The Gitaly writers make it so: they are git processes that the command factory starts with
+// core.fsync covering objects, or helpers that write without fsync and pack their objects into one
+// fsynced pack before the update (see localrepo.WithoutObjectFsync). Migrate then only makes the
+// new names durable, with one fsync per directory. A writer that Gitaly does not control, a custom
+// hook, is reported with MarkUntrustedWriter, and Migrate then fsyncs the files of that quarantine
+// itself.
type Dir struct {
repo *gitalypb.Repository
quarantinedRepo *gitalypb.Repository
dir tempdir.Dir
locator storage.Locator
+ // untrustedWriter is set when a writer that Gitaly does not control may have written into
+ // the quarantine, see MarkUntrustedWriter.
+ untrustedWriter atomic.Bool
+}
+
+// quarantines holds the quarantines of this process by the absolute path of their object
+// directory, so that MarkUntrustedWriter finds the quarantine that a hook was given.
+var quarantines sync.Map
+
+// MarkUntrustedWriter records that a writer that Gitaly does not control, such as a custom hook, ran
+// with objectDir as its object directory. When objectDir is a quarantine of this process, its
+// files are not trusted to be durable, and its migration fsyncs them first.
+func MarkUntrustedWriter(objectDir string) {
+ if d, ok := quarantines.Load(filepath.Clean(objectDir)); ok {
+ d.(*Dir).untrustedWriter.Store(true)
+ }
}
// New creates a new quarantine directory and returns the directory and a cleanup function.
@@ -52,12 +87,20 @@ func New(ctx context.Context, repo *gitalypb.Repository, logger log.Logger, loca
return nil, nil, err
}
- return &Dir{
+ d := &Dir{
repo: repo,
quarantinedRepo: quarantinedRepo,
locator: locator,
dir: quarantineDir,
- }, cleanup, nil
+ }
+
+ key := filepath.Clean(quarantineDir.Path())
+ quarantines.Store(key, d)
+
+ return d, func() {
+ quarantines.CompareAndDelete(key, d)
+ cleanup()
+ }, nil
}
// Apply applies the quarantine on the repository. This is done by setting the quarantineDirectory
@@ -109,67 +152,236 @@ func (d *Dir) Migrate(ctx context.Context) error {
objectDir = "objects"
}
- return migrate(ctx, d.dir.Path(), filepath.Join(repoPath, objectDir))
+ return migrate(ctx, d.dir.Path(), filepath.Join(repoPath, objectDir), d.untrustedWriter.Load())
}
-func migrate(ctx context.Context, sourcePath, targetPath string) error {
+// migrate moves the objects of the quarantine at sourcePath into the object directory at
+// targetPath, in two passes.
+//
+// The first pass links every file into the target directories, in the order of sortEntries, so
+// that a pack's index appears after its data. An empty fan-out directory of the target may be
+// removed at any time: git's housekeeping (prune-packed, repack -d, prune) removes empty fan-out
+// directories, and git's own writers create their temporary file inside the fan-out directory,
+// which pins it. A quarantine is a separate directory, so a target directory that migrate created
+// or found can vanish before its first link. A link that fails because its target directory is
+// gone creates the directory again and links again; once the link succeeds the directory is not
+// empty and stays. Each round needs another removal between the creation and the link, so the loop
+// ends; it also ends with the context.
+//
+// The second pass, unless a transaction manager owns durability (storage.NeedsSync), fsyncs each
+// target directory that received a link, once, through a descriptor opened in the first pass, and
+// the object directory itself. The files are not fsynced: they are durable before the migration
+// (see Dir), unless untrustedWriter is set, in which case they are fsynced in parallel before the
+// first pass.
+//
+// Filesystem assumption (local ext4 in any data mode, or xfs; not NFS): a new hard link is
+// durable once its directory is fsynced; a link is one journaled operation that changes the
+// directory entry and the link count together; the journal commits its transactions in order, and
+// the fsync of a directory forces it up to the directory's last change. So after the second pass
+// every link is durable, and a crash before leaves a prefix of the links in their order.
+func migrate(ctx context.Context, sourcePath, targetPath string, untrustedWriter bool) error {
+ sync := storage.NeedsSync(ctx)
+
+ if sync && untrustedWriter {
+ untrustedMigrationsTotal.Inc()
+ if err := syncFilesInParallel(ctx, sourcePath); err != nil {
+ return fmt.Errorf("syncing the files of an untrusted writer: %w", err)
+ }
+ }
+
+ m := migration{ctx: ctx, dirs: map[string]*os.File{}}
+ defer m.closeDirs()
+
+ if err := m.openDir(targetPath); err != nil {
+ return err
+ }
+ if err := m.link(sourcePath, targetPath); err != nil {
+ return err
+ }
+
+ if !sync {
+ return nil
+ }
+
+ // The object directory last: it holds the entries of the fan-out directories that the first
+ // pass created.
+ for i := len(m.order) - 1; i >= 0; i-- {
+ if err := syncDirFile(m.dirs[m.order[i]]); err != nil {
+ return fmt.Errorf("sync directory %q: %w", m.order[i], err)
+ }
+ }
+
+ return nil
+}
+
+// migration is the state of the first pass of migrate.
+type migration struct {
+ ctx context.Context
+ // dirs holds a descriptor of each target directory, by path.
+ dirs map[string]*os.File
+ // order holds the target directories in the order they were opened, parents first.
+ order []string
+}
+
+// openDir creates the target directory dir if it does not exist and opens a descriptor of it,
+// replacing the descriptor of a directory of the same path that was removed.
+func (m *migration) openDir(dir string) error {
+ if err := os.Mkdir(dir, mode.Directory); err != nil && !errors.Is(err, os.ErrExist) {
+ return fmt.Errorf("creating target directory %q: %w", dir, err)
+ }
+
+ f, err := os.Open(dir)
+ if err != nil {
+ return fmt.Errorf("opening target directory %q: %w", dir, err)
+ }
+
+ if old, ok := m.dirs[dir]; ok {
+ _ = old.Close()
+ } else {
+ m.order = append(m.order, dir)
+ }
+ m.dirs[dir] = f
+
+ return nil
+}
+
+func (m *migration) closeDirs() {
+ for _, f := range m.dirs {
+ _ = f.Close()
+ }
+}
+
+// link links the files of sourcePath into targetPath, recursively, and removes sourcePath.
+func (m *migration) link(sourcePath, targetPath string) error {
entries, err := os.ReadDir(sourcePath)
if err != nil {
return fmt.Errorf("reading directory: %w", err)
}
sortEntries(entries)
- syncer := safe.NewSyncer()
for _, entry := range entries {
- if entry.Name() == "." {
- continue
- }
-
nestedTargetPath := filepath.Join(targetPath, entry.Name())
nestedSourcePath := filepath.Join(sourcePath, entry.Name())
if entry.IsDir() {
- if err := os.Mkdir(nestedTargetPath, mode.Directory); err != nil {
- if !errors.Is(err, os.ErrExist) {
- return fmt.Errorf("creating target directory %q: %w", nestedTargetPath, err)
- }
+ if err := m.openDir(nestedTargetPath); err != nil {
+ return err
}
-
- if err := migrate(ctx, nestedSourcePath, nestedTargetPath); err != nil {
+ if err := m.link(nestedSourcePath, nestedTargetPath); err != nil {
return fmt.Errorf("migrating directory %q: %w", nestedSourcePath, err)
}
-
- if storage.NeedsSync(ctx) {
- if err := syncer.Sync(ctx, nestedTargetPath); err != nil {
- return fmt.Errorf("sync directory: %w", err)
- }
- }
-
continue
}
- if err := finalizeObjectFile(nestedSourcePath, nestedTargetPath); err != nil {
+ if err := m.linkFile(nestedSourcePath, nestedTargetPath); err != nil {
return fmt.Errorf("migrating object file %q: %w", nestedSourcePath, err)
}
+ }
- if storage.NeedsSync(ctx) {
- if err := syncer.Sync(ctx, nestedTargetPath); err != nil {
- return fmt.Errorf("sync object: %w", err)
- }
+ if err := os.Remove(sourcePath); err != nil {
+ return fmt.Errorf("removing source directory: %w", err)
+ }
+
+ return nil
+}
+
+// linkFile links the file at sourcePath to targetPath, creating the target directory again when
+// it was removed (see migrate).
+func (m *migration) linkFile(sourcePath, targetPath string) error {
+ for {
+ beforeLink(targetPath)
+
+ err := finalizeObjectFile(sourcePath, targetPath)
+ if err == nil {
+ return nil
}
+ if !errors.Is(err, fs.ErrNotExist) {
+ return err
+ }
+
+ if _, statErr := os.Lstat(sourcePath); statErr != nil {
+ return fmt.Errorf("%w (source: %w)", err, statErr)
+ }
+ if ctxErr := m.ctx.Err(); ctxErr != nil {
+ return fmt.Errorf("%w (%w)", err, ctxErr)
+ }
+
+ // The source exists, so the target directory is gone.
+ if err := m.openDir(filepath.Dir(targetPath)); err != nil {
+ return err
+ }
+ }
+}
+
+// beforeLink runs before each link of the first pass of migrate. Tests replace it to remove a
+// target directory at that point, as housekeeping may.
+var beforeLink = func(targetPath string) {}
+
+// syncDirFile fsyncs the descriptor of a target directory. Tests replace it to count the fsyncs.
+var syncDirFile = func(f *os.File) error {
+ return f.Sync()
+}
+
+// syncFile fsyncs the file at path. Tests replace it to count the fsyncs.
+var syncFile = func(path string) error {
+ f, err := os.Open(path)
+ if err != nil {
+ return err
}
+ defer f.Close()
+
+ return f.Sync()
+}
- if storage.NeedsSync(ctx) {
- if err := syncer.Sync(ctx, targetPath); err != nil {
- return fmt.Errorf("sync object directory: %w", err)
+// maxParallelSyncs bounds the fsyncs that syncFilesInParallel runs at once.
+const maxParallelSyncs = 16
+
+// syncFilesInParallel fsyncs every file under dir. Concurrent fsyncs of the files of one
+// filesystem share journal commits, which one fsync after another does not.
+func syncFilesInParallel(ctx context.Context, dir string) error {
+ var files []string
+ if err := filepath.WalkDir(dir, func(path string, entry fs.DirEntry, err error) error {
+ if err != nil {
+ return err
+ }
+ if !entry.IsDir() {
+ files = append(files, path)
}
+ return nil
+ }); err != nil {
+ return err
}
- if err := os.Remove(sourcePath); err != nil {
- return fmt.Errorf("removing source directory: %w", err)
+ var (
+ wg sync.WaitGroup
+ mu sync.Mutex
+ firstErr error
+ )
+ slots := make(chan struct{}, maxParallelSyncs)
+ for _, path := range files {
+ if err := ctx.Err(); err != nil {
+ wg.Wait()
+ return err
+ }
+
+ slots <- struct{}{}
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ defer func() { <-slots }()
+
+ if err := syncFile(path); err != nil {
+ mu.Lock()
+ if firstErr == nil {
+ firstErr = fmt.Errorf("sync %q: %w", path, err)
+ }
+ mu.Unlock()
+ }
+ }()
}
+ wg.Wait()
- return nil
+ return firstErr
}
// finalizeObjectFile will move the object file (either a packfile, its metadata or or a loose
diff --git a/internal/git/quarantine/quarantine_test.go b/internal/git/quarantine/quarantine_test.go
index c284930..47fd831 100644
--- a/internal/git/quarantine/quarantine_test.go
+++ b/internal/git/quarantine/quarantine_test.go
@@ -369,7 +369,7 @@ func TestMigrate(t *testing.T) {
target := filepath.Join(dir, "target")
tc.target.create(t, target)
- require.NoError(t, migrate(ctx, source, target))
+ require.NoError(t, migrate(ctx, source, target, false))
require.Equal(t, tc.expected, listEntries(t, target))
require.NoDirExists(t, source)
})
diff --git a/internal/gitaly/hook/custom.go b/internal/gitaly/hook/custom.go
index 31d4b9a..839620c 100644
--- a/internal/gitaly/hook/custom.go
+++ b/internal/gitaly/hook/custom.go
@@ -12,6 +12,7 @@ import (
"github.com/feder-cr/git-ai/internal/cgroups"
"github.com/feder-cr/git-ai/internal/command"
"github.com/feder-cr/git-ai/internal/git/gitcmd"
+ "github.com/feder-cr/git-ai/internal/git/quarantine"
"github.com/feder-cr/git-ai/internal/gitaly/storage"
"github.com/feder-cr/git-ai/internal/helper/env"
"github.com/feder-cr/git-ai/proto/go/gitalypb"
@@ -84,6 +85,17 @@ func (m *GitLabHookManager) newCustomHooksExecutor(ctx context.Context, repo *gi
}
}
+ // A custom hook is code that Gitaly does not control. When it runs with a quarantine as its
+ // object directory, it may write objects there that are not durable, which the migration
+ // of the quarantine then makes durable itself (see quarantine.Dir). Custom hooks run in the
+ // Gitaly process: gitaly-hooks forwards them through the HookService, and updateref calls
+ // the hook manager directly, so the quarantine is one of this process.
+ if len(hookFiles) > 0 {
+ if objectDir := environmentValue(env, "GIT_OBJECT_DIRECTORY"); objectDir != "" {
+ quarantine.MarkUntrustedWriter(objectDir)
+ }
+ }
+
for _, hookFile := range hookFiles {
c, err := command.New(ctx, m.logger, append([]string{hookFile}, args...),
command.WithDir(repoPath),
@@ -212,6 +224,11 @@ func (m *GitLabHookManager) customHooksEnv(ctx context.Context, payload gitcmd.H
// case we're running with bundled Git such that Git can locate its binaries.
customEnvs = append(customEnvs, gitExecEnv.EnvironmentVariables...)
+ // The git commands of a custom hook write objects as durably as the server's own: a
+ // quarantine relies on its files being durable before it is migrated (see quarantine.Dir).
+ // The environment carries no other GIT_CONFIG_* variable (command.AllowedEnvironment).
+ customEnvs = append(customEnvs, gitcmd.ConfigPairsToGitEnvironment(m.fsyncConfiguration())...)
+
return append(customEnvs,
"GIT_DIR="+repoPath,
"GL_REPOSITORY="+payload.Repo.GetGlRepository(),
@@ -222,6 +239,38 @@ func (m *GitLabHookManager) customHooksEnv(ctx context.Context, payload gitcmd.H
), nil
}
+// fsyncConfiguration returns the fsync configuration of the server's git commands: the defaults
+// of the command factory (gitcmd, GlobalConfiguration), overridden by the administrator's [git]
+// configuration, which the command factory applies after them.
+func (m *GitLabHookManager) fsyncConfiguration() []gitcmd.ConfigPair {
+ pairs := []gitcmd.ConfigPair{
+ {Key: "core.fsync", Value: "objects,derived-metadata,reference"},
+ {Key: "core.fsyncMethod", Value: "fsync"},
+ }
+
+ for _, configured := range m.cfg.Git.Config {
+ for i := range pairs {
+ if strings.EqualFold(configured.Key, pairs[i].Key) {
+ pairs[i].Value = configured.Value
+ }
+ }
+ }
+
+ return pairs
+}
+
+// environmentValue returns the value of the variable name in the environment variables envs, the
+// last one if it is set several times.
+func environmentValue(envs []string, name string) string {
+ value := ""
+ for _, variable := range envs {
+ if strings.HasPrefix(variable, name+"=") {
+ value = strings.TrimPrefix(variable, name+"=")
+ }
+ }
+ return value
+}
+
// pushOptionsEnv turns a slice of git push option values into a GIT_PUSH_OPTION_COUNT and individual
// GIT_PUSH_OPTION_0, GIT_PUSH_OPTION_1 etc.
func pushOptionsEnv(options []string) []string {
diff --git a/internal/gitaly/hook/custom_fsync_test.go b/internal/gitaly/hook/custom_fsync_test.go
new file mode 100644
index 0000000..a470c64
--- /dev/null
+++ b/internal/gitaly/hook/custom_fsync_test.go
@@ -0,0 +1,65 @@
+package hook
+
+import (
+ "testing"
+
+ "github.com/feder-cr/git-ai/internal/git/gitcmd"
+ "github.com/feder-cr/git-ai/internal/git/gittest"
+ "github.com/feder-cr/git-ai/internal/gitaly/config"
+ "github.com/feder-cr/git-ai/internal/gitaly/transaction"
+ "github.com/feder-cr/git-ai/internal/gitlab"
+ "github.com/feder-cr/git-ai/internal/testhelper"
+ "github.com/feder-cr/git-ai/internal/testhelper/testcfg"
+ "github.com/stretchr/testify/require"
+)
+
+// TestCustomHooksGetTheFsyncConfiguration checks that the git commands of a custom hook get the
+// fsync configuration of the server's own git commands, as the command factory applies it: its
+// defaults, overridden by the administrator's [git] configuration.
+func TestCustomHooksGetTheFsyncConfiguration(t *testing.T) {
+ t.Parallel()
+
+ ctx := testhelper.Context(t)
+
+ for _, tc := range []struct {
+ desc string
+ admin []config.GitConfig
+ }{
+ {desc: "defaults"},
+ {desc: "administrator override", admin: []config.GitConfig{{Key: "core.fsync", Value: "all"}, {Key: "core.fsyncMethod", Value: "writeout-only"}}},
+ } {
+ t.Run(tc.desc, func(t *testing.T) {
+ t.Parallel()
+
+ cfg := testcfg.Build(t)
+ cfg.Git.Config = tc.admin
+ factory := gittest.NewCommandFactory(t, cfg)
+
+ // What the command factory gives every git command, the administrator last.
+ global, err := factory.GlobalConfiguration(ctx)
+ require.NoError(t, err)
+ want := map[string]string{}
+ for _, pair := range append(global, cfg.Git.Config...) {
+ if pair.Key == "core.fsync" || pair.Key == "core.fsyncMethod" {
+ want[pair.Key] = pair.Value
+ }
+ }
+ require.Len(t, want, 2)
+
+ manager := NewManager(cfg, config.NewLocator(cfg), testhelper.SharedLogger(t), factory, nil, &transaction.MockManager{},
+ gitlab.NewMockClient(t, gitlab.MockAllowed, gitlab.MockPreReceive, gitlab.MockPostReceive), NewPostReceiveRegistry())
+
+ repo, _ := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{SkipCreationViaService: true})
+ env, err := manager.customHooksEnv(ctx, gitcmd.HooksPayload{Repo: repo, UserDetails: &gitcmd.UserDetails{}}, nil, nil)
+ require.NoError(t, err)
+
+ got := map[string]string{}
+ count := environmentValue(env, "GIT_CONFIG_COUNT")
+ require.Equal(t, "2", count)
+ for _, i := range []string{"0", "1"} {
+ got[environmentValue(env, "GIT_CONFIG_KEY_"+i)] = environmentValue(env, "GIT_CONFIG_VALUE_"+i)
+ }
+ require.Equal(t, want, got)
+ })
+ }
+}
diff --git a/internal/gitaly/hook/testhelper_test.go b/internal/gitaly/hook/testhelper_test.go
index 48e890a..24c18b6 100644
--- a/internal/gitaly/hook/testhelper_test.go
+++ b/internal/gitaly/hook/testhelper_test.go
@@ -50,6 +50,16 @@ func getExpectedEnv(tb testing.TB, ctx context.Context, locator storage.Locator,
expectedEnv["PATH"] = fmt.Sprintf("%s:%s", filepath.Dir(execEnv.BinaryPath), os.Getenv("PATH"))
+ // Custom hooks get the fsync configuration of the server's git commands (see customHooksEnv
+ // and TestCustomHooksGetTheFsyncConfiguration).
+ for _, variable := range gitcmd.ConfigPairsToGitEnvironment([]gitcmd.ConfigPair{
+ {Key: "core.fsync", Value: "objects,derived-metadata,reference"},
+ {Key: "core.fsyncMethod", Value: "fsync"},
+ }) {
+ kv := strings.SplitN(variable, "=", 2)
+ expectedEnv[kv[0]] = kv[1]
+ }
+
result := make([]string, 0, len(expectedEnv))
for key, value := range expectedEnv {
result = append(result, fmt.Sprintf("%s=%s", key, value))
diff --git a/internal/gitaly/service/operations/migrate_untrusted_test.go b/internal/gitaly/service/operations/migrate_untrusted_test.go
new file mode 100644
index 0000000..b330a9d
--- /dev/null
+++ b/internal/gitaly/service/operations/migrate_untrusted_test.go
@@ -0,0 +1,108 @@
+package operations
+
+import (
+ "bytes"
+ "compress/zlib"
+ "context"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/feder-cr/git-ai/internal/featureflag"
+ "github.com/feder-cr/git-ai/internal/git/gittest"
+ "github.com/feder-cr/git-ai/internal/gitaly/config"
+ "github.com/feder-cr/git-ai/internal/testhelper"
+ "github.com/feder-cr/git-ai/proto/go/gitalypb"
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/stretchr/testify/require"
+)
+
+// untrustedMigrations returns graft_quarantine_untrusted_migrations_total.
+func untrustedMigrations(t *testing.T) float64 {
+ t.Helper()
+
+ families, err := prometheus.DefaultGatherer.Gather()
+ require.NoError(t, err)
+ for _, family := range families {
+ if family.GetName() == "graft_quarantine_untrusted_migrations_total" {
+ return family.GetMetric()[0].GetCounter().GetValue()
+ }
+ }
+ return 0
+}
+
+// commitAFile writes one file on branch with UserCommitFiles.
+func commitAFile(t *testing.T, ctx context.Context, client gitalypb.OperationServiceClient, repo *gitalypb.Repository, branch, path, content string) {
+ t.Helper()
+
+ stream, err := client.UserCommitFiles(ctx)
+ require.NoError(t, err)
+ header := headerRequest(repo, gittest.TestUser, branch, []byte("commit message"), "", "")
+ setAuthorAndEmail(header, []byte("Author Name"), []byte("[email protected]"))
+ require.NoError(t, stream.Send(header))
+ require.NoError(t, stream.Send(createFileHeaderRequest(path)))
+ require.NoError(t, stream.Send(actionContentRequest(content)))
+ _, err = stream.CloseAndRecv()
+ require.NoError(t, err)
+}
+
+// writeLooseObjectHook writes a custom pre-receive hook that writes a loose object into the
+// quarantine without git: it copies a zlib-compressed object, so no git configuration applies to the
+// write. It returns the ID of the object.
+func writeLooseObjectHook(t *testing.T, cfg config.Cfg, repoPath string) string {
+ t.Helper()
+
+ content := "written by a custom hook\n"
+ oid := strings.TrimSpace(string(gittest.ExecOpts(t, cfg, gittest.ExecConfig{Stdin: strings.NewReader(content)},
+ "-C", repoPath, "hash-object", "--stdin")))
+
+ var compressed bytes.Buffer
+ zw := zlib.NewWriter(&compressed)
+ _, err := fmt.Fprintf(zw, "blob %d\x00%s", len(content), content)
+ require.NoError(t, err)
+ require.NoError(t, zw.Close())
+
+ objectFile := filepath.Join(testhelper.TempDir(t), "object")
+ require.NoError(t, os.WriteFile(objectFile, compressed.Bytes(), 0o644))
+
+ gittest.WriteCustomHook(t, repoPath, "pre-receive", []byte(fmt.Sprintf(`#!/bin/sh
+cat >/dev/null
+mkdir -p "$GIT_OBJECT_DIRECTORY/%[1]s" || exit 1
+cp %[3]q "$GIT_OBJECT_DIRECTORY/%[1]s/%[2]s" || exit 1
+`, oid[:2], oid[2:], objectFile)))
+
+ return oid
+}
+
+// TestUserCommitFiles_migrationOfAnUntrustedWriter is not parallel: it reads a global counter. A
+// custom pre-receive hook writes a loose object into the quarantine of UserCommitFiles without git,
+// so neither the object nor its directory is durable. The migration of that quarantine fsyncs its
+// files, and the object ends up in the repository.
+func TestUserCommitFiles_migrationOfAnUntrustedWriter(t *testing.T) {
+ testhelper.NewFeatureSets(featureflag.GPGSigning).Run(t, func(t *testing.T, ctx context.Context) {
+ ctx, cfg, client := setupOperationsService(t, ctx)
+ repo, repoPath := gittest.CreateRepository(t, ctx, cfg)
+ oid := writeLooseObjectHook(t, cfg, repoPath)
+
+ before := untrustedMigrations(t)
+ commitAFile(t, ctx, client, repo, "untrusted", "file", "content\n")
+ require.InDelta(t, 1, untrustedMigrations(t)-before, 0, "the migration fsyncs the files of the quarantine")
+
+ gittest.Exec(t, cfg, "-C", repoPath, "cat-file", "-e", oid)
+ })
+}
+
+// TestUserCommitFiles_migrationWithoutAnUntrustedWriter is not parallel: it reads a global counter.
+// Without a custom hook, the migration trusts the writers of the quarantine.
+func TestUserCommitFiles_migrationWithoutAnUntrustedWriter(t *testing.T) {
+ testhelper.NewFeatureSets(featureflag.GPGSigning).Run(t, func(t *testing.T, ctx context.Context) {
+ ctx, cfg, client := setupOperationsService(t, ctx)
+ repo, _ := gittest.CreateRepository(t, ctx, cfg)
+
+ before := untrustedMigrations(t)
+ commitAFile(t, ctx, client, repo, "trusted", "file", "content\n")
+ require.InDelta(t, 0, untrustedMigrations(t)-before, 0, "the migration does not fsync the files of the quarantine")
+ })
+}