Skip to content

Git designed for AI agents.

storage, objectpool: Disconnect creates a removed loose object directory again

Federico Elia6f4b7d56f05e
diff --git a/internal/git/objectpool/disconnect.go b/internal/git/objectpool/disconnect.go
index 0692e31..4b746d1 100644
--- a/internal/git/objectpool/disconnect.go
+++ b/internal/git/objectpool/disconnect.go
@@ -102,12 +102,18 @@ func Disconnect(ctx context.Context, f storage.FS, repo *localrepo.Repo, logger
return fmt.Errorf("source relative path: %w", err)
}
targetRelativePath := filepath.Join(repositoryRelativePath, "objects", path)
+ targetDir := filepath.Dir(targetRelativePath)
- if err := storage.MkdirAll(f, filepath.Dir(targetRelativePath)); err != nil {
+ if err := storage.MkdirAll(f, targetDir); err != nil {
return err
}
- if err := storage.Link(f, sourceRelativePath, targetRelativePath); err != nil {
+ // Housekeeping of the repository may remove an empty loose object directory before
+ // the link into it: it is created again.
+ if err := storage.LinkRecreatingDirectory(ctx, filepath.Join(f.Root(), sourceRelativePath), filepath.Join(f.Root(), targetRelativePath), func() error {
+ beforeLink(targetRelativePath)
+ return storage.Link(f, sourceRelativePath, targetRelativePath)
+ }); err != nil {
if errors.Is(err, fs.ErrExist) {
continue
}
@@ -133,6 +139,11 @@ func Disconnect(ctx context.Context, f storage.FS, repo *localrepo.Repo, logger
return removeAlternatesIfOk(ctx, repo, altFile, backupFile, logger, txManager)
}
+// beforeLink runs before each link of an object file of the pool into the repository, with the
+// target path relative to the storage. Tests replace it to remove the target directory at that
+// point, as housekeeping may.
+var beforeLink = func(targetRelativePath string) {}
+
func findObjectFiles(altDir string) ([]string, error) {
var objectFiles []string
if walkErr := filepath.Walk(altDir, func(path string, info os.FileInfo, err error) error {
diff --git a/internal/git/objectpool/disconnect_fanout_test.go b/internal/git/objectpool/disconnect_fanout_test.go
new file mode 100644
index 0000000..830a34b
--- /dev/null
+++ b/internal/git/objectpool/disconnect_fanout_test.go
@@ -0,0 +1,98 @@
+package objectpool
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/feder-cr/git-ai/internal/git/gittest"
+ "github.com/feder-cr/git-ai/internal/gitaly/storage/storagemgr/partition/fsrecorder"
+ "github.com/feder-cr/git-ai/internal/gitaly/transaction"
+ "github.com/feder-cr/git-ai/internal/testhelper"
+ "github.com/feder-cr/git-ai/internal/testhelper/testcfg"
+ "github.com/stretchr/testify/require"
+)
+
+// recordingWAL records the operations of a transaction's FS.
+type recordingWAL struct {
+ directories []string
+ links []string
+}
+
+func (w *recordingWAL) CreateDirectory(relativePath string) { w.directories = append(w.directories, relativePath) }
+func (w *recordingWAL) RemoveDirectoryEntry(string) {}
+func (w *recordingWAL) CreateFile(string, string) error { return nil }
+func (w *recordingWAL) CreateLink(sourcePath, destinationPath string) { w.links = append(w.links, destinationPath) }
+
+// TestDisconnect_fanOutDirectoryRemoved removes each loose object directory of the repository right
+// before the first links into it, three times, as housekeeping may (git prune-packed removes empty
+// ones): each time after Disconnect created it, or created it again. Disconnect creates it again
+// until the link holds, and records each directory and each link once through the transaction's
+// FS, however many rounds a link took.
+func TestDisconnect_fanOutDirectoryRemoved(t *testing.T) {
+ ctx := testhelper.Context(t)
+ cfg := testcfg.Build(t)
+ logger := testhelper.SharedLogger(t)
+
+ pool, repo := setupObjectPoolWithCfg(t, ctx, cfg)
+ require.NoError(t, pool.Link(ctx, repo))
+ poolPath := gittest.RepositoryPath(t, ctx, pool)
+ repoPath := gittest.RepositoryPath(t, ctx, repo)
+
+ // A packed commit and a loose one on top: both kinds of object file are linked.
+ packed := gittest.WriteCommit(t, cfg, poolPath, gittest.WithTreeEntries(
+ gittest.TreeEntry{Path: "packed", Mode: "100644", Content: "packed\n"},
+ ))
+ gittest.Exec(t, cfg, "-C", poolPath, "update-ref", "refs/heads/main", packed.String())
+ gittest.Exec(t, cfg, "-C", poolPath, "repack", "-a", "-d")
+ loose := gittest.WriteCommit(t, cfg, poolPath, gittest.WithParents(packed), gittest.WithTreeEntries(
+ gittest.TreeEntry{Path: "loose", Mode: "100644", Content: "loose\n"},
+ ))
+ gittest.WriteRef(t, cfg, repoPath, "refs/heads/main", loose)
+ looseDirs, err := filepath.Glob(filepath.Join(poolPath, "objects", "[0-9a-f][0-9a-f]"))
+ require.NoError(t, err)
+ require.NotEmpty(t, looseDirs)
+
+ const removals = 3
+ removed := map[string]int{}
+ original := beforeLink
+ beforeLink = func(targetRelativePath string) {
+ dir := filepath.Join(cfg.Storages[0].Path, filepath.Dir(targetRelativePath))
+ if filepath.Base(dir) == "pack" || removed[dir] == removals {
+ return
+ }
+ removed[dir]++
+ require.NoError(t, os.Remove(dir))
+ }
+ t.Cleanup(func() { beforeLink = original })
+
+ wal := &recordingWAL{}
+ require.NoError(t, Disconnect(ctx, fsrecorder.NewFS(cfg.Storages[0].Path, wal), repo, logger, transaction.NewManager(cfg, logger, nil)))
+
+ require.Len(t, removed, len(looseDirs))
+ for dir, times := range removed {
+ require.Equal(t, removals, times, dir)
+ }
+ require.ElementsMatch(t, uniqueStrings(wal.directories), wal.directories, "a directory is recorded once")
+ require.ElementsMatch(t, uniqueStrings(wal.links), wal.links, "a link is recorded once")
+ for _, dir := range looseDirs {
+ require.Contains(t, wal.directories, filepath.Join(repo.GetRelativePath(), "objects", filepath.Base(dir)))
+ }
+
+ require.NoFileExists(t, filepath.Join(repoPath, "objects", "info", "alternates"))
+ gittest.Exec(t, cfg, "-C", repoPath, "cat-file", "-e", loose.String())
+ gittest.Exec(t, cfg, "-C", repoPath, "cat-file", "-e", packed.String())
+ gittest.Exec(t, cfg, "-C", repoPath, "fsck", "--strict")
+}
+
+func uniqueStrings(values []string) []string {
+ seen := map[string]struct{}{}
+ var unique []string
+ for _, value := range values {
+ if _, ok := seen[value]; !ok {
+ seen[value] = struct{}{}
+ unique = append(unique, value)
+ }
+ }
+ return unique
+}
diff --git a/internal/git/quarantine/quarantine.go b/internal/git/quarantine/quarantine.go
index 4e632d1..b443a08 100644
--- a/internal/git/quarantine/quarantine.go
+++ b/internal/git/quarantine/quarantine.go
@@ -334,30 +334,18 @@ func (m *migration) link(sourcePath, targetPath string) error {
// it was removed (see migrate).
func (m *migration) linkFile(sourcePath, targetPath string) error {
dir := filepath.Dir(targetPath)
- for {
- if err := ensureDir(dir); err != nil {
- return err
- }
+ if err := ensureDir(dir); err != nil {
+ return err
+ }
+ if err := storage.LinkRecreatingDirectory(m.ctx, sourcePath, targetPath, func() error {
beforeLink(targetPath)
-
- err := finalizeObjectFile(sourcePath, targetPath)
- if err == nil {
- return m.track(dir)
- }
- if !errors.Is(err, fs.ErrNotExist) {
- return err
- }
-
- if _, statErr := os.Lstat(sourcePath); statErr != nil {
- return fmt.Errorf("%w (source: %w)", err, statErr)
- }
- if ctxErr := m.ctx.Err(); ctxErr != nil {
- return fmt.Errorf("%w (%w)", err, ctxErr)
- }
-
- // The source exists, so the target directory is gone: create it again and link again.
+ return finalizeObjectFile(sourcePath, targetPath)
+ }); err != nil {
+ return err
}
+
+ return m.track(dir)
}
// beforeLink runs before each link of the first pass of migrate. Tests replace it to remove a
diff --git a/internal/gitaly/storage/link.go b/internal/gitaly/storage/link.go
new file mode 100644
index 0000000..ec8eb92
--- /dev/null
+++ b/internal/gitaly/storage/link.go
@@ -0,0 +1,57 @@
+package storage
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io/fs"
+ "os"
+ "path/filepath"
+
+ "github.com/feder-cr/git-ai/internal/gitaly/storage/mode"
+)
+
+// LinkRecreatingDirectory runs link, which links the file at sourcePath to targetPath in a
+// directory of an object directory; the caller created the directory of targetPath. Housekeeping
+// (git prune-packed, repack -d, prune) removes empty loose object directories at any time, so the
+// directory can vanish between its creation and the link. A link that fails with fs.ErrNotExist
+// while the source still exists means that the directory is gone: it is created again and the
+// link runs again. Once the link holds, the directory is not empty and stays. Each round needs
+// another removal between the creation and the link, so the loop ends; it also ends with ctx. A
+// directory that exists but is no directory fails the link at once.
+//
+// The directory is created again with os.Mkdir, not through a transaction's FS. Its creation by
+// the caller is recorded already (or it existed in the snapshot), and its removal was no operation
+// of the transaction: the recorded operations are the same however many rounds the link took. A
+// transaction's snapshot is private, so housekeeping does not remove its directories in the first
+// place.
+func LinkRecreatingDirectory(ctx context.Context, sourcePath, targetPath string, link func() error) error {
+ dir := filepath.Dir(targetPath)
+ for {
+ err := link()
+ if err == nil || !errors.Is(err, fs.ErrNotExist) {
+ return err
+ }
+
+ if _, statErr := os.Lstat(sourcePath); statErr != nil {
+ return fmt.Errorf("%w (source: %w)", err, statErr)
+ }
+ if ctxErr := ctx.Err(); ctxErr != nil {
+ return fmt.Errorf("%w (%w)", err, ctxErr)
+ }
+
+ // The source exists, so the directory is gone: create it again and link again. If it
+ // exists, another writer created it again, unless it is no directory, which no round
+ // would change.
+ mkdirErr := os.Mkdir(dir, mode.Directory)
+ if mkdirErr == nil {
+ continue
+ }
+ if !errors.Is(mkdirErr, fs.ErrExist) {
+ return fmt.Errorf("%w (creating %q again: %w)", err, dir, mkdirErr)
+ }
+ if info, statErr := os.Lstat(dir); statErr == nil && !info.IsDir() {
+ return fmt.Errorf("%w (%q is no directory)", err, dir)
+ }
+ }
+}
diff --git a/internal/gitaly/storage/link_test.go b/internal/gitaly/storage/link_test.go
new file mode 100644
index 0000000..3795e4a
--- /dev/null
+++ b/internal/gitaly/storage/link_test.go
@@ -0,0 +1,74 @@
+package storage
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/feder-cr/git-ai/internal/gitaly/storage/mode"
+ "github.com/feder-cr/git-ai/internal/testhelper"
+ "github.com/stretchr/testify/require"
+)
+
+func TestLinkRecreatingDirectory(t *testing.T) {
+ t.Parallel()
+
+ setup := func(t *testing.T) (string, string) {
+ root := t.TempDir()
+ source := filepath.Join(root, "source")
+ require.NoError(t, os.WriteFile(source, []byte("object"), mode.File))
+ return source, filepath.Join(root, "objects", "0a", "object")
+ }
+
+ t.Run("a directory that vanishes again is created again until the link holds", func(t *testing.T) {
+ t.Parallel()
+
+ source, target := setup(t)
+ require.NoError(t, os.MkdirAll(filepath.Dir(target), mode.Directory))
+
+ links := 0
+ require.NoError(t, LinkRecreatingDirectory(testhelper.Context(t), source, target, func() error {
+ links++
+ if links <= 3 {
+ require.NoError(t, os.Remove(filepath.Dir(target)))
+ }
+ return os.Link(source, target)
+ }))
+ require.Equal(t, 4, links)
+ require.FileExists(t, target)
+ })
+
+ t.Run("a missing source fails at once", func(t *testing.T) {
+ t.Parallel()
+
+ source, target := setup(t)
+ require.NoError(t, os.MkdirAll(filepath.Dir(target), mode.Directory))
+ require.NoError(t, os.Remove(source))
+
+ links := 0
+ err := LinkRecreatingDirectory(testhelper.Context(t), source, target, func() error {
+ links++
+ return os.Link(source, target)
+ })
+ require.ErrorIs(t, err, os.ErrNotExist)
+ require.ErrorContains(t, err, "(source: ")
+ require.Equal(t, 1, links)
+ })
+
+ t.Run("a directory replaced by a dangling symbolic link fails at once", func(t *testing.T) {
+ t.Parallel()
+
+ source, target := setup(t)
+ require.NoError(t, os.MkdirAll(filepath.Dir(filepath.Dir(target)), mode.Directory))
+ require.NoError(t, os.Symlink(filepath.Join(t.TempDir(), "gone"), filepath.Dir(target)))
+
+ links := 0
+ err := LinkRecreatingDirectory(testhelper.Context(t), source, target, func() error {
+ links++
+ return os.Link(source, target)
+ })
+ require.ErrorIs(t, err, os.ErrNotExist)
+ require.ErrorContains(t, err, "is no directory")
+ require.Equal(t, 1, links)
+ })
+}