Git designed for AI agents.
storage, objectpool: Disconnect creates a removed loose object directory again
Federico Elia6f4b7d56f05e
diff --git a/internal/git/objectpool/disconnect.go b/internal/git/objectpool/disconnect.goindex 0692e31..4b746d1 100644--- a/internal/git/objectpool/disconnect.go+++ b/internal/git/objectpool/disconnect.go@@ -102,12 +102,18 @@ func Disconnect(ctx context.Context, f storage.FS, repo *localrepo.Repo, loggerreturn fmt.Errorf("source relative path: %w", err)}targetRelativePath := filepath.Join(repositoryRelativePath, "objects", path)+ targetDir := filepath.Dir(targetRelativePath)- if err := storage.MkdirAll(f, filepath.Dir(targetRelativePath)); err != nil {+ if err := storage.MkdirAll(f, targetDir); err != nil {return err}- if err := storage.Link(f, sourceRelativePath, targetRelativePath); err != nil {+ // Housekeeping of the repository may remove an empty loose object directory before+ // the link into it: it is created again.+ if err := storage.LinkRecreatingDirectory(ctx, filepath.Join(f.Root(), sourceRelativePath), filepath.Join(f.Root(), targetRelativePath), func() error {+ beforeLink(targetRelativePath)+ return storage.Link(f, sourceRelativePath, targetRelativePath)+ }); err != nil {if errors.Is(err, fs.ErrExist) {continue}@@ -133,6 +139,11 @@ func Disconnect(ctx context.Context, f storage.FS, repo *localrepo.Repo, loggerreturn removeAlternatesIfOk(ctx, repo, altFile, backupFile, logger, txManager)}+// beforeLink runs before each link of an object file of the pool into the repository, with the+// target path relative to the storage. Tests replace it to remove the target directory at that+// point, as housekeeping may.+var beforeLink = func(targetRelativePath string) {}+func findObjectFiles(altDir string) ([]string, error) {var objectFiles []stringif walkErr := filepath.Walk(altDir, func(path string, info os.FileInfo, err error) error {diff --git a/internal/git/objectpool/disconnect_fanout_test.go b/internal/git/objectpool/disconnect_fanout_test.gonew file mode 100644index 0000000..830a34b--- /dev/null+++ b/internal/git/objectpool/disconnect_fanout_test.go@@ -0,0 +1,98 @@+package objectpool++import (+ "os"+ "path/filepath"+ "testing"++ "github.com/feder-cr/git-ai/internal/git/gittest"+ "github.com/feder-cr/git-ai/internal/gitaly/storage/storagemgr/partition/fsrecorder"+ "github.com/feder-cr/git-ai/internal/gitaly/transaction"+ "github.com/feder-cr/git-ai/internal/testhelper"+ "github.com/feder-cr/git-ai/internal/testhelper/testcfg"+ "github.com/stretchr/testify/require"+)++// recordingWAL records the operations of a transaction's FS.+type recordingWAL struct {+ directories []string+ links []string+}++func (w *recordingWAL) CreateDirectory(relativePath string) { w.directories = append(w.directories, relativePath) }+func (w *recordingWAL) RemoveDirectoryEntry(string) {}+func (w *recordingWAL) CreateFile(string, string) error { return nil }+func (w *recordingWAL) CreateLink(sourcePath, destinationPath string) { w.links = append(w.links, destinationPath) }++// TestDisconnect_fanOutDirectoryRemoved removes each loose object directory of the repository right+// before the first links into it, three times, as housekeeping may (git prune-packed removes empty+// ones): each time after Disconnect created it, or created it again. Disconnect creates it again+// until the link holds, and records each directory and each link once through the transaction's+// FS, however many rounds a link took.+func TestDisconnect_fanOutDirectoryRemoved(t *testing.T) {+ ctx := testhelper.Context(t)+ cfg := testcfg.Build(t)+ logger := testhelper.SharedLogger(t)++ pool, repo := setupObjectPoolWithCfg(t, ctx, cfg)+ require.NoError(t, pool.Link(ctx, repo))+ poolPath := gittest.RepositoryPath(t, ctx, pool)+ repoPath := gittest.RepositoryPath(t, ctx, repo)++ // A packed commit and a loose one on top: both kinds of object file are linked.+ packed := gittest.WriteCommit(t, cfg, poolPath, gittest.WithTreeEntries(+ gittest.TreeEntry{Path: "packed", Mode: "100644", Content: "packed\n"},+ ))+ gittest.Exec(t, cfg, "-C", poolPath, "update-ref", "refs/heads/main", packed.String())+ gittest.Exec(t, cfg, "-C", poolPath, "repack", "-a", "-d")+ loose := gittest.WriteCommit(t, cfg, poolPath, gittest.WithParents(packed), gittest.WithTreeEntries(+ gittest.TreeEntry{Path: "loose", Mode: "100644", Content: "loose\n"},+ ))+ gittest.WriteRef(t, cfg, repoPath, "refs/heads/main", loose)+ looseDirs, err := filepath.Glob(filepath.Join(poolPath, "objects", "[0-9a-f][0-9a-f]"))+ require.NoError(t, err)+ require.NotEmpty(t, looseDirs)++ const removals = 3+ removed := map[string]int{}+ original := beforeLink+ beforeLink = func(targetRelativePath string) {+ dir := filepath.Join(cfg.Storages[0].Path, filepath.Dir(targetRelativePath))+ if filepath.Base(dir) == "pack" || removed[dir] == removals {+ return+ }+ removed[dir]+++ require.NoError(t, os.Remove(dir))+ }+ t.Cleanup(func() { beforeLink = original })++ wal := &recordingWAL{}+ require.NoError(t, Disconnect(ctx, fsrecorder.NewFS(cfg.Storages[0].Path, wal), repo, logger, transaction.NewManager(cfg, logger, nil)))++ require.Len(t, removed, len(looseDirs))+ for dir, times := range removed {+ require.Equal(t, removals, times, dir)+ }+ require.ElementsMatch(t, uniqueStrings(wal.directories), wal.directories, "a directory is recorded once")+ require.ElementsMatch(t, uniqueStrings(wal.links), wal.links, "a link is recorded once")+ for _, dir := range looseDirs {+ require.Contains(t, wal.directories, filepath.Join(repo.GetRelativePath(), "objects", filepath.Base(dir)))+ }++ require.NoFileExists(t, filepath.Join(repoPath, "objects", "info", "alternates"))+ gittest.Exec(t, cfg, "-C", repoPath, "cat-file", "-e", loose.String())+ gittest.Exec(t, cfg, "-C", repoPath, "cat-file", "-e", packed.String())+ gittest.Exec(t, cfg, "-C", repoPath, "fsck", "--strict")+}++func uniqueStrings(values []string) []string {+ seen := map[string]struct{}{}+ var unique []string+ for _, value := range values {+ if _, ok := seen[value]; !ok {+ seen[value] = struct{}{}+ unique = append(unique, value)+ }+ }+ return unique+}diff --git a/internal/git/quarantine/quarantine.go b/internal/git/quarantine/quarantine.goindex 4e632d1..b443a08 100644--- a/internal/git/quarantine/quarantine.go+++ b/internal/git/quarantine/quarantine.go@@ -334,30 +334,18 @@ func (m *migration) link(sourcePath, targetPath string) error {// it was removed (see migrate).func (m *migration) linkFile(sourcePath, targetPath string) error {dir := filepath.Dir(targetPath)- for {- if err := ensureDir(dir); err != nil {- return err- }+ if err := ensureDir(dir); err != nil {+ return err+ }+ if err := storage.LinkRecreatingDirectory(m.ctx, sourcePath, targetPath, func() error {beforeLink(targetPath)-- err := finalizeObjectFile(sourcePath, targetPath)- if err == nil {- return m.track(dir)- }- if !errors.Is(err, fs.ErrNotExist) {- return err- }-- if _, statErr := os.Lstat(sourcePath); statErr != nil {- return fmt.Errorf("%w (source: %w)", err, statErr)- }- if ctxErr := m.ctx.Err(); ctxErr != nil {- return fmt.Errorf("%w (%w)", err, ctxErr)- }-- // The source exists, so the target directory is gone: create it again and link again.+ return finalizeObjectFile(sourcePath, targetPath)+ }); err != nil {+ return err}++ return m.track(dir)}// beforeLink runs before each link of the first pass of migrate. Tests replace it to remove adiff --git a/internal/gitaly/storage/link.go b/internal/gitaly/storage/link.gonew file mode 100644index 0000000..ec8eb92--- /dev/null+++ b/internal/gitaly/storage/link.go@@ -0,0 +1,57 @@+package storage++import (+ "context"+ "errors"+ "fmt"+ "io/fs"+ "os"+ "path/filepath"++ "github.com/feder-cr/git-ai/internal/gitaly/storage/mode"+)++// LinkRecreatingDirectory runs link, which links the file at sourcePath to targetPath in a+// directory of an object directory; the caller created the directory of targetPath. Housekeeping+// (git prune-packed, repack -d, prune) removes empty loose object directories at any time, so the+// directory can vanish between its creation and the link. A link that fails with fs.ErrNotExist+// while the source still exists means that the directory is gone: it is created again and the+// link runs again. Once the link holds, the directory is not empty and stays. Each round needs+// another removal between the creation and the link, so the loop ends; it also ends with ctx. A+// directory that exists but is no directory fails the link at once.+//+// The directory is created again with os.Mkdir, not through a transaction's FS. Its creation by+// the caller is recorded already (or it existed in the snapshot), and its removal was no operation+// of the transaction: the recorded operations are the same however many rounds the link took. A+// transaction's snapshot is private, so housekeeping does not remove its directories in the first+// place.+func LinkRecreatingDirectory(ctx context.Context, sourcePath, targetPath string, link func() error) error {+ dir := filepath.Dir(targetPath)+ for {+ err := link()+ if err == nil || !errors.Is(err, fs.ErrNotExist) {+ return err+ }++ if _, statErr := os.Lstat(sourcePath); statErr != nil {+ return fmt.Errorf("%w (source: %w)", err, statErr)+ }+ if ctxErr := ctx.Err(); ctxErr != nil {+ return fmt.Errorf("%w (%w)", err, ctxErr)+ }++ // The source exists, so the directory is gone: create it again and link again. If it+ // exists, another writer created it again, unless it is no directory, which no round+ // would change.+ mkdirErr := os.Mkdir(dir, mode.Directory)+ if mkdirErr == nil {+ continue+ }+ if !errors.Is(mkdirErr, fs.ErrExist) {+ return fmt.Errorf("%w (creating %q again: %w)", err, dir, mkdirErr)+ }+ if info, statErr := os.Lstat(dir); statErr == nil && !info.IsDir() {+ return fmt.Errorf("%w (%q is no directory)", err, dir)+ }+ }+}diff --git a/internal/gitaly/storage/link_test.go b/internal/gitaly/storage/link_test.gonew file mode 100644index 0000000..3795e4a--- /dev/null+++ b/internal/gitaly/storage/link_test.go@@ -0,0 +1,74 @@+package storage++import (+ "os"+ "path/filepath"+ "testing"++ "github.com/feder-cr/git-ai/internal/gitaly/storage/mode"+ "github.com/feder-cr/git-ai/internal/testhelper"+ "github.com/stretchr/testify/require"+)++func TestLinkRecreatingDirectory(t *testing.T) {+ t.Parallel()++ setup := func(t *testing.T) (string, string) {+ root := t.TempDir()+ source := filepath.Join(root, "source")+ require.NoError(t, os.WriteFile(source, []byte("object"), mode.File))+ return source, filepath.Join(root, "objects", "0a", "object")+ }++ t.Run("a directory that vanishes again is created again until the link holds", func(t *testing.T) {+ t.Parallel()++ source, target := setup(t)+ require.NoError(t, os.MkdirAll(filepath.Dir(target), mode.Directory))++ links := 0+ require.NoError(t, LinkRecreatingDirectory(testhelper.Context(t), source, target, func() error {+ links+++ if links <= 3 {+ require.NoError(t, os.Remove(filepath.Dir(target)))+ }+ return os.Link(source, target)+ }))+ require.Equal(t, 4, links)+ require.FileExists(t, target)+ })++ t.Run("a missing source fails at once", func(t *testing.T) {+ t.Parallel()++ source, target := setup(t)+ require.NoError(t, os.MkdirAll(filepath.Dir(target), mode.Directory))+ require.NoError(t, os.Remove(source))++ links := 0+ err := LinkRecreatingDirectory(testhelper.Context(t), source, target, func() error {+ links+++ return os.Link(source, target)+ })+ require.ErrorIs(t, err, os.ErrNotExist)+ require.ErrorContains(t, err, "(source: ")+ require.Equal(t, 1, links)+ })++ t.Run("a directory replaced by a dangling symbolic link fails at once", func(t *testing.T) {+ t.Parallel()++ source, target := setup(t)+ require.NoError(t, os.MkdirAll(filepath.Dir(filepath.Dir(target)), mode.Directory))+ require.NoError(t, os.Symlink(filepath.Join(t.TempDir(), "gone"), filepath.Dir(target)))++ links := 0+ err := LinkRecreatingDirectory(testhelper.Context(t), source, target, func() error {+ links+++ return os.Link(source, target)+ })+ require.ErrorIs(t, err, os.ErrNotExist)+ require.ErrorContains(t, err, "is no directory")+ require.Equal(t, 1, links)+ })+}