Git designed for AI agents.
corestore: fixes from the review of the fsync changes
Trigodil65cd52aae3b8
diff --git a/doc/graft/PLAN7.md b/doc/graft/PLAN7.mdindex 652b337..e8a6292 100644--- a/doc/graft/PLAN7.md+++ b/doc/graft/PLAN7.md@@ -716,8 +716,8 @@ repository-wide lock.and nothing is live, the step clears the marker:* It takes git's lock without waiting. The lock is created by linking atemporary file that holds a tag and a random nonce.- * It fsyncs what was written since, marks the state clean, and releases- the lock. The release removes the lock only if it is still the same+ * It fsyncs what was written since, then the watermark and its directory,+ marks the state clean, and releases the lock. The release removes the lock only if it is still the samefile with the same nonce: an inode is reused as soon as it is freed, soa lock that someone else replaced is never removed.* Last, it removes the marker, under a per-repository file lock that@@ -726,6 +726,19 @@ repository-wide lock.* The set of fsynced tables keeps only the tables the stack still lists.* A graceful stop runs a last step for every dirty repository after theservers stopped, so a clean reboot does not rehydrate them.+ * *Watermark.* While a repository of the owner is dirty and has a reftable+ stack, the syncer writes its watermark (`graft/storage/applied`) atomically+ without fsync (`CanDefer`).+ * After a kernel restart a dirty cache is moved aside whatever its watermark+ holds, and after a process restart the page cache kept the write. An+ advance only changes the sequence; the repository ID was written with+ fsync by the hydration, so a lost or torn advance leaves the recovery's+ decision as it was.+ * The step that clears the marker under git's lock, and `Forget` of a+ repository kept in place, fsync the watermark and its directory first. A+ failed fsync makes the repository untrusted, as for the stack.+ * Hydration, import and the gate write it with fsync, as does a write+ whose directory is missing.* *Write failure.* Only a failed fsync means the kernel may have droppedwrites, and the kernel reports a failed writeback once: a later fsync of thesame file succeeds.diff --git a/doc/graft/STORAGE.md b/doc/graft/STORAGE.mdindex 76dfea9..b5cae69 100644--- a/doc/graft/STORAGE.md+++ b/doc/graft/STORAGE.md@@ -591,7 +591,9 @@ or re-import the repository from scratch (seethem durable in a background step a second after a repository goes idle(`doc/graft/PLAN7.md`, LOCAL REFERENCE WRITES RULE). Until then therepository has a dirty marker in `<storage>/+gitaly/graft-ref-durability/`,- which holds the kernel's boot identity.+ which holds the kernel's boot identity. The watermark of a dirty repository is+ written without fsync too, and the background step makes it durable before it+ removes the marker.- *Process crash or restart* (same kernel boot): nothing is lost, because thepage cache kept every write. The restarted node settles the markers.- *Kernel crash or power loss*: the references written since the last durablediff --git a/internal/cli/gitaly/serve.go b/internal/cli/gitaly/serve.goindex 2ee5e8b..99eca7b 100644--- a/internal/cli/gitaly/serve.go+++ b/internal/cli/gitaly/serve.go@@ -990,7 +990,7 @@ func openLocalReferenceWrites(core *corestore.Core, locator storage.Locator, logcore.RefAdmission = localrefs.NewAdmission(lockTimeout, lockTimeout)prometheus.MustRegister(core.RefAdmission)- durability, err := localrefs.NewDurability(locator, localrefs.DurableStepIdle, localrefs.WithWatermark(gitops.WatermarkPath))+ durability, err := localrefs.NewDurability(locator, localrefs.DurableStepIdle, localrefs.WithWmark(gitops.WatermarkPath))if err != nil {return fmt.Errorf("graft storage core: %w", err)}diff --git a/internal/graft/corestore/blobstore/blobstore.go b/internal/graft/corestore/blobstore/blobstore.goindex 30c7356..4524784 100644--- a/internal/graft/corestore/blobstore/blobstore.go+++ b/internal/graft/corestore/blobstore/blobstore.go@@ -36,8 +36,9 @@// anything, and a changeset row in the database (synchronous=FULL) could// outlive a pack whose data never reached the disk. For file buckets Put fsyncs// the content before the link publishes it, and fsyncs the directories that-// hold the new entry (up to the bucket root) after it. The mem and remote-// drivers commit their objects durably themselves.+// hold the new entry after it, up to the first one whose whole chain is durable.+// Bucket directories are never removed while a server uses the bucket: a recreated one could reuse an inode.+// The mem and remote drivers commit their objects durably themselves.//// Listing. List reports objects only. fileblob also reports a writer's// temporary file (<key>.<nanos>.tmp, in the key's directory because file@@ -319,21 +320,17 @@ func syncFile(f *os.File) error {return f.Sync()}-// Fsyncs dir and its ancestors inside the bucket root, up to the first one an earlier-// call made durable that is still the same directory.+// Fsyncs dir and its ancestors inside the bucket root, up to the first one whose whole chain an+// earlier call made durable. The walk caches its directories only once it reached that point.func (s *Store) syncDirs(dir string) error {inside := dir == s.root || strings.HasPrefix(dir, s.root+string(filepath.Separator))- var child string- var childInfo os.FileInfo+ visited := map[string]os.FileInfo{}for d := dir; ; d = filepath.Dir(d) {if err := syncDir(d); err != nil {return err}- if child != "" {- s.durableDirs.Store(child, childInfo)- }if !inside || d == s.root {- return nil+ break}info, err := os.Stat(d)@@ -341,10 +338,15 @@ func (s *Store) syncDirs(dir string) error {return fmt.Errorf("stat directory: %w", err)}if known, ok := s.durableDirs.Load(d); ok && os.SameFile(known.(os.FileInfo), info) {- return nil+ break}- child, childInfo = d, info+ visited[d] = info}++ for d, info := range visited {+ s.durableDirs.Store(d, info)+ }+ return nil}// syncDir fsyncs a directory so that the entries created in it are durable.diff --git a/internal/graft/corestore/blobstore/durable_internal_test.go b/internal/graft/corestore/blobstore/durable_internal_test.goindex b52da3a..859983c 100644--- a/internal/graft/corestore/blobstore/durable_internal_test.go+++ b/internal/graft/corestore/blobstore/durable_internal_test.go@@ -8,6 +8,7 @@ import ("net/url""os""path/filepath"+ "sync""testing""github.com/feder-cr/git-ai/internal/graft/corestore"@@ -70,7 +71,7 @@ func TestPutSyncsFileBeforeCommitAndSyncsDirectories(t *testing.T) {require.Equal(t, content, data)}-func TestPutSyncsOnlyTheDirectoriesWhoseEntriesAreNotDurable(t *testing.T) {+func TestDirs(t *testing.T) {dir := t.TempDir()var dirSyncs []strings := openFileStoreForTest(t, dir, func(path string, isDir bool) error {@@ -103,6 +104,77 @@ func TestPutSyncsOnlyTheDirectoriesWhoseEntriesAreNotDurable(t *testing.T) {require.Equal(t, []string{packsDir, repoDir, filepath.Join(dir, "repos")}, dirSyncs)}+func TestDirErr(t *testing.T) {+ dir := t.TempDir()+ reposDir := filepath.Join(dir, "repos")+ fail := true+ var dirSyncs []string+ s := openFileStoreForTest(t, dir, func(path string, isDir bool) error {+ if !isDir {+ return nil+ }+ if fail && path == reposDir {+ return errors.New("input/output error")+ }+ dirSyncs = append(dirSyncs, path)+ return nil+ })++ content := []byte("pack content")+ require.Error(t, s.Put(testhelper.Context(t), corestore.NewPackKey(1, sha256.Sum256(content)), bytes.NewReader(content)))++ fail = false+ dirSyncs = nil+ require.NoError(t, s.Put(testhelper.Context(t), corestore.NewPackKey(1, sha256.Sum256(content)), bytes.NewReader(content)))+ repoDir := filepath.Join(reposDir, "1")+ require.Equal(t, []string{filepath.Join(repoDir, "packs"), repoDir, reposDir, dir}, dirSyncs)+}++func TestConcur(t *testing.T) {+ dir := t.TempDir()+ reposDir := filepath.Join(dir, "repos")+ reached, release := make(chan struct{}), make(chan struct{})+ var mu sync.Mutex+ held := false+ var dirSyncs []string+ s := openFileStoreForTest(t, dir, func(path string, isDir bool) error {+ if !isDir {+ return nil+ }+ mu.Lock()+ hold := !held && path == reposDir+ held = held || hold+ dirSyncs = append(dirSyncs, path)+ mu.Unlock()+ if hold {+ close(reached)+ <-release+ }+ return nil+ })+ put := func() error {+ content := []byte("pack content")+ return s.Put(testhelper.Context(t), corestore.NewPackKey(1, sha256.Sum256(content)), bytes.NewReader(content))+ }++ // The first Put is held while it fsyncs repos/, after packs/ and repos/1/.+ first := make(chan error, 1)+ go func() { first <- put() }()+ <-reached+ mu.Lock()+ dirSyncs = nil+ mu.Unlock()++ require.NoError(t, put())+ repoDir := filepath.Join(reposDir, "1")+ mu.Lock()+ require.Equal(t, []string{filepath.Join(repoDir, "packs"), repoDir, reposDir, dir}, dirSyncs)+ mu.Unlock()++ close(release)+ require.NoError(t, <-first)+}+func TestPutFailedFileSyncLeavesNoObject(t *testing.T) {dir := t.TempDir()content := []byte("pack content whose sync fails")diff --git a/internal/graft/corestore/coretest/coretest.go b/internal/graft/corestore/coretest/coretest.goindex 612c55b..7daa396 100644--- a/internal/graft/corestore/coretest/coretest.go+++ b/internal/graft/corestore/coretest/coretest.go@@ -213,7 +213,7 @@ func EnableLocalRefs(tb testing.TB, core *corestore.Core, cfg config.Cfg, opts .require.NoError(tb, core.RefDurability.Close())}core.RefAdmission = localrefs.NewAdmission(10*time.Second, 10*time.Second)- durability, err := localrefs.NewDurability(config.NewLocator(cfg), 0, append([]localrefs.DurabilityOption{localrefs.WithBootID(TestBootID), localrefs.WithRecoveryGrace(0), localrefs.WithWatermark(gitops.WatermarkPath)}, opts...)...)+ durability, err := localrefs.NewDurability(config.NewLocator(cfg), 0, append([]localrefs.DurabilityOption{localrefs.WithBootID(TestBootID), localrefs.WithRecoveryGrace(0), localrefs.WithWmark(gitops.WatermarkPath)}, opts...)...)require.NoError(tb, err)// A git process forked at the moment a previous durability closed its lock// holds a copy of it until it execs (close-on-exec): the lock can be busy fordiff --git a/internal/graft/corestore/gitops/gitops.go b/internal/graft/corestore/gitops/gitops.goindex 1199b3c..97d8df7 100644--- a/internal/graft/corestore/gitops/gitops.go+++ b/internal/graft/corestore/gitops/gitops.go@@ -897,15 +897,18 @@ func WriteWatermark(repoPath string, w corestore.Watermark) error {if err := os.MkdirAll(dir, mode.Directory); err != nil {return fmt.Errorf("write watermark: %w", err)}- return writeWatermark(repoPath, w, true)+ return writeWMark(repoPath, w, true)}-// WriteWatermarkNoSync writes the watermark without fsync, into an existing directory.-func WriteWatermarkNoSync(repoPath string, w corestore.Watermark) error {- return writeWatermark(repoPath, w, false)+// WriteLazy writes the watermark without fsync, or with fsync when its directory is missing.+func WriteLazy(repoPath string, w corestore.Watermark) error {+ if _, err := os.Stat(filepath.Dir(WatermarkPath(repoPath))); err != nil {+ return WriteWatermark(repoPath, w)+ }+ return writeWMark(repoPath, w, false)}-func writeWatermark(repoPath string, w corestore.Watermark, sync bool) (returnedErr error) {+func writeWMark(repoPath string, w corestore.Watermark, sync bool) (returnedErr error) {dir := filepath.Dir(WatermarkPath(repoPath))data, err := json.Marshal(w)diff --git a/internal/graft/corestore/gitops/gitops_test.go b/internal/graft/corestore/gitops/gitops_test.goindex a82bdcd..d232c72 100644--- a/internal/graft/corestore/gitops/gitops_test.go+++ b/internal/graft/corestore/gitops/gitops_test.go@@ -533,14 +533,15 @@ func TestWatermarkRoundTrip(t *testing.T) {require.Equal(t, "applied", entries[0].Name())}-func TestWriteWatermarkNoSync(t *testing.T) {+func TestNoSync(t *testing.T) {t.Parallel()repoPath := t.TempDir()- require.Error(t, gitops.WriteWatermarkNoSync(repoPath, corestore.Watermark{RepoID: 7, Seq: 1}), "it does not create the directory")+ // Without its directory the watermark is written as WriteWatermark does.+ require.NoError(t, gitops.WriteLazy(repoPath, corestore.Watermark{RepoID: 7, Seq: 1}))+ require.FileExists(t, gitops.WatermarkPath(repoPath))- require.NoError(t, gitops.WriteWatermark(repoPath, corestore.Watermark{RepoID: 7, Seq: 1}))- require.NoError(t, gitops.WriteWatermarkNoSync(repoPath, corestore.Watermark{RepoID: 7, Seq: 2}))+ require.NoError(t, gitops.WriteLazy(repoPath, corestore.Watermark{RepoID: 7, Seq: 2}))w, ok, err := gitops.ReadWatermark(repoPath)require.NoError(t, err)require.True(t, ok)diff --git a/internal/graft/corestore/localrefs/durability.go b/internal/graft/corestore/localrefs/durability.goindex 0071537..83fee58 100644--- a/internal/graft/corestore/localrefs/durability.go+++ b/internal/graft/corestore/localrefs/durability.go@@ -278,8 +278,8 @@ func WithSync(sync func(f *os.File) error) DurabilityOption {return func(d *Durability) { d.sync = sync }}-// WithWatermark lets a dirty repository's watermark be written without fsync.-func WithWatermark(path func(repoPath string) string) DurabilityOption {+// WithWmark lets a dirty repository's watermark be written without fsync.+func WithWmark(path func(repoPath string) string) DurabilityOption {return func(d *Durability) { d.watermark = path }}@@ -602,13 +602,18 @@ func (d *Durability) Dirty() []string {return paths}-// DefersWatermark reports whether the watermark at path may be written without fsync now: the marker+// CanDefer reports whether the watermark at path may be written without fsync now: the marker// is durable and goes only after the watermark. The caller holds the syncer's repository lock.-func (d *Durability) DefersWatermark(path string) bool {+func (d *Durability) CanDefer(path string) bool {if !d.Deferring() || d.watermark == nil {return false}+ // A marker without a stack goes without a durable step, so its watermark is written with fsync.+ if stack, err := hasStack(path); err != nil || !stack {+ return false+ }+d.mu.Lock()defer d.mu.Unlock()st := d.repos[path]@@ -719,7 +724,7 @@ func (d *Durability) settleLocked(path string, repo RepoRef) error {st := d.lockState(path)defer st.file.Unlock()- if err := d.syncStackAndWatermark(path, st); err != nil {+ if err := d.syncAll(path, st); err != nil {return d.stepErrorLocked(st, repo, path, err)}if d.crashAt == crashDurableStep {@@ -1086,7 +1091,7 @@ func (d *Durability) syncStack(path string, st *refState) error {}// Makes the stack, then the watermark file and its directory, durable.-func (d *Durability) syncStackAndWatermark(path string, st *refState) error {+func (d *Durability) syncAll(path string, st *refState) error {if err := d.syncStack(path, st); err != nil {return err}@@ -1205,7 +1210,7 @@ func (d *Durability) forgetLocked(path string, st *refState, repo RepoRef) (boolreturn false, err}dropped := false- if err = d.syncStackAndWatermark(path, st); err != nil {+ if err = d.syncAll(path, st); err != nil {err = d.stepErrorLocked(st, repo, path, err)} else {dropped, err = d.dropMarker(path, st, true)diff --git a/internal/graft/corestore/localrefs/durability_test.go b/internal/graft/corestore/localrefs/durability_test.goindex 7d0ef1a..49bf6c5 100644--- a/internal/graft/corestore/localrefs/durability_test.go+++ b/internal/graft/corestore/localrefs/durability_test.go@@ -405,11 +405,11 @@ func TestSettleReportsAFailedWriteAsUntrusted(t *testing.T) {require.Equal(t, 0.0, testutil.ToFloat64(d.dirty))}-func testWatermarkPath(repoPath string) string {+func wmPath(repoPath string) string {return filepath.Join(repoPath, "graft", "storage", "applied")}-func TestDefersWatermark(t *testing.T) {+func TestDefers(t *testing.T) {t.Parallel()t.Run("without the option", func(t *testing.T) {@@ -418,59 +418,69 @@ func TestDefersWatermark(t *testing.T) {d := f.durability(t, "boot-a", 0)require.NoError(t, d.MarkDirty(f.ctx, f.repo, f.path))- require.False(t, d.DefersWatermark(f.path))+ require.False(t, d.CanDefer(f.path))})t.Run("while the repository is dirty", func(t *testing.T) {t.Parallel()f := setupDurability(t)- d := f.durability(t, "boot-a", 0, WithWatermark(testWatermarkPath))+ d := f.durability(t, "boot-a", 0, WithWmark(wmPath))- require.False(t, d.DefersWatermark(f.path), "a clean repository")+ require.False(t, d.CanDefer(f.path), "a clean repository")require.NoError(t, d.MarkDirty(f.ctx, f.repo, f.path))- require.True(t, d.DefersWatermark(f.path))+ require.True(t, d.CanDefer(f.path))require.NoError(t, d.Settle(f.path, false))- require.False(t, d.DefersWatermark(f.path), "the marker is gone")+ require.False(t, d.CanDefer(f.path), "the marker is gone")})t.Run("an untrusted repository", func(t *testing.T) {t.Parallel()f := setupDurability(t)- d := f.durability(t, "boot-a", 0, WithWatermark(testWatermarkPath))+ d := f.durability(t, "boot-a", 0, WithWmark(wmPath))require.NoError(t, d.MarkDirty(f.ctx, f.repo, f.path))f.synced.fail(errors.New("input/output error"))require.ErrorIs(t, d.Settle(f.path, false), ErrUntrusted)- require.False(t, d.DefersWatermark(f.path))+ require.False(t, d.CanDefer(f.path))+ })++ t.Run("a repository without a stack", func(t *testing.T) {+ t.Parallel()+ f := setupDurability(t)+ d := f.durability(t, "boot-a", 0, WithWmark(wmPath))++ require.NoError(t, d.MarkDirty(f.ctx, f.repo, f.path))+ require.NoError(t, os.Rename(filepath.Join(f.path, "reftable"), filepath.Join(t.TempDir(), "reftable")))+ require.False(t, d.CanDefer(f.path))})t.Run("a process that does not own the storage", func(t *testing.T) {t.Parallel()f := setupDurability(t)- d := f.unowned(t, "boot-a", 0, WithWatermark(testWatermarkPath))+ d := f.unowned(t, "boot-a", 0, WithWmark(wmPath))- require.False(t, d.DefersWatermark(f.path))+ require.False(t, d.CanDefer(f.path))})t.Run("a nil durability", func(t *testing.T) {t.Parallel()var d *Durability- require.False(t, d.DefersWatermark("/any"))+ require.False(t, d.CanDefer("/any"))})}-func TestSettleMakesTheWatermarkDurableBeforeTheMarkerGoes(t *testing.T) {+func TestSettle(t *testing.T) {t.Parallel()setup := func(t *testing.T) (durabilityFixture, *Durability, string, *bool) {f := setupDurability(t)- watermark := testWatermarkPath(f.path)+ watermark := wmPath(f.path)require.NoError(t, os.MkdirAll(filepath.Dir(watermark), 0o755))require.NoError(t, os.WriteFile(watermark, []byte(`{"repo_id":1,"seq":2}`), 0o644))var markerAtSync boolvar d *Durability- d = f.durability(t, "boot-a", 0, WithWatermark(testWatermarkPath), WithSync(func(file *os.File) error {+ d = f.durability(t, "boot-a", 0, WithWmark(wmPath), WithSync(func(file *os.File) error {if file.Name() == watermark {_, err := os.Stat(f.markerPath(t, d))markerAtSync = err == nildiff --git a/internal/graft/corestore/syncer/syncer.go b/internal/graft/corestore/syncer/syncer.goindex 8995b25..ab7a482 100644--- a/internal/graft/corestore/syncer/syncer.go+++ b/internal/graft/corestore/syncer/syncer.go@@ -885,7 +885,7 @@ func (s *Syncer) advanceLocked(path string, wm corestore.Watermark) corestore.Wa}next := corestore.Watermark{RepoID: wm.RepoID, Seq: newSeq}- if err := s.writeWatermarkLocked(path, next); err != nil {+ if err := s.writeWM(path, next); err != nil {s.logger.WithError(err).WithField("repository_path", path).Error("storage core: write watermark")return wm}@@ -1325,17 +1325,17 @@ func (s *Syncer) applyLocked(ctx context.Context, repo *localrepo.Repo, hash git// reached. The caller holds the repository lock.func (s *Syncer) commitWatermarkLocked(path string, repoID int64, seq uint64) (corestore.Watermark, error) {wm := corestore.Watermark{RepoID: repoID, Seq: seq}- if err := s.writeWatermarkLocked(path, wm); err != nil {+ if err := s.writeWM(path, wm); err != nil {return corestore.Watermark{}, structerr.NewInternal("storage core: write watermark: %w", err)}return s.advanceLocked(path, wm), nil}// Writes the watermark, without fsync while the repository is dirty (caller holds the lock).-func (s *Syncer) writeWatermarkLocked(path string, wm corestore.Watermark) error {- if s.core.RefDurability.DefersWatermark(path) {+func (s *Syncer) writeWM(path string, wm corestore.Watermark) error {+ if s.core.RefDurability.CanDefer(path) {s.hook("watermark deferred")- return gitops.WriteWatermarkNoSync(path, wm)+ return gitops.WriteLazy(path, wm)}s.hook("watermark")return gitops.WriteWatermark(path, wm)diff --git a/internal/graft/corestore/syncer/syncer_localrefs_test.go b/internal/graft/corestore/syncer/syncer_localrefs_test.goindex f8e8008..f375e84 100644--- a/internal/graft/corestore/syncer/syncer_localrefs_test.go+++ b/internal/graft/corestore/syncer/syncer_localrefs_test.go@@ -114,7 +114,7 @@ func TestSyncerCatchUpMarksTheRepositoryDirty(t *testing.T) {e.verifySynced(t, repo)}-func TestSyncerWritesTheWatermarkOfADirtyRepositoryWithoutFsync(t *testing.T) {+func TestDefer(t *testing.T) {t.Parallel()e := newEnv(t, envOptions{})@@ -156,7 +156,7 @@ func TestSyncerWritesTheWatermarkOfADirtyRepositoryWithoutFsync(t *testing.T) {require.Equal(t, []string{"watermark"}, points)}-func TestSyncerMovesAsideADirtyRepositoryWithATornWatermark(t *testing.T) {+func TestTorn(t *testing.T) {t.Parallel()for name, torn := range map[string][]byte{"empty": nil, "garbage": []byte(`{"repo_id":`)} {