Skip to content

Git designed for AI agents.

corestore: fixes from the review of the fsync changes

Trigodil65cd52aae3b8
diff --git a/doc/graft/PLAN7.md b/doc/graft/PLAN7.md
index 652b337..e8a6292 100644
--- a/doc/graft/PLAN7.md
+++ b/doc/graft/PLAN7.md
@@ -716,8 +716,8 @@ repository-wide lock.
and nothing is live, the step clears the marker:
* It takes git's lock without waiting. The lock is created by linking a
temporary file that holds a tag and a random nonce.
- * It fsyncs what was written since, marks the state clean, and releases
- the lock. The release removes the lock only if it is still the same
+ * It fsyncs what was written since, then the watermark and its directory,
+ marks the state clean, and releases the lock. The release removes the lock only if it is still the same
file with the same nonce: an inode is reused as soon as it is freed, so
a lock that someone else replaced is never removed.
* Last, it removes the marker, under a per-repository file lock that
@@ -726,6 +726,19 @@ repository-wide lock.
* The set of fsynced tables keeps only the tables the stack still lists.
* A graceful stop runs a last step for every dirty repository after the
servers stopped, so a clean reboot does not rehydrate them.
+ * *Watermark.* While a repository of the owner is dirty and has a reftable
+ stack, the syncer writes its watermark (`graft/storage/applied`) atomically
+ without fsync (`CanDefer`).
+ * After a kernel restart a dirty cache is moved aside whatever its watermark
+ holds, and after a process restart the page cache kept the write. An
+ advance only changes the sequence; the repository ID was written with
+ fsync by the hydration, so a lost or torn advance leaves the recovery's
+ decision as it was.
+ * The step that clears the marker under git's lock, and `Forget` of a
+ repository kept in place, fsync the watermark and its directory first. A
+ failed fsync makes the repository untrusted, as for the stack.
+ * Hydration, import and the gate write it with fsync, as does a write
+ whose directory is missing.
* *Write failure.* Only a failed fsync means the kernel may have dropped
writes, and the kernel reports a failed writeback once: a later fsync of the
same file succeeds.
diff --git a/doc/graft/STORAGE.md b/doc/graft/STORAGE.md
index 76dfea9..b5cae69 100644
--- a/doc/graft/STORAGE.md
+++ b/doc/graft/STORAGE.md
@@ -591,7 +591,9 @@ or re-import the repository from scratch (see
them durable in a background step a second after a repository goes idle
(`doc/graft/PLAN7.md`, LOCAL REFERENCE WRITES RULE). Until then the
repository has a dirty marker in `<storage>/+gitaly/graft-ref-durability/`,
- which holds the kernel's boot identity.
+ which holds the kernel's boot identity. The watermark of a dirty repository is
+ written without fsync too, and the background step makes it durable before it
+ removes the marker.
- *Process crash or restart* (same kernel boot): nothing is lost, because the
page cache kept every write. The restarted node settles the markers.
- *Kernel crash or power loss*: the references written since the last durable
diff --git a/internal/cli/gitaly/serve.go b/internal/cli/gitaly/serve.go
index 2ee5e8b..99eca7b 100644
--- a/internal/cli/gitaly/serve.go
+++ b/internal/cli/gitaly/serve.go
@@ -990,7 +990,7 @@ func openLocalReferenceWrites(core *corestore.Core, locator storage.Locator, log
core.RefAdmission = localrefs.NewAdmission(lockTimeout, lockTimeout)
prometheus.MustRegister(core.RefAdmission)
- durability, err := localrefs.NewDurability(locator, localrefs.DurableStepIdle, localrefs.WithWatermark(gitops.WatermarkPath))
+ durability, err := localrefs.NewDurability(locator, localrefs.DurableStepIdle, localrefs.WithWmark(gitops.WatermarkPath))
if err != nil {
return fmt.Errorf("graft storage core: %w", err)
}
diff --git a/internal/graft/corestore/blobstore/blobstore.go b/internal/graft/corestore/blobstore/blobstore.go
index 30c7356..4524784 100644
--- a/internal/graft/corestore/blobstore/blobstore.go
+++ b/internal/graft/corestore/blobstore/blobstore.go
@@ -36,8 +36,9 @@
// anything, and a changeset row in the database (synchronous=FULL) could
// outlive a pack whose data never reached the disk. For file buckets Put fsyncs
// the content before the link publishes it, and fsyncs the directories that
-// hold the new entry (up to the bucket root) after it. The mem and remote
-// drivers commit their objects durably themselves.
+// hold the new entry after it, up to the first one whose whole chain is durable.
+// Bucket directories are never removed while a server uses the bucket: a recreated one could reuse an inode.
+// The mem and remote drivers commit their objects durably themselves.
//
// Listing. List reports objects only. fileblob also reports a writer's
// temporary file (<key>.<nanos>.tmp, in the key's directory because file
@@ -319,21 +320,17 @@ func syncFile(f *os.File) error {
return f.Sync()
}
-// Fsyncs dir and its ancestors inside the bucket root, up to the first one an earlier
-// call made durable that is still the same directory.
+// Fsyncs dir and its ancestors inside the bucket root, up to the first one whose whole chain an
+// earlier call made durable. The walk caches its directories only once it reached that point.
func (s *Store) syncDirs(dir string) error {
inside := dir == s.root || strings.HasPrefix(dir, s.root+string(filepath.Separator))
- var child string
- var childInfo os.FileInfo
+ visited := map[string]os.FileInfo{}
for d := dir; ; d = filepath.Dir(d) {
if err := syncDir(d); err != nil {
return err
}
- if child != "" {
- s.durableDirs.Store(child, childInfo)
- }
if !inside || d == s.root {
- return nil
+ break
}
info, err := os.Stat(d)
@@ -341,10 +338,15 @@ func (s *Store) syncDirs(dir string) error {
return fmt.Errorf("stat directory: %w", err)
}
if known, ok := s.durableDirs.Load(d); ok && os.SameFile(known.(os.FileInfo), info) {
- return nil
+ break
}
- child, childInfo = d, info
+ visited[d] = info
}
+
+ for d, info := range visited {
+ s.durableDirs.Store(d, info)
+ }
+ return nil
}
// syncDir fsyncs a directory so that the entries created in it are durable.
diff --git a/internal/graft/corestore/blobstore/durable_internal_test.go b/internal/graft/corestore/blobstore/durable_internal_test.go
index b52da3a..859983c 100644
--- a/internal/graft/corestore/blobstore/durable_internal_test.go
+++ b/internal/graft/corestore/blobstore/durable_internal_test.go
@@ -8,6 +8,7 @@ import (
"net/url"
"os"
"path/filepath"
+ "sync"
"testing"
"github.com/feder-cr/git-ai/internal/graft/corestore"
@@ -70,7 +71,7 @@ func TestPutSyncsFileBeforeCommitAndSyncsDirectories(t *testing.T) {
require.Equal(t, content, data)
}
-func TestPutSyncsOnlyTheDirectoriesWhoseEntriesAreNotDurable(t *testing.T) {
+func TestDirs(t *testing.T) {
dir := t.TempDir()
var dirSyncs []string
s := openFileStoreForTest(t, dir, func(path string, isDir bool) error {
@@ -103,6 +104,77 @@ func TestPutSyncsOnlyTheDirectoriesWhoseEntriesAreNotDurable(t *testing.T) {
require.Equal(t, []string{packsDir, repoDir, filepath.Join(dir, "repos")}, dirSyncs)
}
+func TestDirErr(t *testing.T) {
+ dir := t.TempDir()
+ reposDir := filepath.Join(dir, "repos")
+ fail := true
+ var dirSyncs []string
+ s := openFileStoreForTest(t, dir, func(path string, isDir bool) error {
+ if !isDir {
+ return nil
+ }
+ if fail && path == reposDir {
+ return errors.New("input/output error")
+ }
+ dirSyncs = append(dirSyncs, path)
+ return nil
+ })
+
+ content := []byte("pack content")
+ require.Error(t, s.Put(testhelper.Context(t), corestore.NewPackKey(1, sha256.Sum256(content)), bytes.NewReader(content)))
+
+ fail = false
+ dirSyncs = nil
+ require.NoError(t, s.Put(testhelper.Context(t), corestore.NewPackKey(1, sha256.Sum256(content)), bytes.NewReader(content)))
+ repoDir := filepath.Join(reposDir, "1")
+ require.Equal(t, []string{filepath.Join(repoDir, "packs"), repoDir, reposDir, dir}, dirSyncs)
+}
+
+func TestConcur(t *testing.T) {
+ dir := t.TempDir()
+ reposDir := filepath.Join(dir, "repos")
+ reached, release := make(chan struct{}), make(chan struct{})
+ var mu sync.Mutex
+ held := false
+ var dirSyncs []string
+ s := openFileStoreForTest(t, dir, func(path string, isDir bool) error {
+ if !isDir {
+ return nil
+ }
+ mu.Lock()
+ hold := !held && path == reposDir
+ held = held || hold
+ dirSyncs = append(dirSyncs, path)
+ mu.Unlock()
+ if hold {
+ close(reached)
+ <-release
+ }
+ return nil
+ })
+ put := func() error {
+ content := []byte("pack content")
+ return s.Put(testhelper.Context(t), corestore.NewPackKey(1, sha256.Sum256(content)), bytes.NewReader(content))
+ }
+
+ // The first Put is held while it fsyncs repos/, after packs/ and repos/1/.
+ first := make(chan error, 1)
+ go func() { first <- put() }()
+ <-reached
+ mu.Lock()
+ dirSyncs = nil
+ mu.Unlock()
+
+ require.NoError(t, put())
+ repoDir := filepath.Join(reposDir, "1")
+ mu.Lock()
+ require.Equal(t, []string{filepath.Join(repoDir, "packs"), repoDir, reposDir, dir}, dirSyncs)
+ mu.Unlock()
+
+ close(release)
+ require.NoError(t, <-first)
+}
+
func TestPutFailedFileSyncLeavesNoObject(t *testing.T) {
dir := t.TempDir()
content := []byte("pack content whose sync fails")
diff --git a/internal/graft/corestore/coretest/coretest.go b/internal/graft/corestore/coretest/coretest.go
index 612c55b..7daa396 100644
--- a/internal/graft/corestore/coretest/coretest.go
+++ b/internal/graft/corestore/coretest/coretest.go
@@ -213,7 +213,7 @@ func EnableLocalRefs(tb testing.TB, core *corestore.Core, cfg config.Cfg, opts .
require.NoError(tb, core.RefDurability.Close())
}
core.RefAdmission = localrefs.NewAdmission(10*time.Second, 10*time.Second)
- durability, err := localrefs.NewDurability(config.NewLocator(cfg), 0, append([]localrefs.DurabilityOption{localrefs.WithBootID(TestBootID), localrefs.WithRecoveryGrace(0), localrefs.WithWatermark(gitops.WatermarkPath)}, opts...)...)
+ durability, err := localrefs.NewDurability(config.NewLocator(cfg), 0, append([]localrefs.DurabilityOption{localrefs.WithBootID(TestBootID), localrefs.WithRecoveryGrace(0), localrefs.WithWmark(gitops.WatermarkPath)}, opts...)...)
require.NoError(tb, err)
// A git process forked at the moment a previous durability closed its lock
// holds a copy of it until it execs (close-on-exec): the lock can be busy for
diff --git a/internal/graft/corestore/gitops/gitops.go b/internal/graft/corestore/gitops/gitops.go
index 1199b3c..97d8df7 100644
--- a/internal/graft/corestore/gitops/gitops.go
+++ b/internal/graft/corestore/gitops/gitops.go
@@ -897,15 +897,18 @@ func WriteWatermark(repoPath string, w corestore.Watermark) error {
if err := os.MkdirAll(dir, mode.Directory); err != nil {
return fmt.Errorf("write watermark: %w", err)
}
- return writeWatermark(repoPath, w, true)
+ return writeWMark(repoPath, w, true)
}
-// WriteWatermarkNoSync writes the watermark without fsync, into an existing directory.
-func WriteWatermarkNoSync(repoPath string, w corestore.Watermark) error {
- return writeWatermark(repoPath, w, false)
+// WriteLazy writes the watermark without fsync, or with fsync when its directory is missing.
+func WriteLazy(repoPath string, w corestore.Watermark) error {
+ if _, err := os.Stat(filepath.Dir(WatermarkPath(repoPath))); err != nil {
+ return WriteWatermark(repoPath, w)
+ }
+ return writeWMark(repoPath, w, false)
}
-func writeWatermark(repoPath string, w corestore.Watermark, sync bool) (returnedErr error) {
+func writeWMark(repoPath string, w corestore.Watermark, sync bool) (returnedErr error) {
dir := filepath.Dir(WatermarkPath(repoPath))
data, err := json.Marshal(w)
diff --git a/internal/graft/corestore/gitops/gitops_test.go b/internal/graft/corestore/gitops/gitops_test.go
index a82bdcd..d232c72 100644
--- a/internal/graft/corestore/gitops/gitops_test.go
+++ b/internal/graft/corestore/gitops/gitops_test.go
@@ -533,14 +533,15 @@ func TestWatermarkRoundTrip(t *testing.T) {
require.Equal(t, "applied", entries[0].Name())
}
-func TestWriteWatermarkNoSync(t *testing.T) {
+func TestNoSync(t *testing.T) {
t.Parallel()
repoPath := t.TempDir()
- require.Error(t, gitops.WriteWatermarkNoSync(repoPath, corestore.Watermark{RepoID: 7, Seq: 1}), "it does not create the directory")
+ // Without its directory the watermark is written as WriteWatermark does.
+ require.NoError(t, gitops.WriteLazy(repoPath, corestore.Watermark{RepoID: 7, Seq: 1}))
+ require.FileExists(t, gitops.WatermarkPath(repoPath))
- require.NoError(t, gitops.WriteWatermark(repoPath, corestore.Watermark{RepoID: 7, Seq: 1}))
- require.NoError(t, gitops.WriteWatermarkNoSync(repoPath, corestore.Watermark{RepoID: 7, Seq: 2}))
+ require.NoError(t, gitops.WriteLazy(repoPath, corestore.Watermark{RepoID: 7, Seq: 2}))
w, ok, err := gitops.ReadWatermark(repoPath)
require.NoError(t, err)
require.True(t, ok)
diff --git a/internal/graft/corestore/localrefs/durability.go b/internal/graft/corestore/localrefs/durability.go
index 0071537..83fee58 100644
--- a/internal/graft/corestore/localrefs/durability.go
+++ b/internal/graft/corestore/localrefs/durability.go
@@ -278,8 +278,8 @@ func WithSync(sync func(f *os.File) error) DurabilityOption {
return func(d *Durability) { d.sync = sync }
}
-// WithWatermark lets a dirty repository's watermark be written without fsync.
-func WithWatermark(path func(repoPath string) string) DurabilityOption {
+// WithWmark lets a dirty repository's watermark be written without fsync.
+func WithWmark(path func(repoPath string) string) DurabilityOption {
return func(d *Durability) { d.watermark = path }
}
@@ -602,13 +602,18 @@ func (d *Durability) Dirty() []string {
return paths
}
-// DefersWatermark reports whether the watermark at path may be written without fsync now: the marker
+// CanDefer reports whether the watermark at path may be written without fsync now: the marker
// is durable and goes only after the watermark. The caller holds the syncer's repository lock.
-func (d *Durability) DefersWatermark(path string) bool {
+func (d *Durability) CanDefer(path string) bool {
if !d.Deferring() || d.watermark == nil {
return false
}
+ // A marker without a stack goes without a durable step, so its watermark is written with fsync.
+ if stack, err := hasStack(path); err != nil || !stack {
+ return false
+ }
+
d.mu.Lock()
defer d.mu.Unlock()
st := d.repos[path]
@@ -719,7 +724,7 @@ func (d *Durability) settleLocked(path string, repo RepoRef) error {
st := d.lockState(path)
defer st.file.Unlock()
- if err := d.syncStackAndWatermark(path, st); err != nil {
+ if err := d.syncAll(path, st); err != nil {
return d.stepErrorLocked(st, repo, path, err)
}
if d.crashAt == crashDurableStep {
@@ -1086,7 +1091,7 @@ func (d *Durability) syncStack(path string, st *refState) error {
}
// Makes the stack, then the watermark file and its directory, durable.
-func (d *Durability) syncStackAndWatermark(path string, st *refState) error {
+func (d *Durability) syncAll(path string, st *refState) error {
if err := d.syncStack(path, st); err != nil {
return err
}
@@ -1205,7 +1210,7 @@ func (d *Durability) forgetLocked(path string, st *refState, repo RepoRef) (bool
return false, err
}
dropped := false
- if err = d.syncStackAndWatermark(path, st); err != nil {
+ if err = d.syncAll(path, st); err != nil {
err = d.stepErrorLocked(st, repo, path, err)
} else {
dropped, err = d.dropMarker(path, st, true)
diff --git a/internal/graft/corestore/localrefs/durability_test.go b/internal/graft/corestore/localrefs/durability_test.go
index 7d0ef1a..49bf6c5 100644
--- a/internal/graft/corestore/localrefs/durability_test.go
+++ b/internal/graft/corestore/localrefs/durability_test.go
@@ -405,11 +405,11 @@ func TestSettleReportsAFailedWriteAsUntrusted(t *testing.T) {
require.Equal(t, 0.0, testutil.ToFloat64(d.dirty))
}
-func testWatermarkPath(repoPath string) string {
+func wmPath(repoPath string) string {
return filepath.Join(repoPath, "graft", "storage", "applied")
}
-func TestDefersWatermark(t *testing.T) {
+func TestDefers(t *testing.T) {
t.Parallel()
t.Run("without the option", func(t *testing.T) {
@@ -418,59 +418,69 @@ func TestDefersWatermark(t *testing.T) {
d := f.durability(t, "boot-a", 0)
require.NoError(t, d.MarkDirty(f.ctx, f.repo, f.path))
- require.False(t, d.DefersWatermark(f.path))
+ require.False(t, d.CanDefer(f.path))
})
t.Run("while the repository is dirty", func(t *testing.T) {
t.Parallel()
f := setupDurability(t)
- d := f.durability(t, "boot-a", 0, WithWatermark(testWatermarkPath))
+ d := f.durability(t, "boot-a", 0, WithWmark(wmPath))
- require.False(t, d.DefersWatermark(f.path), "a clean repository")
+ require.False(t, d.CanDefer(f.path), "a clean repository")
require.NoError(t, d.MarkDirty(f.ctx, f.repo, f.path))
- require.True(t, d.DefersWatermark(f.path))
+ require.True(t, d.CanDefer(f.path))
require.NoError(t, d.Settle(f.path, false))
- require.False(t, d.DefersWatermark(f.path), "the marker is gone")
+ require.False(t, d.CanDefer(f.path), "the marker is gone")
})
t.Run("an untrusted repository", func(t *testing.T) {
t.Parallel()
f := setupDurability(t)
- d := f.durability(t, "boot-a", 0, WithWatermark(testWatermarkPath))
+ d := f.durability(t, "boot-a", 0, WithWmark(wmPath))
require.NoError(t, d.MarkDirty(f.ctx, f.repo, f.path))
f.synced.fail(errors.New("input/output error"))
require.ErrorIs(t, d.Settle(f.path, false), ErrUntrusted)
- require.False(t, d.DefersWatermark(f.path))
+ require.False(t, d.CanDefer(f.path))
+ })
+
+ t.Run("a repository without a stack", func(t *testing.T) {
+ t.Parallel()
+ f := setupDurability(t)
+ d := f.durability(t, "boot-a", 0, WithWmark(wmPath))
+
+ require.NoError(t, d.MarkDirty(f.ctx, f.repo, f.path))
+ require.NoError(t, os.Rename(filepath.Join(f.path, "reftable"), filepath.Join(t.TempDir(), "reftable")))
+ require.False(t, d.CanDefer(f.path))
})
t.Run("a process that does not own the storage", func(t *testing.T) {
t.Parallel()
f := setupDurability(t)
- d := f.unowned(t, "boot-a", 0, WithWatermark(testWatermarkPath))
+ d := f.unowned(t, "boot-a", 0, WithWmark(wmPath))
- require.False(t, d.DefersWatermark(f.path))
+ require.False(t, d.CanDefer(f.path))
})
t.Run("a nil durability", func(t *testing.T) {
t.Parallel()
var d *Durability
- require.False(t, d.DefersWatermark("/any"))
+ require.False(t, d.CanDefer("/any"))
})
}
-func TestSettleMakesTheWatermarkDurableBeforeTheMarkerGoes(t *testing.T) {
+func TestSettle(t *testing.T) {
t.Parallel()
setup := func(t *testing.T) (durabilityFixture, *Durability, string, *bool) {
f := setupDurability(t)
- watermark := testWatermarkPath(f.path)
+ watermark := wmPath(f.path)
require.NoError(t, os.MkdirAll(filepath.Dir(watermark), 0o755))
require.NoError(t, os.WriteFile(watermark, []byte(`{"repo_id":1,"seq":2}`), 0o644))
var markerAtSync bool
var d *Durability
- d = f.durability(t, "boot-a", 0, WithWatermark(testWatermarkPath), WithSync(func(file *os.File) error {
+ d = f.durability(t, "boot-a", 0, WithWmark(wmPath), WithSync(func(file *os.File) error {
if file.Name() == watermark {
_, err := os.Stat(f.markerPath(t, d))
markerAtSync = err == nil
diff --git a/internal/graft/corestore/syncer/syncer.go b/internal/graft/corestore/syncer/syncer.go
index 8995b25..ab7a482 100644
--- a/internal/graft/corestore/syncer/syncer.go
+++ b/internal/graft/corestore/syncer/syncer.go
@@ -885,7 +885,7 @@ func (s *Syncer) advanceLocked(path string, wm corestore.Watermark) corestore.Wa
}
next := corestore.Watermark{RepoID: wm.RepoID, Seq: newSeq}
- if err := s.writeWatermarkLocked(path, next); err != nil {
+ if err := s.writeWM(path, next); err != nil {
s.logger.WithError(err).WithField("repository_path", path).Error("storage core: write watermark")
return wm
}
@@ -1325,17 +1325,17 @@ func (s *Syncer) applyLocked(ctx context.Context, repo *localrepo.Repo, hash git
// reached. The caller holds the repository lock.
func (s *Syncer) commitWatermarkLocked(path string, repoID int64, seq uint64) (corestore.Watermark, error) {
wm := corestore.Watermark{RepoID: repoID, Seq: seq}
- if err := s.writeWatermarkLocked(path, wm); err != nil {
+ if err := s.writeWM(path, wm); err != nil {
return corestore.Watermark{}, structerr.NewInternal("storage core: write watermark: %w", err)
}
return s.advanceLocked(path, wm), nil
}
// Writes the watermark, without fsync while the repository is dirty (caller holds the lock).
-func (s *Syncer) writeWatermarkLocked(path string, wm corestore.Watermark) error {
- if s.core.RefDurability.DefersWatermark(path) {
+func (s *Syncer) writeWM(path string, wm corestore.Watermark) error {
+ if s.core.RefDurability.CanDefer(path) {
s.hook("watermark deferred")
- return gitops.WriteWatermarkNoSync(path, wm)
+ return gitops.WriteLazy(path, wm)
}
s.hook("watermark")
return gitops.WriteWatermark(path, wm)
diff --git a/internal/graft/corestore/syncer/syncer_localrefs_test.go b/internal/graft/corestore/syncer/syncer_localrefs_test.go
index f8e8008..f375e84 100644
--- a/internal/graft/corestore/syncer/syncer_localrefs_test.go
+++ b/internal/graft/corestore/syncer/syncer_localrefs_test.go
@@ -114,7 +114,7 @@ func TestSyncerCatchUpMarksTheRepositoryDirty(t *testing.T) {
e.verifySynced(t, repo)
}
-func TestSyncerWritesTheWatermarkOfADirtyRepositoryWithoutFsync(t *testing.T) {
+func TestDefer(t *testing.T) {
t.Parallel()
e := newEnv(t, envOptions{})
@@ -156,7 +156,7 @@ func TestSyncerWritesTheWatermarkOfADirtyRepositoryWithoutFsync(t *testing.T) {
require.Equal(t, []string{"watermark"}, points)
}
-func TestSyncerMovesAsideADirtyRepositoryWithATornWatermark(t *testing.T) {
+func TestTorn(t *testing.T) {
t.Parallel()
for name, torn := range map[string][]byte{"empty": nil, "garbage": []byte(`{"repo_id":`)} {