Skip to content

Git designed for AI agents.

corestore: fsync the deferred watermark outside git's lock, and before every marker removal

Federico Elia50194ed8768a
diff --git a/internal/cli/gitaly/serve.go b/internal/cli/gitaly/serve.go
index f194822..dcd1c5f 100644
--- a/internal/cli/gitaly/serve.go
+++ b/internal/cli/gitaly/serve.go
@@ -1006,7 +1006,8 @@ func openLocalReferenceWrites(core *corestore.Core, locator storage.Locator, log
core.RefAdmission = localrefs.NewAdmission(lockTimeout, lockTimeout)
prometheus.MustRegister(core.RefAdmission)
- durability, err := localrefs.NewDurability(locator, localrefs.DurableStepIdle, localrefs.WithWmark(gitops.WatermarkPath))
+ durability, err := localrefs.NewDurability(locator, localrefs.DurableStepIdle,
+ localrefs.WithWatermark(gitops.WatermarkPath))
if err != nil {
return fmt.Errorf("graft storage core: %w", err)
}
diff --git a/internal/graft/corestore/blobstore/blobstore.go b/internal/graft/corestore/blobstore/blobstore.go
index 4524784..1d8a627 100644
--- a/internal/graft/corestore/blobstore/blobstore.go
+++ b/internal/graft/corestore/blobstore/blobstore.go
@@ -36,9 +36,12 @@
// anything, and a changeset row in the database (synchronous=FULL) could
// outlive a pack whose data never reached the disk. For file buckets Put fsyncs
// the content before the link publishes it, and fsyncs the directories that
-// hold the new entry after it, up to the first one whose whole chain is durable.
-// Bucket directories are never removed while a server uses the bucket: a recreated one could reuse an inode.
-// The mem and remote drivers commit their objects durably themselves.
+// hold the new entry after it, up to the first one whose whole chain up to the
+// bucket root an earlier Put made durable. That chain is recognized by the
+// directory's identity (device and inode), so bucket directories are never
+// removed while a server uses the bucket: a directory removed and created again
+// could reuse the inode of one that was durable. The mem and remote drivers
+// commit their objects durably themselves.
//
// Listing. List reports objects only. fileblob also reports a writer's
// temporary file (<key>.<nanos>.tmp, in the key's directory because file
@@ -87,7 +90,8 @@ type Store struct {
// root is the cleaned directory of a file bucket and empty for any other
// driver. Put makes the writes of a file bucket durable from it.
root string
- // Directories whose entry in their parent was made durable, by path.
+ // durableDirs holds, by path, the os.FileInfo of the directories of a file
+ // bucket whose entries up to the root were made durable (syncDirs).
durableDirs sync.Map
}
@@ -320,8 +324,13 @@ func syncFile(f *os.File) error {
return f.Sync()
}
-// Fsyncs dir and its ancestors inside the bucket root, up to the first one whose whole chain an
-// earlier call made durable. The walk caches its directories only once it reached that point.
+// syncDirs fsyncs dir and, when dir is inside the bucket root, each of its
+// ancestors: the directories that the driver may have created, and the entry
+// that the link created. The walk stops at the root, or after the first
+// ancestor whose entries up to the root an earlier walk made durable. The
+// directories it visited are remembered only when it got there without an
+// error, so that a concurrent Put never stops at a directory whose own entry is
+// not durable yet.
func (s *Store) syncDirs(dir string) error {
inside := dir == s.root || strings.HasPrefix(dir, s.root+string(filepath.Separator))
visited := map[string]os.FileInfo{}
diff --git a/internal/graft/corestore/blobstore/durable_internal_test.go b/internal/graft/corestore/blobstore/durable_internal_test.go
index 859983c..a3994d9 100644
--- a/internal/graft/corestore/blobstore/durable_internal_test.go
+++ b/internal/graft/corestore/blobstore/durable_internal_test.go
@@ -71,7 +71,10 @@ func TestPutSyncsFileBeforeCommitAndSyncsDirectories(t *testing.T) {
require.Equal(t, content, data)
}
-func TestDirs(t *testing.T) {
+// TestPutSyncsOnlyTheDirectoriesThatAreNotDurable checks that a Put fsyncs the
+// directories up to the bucket root once, then only the directory of its key,
+// and that a directory created again (another inode) is fsynced again.
+func TestPutSyncsOnlyTheDirectoriesThatAreNotDurable(t *testing.T) {
dir := t.TempDir()
var dirSyncs []string
s := openFileStoreForTest(t, dir, func(path string, isDir bool) error {
@@ -84,7 +87,8 @@ func TestDirs(t *testing.T) {
put := func(repoID int64) {
dirSyncs = nil
content := []byte("pack content")
- require.NoError(t, s.Put(testhelper.Context(t), corestore.NewPackKey(repoID, sha256.Sum256(content)), bytes.NewReader(content)))
+ key := corestore.NewPackKey(repoID, sha256.Sum256(content))
+ require.NoError(t, s.Put(testhelper.Context(t), key, bytes.NewReader(content)))
}
repoDir := filepath.Join(dir, "repos", "1")
packsDir := filepath.Join(repoDir, "packs")
@@ -96,7 +100,8 @@ func TestDirs(t *testing.T) {
require.Equal(t, []string{packsDir}, dirSyncs)
put(2)
- require.Equal(t, []string{filepath.Join(dir, "repos", "2", "packs"), filepath.Join(dir, "repos", "2"), filepath.Join(dir, "repos")}, dirSyncs)
+ repo2Dir := filepath.Join(dir, "repos", "2")
+ require.Equal(t, []string{filepath.Join(repo2Dir, "packs"), repo2Dir, filepath.Join(dir, "repos")}, dirSyncs)
// Moving the directory aside keeps its inode in use, so the new directory gets another one.
require.NoError(t, os.Rename(repoDir, filepath.Join(t.TempDir(), "old")))
@@ -104,7 +109,10 @@ func TestDirs(t *testing.T) {
require.Equal(t, []string{packsDir, repoDir, filepath.Join(dir, "repos")}, dirSyncs)
}
-func TestDirErr(t *testing.T) {
+// TestPutSyncsAgainAfterAFailedDirectorySync checks that a Put whose fsync of an
+// ancestor failed remembers none of its directories: the next Put fsyncs them
+// all up to the root.
+func TestPutSyncsAgainAfterAFailedDirectorySync(t *testing.T) {
dir := t.TempDir()
reposDir := filepath.Join(dir, "repos")
fail := true
@@ -121,16 +129,21 @@ func TestDirErr(t *testing.T) {
})
content := []byte("pack content")
- require.Error(t, s.Put(testhelper.Context(t), corestore.NewPackKey(1, sha256.Sum256(content)), bytes.NewReader(content)))
+ key := corestore.NewPackKey(1, sha256.Sum256(content))
+ require.Error(t, s.Put(testhelper.Context(t), key, bytes.NewReader(content)))
fail = false
dirSyncs = nil
- require.NoError(t, s.Put(testhelper.Context(t), corestore.NewPackKey(1, sha256.Sum256(content)), bytes.NewReader(content)))
+ key = corestore.NewPackKey(1, sha256.Sum256(content))
+ require.NoError(t, s.Put(testhelper.Context(t), key, bytes.NewReader(content)))
repoDir := filepath.Join(reposDir, "1")
require.Equal(t, []string{filepath.Join(repoDir, "packs"), repoDir, reposDir, dir}, dirSyncs)
}
-func TestConcur(t *testing.T) {
+// TestPutConcurrentWithTheFirstPutSyncsUpToTheRoot checks that a Put that runs
+// while the first Put of a repository is still fsyncing its directories does not
+// stop at one of them: it fsyncs its own way up to the root.
+func TestPutConcurrentWithTheFirstPutSyncsUpToTheRoot(t *testing.T) {
dir := t.TempDir()
reposDir := filepath.Join(dir, "repos")
reached, release := make(chan struct{}), make(chan struct{})
diff --git a/internal/graft/corestore/coretest/coretest.go b/internal/graft/corestore/coretest/coretest.go
index 7daa396..54eecca 100644
--- a/internal/graft/corestore/coretest/coretest.go
+++ b/internal/graft/corestore/coretest/coretest.go
@@ -213,7 +213,11 @@ func EnableLocalRefs(tb testing.TB, core *corestore.Core, cfg config.Cfg, opts .
require.NoError(tb, core.RefDurability.Close())
}
core.RefAdmission = localrefs.NewAdmission(10*time.Second, 10*time.Second)
- durability, err := localrefs.NewDurability(config.NewLocator(cfg), 0, append([]localrefs.DurabilityOption{localrefs.WithBootID(TestBootID), localrefs.WithRecoveryGrace(0), localrefs.WithWmark(gitops.WatermarkPath)}, opts...)...)
+ durability, err := localrefs.NewDurability(config.NewLocator(cfg), 0, append([]localrefs.DurabilityOption{
+ localrefs.WithBootID(TestBootID),
+ localrefs.WithRecoveryGrace(0),
+ localrefs.WithWatermark(gitops.WatermarkPath),
+ }, opts...)...)
require.NoError(tb, err)
// A git process forked at the moment a previous durability closed its lock
// holds a copy of it until it execs (close-on-exec): the lock can be busy for
diff --git a/internal/graft/corestore/gitops/gitops.go b/internal/graft/corestore/gitops/gitops.go
index ad9b5dc..a2198ca 100644
--- a/internal/graft/corestore/gitops/gitops.go
+++ b/internal/graft/corestore/gitops/gitops.go
@@ -935,18 +935,24 @@ func WriteWatermark(repoPath string, w corestore.Watermark) error {
if err := os.MkdirAll(dir, mode.Directory); err != nil {
return fmt.Errorf("write watermark: %w", err)
}
- return writeWMark(repoPath, w, true)
+ return writeWatermarkFile(repoPath, w, true)
}
-// WriteLazy writes the watermark without fsync, or with fsync when its directory is missing.
-func WriteLazy(repoPath string, w corestore.Watermark) error {
+// WriteWatermarkWithoutSync writes the watermark atomically (temporary file,
+// rename) without fsync, for a repository whose durable step makes it durable
+// later (localrefs.Durability.DefersWatermark). When graft/storage is missing it
+// writes as WriteWatermark does, so that the directory it creates is durable.
+func WriteWatermarkWithoutSync(repoPath string, w corestore.Watermark) error {
if _, err := os.Stat(filepath.Dir(WatermarkPath(repoPath))); err != nil {
return WriteWatermark(repoPath, w)
}
- return writeWMark(repoPath, w, false)
+ return writeWatermarkFile(repoPath, w, false)
}
-func writeWMark(repoPath string, w corestore.Watermark, sync bool) (returnedErr error) {
+// writeWatermarkFile writes the watermark into graft/storage, which exists,
+// through a temporary file and a rename; sync fsyncs the file before the rename
+// and the directory after it.
+func writeWatermarkFile(repoPath string, w corestore.Watermark, sync bool) (returnedErr error) {
dir := filepath.Dir(WatermarkPath(repoPath))
data, err := json.Marshal(w)
diff --git a/internal/graft/corestore/gitops/gitops_test.go b/internal/graft/corestore/gitops/gitops_test.go
index 881e8b2..26ebe4e 100644
--- a/internal/graft/corestore/gitops/gitops_test.go
+++ b/internal/graft/corestore/gitops/gitops_test.go
@@ -536,15 +536,18 @@ func TestWatermarkRoundTrip(t *testing.T) {
require.Equal(t, "applied", entries[0].Name())
}
-func TestNoSync(t *testing.T) {
+// TestWriteWatermarkWithoutSync checks that the watermark written without fsync
+// reads back, leaves no temporary file, and is written as WriteWatermark does
+// when its directory is missing.
+func TestWriteWatermarkWithoutSync(t *testing.T) {
t.Parallel()
repoPath := t.TempDir()
// Without its directory the watermark is written as WriteWatermark does.
- require.NoError(t, gitops.WriteLazy(repoPath, corestore.Watermark{RepoID: 7, Seq: 1}))
+ require.NoError(t, gitops.WriteWatermarkWithoutSync(repoPath, corestore.Watermark{RepoID: 7, Seq: 1}))
require.FileExists(t, gitops.WatermarkPath(repoPath))
- require.NoError(t, gitops.WriteLazy(repoPath, corestore.Watermark{RepoID: 7, Seq: 2}))
+ require.NoError(t, gitops.WriteWatermarkWithoutSync(repoPath, corestore.Watermark{RepoID: 7, Seq: 2}))
w, ok, err := gitops.ReadWatermark(repoPath)
require.NoError(t, err)
require.True(t, ok)
diff --git a/internal/graft/corestore/localrefs/durability.go b/internal/graft/corestore/localrefs/durability.go
index 83fee58..ab0b8ae 100644
--- a/internal/graft/corestore/localrefs/durability.go
+++ b/internal/graft/corestore/localrefs/durability.go
@@ -165,7 +165,8 @@ type Durability struct {
sync func(f *os.File) error
// crashAt is the crash point selected with CrashEnv.
crashAt string
- // Returns the path of a repository's watermark.
+ // watermark returns the path of the watermark of the repository at repoPath;
+ // nil unless the syncer may write it without fsync (WithWatermark).
watermark func(repoPath string) string
// ownerFiles hold the owner locks, taken by TryOwn. owner is set by
@@ -278,8 +279,10 @@ func WithSync(sync func(f *os.File) error) DurabilityOption {
return func(d *Durability) { d.sync = sync }
}
-// WithWmark lets a dirty repository's watermark be written without fsync.
-func WithWmark(path func(repoPath string) string) DurabilityOption {
+// WithWatermark lets the syncer write the watermark of a dirty repository
+// without fsync (DefersWatermark); path returns the watermark of a repository,
+// which the durable step then makes durable before the marker goes.
+func WithWatermark(path func(repoPath string) string) DurabilityOption {
return func(d *Durability) { d.watermark = path }
}
@@ -602,14 +605,17 @@ func (d *Durability) Dirty() []string {
return paths
}
-// CanDefer reports whether the watermark at path may be written without fsync now: the marker
-// is durable and goes only after the watermark. The caller holds the syncer's repository lock.
-func (d *Durability) CanDefer(path string) bool {
+// DefersWatermark reports whether the watermark of the repository at path may be
+// written without fsync now: the repository is dirty and trusted, so its marker
+// is durable, and the marker goes only after a durable step made the watermark
+// durable (syncWatermark). The caller holds the syncer's repository lock.
+func (d *Durability) DefersWatermark(path string) bool {
if !d.Deferring() || d.watermark == nil {
return false
}
- // A marker without a stack goes without a durable step, so its watermark is written with fsync.
+ // A repository without a stack has no durable step of its references: its
+ // watermark is written with fsync.
if stack, err := hasStack(path); err != nil || !stack {
return false
}
@@ -683,12 +689,19 @@ func (d *Durability) Settle(path string, final bool) error {
return err
}
if !dirty || !stack {
- // A repository without a stack (gone, or not reftable) has nothing to
- // make durable; a clean state only leaves the map. A transaction that
+ // A repository without a stack (gone, or not reftable) has no references
+ // to make durable; a clean state only leaves the map. A transaction that
// marked a repository with a stack meanwhile leaves it to the next step,
// under git's reference lock (dropMarker refuses otherwise).
st := d.lockState(path)
defer st.file.Unlock()
+ if dirty {
+ // Its watermark may have been written without fsync while it had a
+ // stack.
+ if err := d.syncWatermark(path); err != nil {
+ return d.stepErrorLocked(st, repo, path, err)
+ }
+ }
_, err := d.dropMarker(path, st, false)
return err
}
@@ -702,6 +715,13 @@ func (d *Durability) Settle(path string, final bool) error {
return nil
}
+ // The watermark is made durable before git's reference lock is taken, so that
+ // its fsync does not hold the transactions of the repository.
+ if err := d.syncWatermark(path); err != nil {
+ st := d.lockState(path)
+ defer st.file.Unlock()
+ return d.stepErrorLocked(st, repo, path, err)
+ }
lock, ok, err := lockStack(path)
if err != nil {
return err
@@ -724,7 +744,7 @@ func (d *Durability) settleLocked(path string, repo RepoRef) error {
st := d.lockState(path)
defer st.file.Unlock()
- if err := d.syncAll(path, st); err != nil {
+ if err := d.syncStack(path, st); err != nil {
return d.stepErrorLocked(st, repo, path, err)
}
if d.crashAt == crashDurableStep {
@@ -1090,11 +1110,12 @@ func (d *Durability) syncStack(path string, st *refState) error {
return nil
}
-// Makes the stack, then the watermark file and its directory, durable.
-func (d *Durability) syncAll(path string, st *refState) error {
- if err := d.syncStack(path, st); err != nil {
- return err
- }
+// syncWatermark makes the watermark of the repository at path and its directory
+// durable, when the syncer writes it without fsync (WithWatermark) and the
+// repository has one. The callers decide the marker after it, and hold the
+// syncer's repository lock, under which the syncer writes the watermark: it
+// cannot change before the marker goes.
+func (d *Durability) syncWatermark(path string) error {
if d.watermark == nil {
return nil
}
@@ -1193,14 +1214,19 @@ func (d *Durability) Forget(ctx context.Context, repo RepoRef, path string) erro
}
// forgetLocked lets dropMarker decide the marker of a forgotten repository.
-// One kept in place is made durable under git's reference lock first, as a
-// durable step does; when the lock is busy the marker stays for the next step.
-// The caller holds the state's file lock.
+// Its watermark is made durable first. One kept in place is then made durable
+// under git's reference lock, as a durable step does; when the lock is busy the
+// marker stays for the next step. The caller holds the state's file lock and the
+// syncer's repository lock.
func (d *Durability) forgetLocked(path string, st *refState, repo RepoRef) (bool, error) {
stack, err := hasStack(path)
if err != nil {
return false, err
}
+ // The watermark goes first, outside git's reference lock, as in Settle.
+ if err := d.syncWatermark(path); err != nil {
+ return false, d.stepErrorLocked(st, repo, path, err)
+ }
if !stack {
return d.dropMarker(path, st, false)
}
@@ -1210,7 +1236,7 @@ func (d *Durability) forgetLocked(path string, st *refState, repo RepoRef) (bool
return false, err
}
dropped := false
- if err = d.syncAll(path, st); err != nil {
+ if err = d.syncStack(path, st); err != nil {
err = d.stepErrorLocked(st, repo, path, err)
} else {
dropped, err = d.dropMarker(path, st, true)
diff --git a/internal/graft/corestore/localrefs/durability_test.go b/internal/graft/corestore/localrefs/durability_test.go
index 49bf6c5..f142f11 100644
--- a/internal/graft/corestore/localrefs/durability_test.go
+++ b/internal/graft/corestore/localrefs/durability_test.go
@@ -405,11 +405,16 @@ func TestSettleReportsAFailedWriteAsUntrusted(t *testing.T) {
require.Equal(t, 0.0, testutil.ToFloat64(d.dirty))
}
-func wmPath(repoPath string) string {
+// testWatermarkPath is gitops.WatermarkPath, the watermark of the syncer.
+func testWatermarkPath(repoPath string) string {
return filepath.Join(repoPath, "graft", "storage", "applied")
}
-func TestDefers(t *testing.T) {
+// TestDurabilityDefersTheWatermarkOfADirtyRepository checks that the watermark
+// is written without fsync only by the owner configured with WithWatermark, and
+// only for a trusted dirty repository with a reftable stack, whose durable step
+// makes it durable.
+func TestDurabilityDefersTheWatermarkOfADirtyRepository(t *testing.T) {
t.Parallel()
t.Run("without the option", func(t *testing.T) {
@@ -418,81 +423,96 @@ func TestDefers(t *testing.T) {
d := f.durability(t, "boot-a", 0)
require.NoError(t, d.MarkDirty(f.ctx, f.repo, f.path))
- require.False(t, d.CanDefer(f.path))
+ require.False(t, d.DefersWatermark(f.path))
})
t.Run("while the repository is dirty", func(t *testing.T) {
t.Parallel()
f := setupDurability(t)
- d := f.durability(t, "boot-a", 0, WithWmark(wmPath))
+ d := f.durability(t, "boot-a", 0, WithWatermark(testWatermarkPath))
- require.False(t, d.CanDefer(f.path), "a clean repository")
+ require.False(t, d.DefersWatermark(f.path), "a clean repository")
require.NoError(t, d.MarkDirty(f.ctx, f.repo, f.path))
- require.True(t, d.CanDefer(f.path))
+ require.True(t, d.DefersWatermark(f.path))
require.NoError(t, d.Settle(f.path, false))
- require.False(t, d.CanDefer(f.path), "the marker is gone")
+ require.False(t, d.DefersWatermark(f.path), "the marker is gone")
})
t.Run("an untrusted repository", func(t *testing.T) {
t.Parallel()
f := setupDurability(t)
- d := f.durability(t, "boot-a", 0, WithWmark(wmPath))
+ d := f.durability(t, "boot-a", 0, WithWatermark(testWatermarkPath))
require.NoError(t, d.MarkDirty(f.ctx, f.repo, f.path))
f.synced.fail(errors.New("input/output error"))
require.ErrorIs(t, d.Settle(f.path, false), ErrUntrusted)
- require.False(t, d.CanDefer(f.path))
+ require.False(t, d.DefersWatermark(f.path))
})
t.Run("a repository without a stack", func(t *testing.T) {
t.Parallel()
f := setupDurability(t)
- d := f.durability(t, "boot-a", 0, WithWmark(wmPath))
+ d := f.durability(t, "boot-a", 0, WithWatermark(testWatermarkPath))
require.NoError(t, d.MarkDirty(f.ctx, f.repo, f.path))
require.NoError(t, os.Rename(filepath.Join(f.path, "reftable"), filepath.Join(t.TempDir(), "reftable")))
- require.False(t, d.CanDefer(f.path))
+ require.False(t, d.DefersWatermark(f.path))
})
t.Run("a process that does not own the storage", func(t *testing.T) {
t.Parallel()
f := setupDurability(t)
- d := f.unowned(t, "boot-a", 0, WithWmark(wmPath))
+ d := f.unowned(t, "boot-a", 0, WithWatermark(testWatermarkPath))
- require.False(t, d.CanDefer(f.path))
+ require.False(t, d.DefersWatermark(f.path))
})
t.Run("a nil durability", func(t *testing.T) {
t.Parallel()
var d *Durability
- require.False(t, d.CanDefer("/any"))
+ require.False(t, d.DefersWatermark("/any"))
})
}
-func TestSettle(t *testing.T) {
+// TestDurableStepMakesTheWatermarkDurableBeforeTheMarkerGoes checks that a
+// durable step and Forget fsync the watermark that the syncer wrote without
+// fsync, and its directory, while the marker is still there, and outside git's
+// reference lock, which the transactions of the repository wait for. A
+// repository without a stack gets the same.
+func TestDurableStepMakesTheWatermarkDurableBeforeTheMarkerGoes(t *testing.T) {
t.Parallel()
- setup := func(t *testing.T) (durabilityFixture, *Durability, string, *bool) {
+ // watermarkSync is what the fsync of the watermark found.
+ type watermarkSync struct {
+ synced bool
+ // marker and gitLock tell whether the dirty marker and git's reference
+ // lock existed then.
+ marker, gitLock bool
+ }
+ setup := func(t *testing.T) (durabilityFixture, *Durability, string, *watermarkSync) {
f := setupDurability(t)
- watermark := wmPath(f.path)
+ watermark := testWatermarkPath(f.path)
require.NoError(t, os.MkdirAll(filepath.Dir(watermark), 0o755))
require.NoError(t, os.WriteFile(watermark, []byte(`{"repo_id":1,"seq":2}`), 0o644))
- var markerAtSync bool
+ var found watermarkSync
var d *Durability
- d = f.durability(t, "boot-a", 0, WithWmark(wmPath), WithSync(func(file *os.File) error {
+ d = f.durability(t, "boot-a", 0, WithWatermark(testWatermarkPath), WithSync(func(file *os.File) error {
if file.Name() == watermark {
+ found.synced = true
_, err := os.Stat(f.markerPath(t, d))
- markerAtSync = err == nil
+ found.marker = err == nil
+ _, err = os.Stat(filepath.Join(f.path, "reftable", "tables.list.lock"))
+ found.gitLock = err == nil
}
return f.synced.sync(file)
}))
- return f, d, watermark, &markerAtSync
+ return f, d, watermark, &found
}
t.Run("settle", func(t *testing.T) {
t.Parallel()
- f, d, watermark, markerAtSync := setup(t)
+ f, d, watermark, found := setup(t)
require.NoError(t, d.MarkDirty(f.ctx, f.repo, f.path))
require.NoError(t, d.Settle(f.path, false))
@@ -500,22 +520,65 @@ func TestSettle(t *testing.T) {
recorded := f.synced.recorded()
require.Contains(t, recorded, watermark)
require.Contains(t, recorded, filepath.Dir(watermark))
- require.True(t, *markerAtSync, "the watermark is durable before the marker goes")
+ require.Equal(t, watermarkSync{synced: true, marker: true}, *found,
+ "the watermark is durable before the marker goes, and its fsync does not hold git's lock")
require.NoFileExists(t, f.markerPath(t, d))
})
t.Run("forget", func(t *testing.T) {
t.Parallel()
- f, d, watermark, markerAtSync := setup(t)
+ f, d, watermark, found := setup(t)
require.NoError(t, d.MarkDirty(f.ctx, f.repo, f.path))
require.NoError(t, d.Forget(f.ctx, f.repo, f.path))
require.Contains(t, f.synced.recorded(), watermark)
- require.True(t, *markerAtSync)
+ require.Equal(t, watermarkSync{synced: true, marker: true}, *found)
require.NoFileExists(t, f.markerPath(t, d))
})
+ t.Run("a step that is not due", func(t *testing.T) {
+ t.Parallel()
+ f := setupDurability(t)
+ watermark := testWatermarkPath(f.path)
+ require.NoError(t, os.MkdirAll(filepath.Dir(watermark), 0o755))
+ require.NoError(t, os.WriteFile(watermark, []byte(`{"repo_id":1,"seq":2}`), 0o644))
+ d := f.durability(t, "boot-a", time.Hour, WithWatermark(testWatermarkPath))
+
+ require.NoError(t, d.MarkDirty(f.ctx, f.repo, f.path))
+ require.NoError(t, d.Settle(f.path, false))
+ require.NotContains(t, f.synced.recorded(), watermark, "the marker stays, so the watermark need not be durable yet")
+ require.FileExists(t, f.markerPath(t, d))
+ })
+
+ for _, forget := range []bool{false, true} {
+ desc := "a repository without a stack, settled"
+ if forget {
+ desc = "a repository without a stack, forgotten"
+ }
+ t.Run(desc, func(t *testing.T) {
+ t.Parallel()
+ f, d, watermark, found := setup(t)
+
+ // The syncer wrote the watermark without fsync while the repository had a
+ // stack, which has gone since.
+ require.NoError(t, d.MarkDirty(f.ctx, f.repo, f.path))
+ require.True(t, d.DefersWatermark(f.path))
+ require.NoError(t, os.Rename(filepath.Join(f.path, "reftable"), filepath.Join(t.TempDir(), "reftable")))
+
+ if forget {
+ require.NoError(t, d.Forget(f.ctx, f.repo, f.path))
+ } else {
+ require.NoError(t, d.Settle(f.path, false))
+ }
+
+ require.Contains(t, f.synced.recorded(), watermark)
+ require.Contains(t, f.synced.recorded(), filepath.Dir(watermark))
+ require.Equal(t, watermarkSync{synced: true, marker: true}, *found)
+ require.NoFileExists(t, f.markerPath(t, d))
+ })
+ }
+
t.Run("a failed fsync of the watermark", func(t *testing.T) {
t.Parallel()
f, d, _, _ := setup(t)
diff --git a/internal/graft/corestore/syncer/syncer.go b/internal/graft/corestore/syncer/syncer.go
index ab7a482..c6e1d8a 100644
--- a/internal/graft/corestore/syncer/syncer.go
+++ b/internal/graft/corestore/syncer/syncer.go
@@ -885,7 +885,7 @@ func (s *Syncer) advanceLocked(path string, wm corestore.Watermark) corestore.Wa
}
next := corestore.Watermark{RepoID: wm.RepoID, Seq: newSeq}
- if err := s.writeWM(path, next); err != nil {
+ if err := s.writeWatermark(path, next); err != nil {
s.logger.WithError(err).WithField("repository_path", path).Error("storage core: write watermark")
return wm
}
@@ -1325,17 +1325,20 @@ func (s *Syncer) applyLocked(ctx context.Context, repo *localrepo.Repo, hash git
// reached. The caller holds the repository lock.
func (s *Syncer) commitWatermarkLocked(path string, repoID int64, seq uint64) (corestore.Watermark, error) {
wm := corestore.Watermark{RepoID: repoID, Seq: seq}
- if err := s.writeWM(path, wm); err != nil {
+ if err := s.writeWatermark(path, wm); err != nil {
return corestore.Watermark{}, structerr.NewInternal("storage core: write watermark: %w", err)
}
return s.advanceLocked(path, wm), nil
}
-// Writes the watermark, without fsync while the repository is dirty (caller holds the lock).
-func (s *Syncer) writeWM(path string, wm corestore.Watermark) error {
- if s.core.RefDurability.CanDefer(path) {
+// writeWatermark writes the watermark of the repository at path: without fsync
+// while the repository is dirty (localrefs.Durability.DefersWatermark), whose
+// durable step makes it durable before the marker goes, and with fsync
+// otherwise. The caller holds the repository lock.
+func (s *Syncer) writeWatermark(path string, wm corestore.Watermark) error {
+ if s.core.RefDurability.DefersWatermark(path) {
s.hook("watermark deferred")
- return gitops.WriteLazy(path, wm)
+ return gitops.WriteWatermarkWithoutSync(path, wm)
}
s.hook("watermark")
return gitops.WriteWatermark(path, wm)
diff --git a/internal/graft/corestore/syncer/syncer_localrefs_test.go b/internal/graft/corestore/syncer/syncer_localrefs_test.go
index f375e84..d140950 100644
--- a/internal/graft/corestore/syncer/syncer_localrefs_test.go
+++ b/internal/graft/corestore/syncer/syncer_localrefs_test.go
@@ -114,7 +114,10 @@ func TestSyncerCatchUpMarksTheRepositoryDirty(t *testing.T) {
e.verifySynced(t, repo)
}
-func TestDefer(t *testing.T) {
+// TestSyncerDefersTheWatermarkOfADirtyRepository checks that the syncer writes
+// the watermark without fsync while the repository is dirty, and with fsync once
+// the durable step removed the marker.
+func TestSyncerDefersTheWatermarkOfADirtyRepository(t *testing.T) {
t.Parallel()
e := newEnv(t, envOptions{})
@@ -156,7 +159,10 @@ func TestDefer(t *testing.T) {
require.Equal(t, []string{"watermark"}, points)
}
-func TestTorn(t *testing.T) {
+// TestSyncerMovesAsideATornWatermarkAfterAKernelRestart checks that a dirty
+// repository whose watermark, written without fsync, is empty or torn after a
+// kernel restart is moved aside and hydrated again like any dirty repository.
+func TestSyncerMovesAsideATornWatermarkAfterAKernelRestart(t *testing.T) {
t.Parallel()
for name, torn := range map[string][]byte{"empty": nil, "garbage": []byte(`{"repo_id":`)} {