Git designed for AI agents.
corestore: fsync the deferred watermark outside git's lock, and before every marker removal
Federico Elia50194ed8768a
diff --git a/internal/cli/gitaly/serve.go b/internal/cli/gitaly/serve.goindex f194822..dcd1c5f 100644--- a/internal/cli/gitaly/serve.go+++ b/internal/cli/gitaly/serve.go@@ -1006,7 +1006,8 @@ func openLocalReferenceWrites(core *corestore.Core, locator storage.Locator, logcore.RefAdmission = localrefs.NewAdmission(lockTimeout, lockTimeout)prometheus.MustRegister(core.RefAdmission)- durability, err := localrefs.NewDurability(locator, localrefs.DurableStepIdle, localrefs.WithWmark(gitops.WatermarkPath))+ durability, err := localrefs.NewDurability(locator, localrefs.DurableStepIdle,+ localrefs.WithWatermark(gitops.WatermarkPath))if err != nil {return fmt.Errorf("graft storage core: %w", err)}diff --git a/internal/graft/corestore/blobstore/blobstore.go b/internal/graft/corestore/blobstore/blobstore.goindex 4524784..1d8a627 100644--- a/internal/graft/corestore/blobstore/blobstore.go+++ b/internal/graft/corestore/blobstore/blobstore.go@@ -36,9 +36,12 @@// anything, and a changeset row in the database (synchronous=FULL) could// outlive a pack whose data never reached the disk. For file buckets Put fsyncs// the content before the link publishes it, and fsyncs the directories that-// hold the new entry after it, up to the first one whose whole chain is durable.-// Bucket directories are never removed while a server uses the bucket: a recreated one could reuse an inode.-// The mem and remote drivers commit their objects durably themselves.+// hold the new entry after it, up to the first one whose whole chain up to the+// bucket root an earlier Put made durable. That chain is recognized by the+// directory's identity (device and inode), so bucket directories are never+// removed while a server uses the bucket: a directory removed and created again+// could reuse the inode of one that was durable. The mem and remote drivers+// commit their objects durably themselves.//// Listing. List reports objects only. fileblob also reports a writer's// temporary file (<key>.<nanos>.tmp, in the key's directory because file@@ -87,7 +90,8 @@ type Store struct {// root is the cleaned directory of a file bucket and empty for any other// driver. Put makes the writes of a file bucket durable from it.root string- // Directories whose entry in their parent was made durable, by path.+ // durableDirs holds, by path, the os.FileInfo of the directories of a file+ // bucket whose entries up to the root were made durable (syncDirs).durableDirs sync.Map}@@ -320,8 +324,13 @@ func syncFile(f *os.File) error {return f.Sync()}-// Fsyncs dir and its ancestors inside the bucket root, up to the first one whose whole chain an-// earlier call made durable. The walk caches its directories only once it reached that point.+// syncDirs fsyncs dir and, when dir is inside the bucket root, each of its+// ancestors: the directories that the driver may have created, and the entry+// that the link created. The walk stops at the root, or after the first+// ancestor whose entries up to the root an earlier walk made durable. The+// directories it visited are remembered only when it got there without an+// error, so that a concurrent Put never stops at a directory whose own entry is+// not durable yet.func (s *Store) syncDirs(dir string) error {inside := dir == s.root || strings.HasPrefix(dir, s.root+string(filepath.Separator))visited := map[string]os.FileInfo{}diff --git a/internal/graft/corestore/blobstore/durable_internal_test.go b/internal/graft/corestore/blobstore/durable_internal_test.goindex 859983c..a3994d9 100644--- a/internal/graft/corestore/blobstore/durable_internal_test.go+++ b/internal/graft/corestore/blobstore/durable_internal_test.go@@ -71,7 +71,10 @@ func TestPutSyncsFileBeforeCommitAndSyncsDirectories(t *testing.T) {require.Equal(t, content, data)}-func TestDirs(t *testing.T) {+// TestPutSyncsOnlyTheDirectoriesThatAreNotDurable checks that a Put fsyncs the+// directories up to the bucket root once, then only the directory of its key,+// and that a directory created again (another inode) is fsynced again.+func TestPutSyncsOnlyTheDirectoriesThatAreNotDurable(t *testing.T) {dir := t.TempDir()var dirSyncs []strings := openFileStoreForTest(t, dir, func(path string, isDir bool) error {@@ -84,7 +87,8 @@ func TestDirs(t *testing.T) {put := func(repoID int64) {dirSyncs = nilcontent := []byte("pack content")- require.NoError(t, s.Put(testhelper.Context(t), corestore.NewPackKey(repoID, sha256.Sum256(content)), bytes.NewReader(content)))+ key := corestore.NewPackKey(repoID, sha256.Sum256(content))+ require.NoError(t, s.Put(testhelper.Context(t), key, bytes.NewReader(content)))}repoDir := filepath.Join(dir, "repos", "1")packsDir := filepath.Join(repoDir, "packs")@@ -96,7 +100,8 @@ func TestDirs(t *testing.T) {require.Equal(t, []string{packsDir}, dirSyncs)put(2)- require.Equal(t, []string{filepath.Join(dir, "repos", "2", "packs"), filepath.Join(dir, "repos", "2"), filepath.Join(dir, "repos")}, dirSyncs)+ repo2Dir := filepath.Join(dir, "repos", "2")+ require.Equal(t, []string{filepath.Join(repo2Dir, "packs"), repo2Dir, filepath.Join(dir, "repos")}, dirSyncs)// Moving the directory aside keeps its inode in use, so the new directory gets another one.require.NoError(t, os.Rename(repoDir, filepath.Join(t.TempDir(), "old")))@@ -104,7 +109,10 @@ func TestDirs(t *testing.T) {require.Equal(t, []string{packsDir, repoDir, filepath.Join(dir, "repos")}, dirSyncs)}-func TestDirErr(t *testing.T) {+// TestPutSyncsAgainAfterAFailedDirectorySync checks that a Put whose fsync of an+// ancestor failed remembers none of its directories: the next Put fsyncs them+// all up to the root.+func TestPutSyncsAgainAfterAFailedDirectorySync(t *testing.T) {dir := t.TempDir()reposDir := filepath.Join(dir, "repos")fail := true@@ -121,16 +129,21 @@ func TestDirErr(t *testing.T) {})content := []byte("pack content")- require.Error(t, s.Put(testhelper.Context(t), corestore.NewPackKey(1, sha256.Sum256(content)), bytes.NewReader(content)))+ key := corestore.NewPackKey(1, sha256.Sum256(content))+ require.Error(t, s.Put(testhelper.Context(t), key, bytes.NewReader(content)))fail = falsedirSyncs = nil- require.NoError(t, s.Put(testhelper.Context(t), corestore.NewPackKey(1, sha256.Sum256(content)), bytes.NewReader(content)))+ key = corestore.NewPackKey(1, sha256.Sum256(content))+ require.NoError(t, s.Put(testhelper.Context(t), key, bytes.NewReader(content)))repoDir := filepath.Join(reposDir, "1")require.Equal(t, []string{filepath.Join(repoDir, "packs"), repoDir, reposDir, dir}, dirSyncs)}-func TestConcur(t *testing.T) {+// TestPutConcurrentWithTheFirstPutSyncsUpToTheRoot checks that a Put that runs+// while the first Put of a repository is still fsyncing its directories does not+// stop at one of them: it fsyncs its own way up to the root.+func TestPutConcurrentWithTheFirstPutSyncsUpToTheRoot(t *testing.T) {dir := t.TempDir()reposDir := filepath.Join(dir, "repos")reached, release := make(chan struct{}), make(chan struct{})diff --git a/internal/graft/corestore/coretest/coretest.go b/internal/graft/corestore/coretest/coretest.goindex 7daa396..54eecca 100644--- a/internal/graft/corestore/coretest/coretest.go+++ b/internal/graft/corestore/coretest/coretest.go@@ -213,7 +213,11 @@ func EnableLocalRefs(tb testing.TB, core *corestore.Core, cfg config.Cfg, opts .require.NoError(tb, core.RefDurability.Close())}core.RefAdmission = localrefs.NewAdmission(10*time.Second, 10*time.Second)- durability, err := localrefs.NewDurability(config.NewLocator(cfg), 0, append([]localrefs.DurabilityOption{localrefs.WithBootID(TestBootID), localrefs.WithRecoveryGrace(0), localrefs.WithWmark(gitops.WatermarkPath)}, opts...)...)+ durability, err := localrefs.NewDurability(config.NewLocator(cfg), 0, append([]localrefs.DurabilityOption{+ localrefs.WithBootID(TestBootID),+ localrefs.WithRecoveryGrace(0),+ localrefs.WithWatermark(gitops.WatermarkPath),+ }, opts...)...)require.NoError(tb, err)// A git process forked at the moment a previous durability closed its lock// holds a copy of it until it execs (close-on-exec): the lock can be busy fordiff --git a/internal/graft/corestore/gitops/gitops.go b/internal/graft/corestore/gitops/gitops.goindex ad9b5dc..a2198ca 100644--- a/internal/graft/corestore/gitops/gitops.go+++ b/internal/graft/corestore/gitops/gitops.go@@ -935,18 +935,24 @@ func WriteWatermark(repoPath string, w corestore.Watermark) error {if err := os.MkdirAll(dir, mode.Directory); err != nil {return fmt.Errorf("write watermark: %w", err)}- return writeWMark(repoPath, w, true)+ return writeWatermarkFile(repoPath, w, true)}-// WriteLazy writes the watermark without fsync, or with fsync when its directory is missing.-func WriteLazy(repoPath string, w corestore.Watermark) error {+// WriteWatermarkWithoutSync writes the watermark atomically (temporary file,+// rename) without fsync, for a repository whose durable step makes it durable+// later (localrefs.Durability.DefersWatermark). When graft/storage is missing it+// writes as WriteWatermark does, so that the directory it creates is durable.+func WriteWatermarkWithoutSync(repoPath string, w corestore.Watermark) error {if _, err := os.Stat(filepath.Dir(WatermarkPath(repoPath))); err != nil {return WriteWatermark(repoPath, w)}- return writeWMark(repoPath, w, false)+ return writeWatermarkFile(repoPath, w, false)}-func writeWMark(repoPath string, w corestore.Watermark, sync bool) (returnedErr error) {+// writeWatermarkFile writes the watermark into graft/storage, which exists,+// through a temporary file and a rename; sync fsyncs the file before the rename+// and the directory after it.+func writeWatermarkFile(repoPath string, w corestore.Watermark, sync bool) (returnedErr error) {dir := filepath.Dir(WatermarkPath(repoPath))data, err := json.Marshal(w)diff --git a/internal/graft/corestore/gitops/gitops_test.go b/internal/graft/corestore/gitops/gitops_test.goindex 881e8b2..26ebe4e 100644--- a/internal/graft/corestore/gitops/gitops_test.go+++ b/internal/graft/corestore/gitops/gitops_test.go@@ -536,15 +536,18 @@ func TestWatermarkRoundTrip(t *testing.T) {require.Equal(t, "applied", entries[0].Name())}-func TestNoSync(t *testing.T) {+// TestWriteWatermarkWithoutSync checks that the watermark written without fsync+// reads back, leaves no temporary file, and is written as WriteWatermark does+// when its directory is missing.+func TestWriteWatermarkWithoutSync(t *testing.T) {t.Parallel()repoPath := t.TempDir()// Without its directory the watermark is written as WriteWatermark does.- require.NoError(t, gitops.WriteLazy(repoPath, corestore.Watermark{RepoID: 7, Seq: 1}))+ require.NoError(t, gitops.WriteWatermarkWithoutSync(repoPath, corestore.Watermark{RepoID: 7, Seq: 1}))require.FileExists(t, gitops.WatermarkPath(repoPath))- require.NoError(t, gitops.WriteLazy(repoPath, corestore.Watermark{RepoID: 7, Seq: 2}))+ require.NoError(t, gitops.WriteWatermarkWithoutSync(repoPath, corestore.Watermark{RepoID: 7, Seq: 2}))w, ok, err := gitops.ReadWatermark(repoPath)require.NoError(t, err)require.True(t, ok)diff --git a/internal/graft/corestore/localrefs/durability.go b/internal/graft/corestore/localrefs/durability.goindex 83fee58..ab0b8ae 100644--- a/internal/graft/corestore/localrefs/durability.go+++ b/internal/graft/corestore/localrefs/durability.go@@ -165,7 +165,8 @@ type Durability struct {sync func(f *os.File) error// crashAt is the crash point selected with CrashEnv.crashAt string- // Returns the path of a repository's watermark.+ // watermark returns the path of the watermark of the repository at repoPath;+ // nil unless the syncer may write it without fsync (WithWatermark).watermark func(repoPath string) string// ownerFiles hold the owner locks, taken by TryOwn. owner is set by@@ -278,8 +279,10 @@ func WithSync(sync func(f *os.File) error) DurabilityOption {return func(d *Durability) { d.sync = sync }}-// WithWmark lets a dirty repository's watermark be written without fsync.-func WithWmark(path func(repoPath string) string) DurabilityOption {+// WithWatermark lets the syncer write the watermark of a dirty repository+// without fsync (DefersWatermark); path returns the watermark of a repository,+// which the durable step then makes durable before the marker goes.+func WithWatermark(path func(repoPath string) string) DurabilityOption {return func(d *Durability) { d.watermark = path }}@@ -602,14 +605,17 @@ func (d *Durability) Dirty() []string {return paths}-// CanDefer reports whether the watermark at path may be written without fsync now: the marker-// is durable and goes only after the watermark. The caller holds the syncer's repository lock.-func (d *Durability) CanDefer(path string) bool {+// DefersWatermark reports whether the watermark of the repository at path may be+// written without fsync now: the repository is dirty and trusted, so its marker+// is durable, and the marker goes only after a durable step made the watermark+// durable (syncWatermark). The caller holds the syncer's repository lock.+func (d *Durability) DefersWatermark(path string) bool {if !d.Deferring() || d.watermark == nil {return false}- // A marker without a stack goes without a durable step, so its watermark is written with fsync.+ // A repository without a stack has no durable step of its references: its+ // watermark is written with fsync.if stack, err := hasStack(path); err != nil || !stack {return false}@@ -683,12 +689,19 @@ func (d *Durability) Settle(path string, final bool) error {return err}if !dirty || !stack {- // A repository without a stack (gone, or not reftable) has nothing to- // make durable; a clean state only leaves the map. A transaction that+ // A repository without a stack (gone, or not reftable) has no references+ // to make durable; a clean state only leaves the map. A transaction that// marked a repository with a stack meanwhile leaves it to the next step,// under git's reference lock (dropMarker refuses otherwise).st := d.lockState(path)defer st.file.Unlock()+ if dirty {+ // Its watermark may have been written without fsync while it had a+ // stack.+ if err := d.syncWatermark(path); err != nil {+ return d.stepErrorLocked(st, repo, path, err)+ }+ }_, err := d.dropMarker(path, st, false)return err}@@ -702,6 +715,13 @@ func (d *Durability) Settle(path string, final bool) error {return nil}+ // The watermark is made durable before git's reference lock is taken, so that+ // its fsync does not hold the transactions of the repository.+ if err := d.syncWatermark(path); err != nil {+ st := d.lockState(path)+ defer st.file.Unlock()+ return d.stepErrorLocked(st, repo, path, err)+ }lock, ok, err := lockStack(path)if err != nil {return err@@ -724,7 +744,7 @@ func (d *Durability) settleLocked(path string, repo RepoRef) error {st := d.lockState(path)defer st.file.Unlock()- if err := d.syncAll(path, st); err != nil {+ if err := d.syncStack(path, st); err != nil {return d.stepErrorLocked(st, repo, path, err)}if d.crashAt == crashDurableStep {@@ -1090,11 +1110,12 @@ func (d *Durability) syncStack(path string, st *refState) error {return nil}-// Makes the stack, then the watermark file and its directory, durable.-func (d *Durability) syncAll(path string, st *refState) error {- if err := d.syncStack(path, st); err != nil {- return err- }+// syncWatermark makes the watermark of the repository at path and its directory+// durable, when the syncer writes it without fsync (WithWatermark) and the+// repository has one. The callers decide the marker after it, and hold the+// syncer's repository lock, under which the syncer writes the watermark: it+// cannot change before the marker goes.+func (d *Durability) syncWatermark(path string) error {if d.watermark == nil {return nil}@@ -1193,14 +1214,19 @@ func (d *Durability) Forget(ctx context.Context, repo RepoRef, path string) erro}// forgetLocked lets dropMarker decide the marker of a forgotten repository.-// One kept in place is made durable under git's reference lock first, as a-// durable step does; when the lock is busy the marker stays for the next step.-// The caller holds the state's file lock.+// Its watermark is made durable first. One kept in place is then made durable+// under git's reference lock, as a durable step does; when the lock is busy the+// marker stays for the next step. The caller holds the state's file lock and the+// syncer's repository lock.func (d *Durability) forgetLocked(path string, st *refState, repo RepoRef) (bool, error) {stack, err := hasStack(path)if err != nil {return false, err}+ // The watermark goes first, outside git's reference lock, as in Settle.+ if err := d.syncWatermark(path); err != nil {+ return false, d.stepErrorLocked(st, repo, path, err)+ }if !stack {return d.dropMarker(path, st, false)}@@ -1210,7 +1236,7 @@ func (d *Durability) forgetLocked(path string, st *refState, repo RepoRef) (boolreturn false, err}dropped := false- if err = d.syncAll(path, st); err != nil {+ if err = d.syncStack(path, st); err != nil {err = d.stepErrorLocked(st, repo, path, err)} else {dropped, err = d.dropMarker(path, st, true)diff --git a/internal/graft/corestore/localrefs/durability_test.go b/internal/graft/corestore/localrefs/durability_test.goindex 49bf6c5..f142f11 100644--- a/internal/graft/corestore/localrefs/durability_test.go+++ b/internal/graft/corestore/localrefs/durability_test.go@@ -405,11 +405,16 @@ func TestSettleReportsAFailedWriteAsUntrusted(t *testing.T) {require.Equal(t, 0.0, testutil.ToFloat64(d.dirty))}-func wmPath(repoPath string) string {+// testWatermarkPath is gitops.WatermarkPath, the watermark of the syncer.+func testWatermarkPath(repoPath string) string {return filepath.Join(repoPath, "graft", "storage", "applied")}-func TestDefers(t *testing.T) {+// TestDurabilityDefersTheWatermarkOfADirtyRepository checks that the watermark+// is written without fsync only by the owner configured with WithWatermark, and+// only for a trusted dirty repository with a reftable stack, whose durable step+// makes it durable.+func TestDurabilityDefersTheWatermarkOfADirtyRepository(t *testing.T) {t.Parallel()t.Run("without the option", func(t *testing.T) {@@ -418,81 +423,96 @@ func TestDefers(t *testing.T) {d := f.durability(t, "boot-a", 0)require.NoError(t, d.MarkDirty(f.ctx, f.repo, f.path))- require.False(t, d.CanDefer(f.path))+ require.False(t, d.DefersWatermark(f.path))})t.Run("while the repository is dirty", func(t *testing.T) {t.Parallel()f := setupDurability(t)- d := f.durability(t, "boot-a", 0, WithWmark(wmPath))+ d := f.durability(t, "boot-a", 0, WithWatermark(testWatermarkPath))- require.False(t, d.CanDefer(f.path), "a clean repository")+ require.False(t, d.DefersWatermark(f.path), "a clean repository")require.NoError(t, d.MarkDirty(f.ctx, f.repo, f.path))- require.True(t, d.CanDefer(f.path))+ require.True(t, d.DefersWatermark(f.path))require.NoError(t, d.Settle(f.path, false))- require.False(t, d.CanDefer(f.path), "the marker is gone")+ require.False(t, d.DefersWatermark(f.path), "the marker is gone")})t.Run("an untrusted repository", func(t *testing.T) {t.Parallel()f := setupDurability(t)- d := f.durability(t, "boot-a", 0, WithWmark(wmPath))+ d := f.durability(t, "boot-a", 0, WithWatermark(testWatermarkPath))require.NoError(t, d.MarkDirty(f.ctx, f.repo, f.path))f.synced.fail(errors.New("input/output error"))require.ErrorIs(t, d.Settle(f.path, false), ErrUntrusted)- require.False(t, d.CanDefer(f.path))+ require.False(t, d.DefersWatermark(f.path))})t.Run("a repository without a stack", func(t *testing.T) {t.Parallel()f := setupDurability(t)- d := f.durability(t, "boot-a", 0, WithWmark(wmPath))+ d := f.durability(t, "boot-a", 0, WithWatermark(testWatermarkPath))require.NoError(t, d.MarkDirty(f.ctx, f.repo, f.path))require.NoError(t, os.Rename(filepath.Join(f.path, "reftable"), filepath.Join(t.TempDir(), "reftable")))- require.False(t, d.CanDefer(f.path))+ require.False(t, d.DefersWatermark(f.path))})t.Run("a process that does not own the storage", func(t *testing.T) {t.Parallel()f := setupDurability(t)- d := f.unowned(t, "boot-a", 0, WithWmark(wmPath))+ d := f.unowned(t, "boot-a", 0, WithWatermark(testWatermarkPath))- require.False(t, d.CanDefer(f.path))+ require.False(t, d.DefersWatermark(f.path))})t.Run("a nil durability", func(t *testing.T) {t.Parallel()var d *Durability- require.False(t, d.CanDefer("/any"))+ require.False(t, d.DefersWatermark("/any"))})}-func TestSettle(t *testing.T) {+// TestDurableStepMakesTheWatermarkDurableBeforeTheMarkerGoes checks that a+// durable step and Forget fsync the watermark that the syncer wrote without+// fsync, and its directory, while the marker is still there, and outside git's+// reference lock, which the transactions of the repository wait for. A+// repository without a stack gets the same.+func TestDurableStepMakesTheWatermarkDurableBeforeTheMarkerGoes(t *testing.T) {t.Parallel()- setup := func(t *testing.T) (durabilityFixture, *Durability, string, *bool) {+ // watermarkSync is what the fsync of the watermark found.+ type watermarkSync struct {+ synced bool+ // marker and gitLock tell whether the dirty marker and git's reference+ // lock existed then.+ marker, gitLock bool+ }+ setup := func(t *testing.T) (durabilityFixture, *Durability, string, *watermarkSync) {f := setupDurability(t)- watermark := wmPath(f.path)+ watermark := testWatermarkPath(f.path)require.NoError(t, os.MkdirAll(filepath.Dir(watermark), 0o755))require.NoError(t, os.WriteFile(watermark, []byte(`{"repo_id":1,"seq":2}`), 0o644))- var markerAtSync bool+ var found watermarkSyncvar d *Durability- d = f.durability(t, "boot-a", 0, WithWmark(wmPath), WithSync(func(file *os.File) error {+ d = f.durability(t, "boot-a", 0, WithWatermark(testWatermarkPath), WithSync(func(file *os.File) error {if file.Name() == watermark {+ found.synced = true_, err := os.Stat(f.markerPath(t, d))- markerAtSync = err == nil+ found.marker = err == nil+ _, err = os.Stat(filepath.Join(f.path, "reftable", "tables.list.lock"))+ found.gitLock = err == nil}return f.synced.sync(file)}))- return f, d, watermark, &markerAtSync+ return f, d, watermark, &found}t.Run("settle", func(t *testing.T) {t.Parallel()- f, d, watermark, markerAtSync := setup(t)+ f, d, watermark, found := setup(t)require.NoError(t, d.MarkDirty(f.ctx, f.repo, f.path))require.NoError(t, d.Settle(f.path, false))@@ -500,22 +520,65 @@ func TestSettle(t *testing.T) {recorded := f.synced.recorded()require.Contains(t, recorded, watermark)require.Contains(t, recorded, filepath.Dir(watermark))- require.True(t, *markerAtSync, "the watermark is durable before the marker goes")+ require.Equal(t, watermarkSync{synced: true, marker: true}, *found,+ "the watermark is durable before the marker goes, and its fsync does not hold git's lock")require.NoFileExists(t, f.markerPath(t, d))})t.Run("forget", func(t *testing.T) {t.Parallel()- f, d, watermark, markerAtSync := setup(t)+ f, d, watermark, found := setup(t)require.NoError(t, d.MarkDirty(f.ctx, f.repo, f.path))require.NoError(t, d.Forget(f.ctx, f.repo, f.path))require.Contains(t, f.synced.recorded(), watermark)- require.True(t, *markerAtSync)+ require.Equal(t, watermarkSync{synced: true, marker: true}, *found)require.NoFileExists(t, f.markerPath(t, d))})+ t.Run("a step that is not due", func(t *testing.T) {+ t.Parallel()+ f := setupDurability(t)+ watermark := testWatermarkPath(f.path)+ require.NoError(t, os.MkdirAll(filepath.Dir(watermark), 0o755))+ require.NoError(t, os.WriteFile(watermark, []byte(`{"repo_id":1,"seq":2}`), 0o644))+ d := f.durability(t, "boot-a", time.Hour, WithWatermark(testWatermarkPath))++ require.NoError(t, d.MarkDirty(f.ctx, f.repo, f.path))+ require.NoError(t, d.Settle(f.path, false))+ require.NotContains(t, f.synced.recorded(), watermark, "the marker stays, so the watermark need not be durable yet")+ require.FileExists(t, f.markerPath(t, d))+ })++ for _, forget := range []bool{false, true} {+ desc := "a repository without a stack, settled"+ if forget {+ desc = "a repository without a stack, forgotten"+ }+ t.Run(desc, func(t *testing.T) {+ t.Parallel()+ f, d, watermark, found := setup(t)++ // The syncer wrote the watermark without fsync while the repository had a+ // stack, which has gone since.+ require.NoError(t, d.MarkDirty(f.ctx, f.repo, f.path))+ require.True(t, d.DefersWatermark(f.path))+ require.NoError(t, os.Rename(filepath.Join(f.path, "reftable"), filepath.Join(t.TempDir(), "reftable")))++ if forget {+ require.NoError(t, d.Forget(f.ctx, f.repo, f.path))+ } else {+ require.NoError(t, d.Settle(f.path, false))+ }++ require.Contains(t, f.synced.recorded(), watermark)+ require.Contains(t, f.synced.recorded(), filepath.Dir(watermark))+ require.Equal(t, watermarkSync{synced: true, marker: true}, *found)+ require.NoFileExists(t, f.markerPath(t, d))+ })+ }+t.Run("a failed fsync of the watermark", func(t *testing.T) {t.Parallel()f, d, _, _ := setup(t)diff --git a/internal/graft/corestore/syncer/syncer.go b/internal/graft/corestore/syncer/syncer.goindex ab7a482..c6e1d8a 100644--- a/internal/graft/corestore/syncer/syncer.go+++ b/internal/graft/corestore/syncer/syncer.go@@ -885,7 +885,7 @@ func (s *Syncer) advanceLocked(path string, wm corestore.Watermark) corestore.Wa}next := corestore.Watermark{RepoID: wm.RepoID, Seq: newSeq}- if err := s.writeWM(path, next); err != nil {+ if err := s.writeWatermark(path, next); err != nil {s.logger.WithError(err).WithField("repository_path", path).Error("storage core: write watermark")return wm}@@ -1325,17 +1325,20 @@ func (s *Syncer) applyLocked(ctx context.Context, repo *localrepo.Repo, hash git// reached. The caller holds the repository lock.func (s *Syncer) commitWatermarkLocked(path string, repoID int64, seq uint64) (corestore.Watermark, error) {wm := corestore.Watermark{RepoID: repoID, Seq: seq}- if err := s.writeWM(path, wm); err != nil {+ if err := s.writeWatermark(path, wm); err != nil {return corestore.Watermark{}, structerr.NewInternal("storage core: write watermark: %w", err)}return s.advanceLocked(path, wm), nil}-// Writes the watermark, without fsync while the repository is dirty (caller holds the lock).-func (s *Syncer) writeWM(path string, wm corestore.Watermark) error {- if s.core.RefDurability.CanDefer(path) {+// writeWatermark writes the watermark of the repository at path: without fsync+// while the repository is dirty (localrefs.Durability.DefersWatermark), whose+// durable step makes it durable before the marker goes, and with fsync+// otherwise. The caller holds the repository lock.+func (s *Syncer) writeWatermark(path string, wm corestore.Watermark) error {+ if s.core.RefDurability.DefersWatermark(path) {s.hook("watermark deferred")- return gitops.WriteLazy(path, wm)+ return gitops.WriteWatermarkWithoutSync(path, wm)}s.hook("watermark")return gitops.WriteWatermark(path, wm)diff --git a/internal/graft/corestore/syncer/syncer_localrefs_test.go b/internal/graft/corestore/syncer/syncer_localrefs_test.goindex f375e84..d140950 100644--- a/internal/graft/corestore/syncer/syncer_localrefs_test.go+++ b/internal/graft/corestore/syncer/syncer_localrefs_test.go@@ -114,7 +114,10 @@ func TestSyncerCatchUpMarksTheRepositoryDirty(t *testing.T) {e.verifySynced(t, repo)}-func TestDefer(t *testing.T) {+// TestSyncerDefersTheWatermarkOfADirtyRepository checks that the syncer writes+// the watermark without fsync while the repository is dirty, and with fsync once+// the durable step removed the marker.+func TestSyncerDefersTheWatermarkOfADirtyRepository(t *testing.T) {t.Parallel()e := newEnv(t, envOptions{})@@ -156,7 +159,10 @@ func TestDefer(t *testing.T) {require.Equal(t, []string{"watermark"}, points)}-func TestTorn(t *testing.T) {+// TestSyncerMovesAsideATornWatermarkAfterAKernelRestart checks that a dirty+// repository whose watermark, written without fsync, is empty or torn after a+// kernel restart is moved aside and hydrated again like any dirty repository.+func TestSyncerMovesAsideATornWatermarkAfterAKernelRestart(t *testing.T) {t.Parallel()for name, torn := range map[string][]byte{"empty": nil, "garbage": []byte(`{"repo_id":`)} {