Skip to content

Git designed for AI agents.

syncer, verify: hydrate from checkpoints, decide by C_sync (PLAN7 WP76)

Federico Elia3f994c67ca9e
diff --git a/doc/graft/PLAN7.md b/doc/graft/PLAN7.md
index 9d59e22..d23e7fd 100644
--- a/doc/graft/PLAN7.md
+++ b/doc/graft/PLAN7.md
@@ -1032,6 +1032,14 @@ next run finishes it. No lease (the repository is gone and its ID is never reuse
A missing checkpoint pack or a missing pack `> S` is an Internal error naming the
key (corruption, GC BOUNDARY rule).
+*As implemented (WP76).* `syncer.seed` reads the newest checkpoint and its
+snapshot. It checks the pack key's repository ID and validates every name and
+target like a changeset's (names, targets, symbolic targets), then indexes the
+pack through the existing `indexPack` (SHA-256 against the key). Final
+references = snapshot overlaid by the fetched final values, where an absent
+value deletes. `graft_storage_sync_checkpoint_seeds_total` counts the seeded
+hydrations.
+
### Catch-up and the sync boundary (syncer, WP76, change C)
In `sync()`, each round reads `C_sync = SyncBoundary(r.ID, gc_grace − sync_timeout −
@@ -1039,6 +1047,26 @@ gc_margin)` and hydrates when `!ok || wm.RepoID != r.ID || !localCurrent ||
wm.Seq < C_sync`; otherwise it catches up as in phase 6 (the WP55 RULE unchanged).
`checkReplaceable` still runs before any hydration.
+*As implemented (WP76).*
+* `C_sync` is read only when a catch-up is needed: after the node's own noted
+ sequences advanced the watermark, and only when it is below the head. So a
+ fresh repository costs no extra query, and a node whose own writes carried it
+ past `C_sync` is not re-hydrated.
+* `install`'s errRestart condition is the exact negation of the new hydrate
+ condition (`wm.Seq >= C_sync` added). Without it, a current but lagging copy
+ would be sent back to sync forever.
+* The age `gc_grace − sync_timeout − gc_margin` is taken as 0 when it is not
+ positive (the configuration refuses that; only test cores have such
+ times). With age 0 every checkpoint counts, which is the conservative end:
+ more hydrations, never a collected pack.
+* Audit of the bound: every pack read and git process of a round (`fetch`,
+ `indexPack`, `applyBuild`, the catch-up loop, `install`'s lock) runs under
+ the round's context, which `EnsureFresh` bounds by `sync_timeout` per flight.
+ The only work outside it is the best-effort `ReportApplied` after the round
+ (at most 1 s, no pack read).
+* `TestSyncerRoundBoundedBySyncTimeout` shows that a bucket read is cancelled by
+ the round's deadline and nothing is installed.
+
### Verify after compaction (`verify_objects.go`, WP76)
`checkStoredObjects` indexes the newest checkpoint's pack and the changeset packs
@@ -1048,6 +1076,16 @@ under a deadline of `W` from its start (a dry hydration that read checkpoint `S`
must end before a checkpoint newer than `S` can become `C_gc`); a missing pack is
reported as a connectivity failure starting with `storage core objects:`.
+*As implemented (WP76).*
+* `checkStoredObjects` reads the newest checkpoint before the references. A
+ checkpoint never exceeds the head at its creation, so the head read after it
+ is at or above it.
+* The open reviews' commits are checked through their keep-around references
+ (REVIEW ROOTS RULE), which are database references; there is no separate
+ review check.
+* Running out of `W` is `Unavailable` ("did not finish within the write
+ deadline").
+
### Background maintenance (`internal/graft/corestore/maintenance`, WP77)
```go
diff --git a/internal/gitaly/service/storagecore/verify.go b/internal/gitaly/service/storagecore/verify.go
index 8ed5617..b417ce0 100644
--- a/internal/gitaly/service/storagecore/verify.go
+++ b/internal/gitaly/service/storagecore/verify.go
@@ -95,7 +95,7 @@ func (s *server) VerifyRepository(ctx context.Context, req *gitalypb.VerifyRepos
// Equal references are not enough: the object storage must also hold
// every object they reach, or a hydration on another node fails.
- storedOK, storedMsg, err := s.checkStoredObjects(ctx, core, repo, record, dbRefs, headSeq)
+ storedOK, storedMsg, err := s.checkStoredObjects(ctx, core, repo, record)
if err != nil {
return nil, err
}
diff --git a/internal/gitaly/service/storagecore/verify_objects.go b/internal/gitaly/service/storagecore/verify_objects.go
index d4a4d29..9cc499e 100644
--- a/internal/gitaly/service/storagecore/verify_objects.go
+++ b/internal/gitaly/service/storagecore/verify_objects.go
@@ -8,6 +8,7 @@ import (
"os"
"path/filepath"
"strings"
+ "time"
"github.com/feder-cr/git-ai/internal/command"
"github.com/feder-cr/git-ai/internal/git/gitcmd"
@@ -23,10 +24,20 @@ import (
const verifyChangesetBatch = 500
// checkStoredObjects checks that the object storage can rebuild the database
-// state of the repository: it indexes the pack of every changeset up to
-// headSeq into a new empty repository in the storage's temporary directory (a
-// dry hydration, nothing is installed) and checks that every object reachable
-// from the object IDs of dbRefs is there (`git rev-list --objects --quiet`).
+// state of the repository the way a hydration does: into a new empty
+// repository in the storage's temporary directory (a dry hydration, nothing is
+// installed) it indexes the pack of the newest checkpoint, if any, and the pack
+// of every changeset after it up to the head, then checks that every object
+// reachable from the object IDs of the database references is there (`git
+// rev-list --objects --quiet`). The keep-around references of the open reviews
+// are among them, so their commits are checked too (doc/graft/PLAN7.md, "Verify
+// after compaction").
+//
+// The checkpoint is read before the references, so the head they belong to is
+// at or above it. The check runs under a deadline of W (WriteDeadline) from its
+// start: a checkpoint newer than the one it read can become C_gc, and so let
+// garbage collection delete the packs this check reads, only gc_grace after its
+// creation, which is after the start of the check.
//
// The local connectivity check cannot see this: a git write that shadow mode
// did not record leaves objects that are in the local repository but in no
@@ -34,7 +45,14 @@ const verifyChangesetBatch = 500
// references of git and of the database agree while a hydration on another
// node fails. It returns whether the check passed and, when it did not, what
// is missing; an error only when the check could not run.
-func (s *server) checkStoredObjects(ctx context.Context, core *corestore.Core, repo *gitalypb.Repository, record corestore.Repository, dbRefs map[string]corestore.Target, headSeq uint64) (bool, string, error) {
+func (s *server) checkStoredObjects(ctx context.Context, core *corestore.Core, repo *gitalypb.Repository, record corestore.Repository) (bool, string, error) {
+ deadline := core.WriteDeadline()
+ if deadline <= 0 {
+ return false, "", structerr.NewFailedPrecondition("storage core: invalid write deadline %s", deadline)
+ }
+ ctx, cancel := context.WithTimeout(ctx, deadline)
+ defer cancel()
+
storageName := repo.GetStorageName()
storagePath, err := s.deps.GetLocator().GetStorageByName(ctx, storageName)
if err != nil {
@@ -63,11 +81,32 @@ func (s *server) checkStoredObjects(ctx context.Context, core *corestore.Core, r
}
stored := s.localRepo(&gitalypb.Repository{StorageName: storageName, RelativePath: rel})
- for after := uint64(0); after < headSeq; {
+ cp, seeded, err := core.Meta.NewestCheckpoint(ctx, record.ID)
+ if err != nil {
+ return false, "", deadlineOr(ctx, deadline, structerr.NewUnavailable("storage core: newest checkpoint: %w", err))
+ }
+ dbRefs, headSeq, err := core.Meta.Refs(ctx, record.ID)
+ if err != nil {
+ return false, "", deadlineOr(ctx, deadline, structerr.NewUnavailable("storage core: references: %w", err))
+ }
+
+ after := uint64(0)
+ if seeded {
+ if id, _, ok := corestore.ParsePackKey(cp.PackKey); !ok || id != record.ID {
+ return false, fmt.Sprintf("checkpoint %d references %q, which is not a pack of the repository", cp.Seq, cp.PackKey), nil
+ }
+ problem, err := indexStoredPack(ctx, core, stored, cp.PackKey, fmt.Sprintf("checkpoint %d", cp.Seq))
+ if err != nil || problem != "" {
+ return false, problem, deadlineOr(ctx, deadline, err)
+ }
+ after = cp.Seq
+ }
+
+ for after < headSeq {
upTo := min(after+verifyChangesetBatch, headSeq)
changesets, err := core.Meta.ListChangesets(ctx, record.ID, after, upTo)
if err != nil {
- return false, "", structerr.NewUnavailable("storage core: list changesets: %w", err)
+ return false, "", deadlineOr(ctx, deadline, structerr.NewUnavailable("storage core: list changesets: %w", err))
}
if len(changesets) == 0 {
return false, fmt.Sprintf("the database has no changesets after %d up to %d", after, upTo), nil
@@ -82,9 +121,9 @@ func (s *server) checkStoredObjects(ctx context.Context, core *corestore.Core, r
return false, fmt.Sprintf("changeset %d references %q, which is not a pack of the repository", cs.Seq, cs.PackKey), nil
}
- problem, err := indexStoredPack(ctx, core, stored, cs)
+ problem, err := indexStoredPack(ctx, core, stored, cs.PackKey, fmt.Sprintf("changeset %d", cs.Seq))
if err != nil || problem != "" {
- return false, problem, err
+ return false, problem, deadlineOr(ctx, deadline, err)
}
}
}
@@ -114,7 +153,7 @@ func (s *server) checkStoredObjects(ctx context.Context, core *corestore.Core, r
return true, "", nil
}
if ctxErr := ctx.Err(); ctxErr != nil {
- return false, "", structerr.NewUnavailable("stored objects check: %w", ctxErr)
+ return false, "", deadlineOr(ctx, deadline, structerr.NewUnavailable("stored objects check: %w", ctxErr))
}
if _, exited := command.ExitStatus(err); !exited {
return false, "", structerr.NewInternal("stored objects check: %w", err)
@@ -127,13 +166,14 @@ func (s *server) checkStoredObjects(ctx context.Context, core *corestore.Core, r
return false, "the packs of the database do not hold every object of its references: " + msg, nil
}
-// indexStoredPack downloads the pack of cs (whose key was checked) and indexes
-// it into repo. It returns a description of a missing or corrupt pack, or an
-// error when the check could not run.
-func indexStoredPack(ctx context.Context, core *corestore.Core, repo *localrepo.Repo, cs corestore.Changeset) (string, error) {
- rc, err := core.Blobs.Get(ctx, cs.PackKey)
+// indexStoredPack downloads the pack of key (which was checked), the pack of
+// owner ("checkpoint 4", "changeset 7"), and indexes it into repo. It returns a
+// description of a missing or corrupt pack, or an error when the check could
+// not run.
+func indexStoredPack(ctx context.Context, core *corestore.Core, repo *localrepo.Repo, key, owner string) (string, error) {
+ rc, err := core.Blobs.Get(ctx, key)
if errors.Is(err, corestore.ErrBlobNotFound) {
- return fmt.Sprintf("the pack %q of changeset %d is missing from the object storage", cs.PackKey, cs.Seq), nil
+ return fmt.Sprintf("the pack %q of %s is missing from the object storage", key, owner), nil
}
if err != nil {
return "", structerr.NewUnavailable("storage core: download pack: %w", err)
@@ -144,7 +184,16 @@ func indexStoredPack(ctx context.Context, core *corestore.Core, repo *localrepo.
if ctxErr := ctx.Err(); ctxErr != nil {
return "", structerr.NewUnavailable("stored objects check: %w", ctxErr)
}
- return fmt.Sprintf("the pack %q of changeset %d cannot be indexed: %v", cs.PackKey, cs.Seq, err), nil
+ return fmt.Sprintf("the pack %q of %s cannot be indexed: %v", key, owner, err), nil
}
return "", nil
}
+
+// deadlineOr returns the error of a check that ran out of its deadline when
+// ctx's deadline has passed, and err otherwise.
+func deadlineOr(ctx context.Context, deadline time.Duration, err error) error {
+ if err != nil && errors.Is(ctx.Err(), context.DeadlineExceeded) {
+ return structerr.NewUnavailable("storage core: the stored objects check did not finish within the write deadline %s: %w", deadline, err)
+ }
+ return err
+}
diff --git a/internal/gitaly/service/storagecore/verify_objects_test.go b/internal/gitaly/service/storagecore/verify_objects_test.go
index 3e7e702..8457ae2 100644
--- a/internal/gitaly/service/storagecore/verify_objects_test.go
+++ b/internal/gitaly/service/storagecore/verify_objects_test.go
@@ -1,9 +1,15 @@
package storagecore
import (
+ "context"
+ "io"
+ "sync/atomic"
"testing"
+ "time"
"github.com/stretchr/testify/require"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
"github.com/feder-cr/git-ai/internal/git"
"github.com/feder-cr/git-ai/internal/git/gittest"
@@ -11,6 +17,7 @@ import (
"github.com/feder-cr/git-ai/internal/graft/corestore"
"github.com/feder-cr/git-ai/internal/graft/corestore/gitops"
"github.com/feder-cr/git-ai/internal/graft/corestore/sqlstore"
+ "github.com/feder-cr/git-ai/proto/go/gitalypb"
)
// A git write that the storage core did not record leaves objects that no pack
@@ -69,3 +76,98 @@ func TestVerifyRepositoryReportsObjectsMissingFromStorage(t *testing.T) {
require.NotContains(t, verify.GetConnectivityError(), "\nstorage core objects", "the local check passed: only the storage message")
})
}
+
+// checkpointAtHead records a checkpoint of the repository at its database
+// head, with a pack of everything its references reach built from the local
+// repository, as checkpoint.Create does.
+func (h *harness) checkpointAtHead(repoID int64) corestore.Checkpoint {
+ h.t.Helper()
+
+ refs, seq, err := h.core.Meta.Refs(h.ctx, repoID)
+ require.NoError(h.t, err)
+ var tips []git.ObjectID
+ for _, target := range refs {
+ if !target.IsSymbolic() {
+ tips = append(tips, git.ObjectID(target))
+ }
+ }
+ pack, err := gitops.BuildPackFrom(h.ctx, localrepo.NewTestRepo(h.t, h.cfg, h.repo), tips, h.t.TempDir())
+ require.NoError(h.t, err)
+ defer func() { require.NoError(h.t, pack.Remove()) }()
+ key := corestore.NewPackKey(repoID, pack.SHA256)
+ f, err := pack.Open()
+ require.NoError(h.t, err)
+ require.NoError(h.t, h.core.Blobs.Put(h.ctx, key, f))
+ require.NoError(h.t, f.Close())
+
+ cp, created, err := h.core.Meta.CreateCheckpoint(h.ctx, repoID, corestore.CheckpointInput{Seq: seq, Refs: refs, PackKey: key, PackBytes: pack.Size})
+ require.NoError(h.t, err)
+ require.True(h.t, created)
+ return cp
+}
+
+// TestVerifyRepositoryAfterCompaction: the stored objects check rebuilds the
+// repository the way a hydration does, from the newest checkpoint and the
+// changesets after it, so it passes once garbage collection deleted the packs
+// the checkpoint covers; a missing checkpoint pack is reported.
+func TestVerifyRepositoryAfterCompaction(t *testing.T) {
+ forEachDialect(t, func(t *testing.T, dialect sqlstore.Dialect) {
+ h := setup(t, corestore.ModeShadow, dialect)
+ h.seed()
+ repoID := h.importRepo()
+ cp := h.checkpointAtHead(repoID)
+
+ // Collected: every changeset pack up to the checkpoint.
+ st, err := h.core.Meta.GCSnapshot(h.ctx, repoID)
+ require.NoError(t, err)
+ require.NotEmpty(t, st.Packs)
+ for key, seq := range st.Packs {
+ require.LessOrEqual(t, seq, cp.Seq)
+ require.NoError(t, h.core.Blobs.Delete(h.ctx, key))
+ }
+
+ verify := h.verify(true)
+ require.True(t, verify.GetConnectivityOk(), verify.GetConnectivityError())
+
+ require.NoError(t, h.core.Blobs.Delete(h.ctx, cp.PackKey))
+ verify = h.verify(true)
+ require.False(t, verify.GetConnectivityOk())
+ require.Contains(t, verify.GetConnectivityError(), "storage core objects: the pack \""+cp.PackKey+"\" of checkpoint")
+ })
+}
+
+// slowReads is a bucket whose reads, once enabled, last until the end of their
+// context. It is installed before the server starts and switched afterwards,
+// so the core is never changed while the server reads it.
+type slowReads struct {
+ corestore.BlobStore
+ enabled *atomic.Bool
+}
+
+func (b slowReads) Get(ctx context.Context, key string) (io.ReadCloser, error) {
+ if b.enabled.Load() {
+ <-ctx.Done()
+ return nil, ctx.Err()
+ }
+ return b.BlobStore.Get(ctx, key)
+}
+
+// TestVerifyRepositoryStoredObjectsCheckIsBounded: the dry hydration runs under
+// the write deadline W from its start (GC BOUNDARY rule, enforcement): one that
+// outlives it is cancelled and fails as Unavailable.
+func TestVerifyRepositoryStoredObjectsCheckIsBounded(t *testing.T) {
+ var enabled atomic.Bool
+ h := setup(t, corestore.ModeShadow, sqlstore.SQLite, func(core *corestore.Core) {
+ // W = gc_grace - gc_margin = 3 s.
+ core.GCGrace = 4 * time.Second
+ core.GCMargin = time.Second
+ core.Blobs = slowReads{BlobStore: core.Blobs, enabled: &enabled}
+ })
+ h.seed()
+ h.importRepo()
+
+ enabled.Store(true)
+ _, err := h.client.VerifyRepository(h.ctx, &gitalypb.VerifyRepositoryRequest{Repository: h.repo, CheckConnectivity: true})
+ require.Equal(t, codes.Unavailable, status.Code(err), "%v", err)
+ require.ErrorContains(t, err, "did not finish within the write deadline 3s")
+}
diff --git a/internal/graft/corestore/syncer/syncer.go b/internal/graft/corestore/syncer/syncer.go
index 74bb43d..3c91015 100644
--- a/internal/graft/corestore/syncer/syncer.go
+++ b/internal/graft/corestore/syncer/syncer.go
@@ -130,6 +130,7 @@ type Syncer struct {
syncSeconds prometheus.Histogram
syncChangesets prometheus.Counter
syncBytes prometheus.Counter
+ syncSeeds prometheus.Counter
}
type repoLock struct {
@@ -176,6 +177,10 @@ func New(core *corestore.Core, logger log.Logger, locator storage.Locator, repoF
Name: "graft_storage_sync_bytes_total",
Help: "Bytes of packs downloaded by catch-up and hydration.",
}),
+ syncSeeds: prometheus.NewCounter(prometheus.CounterOpts{
+ Name: "graft_storage_sync_checkpoint_seeds_total",
+ Help: "Number of hydrations that started from a checkpoint instead of changeset 1.",
+ }),
}
for _, result := range []string{resultFresh, resultCaughtUp, resultHydrated, resultError} {
s.syncTotal.WithLabelValues(result)
@@ -300,6 +305,7 @@ func (s *Syncer) Describe(descs chan<- *prometheus.Desc) {
s.syncSeconds.Describe(descs)
s.syncChangesets.Describe(descs)
s.syncBytes.Describe(descs)
+ s.syncSeeds.Describe(descs)
}
// Collect implements prometheus.Collector.
@@ -311,6 +317,7 @@ func (s *Syncer) Collect(metrics chan<- prometheus.Metric) {
s.syncSeconds.Collect(metrics)
s.syncChangesets.Collect(metrics)
s.syncBytes.Collect(metrics)
+ s.syncSeeds.Collect(metrics)
}
func (s *Syncer) syncTimeout() time.Duration {
@@ -354,29 +361,44 @@ func (s *Syncer) sync(ctx context.Context, key corestore.RepoKey) error {
}
wm, ok := s.readWatermark(path)
- if !ok || wm.RepoID != r.ID || !s.localCurrent(ctx, repo, path) {
- if err := s.checkReplaceable(ctx, r, path, wm, ok); err != nil {
+ stale := !ok || wm.RepoID != r.ID || !s.localCurrent(ctx, repo, path)
+ // boundary is C_sync: a local repository below it is hydrated, because
+ // the changeset packs it would catch up from may be collected
+ // (doc/graft/PLAN7.md, GC BOUNDARY rule, change C).
+ var boundary uint64
+ if !stale {
+ // Advance through the sequences this node committed itself before
+ // downloading anything.
+ wm, err = s.advance(ctx, path, wm)
+ if err != nil {
return err
}
- err := s.hydrate(ctx, repo, r, hash, path)
+ if wm.Seq >= r.HeadSeq {
+ result = resultFresh
+ return nil
+ }
+ boundary, err = s.syncBoundary(ctx, r.ID)
if errors.Is(err, errRestart) {
continue
}
if err != nil {
return err
}
- result = resultHydrated
- return nil
+ stale = wm.Seq < boundary
}
- // Advance through the sequences this node committed itself before
- // downloading anything.
- wm, err = s.advance(ctx, path, wm)
- if err != nil {
- return err
- }
- if wm.Seq >= r.HeadSeq {
- result = resultFresh
+ if stale {
+ if err := s.checkReplaceable(ctx, r, path, wm, ok); err != nil {
+ return err
+ }
+ err := s.hydrate(ctx, repo, r, hash, path, boundary)
+ if errors.Is(err, errRestart) {
+ continue
+ }
+ if err != nil {
+ return err
+ }
+ result = resultHydrated
return nil
}
@@ -396,6 +418,26 @@ func (s *Syncer) sync(ctx context.Context, key corestore.RepoKey) error {
}
}
+// syncBoundary returns C_sync of the repository: the sequence of the newest
+// checkpoint created at least gc_grace − sync_timeout − gc_margin ago on the
+// database clock, 0 when there is none (doc/graft/PLAN7.md, GC BOUNDARY rule).
+// A sync round, bounded by sync_timeout, never needs a pack that garbage
+// collection deletes meanwhile when its watermark is at or above C_sync. The
+// configuration requires gc_grace > sync_timeout + gc_margin; with times that
+// do not (only test cores), the age is 0 and every checkpoint counts, which is
+// the conservative end: more hydrations, never a collected pack.
+func (s *Syncer) syncBoundary(ctx context.Context, repoID int64) (uint64, error) {
+ age := max(s.core.GCGrace-s.syncTimeout()-s.core.GCMargin, 0)
+ boundary, err := s.core.Meta.SyncBoundary(ctx, repoID, age)
+ if err != nil {
+ if errors.Is(err, corestore.ErrRepositoryNotFound) {
+ return 0, errRestart
+ }
+ return 0, structerr.NewUnavailable("storage core: sync boundary: %w", err)
+ }
+ return boundary, nil
+}
+
// record reads the database record of key and the object hash of its format.
// For a key the database does not have it returns the error of untracked
// (errRestart when the repository was registered meanwhile).
@@ -1362,12 +1404,14 @@ func refTransaction(ctx context.Context, repo *localrepo.Repo, commands []string
}
}
-// hydrate builds a bare repository from changeset 1 up to the head in the
-// temporary directory of the storage, then moves it to path. A local
-// repository at path (another repository ID, or invalid) is moved away first
-// and removed in the background. Only the final moves are done under the
-// repository lock.
-func (s *Syncer) hydrate(ctx context.Context, repo *gitalypb.Repository, r corestore.Repository, hash git.ObjectHash, path string) error {
+// hydrate builds a bare repository in the temporary directory of the storage,
+// then moves it to path. It starts from the newest checkpoint when there is one
+// (its pack and its reference snapshot) and replays the changesets after it,
+// otherwise from changeset 1 (doc/graft/PLAN7.md, "Hydration from a
+// checkpoint"). A local repository at path (another repository ID, invalid, or
+// below boundary, C_sync) is moved away first and removed in the background.
+// Only the final moves are done under the repository lock.
+func (s *Syncer) hydrate(ctx context.Context, repo *gitalypb.Repository, r corestore.Repository, hash git.ObjectHash, path string, boundary uint64) error {
storageName := repo.GetStorageName()
storagePath, err := s.locator.GetStorageByName(ctx, storageName)
@@ -1396,18 +1440,28 @@ func (s *Syncer) hydrate(ctx context.Context, repo *gitalypb.Repository, r cores
}
buildRepo := s.repoFactory.Build(&gitalypb.Repository{StorageName: storageName, RelativePath: rel})
- f := newFetched(0)
+ refs, f, err := s.seed(ctx, buildRepo, tmpDir, r, hash)
+ if err != nil {
+ return err
+ }
if err := s.fetch(ctx, buildRepo, tmpDir, r.ID, hash, f, r.HeadSeq); err != nil {
return err
}
- if err := s.applyBuild(ctx, buildRepo, f.finalRefs()); err != nil {
+ for name, target := range f.finalRefs() {
+ if target.Absent() {
+ delete(refs, name)
+ } else {
+ refs[name] = target
+ }
+ }
+ if err := s.applyBuild(ctx, buildRepo, refs); err != nil {
return err
}
if err := gitops.WriteWatermark(buildPath, corestore.Watermark{RepoID: r.ID, Seq: f.head}); err != nil {
return structerr.NewInternal("storage core: write watermark: %w", err)
}
- if err := s.install(ctx, repo, r, path, buildPath, tmpDir); err != nil {
+ if err := s.install(ctx, repo, r, path, buildPath, tmpDir, boundary); err != nil {
return err
}
@@ -1417,23 +1471,71 @@ func (s *Syncer) hydrate(ctx context.Context, repo *gitalypb.Repository, r cores
return nil
}
+// seed starts a hydration: from the newest checkpoint when there is one (its
+// pack, verified against its key and indexed into repo, and its validated
+// reference snapshot), else from nothing. It returns the references to start
+// from and the fetch state positioned after the seed.
+func (s *Syncer) seed(ctx context.Context, repo *localrepo.Repo, tmpDir string, r corestore.Repository, hash git.ObjectHash) (map[string]corestore.Target, *fetched, error) {
+ cp, ok, err := s.core.Meta.NewestCheckpoint(ctx, r.ID)
+ if err != nil {
+ return nil, nil, structerr.NewUnavailable("storage core: newest checkpoint: %w", err)
+ }
+ if !ok {
+ return map[string]corestore.Target{}, newFetched(0), nil
+ }
+
+ // Data read from the database is validated before it touches the
+ // repository (RULE 22).
+ if id, _, ok := corestore.ParsePackKey(cp.PackKey); !ok || id != r.ID {
+ return nil, nil, structerr.NewInternal("storage core: checkpoint %d has an invalid pack key", cp.Seq)
+ }
+ refs, err := s.core.Meta.CheckpointRefs(ctx, r.ID, cp.Seq)
+ if err != nil {
+ return nil, nil, structerr.NewUnavailable("storage core: checkpoint %d references: %w", cp.Seq, err)
+ }
+ for name, target := range refs {
+ if target.Absent() {
+ return nil, nil, structerr.NewInternal("storage core: checkpoint %d: reference %q has no value", cp.Seq, name)
+ }
+ if err := gitops.ValidateName(name); err != nil {
+ return nil, nil, structerr.NewInternal("storage core: checkpoint %d: invalid reference name %q: %w", cp.Seq, name, err)
+ }
+ if err := gitops.ValidateTarget(target, hash); err != nil {
+ return nil, nil, structerr.NewInternal("storage core: checkpoint %d: invalid target of %q: %w", cp.Seq, name, err)
+ }
+ if err := validateSymbolicTarget(name, target); err != nil {
+ return nil, nil, structerr.NewInternal("storage core: checkpoint %d: invalid target of %q: %w", cp.Seq, name, err)
+ }
+ }
+
+ f := newFetched(cp.Seq)
+ n, err := s.indexPack(ctx, repo, tmpDir, cp.PackKey)
+ if err != nil {
+ return nil, nil, err
+ }
+ f.bytes += n
+ s.syncSeeds.Inc()
+ return refs, f, nil
+}
+
// install moves the hydrated repository at buildPath to path under the
// repository lock, moving a local repository that is in the way into the
// temporary directory first. It returns errRestart when, meanwhile, the local
// repository became one that a catch-up can bring up to date (a reftable
-// repository with a watermark of the repository: it was imported or created
-// here): it is caught up instead of being replaced. The condition is the
-// negation of the one that made sync hydrate, so a repository that is still
-// stale (for example a files-backend one with a matching watermark) is
-// replaced, never sent back to sync.
-func (s *Syncer) install(ctx context.Context, repo *gitalypb.Repository, r corestore.Repository, path, buildPath, tmpDir string) error {
+// repository with a watermark of the repository at or above boundary, C_sync:
+// it was imported, created or caught up here): it is caught up instead of
+// being replaced. The condition is the negation of the one that made sync
+// hydrate, so a repository that is still stale (for example a files-backend
+// one with a matching watermark, or one below C_sync) is replaced, never sent
+// back to sync.
+func (s *Syncer) install(ctx context.Context, repo *gitalypb.Repository, r corestore.Repository, path, buildPath, tmpDir string, boundary uint64) error {
release, err := s.lock(ctx, path)
if err != nil {
return err
}
defer release()
- if wm, ok := s.readWatermark(path); ok && wm.RepoID == r.ID && s.localCurrent(ctx, repo, path) {
+ if wm, ok := s.readWatermark(path); ok && wm.RepoID == r.ID && wm.Seq >= boundary && s.localCurrent(ctx, repo, path) {
return errRestart
}
diff --git a/internal/graft/corestore/syncer/syncer_checkpoint_test.go b/internal/graft/corestore/syncer/syncer_checkpoint_test.go
new file mode 100644
index 0000000..27015a5
--- /dev/null
+++ b/internal/graft/corestore/syncer/syncer_checkpoint_test.go
@@ -0,0 +1,230 @@
+package syncer_test
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
+
+ "github.com/feder-cr/git-ai/internal/git"
+ "github.com/feder-cr/git-ai/internal/graft/corestore"
+ "github.com/feder-cr/git-ai/internal/graft/corestore/gitops"
+ "github.com/feder-cr/git-ai/internal/graft/corestore/sqlstore"
+)
+
+// checkpoint records a checkpoint of repository id at its head, with a pack of
+// everything its references reach built from src, as checkpoint.Create does.
+func (e *env) checkpoint(t *testing.T, id int64, src source) corestore.Checkpoint {
+ t.Helper()
+
+ refs, seq, err := e.core.Meta.Refs(e.ctx, id)
+ require.NoError(t, err)
+ var tips []git.ObjectID
+ for _, target := range refs {
+ if !target.IsSymbolic() {
+ tips = append(tips, git.ObjectID(target))
+ }
+ }
+ pack, err := gitops.BuildPackFrom(e.ctx, src.repo, tips, t.TempDir())
+ require.NoError(t, err)
+ defer func() { require.NoError(t, pack.Remove()) }()
+
+ key := corestore.NewPackKey(id, pack.SHA256)
+ rc, err := pack.Open()
+ require.NoError(t, err)
+ require.NoError(t, e.core.Blobs.Put(e.ctx, key, rc))
+ require.NoError(t, rc.Close())
+
+ cp, created, err := e.core.Meta.CreateCheckpoint(e.ctx, id, corestore.CheckpointInput{Seq: seq, Refs: refs, PackKey: key, PackBytes: pack.Size})
+ require.NoError(t, err)
+ require.True(t, created)
+ return cp
+}
+
+// collectCovered deletes the changeset packs up to seq from the bucket, as
+// garbage collection does once a checkpoint at seq is C_gc.
+func (e *env) collectCovered(t *testing.T, id int64, seq uint64) int {
+ t.Helper()
+
+ st, err := e.core.Meta.GCSnapshot(e.ctx, id)
+ require.NoError(t, err)
+ deleted := 0
+ for key, last := range st.Packs {
+ if last <= seq {
+ require.NoError(t, e.core.Blobs.Delete(e.ctx, key))
+ deleted++
+ }
+ }
+ return deleted
+}
+
+// ageCheckpoint moves the creation of the checkpoint at seq d into the past on
+// the database clock.
+func (e *env) ageCheckpoint(t *testing.T, id int64, seq uint64, d time.Duration) {
+ t.Helper()
+
+ store := e.core.Meta.(*sqlstore.Store)
+ when := time.Now().Add(-d).UTC()
+ var value any = when
+ query := "UPDATE graft_checkpoints SET created_at = ? WHERE repo_id = ? AND seq = ?"
+ if store.Dialect() == sqlstore.SQLite {
+ value = when.Format(time.RFC3339Nano)
+ } else {
+ query = "UPDATE graft_checkpoints SET created_at = $1 WHERE repo_id = $2 AND seq = $3"
+ }
+ _, err := store.DB().ExecContext(e.ctx, query, value, id, int64(seq))
+ require.NoError(t, err)
+}
+
+// TestSyncerHydratesFromCheckpoint: a node with no copy hydrates from the
+// newest checkpoint and replays only the changesets after it, so it works after
+// garbage collection deleted the packs the checkpoint covers (CKPT-1).
+func TestSyncerHydratesFromCheckpoint(t *testing.T) {
+ forEachDialect(t, func(t *testing.T, dialect sqlstore.Dialect) {
+ a := newEnv(t, envOptions{dialect: dialect})
+ src := a.newSource(t)
+ c1 := a.writeCommit(t, src)
+ repo := a.register(t, a.key(localRelPath), src, c1)
+ c2 := a.advanceMain(t, repo.ID, src, c1)
+ a.changeset(t, repo.ID, src, []corestore.RefUpdate{
+ {Name: "refs/tags/v1", Expect: corestore.ExpectValue, New: corestore.Target(c2)},
+ }, c2)
+ cp := a.checkpoint(t, repo.ID, src)
+ require.Equal(t, uint64(3), cp.Seq)
+ require.Equal(t, 3, a.collectCovered(t, repo.ID, cp.Seq), "every pack up to the checkpoint is gone")
+
+ // After the checkpoint: a move of main and the deletion of the tag.
+ c3 := a.advanceMain(t, repo.ID, src, c2)
+ a.changeset(t, repo.ID, src, []corestore.RefUpdate{
+ {Name: "refs/tags/v1", Expect: corestore.ExpectValue, Old: corestore.Target(c2), New: ""},
+ })
+
+ b := a.secondNode(t, dialect)
+ require.NoError(t, b.sy.EnsureFresh(b.ctx, b.pbRepo(localRelPath)))
+ repo, err := b.core.Meta.GetRepository(b.ctx, repo.Key)
+ require.NoError(t, err)
+ b.verifySynced(t, repo)
+
+ refs, err := gitops.ReadRefs(b.ctx, b.localRepo())
+ require.NoError(t, err)
+ require.Equal(t, corestore.Target(c3), refs["refs/heads/main"])
+ require.NotContains(t, refs, "refs/tags/v1", "a reference the replay deletes is not in the result")
+
+ require.Equal(t, float64(1), counter(t, b.sy, "graft_storage_sync_checkpoint_seeds_total", ""))
+ require.Equal(t, float64(2), counter(t, b.sy, "graft_storage_sync_changesets_total", ""), "only the changesets after the checkpoint")
+ require.Equal(t, float64(1), syncCount(t, b.sy, "hydrated"))
+ })
+}
+
+// TestSyncerSyncBoundary covers change C: a node a few changesets behind a new
+// checkpoint catches up (case 7); a node below C_sync, the newest checkpoint
+// at least gc_grace − sync_timeout − gc_margin old, hydrates from the
+// checkpoint instead (case 8), because the packs it would catch up from may be
+// collected.
+func TestSyncerSyncBoundary(t *testing.T) {
+ forEachDialect(t, func(t *testing.T, dialect sqlstore.Dialect) {
+ a := newEnv(t, envOptions{dialect: dialect})
+ src := a.newSource(t)
+ c1 := a.writeCommit(t, src)
+ repo := a.register(t, a.key(localRelPath), src, c1)
+
+ // B hydrates at 1, C at 2.
+ b := a.secondNode(t, dialect)
+ require.NoError(t, b.sy.EnsureFresh(b.ctx, b.pbRepo(localRelPath)))
+ c2 := a.advanceMain(t, repo.ID, src, c1)
+ c := a.secondNode(t, dialect)
+ require.NoError(t, c.sy.EnsureFresh(c.ctx, c.pbRepo(localRelPath)))
+
+ c3 := a.advanceMain(t, repo.ID, src, c2)
+ cp := a.checkpoint(t, repo.ID, src)
+ require.Equal(t, uint64(3), cp.Seq)
+
+ // Case 7: a new checkpoint is not C_sync yet, so B catches up to 3.
+ require.NoError(t, b.sy.EnsureFresh(b.ctx, b.pbRepo(localRelPath)))
+ require.Equal(t, float64(1), syncCount(t, b.sy, "caught_up"))
+ require.Equal(t, float64(1), syncCount(t, b.sy, "hydrated"))
+
+ // The checkpoint becomes C_sync, and the head moves on.
+ age := a.core.GCGrace - a.core.SyncTimeout - a.core.GCMargin
+ a.ageCheckpoint(t, repo.ID, cp.Seq, age+time.Minute)
+ a.advanceMain(t, repo.ID, src, c3)
+ head, err := a.core.Meta.GetRepository(a.ctx, repo.Key)
+ require.NoError(t, err)
+
+ // Case 8: C, at 2, is below C_sync 3 and hydrates from the checkpoint.
+ require.NoError(t, c.sy.EnsureFresh(c.ctx, c.pbRepo(localRelPath)))
+ require.Equal(t, float64(2), syncCount(t, c.sy, "hydrated"))
+ require.Zero(t, syncCount(t, c.sy, "caught_up"))
+ require.Equal(t, float64(1), counter(t, c.sy, "graft_storage_sync_checkpoint_seeds_total", ""))
+ c.verifySynced(t, head)
+
+ // B, at 3, is not below C_sync 3: it catches up.
+ require.NoError(t, b.sy.EnsureFresh(b.ctx, b.pbRepo(localRelPath)))
+ require.Equal(t, float64(2), syncCount(t, b.sy, "caught_up"))
+ require.Equal(t, float64(1), syncCount(t, b.sy, "hydrated"))
+ b.verifySynced(t, head)
+ })
+}
+
+// blockingBlobs is a bucket whose reads last until the end of their context.
+type blockingBlobs struct {
+ corestore.BlobStore
+ cancelled chan error
+}
+
+func (b blockingBlobs) Get(ctx context.Context, key string) (io.ReadCloser, error) {
+ <-ctx.Done()
+ b.cancelled <- ctx.Err()
+ return nil, ctx.Err()
+}
+
+// TestSyncerRoundBoundedBySyncTimeout: a sync round is bounded by sync_timeout,
+// which cancels its bucket reads: a hydration whose download outlives it fails
+// as Unavailable and its read is cancelled (GC BOUNDARY rule, enforcement).
+func TestSyncerRoundBoundedBySyncTimeout(t *testing.T) {
+ a := newEnv(t, envOptions{})
+ src := a.newSource(t)
+ c1 := a.writeCommit(t, src)
+ a.register(t, a.key(localRelPath), src, c1)
+
+ b := a.secondNode(t, sqlstore.SQLite)
+ cancelled := make(chan error, 1)
+ b.core.Blobs = blockingBlobs{BlobStore: b.core.Blobs, cancelled: cancelled}
+ b.core.SyncTimeout = 200 * time.Millisecond
+
+ err := b.sy.EnsureFresh(b.ctx, b.pbRepo(localRelPath))
+ require.Equal(t, codes.Unavailable, status.Code(err), "%v", err)
+ require.ErrorIs(t, <-cancelled, context.DeadlineExceeded, "the bucket read was cancelled by the round's deadline")
+
+ _, ok, err := b.sy.Applied(b.ctx, b.pbRepo(localRelPath))
+ require.NoError(t, err)
+ require.False(t, ok, "nothing was installed")
+}
+
+// TestSyncerMissingCheckpointPack: a checkpoint whose pack is gone is
+// corruption (the newest checkpoint is never collected): Internal naming the
+// key, nothing installed.
+func TestSyncerMissingCheckpointPack(t *testing.T) {
+ forEachDialect(t, func(t *testing.T, dialect sqlstore.Dialect) {
+ a := newEnv(t, envOptions{dialect: dialect})
+ src := a.newSource(t)
+ c1 := a.writeCommit(t, src)
+ repo := a.register(t, a.key(localRelPath), src, c1)
+ cp := a.checkpoint(t, repo.ID, src)
+ require.NoError(t, a.core.Blobs.Delete(a.ctx, cp.PackKey))
+
+ b := a.secondNode(t, dialect)
+ err := b.sy.EnsureFresh(b.ctx, b.pbRepo(localRelPath))
+ require.Equal(t, codes.Internal, status.Code(err), "%v", err)
+ require.True(t, strings.Contains(err.Error(), cp.PackKey), fmt.Sprintf("the error names the pack: %v", err))
+ _, ok, err := b.sy.Applied(b.ctx, b.pbRepo(localRelPath))
+ require.NoError(t, err)
+ require.False(t, ok)
+ })
+}