Git designed for AI agents.
syncer, verify: hydrate from checkpoints, decide by C_sync (PLAN7 WP76)
Federico Elia3f994c67ca9e
diff --git a/doc/graft/PLAN7.md b/doc/graft/PLAN7.mdindex 9d59e22..d23e7fd 100644--- a/doc/graft/PLAN7.md+++ b/doc/graft/PLAN7.md@@ -1032,6 +1032,14 @@ next run finishes it. No lease (the repository is gone and its ID is never reuseA missing checkpoint pack or a missing pack `> S` is an Internal error naming thekey (corruption, GC BOUNDARY rule).+*As implemented (WP76).* `syncer.seed` reads the newest checkpoint and its+snapshot. It checks the pack key's repository ID and validates every name and+target like a changeset's (names, targets, symbolic targets), then indexes the+pack through the existing `indexPack` (SHA-256 against the key). Final+references = snapshot overlaid by the fetched final values, where an absent+value deletes. `graft_storage_sync_checkpoint_seeds_total` counts the seeded+hydrations.+### Catch-up and the sync boundary (syncer, WP76, change C)In `sync()`, each round reads `C_sync = SyncBoundary(r.ID, gc_grace − sync_timeout −@@ -1039,6 +1047,26 @@ gc_margin)` and hydrates when `!ok || wm.RepoID != r.ID || !localCurrent ||wm.Seq < C_sync`; otherwise it catches up as in phase 6 (the WP55 RULE unchanged).`checkReplaceable` still runs before any hydration.+*As implemented (WP76).*+* `C_sync` is read only when a catch-up is needed: after the node's own noted+ sequences advanced the watermark, and only when it is below the head. So a+ fresh repository costs no extra query, and a node whose own writes carried it+ past `C_sync` is not re-hydrated.+* `install`'s errRestart condition is the exact negation of the new hydrate+ condition (`wm.Seq >= C_sync` added). Without it, a current but lagging copy+ would be sent back to sync forever.+* The age `gc_grace − sync_timeout − gc_margin` is taken as 0 when it is not+ positive (the configuration refuses that; only test cores have such+ times). With age 0 every checkpoint counts, which is the conservative end:+ more hydrations, never a collected pack.+* Audit of the bound: every pack read and git process of a round (`fetch`,+ `indexPack`, `applyBuild`, the catch-up loop, `install`'s lock) runs under+ the round's context, which `EnsureFresh` bounds by `sync_timeout` per flight.+ The only work outside it is the best-effort `ReportApplied` after the round+ (at most 1 s, no pack read).+* `TestSyncerRoundBoundedBySyncTimeout` shows that a bucket read is cancelled by+ the round's deadline and nothing is installed.+### Verify after compaction (`verify_objects.go`, WP76)`checkStoredObjects` indexes the newest checkpoint's pack and the changeset packs@@ -1048,6 +1076,16 @@ under a deadline of `W` from its start (a dry hydration that read checkpoint `S`must end before a checkpoint newer than `S` can become `C_gc`); a missing pack isreported as a connectivity failure starting with `storage core objects:`.+*As implemented (WP76).*+* `checkStoredObjects` reads the newest checkpoint before the references. A+ checkpoint never exceeds the head at its creation, so the head read after it+ is at or above it.+* The open reviews' commits are checked through their keep-around references+ (REVIEW ROOTS RULE), which are database references; there is no separate+ review check.+* Running out of `W` is `Unavailable` ("did not finish within the write+ deadline").+### Background maintenance (`internal/graft/corestore/maintenance`, WP77)```godiff --git a/internal/gitaly/service/storagecore/verify.go b/internal/gitaly/service/storagecore/verify.goindex 8ed5617..b417ce0 100644--- a/internal/gitaly/service/storagecore/verify.go+++ b/internal/gitaly/service/storagecore/verify.go@@ -95,7 +95,7 @@ func (s *server) VerifyRepository(ctx context.Context, req *gitalypb.VerifyRepos// Equal references are not enough: the object storage must also hold// every object they reach, or a hydration on another node fails.- storedOK, storedMsg, err := s.checkStoredObjects(ctx, core, repo, record, dbRefs, headSeq)+ storedOK, storedMsg, err := s.checkStoredObjects(ctx, core, repo, record)if err != nil {return nil, err}diff --git a/internal/gitaly/service/storagecore/verify_objects.go b/internal/gitaly/service/storagecore/verify_objects.goindex d4a4d29..9cc499e 100644--- a/internal/gitaly/service/storagecore/verify_objects.go+++ b/internal/gitaly/service/storagecore/verify_objects.go@@ -8,6 +8,7 @@ import ("os""path/filepath""strings"+ "time""github.com/feder-cr/git-ai/internal/command""github.com/feder-cr/git-ai/internal/git/gitcmd"@@ -23,10 +24,20 @@ import (const verifyChangesetBatch = 500// checkStoredObjects checks that the object storage can rebuild the database-// state of the repository: it indexes the pack of every changeset up to-// headSeq into a new empty repository in the storage's temporary directory (a-// dry hydration, nothing is installed) and checks that every object reachable-// from the object IDs of dbRefs is there (`git rev-list --objects --quiet`).+// state of the repository the way a hydration does: into a new empty+// repository in the storage's temporary directory (a dry hydration, nothing is+// installed) it indexes the pack of the newest checkpoint, if any, and the pack+// of every changeset after it up to the head, then checks that every object+// reachable from the object IDs of the database references is there (`git+// rev-list --objects --quiet`). The keep-around references of the open reviews+// are among them, so their commits are checked too (doc/graft/PLAN7.md, "Verify+// after compaction").+//+// The checkpoint is read before the references, so the head they belong to is+// at or above it. The check runs under a deadline of W (WriteDeadline) from its+// start: a checkpoint newer than the one it read can become C_gc, and so let+// garbage collection delete the packs this check reads, only gc_grace after its+// creation, which is after the start of the check.//// The local connectivity check cannot see this: a git write that shadow mode// did not record leaves objects that are in the local repository but in no@@ -34,7 +45,14 @@ const verifyChangesetBatch = 500// references of git and of the database agree while a hydration on another// node fails. It returns whether the check passed and, when it did not, what// is missing; an error only when the check could not run.-func (s *server) checkStoredObjects(ctx context.Context, core *corestore.Core, repo *gitalypb.Repository, record corestore.Repository, dbRefs map[string]corestore.Target, headSeq uint64) (bool, string, error) {+func (s *server) checkStoredObjects(ctx context.Context, core *corestore.Core, repo *gitalypb.Repository, record corestore.Repository) (bool, string, error) {+ deadline := core.WriteDeadline()+ if deadline <= 0 {+ return false, "", structerr.NewFailedPrecondition("storage core: invalid write deadline %s", deadline)+ }+ ctx, cancel := context.WithTimeout(ctx, deadline)+ defer cancel()+storageName := repo.GetStorageName()storagePath, err := s.deps.GetLocator().GetStorageByName(ctx, storageName)if err != nil {@@ -63,11 +81,32 @@ func (s *server) checkStoredObjects(ctx context.Context, core *corestore.Core, r}stored := s.localRepo(&gitalypb.Repository{StorageName: storageName, RelativePath: rel})- for after := uint64(0); after < headSeq; {+ cp, seeded, err := core.Meta.NewestCheckpoint(ctx, record.ID)+ if err != nil {+ return false, "", deadlineOr(ctx, deadline, structerr.NewUnavailable("storage core: newest checkpoint: %w", err))+ }+ dbRefs, headSeq, err := core.Meta.Refs(ctx, record.ID)+ if err != nil {+ return false, "", deadlineOr(ctx, deadline, structerr.NewUnavailable("storage core: references: %w", err))+ }++ after := uint64(0)+ if seeded {+ if id, _, ok := corestore.ParsePackKey(cp.PackKey); !ok || id != record.ID {+ return false, fmt.Sprintf("checkpoint %d references %q, which is not a pack of the repository", cp.Seq, cp.PackKey), nil+ }+ problem, err := indexStoredPack(ctx, core, stored, cp.PackKey, fmt.Sprintf("checkpoint %d", cp.Seq))+ if err != nil || problem != "" {+ return false, problem, deadlineOr(ctx, deadline, err)+ }+ after = cp.Seq+ }++ for after < headSeq {upTo := min(after+verifyChangesetBatch, headSeq)changesets, err := core.Meta.ListChangesets(ctx, record.ID, after, upTo)if err != nil {- return false, "", structerr.NewUnavailable("storage core: list changesets: %w", err)+ return false, "", deadlineOr(ctx, deadline, structerr.NewUnavailable("storage core: list changesets: %w", err))}if len(changesets) == 0 {return false, fmt.Sprintf("the database has no changesets after %d up to %d", after, upTo), nil@@ -82,9 +121,9 @@ func (s *server) checkStoredObjects(ctx context.Context, core *corestore.Core, rreturn false, fmt.Sprintf("changeset %d references %q, which is not a pack of the repository", cs.Seq, cs.PackKey), nil}- problem, err := indexStoredPack(ctx, core, stored, cs)+ problem, err := indexStoredPack(ctx, core, stored, cs.PackKey, fmt.Sprintf("changeset %d", cs.Seq))if err != nil || problem != "" {- return false, problem, err+ return false, problem, deadlineOr(ctx, deadline, err)}}}@@ -114,7 +153,7 @@ func (s *server) checkStoredObjects(ctx context.Context, core *corestore.Core, rreturn true, "", nil}if ctxErr := ctx.Err(); ctxErr != nil {- return false, "", structerr.NewUnavailable("stored objects check: %w", ctxErr)+ return false, "", deadlineOr(ctx, deadline, structerr.NewUnavailable("stored objects check: %w", ctxErr))}if _, exited := command.ExitStatus(err); !exited {return false, "", structerr.NewInternal("stored objects check: %w", err)@@ -127,13 +166,14 @@ func (s *server) checkStoredObjects(ctx context.Context, core *corestore.Core, rreturn false, "the packs of the database do not hold every object of its references: " + msg, nil}-// indexStoredPack downloads the pack of cs (whose key was checked) and indexes-// it into repo. It returns a description of a missing or corrupt pack, or an-// error when the check could not run.-func indexStoredPack(ctx context.Context, core *corestore.Core, repo *localrepo.Repo, cs corestore.Changeset) (string, error) {- rc, err := core.Blobs.Get(ctx, cs.PackKey)+// indexStoredPack downloads the pack of key (which was checked), the pack of+// owner ("checkpoint 4", "changeset 7"), and indexes it into repo. It returns a+// description of a missing or corrupt pack, or an error when the check could+// not run.+func indexStoredPack(ctx context.Context, core *corestore.Core, repo *localrepo.Repo, key, owner string) (string, error) {+ rc, err := core.Blobs.Get(ctx, key)if errors.Is(err, corestore.ErrBlobNotFound) {- return fmt.Sprintf("the pack %q of changeset %d is missing from the object storage", cs.PackKey, cs.Seq), nil+ return fmt.Sprintf("the pack %q of %s is missing from the object storage", key, owner), nil}if err != nil {return "", structerr.NewUnavailable("storage core: download pack: %w", err)@@ -144,7 +184,16 @@ func indexStoredPack(ctx context.Context, core *corestore.Core, repo *localrepo.if ctxErr := ctx.Err(); ctxErr != nil {return "", structerr.NewUnavailable("stored objects check: %w", ctxErr)}- return fmt.Sprintf("the pack %q of changeset %d cannot be indexed: %v", cs.PackKey, cs.Seq, err), nil+ return fmt.Sprintf("the pack %q of %s cannot be indexed: %v", key, owner, err), nil}return "", nil}++// deadlineOr returns the error of a check that ran out of its deadline when+// ctx's deadline has passed, and err otherwise.+func deadlineOr(ctx context.Context, deadline time.Duration, err error) error {+ if err != nil && errors.Is(ctx.Err(), context.DeadlineExceeded) {+ return structerr.NewUnavailable("storage core: the stored objects check did not finish within the write deadline %s: %w", deadline, err)+ }+ return err+}diff --git a/internal/gitaly/service/storagecore/verify_objects_test.go b/internal/gitaly/service/storagecore/verify_objects_test.goindex 3e7e702..8457ae2 100644--- a/internal/gitaly/service/storagecore/verify_objects_test.go+++ b/internal/gitaly/service/storagecore/verify_objects_test.go@@ -1,9 +1,15 @@package storagecoreimport (+ "context"+ "io"+ "sync/atomic""testing"+ "time""github.com/stretchr/testify/require"+ "google.golang.org/grpc/codes"+ "google.golang.org/grpc/status""github.com/feder-cr/git-ai/internal/git""github.com/feder-cr/git-ai/internal/git/gittest"@@ -11,6 +17,7 @@ import ("github.com/feder-cr/git-ai/internal/graft/corestore""github.com/feder-cr/git-ai/internal/graft/corestore/gitops""github.com/feder-cr/git-ai/internal/graft/corestore/sqlstore"+ "github.com/feder-cr/git-ai/proto/go/gitalypb")// A git write that the storage core did not record leaves objects that no pack@@ -69,3 +76,98 @@ func TestVerifyRepositoryReportsObjectsMissingFromStorage(t *testing.T) {require.NotContains(t, verify.GetConnectivityError(), "\nstorage core objects", "the local check passed: only the storage message")})}++// checkpointAtHead records a checkpoint of the repository at its database+// head, with a pack of everything its references reach built from the local+// repository, as checkpoint.Create does.+func (h *harness) checkpointAtHead(repoID int64) corestore.Checkpoint {+ h.t.Helper()++ refs, seq, err := h.core.Meta.Refs(h.ctx, repoID)+ require.NoError(h.t, err)+ var tips []git.ObjectID+ for _, target := range refs {+ if !target.IsSymbolic() {+ tips = append(tips, git.ObjectID(target))+ }+ }+ pack, err := gitops.BuildPackFrom(h.ctx, localrepo.NewTestRepo(h.t, h.cfg, h.repo), tips, h.t.TempDir())+ require.NoError(h.t, err)+ defer func() { require.NoError(h.t, pack.Remove()) }()+ key := corestore.NewPackKey(repoID, pack.SHA256)+ f, err := pack.Open()+ require.NoError(h.t, err)+ require.NoError(h.t, h.core.Blobs.Put(h.ctx, key, f))+ require.NoError(h.t, f.Close())++ cp, created, err := h.core.Meta.CreateCheckpoint(h.ctx, repoID, corestore.CheckpointInput{Seq: seq, Refs: refs, PackKey: key, PackBytes: pack.Size})+ require.NoError(h.t, err)+ require.True(h.t, created)+ return cp+}++// TestVerifyRepositoryAfterCompaction: the stored objects check rebuilds the+// repository the way a hydration does, from the newest checkpoint and the+// changesets after it, so it passes once garbage collection deleted the packs+// the checkpoint covers; a missing checkpoint pack is reported.+func TestVerifyRepositoryAfterCompaction(t *testing.T) {+ forEachDialect(t, func(t *testing.T, dialect sqlstore.Dialect) {+ h := setup(t, corestore.ModeShadow, dialect)+ h.seed()+ repoID := h.importRepo()+ cp := h.checkpointAtHead(repoID)++ // Collected: every changeset pack up to the checkpoint.+ st, err := h.core.Meta.GCSnapshot(h.ctx, repoID)+ require.NoError(t, err)+ require.NotEmpty(t, st.Packs)+ for key, seq := range st.Packs {+ require.LessOrEqual(t, seq, cp.Seq)+ require.NoError(t, h.core.Blobs.Delete(h.ctx, key))+ }++ verify := h.verify(true)+ require.True(t, verify.GetConnectivityOk(), verify.GetConnectivityError())++ require.NoError(t, h.core.Blobs.Delete(h.ctx, cp.PackKey))+ verify = h.verify(true)+ require.False(t, verify.GetConnectivityOk())+ require.Contains(t, verify.GetConnectivityError(), "storage core objects: the pack \""+cp.PackKey+"\" of checkpoint")+ })+}++// slowReads is a bucket whose reads, once enabled, last until the end of their+// context. It is installed before the server starts and switched afterwards,+// so the core is never changed while the server reads it.+type slowReads struct {+ corestore.BlobStore+ enabled *atomic.Bool+}++func (b slowReads) Get(ctx context.Context, key string) (io.ReadCloser, error) {+ if b.enabled.Load() {+ <-ctx.Done()+ return nil, ctx.Err()+ }+ return b.BlobStore.Get(ctx, key)+}++// TestVerifyRepositoryStoredObjectsCheckIsBounded: the dry hydration runs under+// the write deadline W from its start (GC BOUNDARY rule, enforcement): one that+// outlives it is cancelled and fails as Unavailable.+func TestVerifyRepositoryStoredObjectsCheckIsBounded(t *testing.T) {+ var enabled atomic.Bool+ h := setup(t, corestore.ModeShadow, sqlstore.SQLite, func(core *corestore.Core) {+ // W = gc_grace - gc_margin = 3 s.+ core.GCGrace = 4 * time.Second+ core.GCMargin = time.Second+ core.Blobs = slowReads{BlobStore: core.Blobs, enabled: &enabled}+ })+ h.seed()+ h.importRepo()++ enabled.Store(true)+ _, err := h.client.VerifyRepository(h.ctx, &gitalypb.VerifyRepositoryRequest{Repository: h.repo, CheckConnectivity: true})+ require.Equal(t, codes.Unavailable, status.Code(err), "%v", err)+ require.ErrorContains(t, err, "did not finish within the write deadline 3s")+}diff --git a/internal/graft/corestore/syncer/syncer.go b/internal/graft/corestore/syncer/syncer.goindex 74bb43d..3c91015 100644--- a/internal/graft/corestore/syncer/syncer.go+++ b/internal/graft/corestore/syncer/syncer.go@@ -130,6 +130,7 @@ type Syncer struct {syncSeconds prometheus.HistogramsyncChangesets prometheus.CountersyncBytes prometheus.Counter+ syncSeeds prometheus.Counter}type repoLock struct {@@ -176,6 +177,10 @@ func New(core *corestore.Core, logger log.Logger, locator storage.Locator, repoFName: "graft_storage_sync_bytes_total",Help: "Bytes of packs downloaded by catch-up and hydration.",}),+ syncSeeds: prometheus.NewCounter(prometheus.CounterOpts{+ Name: "graft_storage_sync_checkpoint_seeds_total",+ Help: "Number of hydrations that started from a checkpoint instead of changeset 1.",+ }),}for _, result := range []string{resultFresh, resultCaughtUp, resultHydrated, resultError} {s.syncTotal.WithLabelValues(result)@@ -300,6 +305,7 @@ func (s *Syncer) Describe(descs chan<- *prometheus.Desc) {s.syncSeconds.Describe(descs)s.syncChangesets.Describe(descs)s.syncBytes.Describe(descs)+ s.syncSeeds.Describe(descs)}// Collect implements prometheus.Collector.@@ -311,6 +317,7 @@ func (s *Syncer) Collect(metrics chan<- prometheus.Metric) {s.syncSeconds.Collect(metrics)s.syncChangesets.Collect(metrics)s.syncBytes.Collect(metrics)+ s.syncSeeds.Collect(metrics)}func (s *Syncer) syncTimeout() time.Duration {@@ -354,29 +361,44 @@ func (s *Syncer) sync(ctx context.Context, key corestore.RepoKey) error {}wm, ok := s.readWatermark(path)- if !ok || wm.RepoID != r.ID || !s.localCurrent(ctx, repo, path) {- if err := s.checkReplaceable(ctx, r, path, wm, ok); err != nil {+ stale := !ok || wm.RepoID != r.ID || !s.localCurrent(ctx, repo, path)+ // boundary is C_sync: a local repository below it is hydrated, because+ // the changeset packs it would catch up from may be collected+ // (doc/graft/PLAN7.md, GC BOUNDARY rule, change C).+ var boundary uint64+ if !stale {+ // Advance through the sequences this node committed itself before+ // downloading anything.+ wm, err = s.advance(ctx, path, wm)+ if err != nil {return err}- err := s.hydrate(ctx, repo, r, hash, path)+ if wm.Seq >= r.HeadSeq {+ result = resultFresh+ return nil+ }+ boundary, err = s.syncBoundary(ctx, r.ID)if errors.Is(err, errRestart) {continue}if err != nil {return err}- result = resultHydrated- return nil+ stale = wm.Seq < boundary}- // Advance through the sequences this node committed itself before- // downloading anything.- wm, err = s.advance(ctx, path, wm)- if err != nil {- return err- }- if wm.Seq >= r.HeadSeq {- result = resultFresh+ if stale {+ if err := s.checkReplaceable(ctx, r, path, wm, ok); err != nil {+ return err+ }+ err := s.hydrate(ctx, repo, r, hash, path, boundary)+ if errors.Is(err, errRestart) {+ continue+ }+ if err != nil {+ return err+ }+ result = resultHydratedreturn nil}@@ -396,6 +418,26 @@ func (s *Syncer) sync(ctx context.Context, key corestore.RepoKey) error {}}+// syncBoundary returns C_sync of the repository: the sequence of the newest+// checkpoint created at least gc_grace − sync_timeout − gc_margin ago on the+// database clock, 0 when there is none (doc/graft/PLAN7.md, GC BOUNDARY rule).+// A sync round, bounded by sync_timeout, never needs a pack that garbage+// collection deletes meanwhile when its watermark is at or above C_sync. The+// configuration requires gc_grace > sync_timeout + gc_margin; with times that+// do not (only test cores), the age is 0 and every checkpoint counts, which is+// the conservative end: more hydrations, never a collected pack.+func (s *Syncer) syncBoundary(ctx context.Context, repoID int64) (uint64, error) {+ age := max(s.core.GCGrace-s.syncTimeout()-s.core.GCMargin, 0)+ boundary, err := s.core.Meta.SyncBoundary(ctx, repoID, age)+ if err != nil {+ if errors.Is(err, corestore.ErrRepositoryNotFound) {+ return 0, errRestart+ }+ return 0, structerr.NewUnavailable("storage core: sync boundary: %w", err)+ }+ return boundary, nil+}+// record reads the database record of key and the object hash of its format.// For a key the database does not have it returns the error of untracked// (errRestart when the repository was registered meanwhile).@@ -1362,12 +1404,14 @@ func refTransaction(ctx context.Context, repo *localrepo.Repo, commands []string}}-// hydrate builds a bare repository from changeset 1 up to the head in the-// temporary directory of the storage, then moves it to path. A local-// repository at path (another repository ID, or invalid) is moved away first-// and removed in the background. Only the final moves are done under the-// repository lock.-func (s *Syncer) hydrate(ctx context.Context, repo *gitalypb.Repository, r corestore.Repository, hash git.ObjectHash, path string) error {+// hydrate builds a bare repository in the temporary directory of the storage,+// then moves it to path. It starts from the newest checkpoint when there is one+// (its pack and its reference snapshot) and replays the changesets after it,+// otherwise from changeset 1 (doc/graft/PLAN7.md, "Hydration from a+// checkpoint"). A local repository at path (another repository ID, invalid, or+// below boundary, C_sync) is moved away first and removed in the background.+// Only the final moves are done under the repository lock.+func (s *Syncer) hydrate(ctx context.Context, repo *gitalypb.Repository, r corestore.Repository, hash git.ObjectHash, path string, boundary uint64) error {storageName := repo.GetStorageName()storagePath, err := s.locator.GetStorageByName(ctx, storageName)@@ -1396,18 +1440,28 @@ func (s *Syncer) hydrate(ctx context.Context, repo *gitalypb.Repository, r cores}buildRepo := s.repoFactory.Build(&gitalypb.Repository{StorageName: storageName, RelativePath: rel})- f := newFetched(0)+ refs, f, err := s.seed(ctx, buildRepo, tmpDir, r, hash)+ if err != nil {+ return err+ }if err := s.fetch(ctx, buildRepo, tmpDir, r.ID, hash, f, r.HeadSeq); err != nil {return err}- if err := s.applyBuild(ctx, buildRepo, f.finalRefs()); err != nil {+ for name, target := range f.finalRefs() {+ if target.Absent() {+ delete(refs, name)+ } else {+ refs[name] = target+ }+ }+ if err := s.applyBuild(ctx, buildRepo, refs); err != nil {return err}if err := gitops.WriteWatermark(buildPath, corestore.Watermark{RepoID: r.ID, Seq: f.head}); err != nil {return structerr.NewInternal("storage core: write watermark: %w", err)}- if err := s.install(ctx, repo, r, path, buildPath, tmpDir); err != nil {+ if err := s.install(ctx, repo, r, path, buildPath, tmpDir, boundary); err != nil {return err}@@ -1417,23 +1471,71 @@ func (s *Syncer) hydrate(ctx context.Context, repo *gitalypb.Repository, r coresreturn nil}+// seed starts a hydration: from the newest checkpoint when there is one (its+// pack, verified against its key and indexed into repo, and its validated+// reference snapshot), else from nothing. It returns the references to start+// from and the fetch state positioned after the seed.+func (s *Syncer) seed(ctx context.Context, repo *localrepo.Repo, tmpDir string, r corestore.Repository, hash git.ObjectHash) (map[string]corestore.Target, *fetched, error) {+ cp, ok, err := s.core.Meta.NewestCheckpoint(ctx, r.ID)+ if err != nil {+ return nil, nil, structerr.NewUnavailable("storage core: newest checkpoint: %w", err)+ }+ if !ok {+ return map[string]corestore.Target{}, newFetched(0), nil+ }++ // Data read from the database is validated before it touches the+ // repository (RULE 22).+ if id, _, ok := corestore.ParsePackKey(cp.PackKey); !ok || id != r.ID {+ return nil, nil, structerr.NewInternal("storage core: checkpoint %d has an invalid pack key", cp.Seq)+ }+ refs, err := s.core.Meta.CheckpointRefs(ctx, r.ID, cp.Seq)+ if err != nil {+ return nil, nil, structerr.NewUnavailable("storage core: checkpoint %d references: %w", cp.Seq, err)+ }+ for name, target := range refs {+ if target.Absent() {+ return nil, nil, structerr.NewInternal("storage core: checkpoint %d: reference %q has no value", cp.Seq, name)+ }+ if err := gitops.ValidateName(name); err != nil {+ return nil, nil, structerr.NewInternal("storage core: checkpoint %d: invalid reference name %q: %w", cp.Seq, name, err)+ }+ if err := gitops.ValidateTarget(target, hash); err != nil {+ return nil, nil, structerr.NewInternal("storage core: checkpoint %d: invalid target of %q: %w", cp.Seq, name, err)+ }+ if err := validateSymbolicTarget(name, target); err != nil {+ return nil, nil, structerr.NewInternal("storage core: checkpoint %d: invalid target of %q: %w", cp.Seq, name, err)+ }+ }++ f := newFetched(cp.Seq)+ n, err := s.indexPack(ctx, repo, tmpDir, cp.PackKey)+ if err != nil {+ return nil, nil, err+ }+ f.bytes += n+ s.syncSeeds.Inc()+ return refs, f, nil+}+// install moves the hydrated repository at buildPath to path under the// repository lock, moving a local repository that is in the way into the// temporary directory first. It returns errRestart when, meanwhile, the local// repository became one that a catch-up can bring up to date (a reftable-// repository with a watermark of the repository: it was imported or created-// here): it is caught up instead of being replaced. The condition is the-// negation of the one that made sync hydrate, so a repository that is still-// stale (for example a files-backend one with a matching watermark) is-// replaced, never sent back to sync.-func (s *Syncer) install(ctx context.Context, repo *gitalypb.Repository, r corestore.Repository, path, buildPath, tmpDir string) error {+// repository with a watermark of the repository at or above boundary, C_sync:+// it was imported, created or caught up here): it is caught up instead of+// being replaced. The condition is the negation of the one that made sync+// hydrate, so a repository that is still stale (for example a files-backend+// one with a matching watermark, or one below C_sync) is replaced, never sent+// back to sync.+func (s *Syncer) install(ctx context.Context, repo *gitalypb.Repository, r corestore.Repository, path, buildPath, tmpDir string, boundary uint64) error {release, err := s.lock(ctx, path)if err != nil {return err}defer release()- if wm, ok := s.readWatermark(path); ok && wm.RepoID == r.ID && s.localCurrent(ctx, repo, path) {+ if wm, ok := s.readWatermark(path); ok && wm.RepoID == r.ID && wm.Seq >= boundary && s.localCurrent(ctx, repo, path) {return errRestart}diff --git a/internal/graft/corestore/syncer/syncer_checkpoint_test.go b/internal/graft/corestore/syncer/syncer_checkpoint_test.gonew file mode 100644index 0000000..27015a5--- /dev/null+++ b/internal/graft/corestore/syncer/syncer_checkpoint_test.go@@ -0,0 +1,230 @@+package syncer_test++import (+ "context"+ "fmt"+ "io"+ "strings"+ "testing"+ "time"++ "github.com/stretchr/testify/require"+ "google.golang.org/grpc/codes"+ "google.golang.org/grpc/status"++ "github.com/feder-cr/git-ai/internal/git"+ "github.com/feder-cr/git-ai/internal/graft/corestore"+ "github.com/feder-cr/git-ai/internal/graft/corestore/gitops"+ "github.com/feder-cr/git-ai/internal/graft/corestore/sqlstore"+)++// checkpoint records a checkpoint of repository id at its head, with a pack of+// everything its references reach built from src, as checkpoint.Create does.+func (e *env) checkpoint(t *testing.T, id int64, src source) corestore.Checkpoint {+ t.Helper()++ refs, seq, err := e.core.Meta.Refs(e.ctx, id)+ require.NoError(t, err)+ var tips []git.ObjectID+ for _, target := range refs {+ if !target.IsSymbolic() {+ tips = append(tips, git.ObjectID(target))+ }+ }+ pack, err := gitops.BuildPackFrom(e.ctx, src.repo, tips, t.TempDir())+ require.NoError(t, err)+ defer func() { require.NoError(t, pack.Remove()) }()++ key := corestore.NewPackKey(id, pack.SHA256)+ rc, err := pack.Open()+ require.NoError(t, err)+ require.NoError(t, e.core.Blobs.Put(e.ctx, key, rc))+ require.NoError(t, rc.Close())++ cp, created, err := e.core.Meta.CreateCheckpoint(e.ctx, id, corestore.CheckpointInput{Seq: seq, Refs: refs, PackKey: key, PackBytes: pack.Size})+ require.NoError(t, err)+ require.True(t, created)+ return cp+}++// collectCovered deletes the changeset packs up to seq from the bucket, as+// garbage collection does once a checkpoint at seq is C_gc.+func (e *env) collectCovered(t *testing.T, id int64, seq uint64) int {+ t.Helper()++ st, err := e.core.Meta.GCSnapshot(e.ctx, id)+ require.NoError(t, err)+ deleted := 0+ for key, last := range st.Packs {+ if last <= seq {+ require.NoError(t, e.core.Blobs.Delete(e.ctx, key))+ deleted+++ }+ }+ return deleted+}++// ageCheckpoint moves the creation of the checkpoint at seq d into the past on+// the database clock.+func (e *env) ageCheckpoint(t *testing.T, id int64, seq uint64, d time.Duration) {+ t.Helper()++ store := e.core.Meta.(*sqlstore.Store)+ when := time.Now().Add(-d).UTC()+ var value any = when+ query := "UPDATE graft_checkpoints SET created_at = ? WHERE repo_id = ? AND seq = ?"+ if store.Dialect() == sqlstore.SQLite {+ value = when.Format(time.RFC3339Nano)+ } else {+ query = "UPDATE graft_checkpoints SET created_at = $1 WHERE repo_id = $2 AND seq = $3"+ }+ _, err := store.DB().ExecContext(e.ctx, query, value, id, int64(seq))+ require.NoError(t, err)+}++// TestSyncerHydratesFromCheckpoint: a node with no copy hydrates from the+// newest checkpoint and replays only the changesets after it, so it works after+// garbage collection deleted the packs the checkpoint covers (CKPT-1).+func TestSyncerHydratesFromCheckpoint(t *testing.T) {+ forEachDialect(t, func(t *testing.T, dialect sqlstore.Dialect) {+ a := newEnv(t, envOptions{dialect: dialect})+ src := a.newSource(t)+ c1 := a.writeCommit(t, src)+ repo := a.register(t, a.key(localRelPath), src, c1)+ c2 := a.advanceMain(t, repo.ID, src, c1)+ a.changeset(t, repo.ID, src, []corestore.RefUpdate{+ {Name: "refs/tags/v1", Expect: corestore.ExpectValue, New: corestore.Target(c2)},+ }, c2)+ cp := a.checkpoint(t, repo.ID, src)+ require.Equal(t, uint64(3), cp.Seq)+ require.Equal(t, 3, a.collectCovered(t, repo.ID, cp.Seq), "every pack up to the checkpoint is gone")++ // After the checkpoint: a move of main and the deletion of the tag.+ c3 := a.advanceMain(t, repo.ID, src, c2)+ a.changeset(t, repo.ID, src, []corestore.RefUpdate{+ {Name: "refs/tags/v1", Expect: corestore.ExpectValue, Old: corestore.Target(c2), New: ""},+ })++ b := a.secondNode(t, dialect)+ require.NoError(t, b.sy.EnsureFresh(b.ctx, b.pbRepo(localRelPath)))+ repo, err := b.core.Meta.GetRepository(b.ctx, repo.Key)+ require.NoError(t, err)+ b.verifySynced(t, repo)++ refs, err := gitops.ReadRefs(b.ctx, b.localRepo())+ require.NoError(t, err)+ require.Equal(t, corestore.Target(c3), refs["refs/heads/main"])+ require.NotContains(t, refs, "refs/tags/v1", "a reference the replay deletes is not in the result")++ require.Equal(t, float64(1), counter(t, b.sy, "graft_storage_sync_checkpoint_seeds_total", ""))+ require.Equal(t, float64(2), counter(t, b.sy, "graft_storage_sync_changesets_total", ""), "only the changesets after the checkpoint")+ require.Equal(t, float64(1), syncCount(t, b.sy, "hydrated"))+ })+}++// TestSyncerSyncBoundary covers change C: a node a few changesets behind a new+// checkpoint catches up (case 7); a node below C_sync, the newest checkpoint+// at least gc_grace − sync_timeout − gc_margin old, hydrates from the+// checkpoint instead (case 8), because the packs it would catch up from may be+// collected.+func TestSyncerSyncBoundary(t *testing.T) {+ forEachDialect(t, func(t *testing.T, dialect sqlstore.Dialect) {+ a := newEnv(t, envOptions{dialect: dialect})+ src := a.newSource(t)+ c1 := a.writeCommit(t, src)+ repo := a.register(t, a.key(localRelPath), src, c1)++ // B hydrates at 1, C at 2.+ b := a.secondNode(t, dialect)+ require.NoError(t, b.sy.EnsureFresh(b.ctx, b.pbRepo(localRelPath)))+ c2 := a.advanceMain(t, repo.ID, src, c1)+ c := a.secondNode(t, dialect)+ require.NoError(t, c.sy.EnsureFresh(c.ctx, c.pbRepo(localRelPath)))++ c3 := a.advanceMain(t, repo.ID, src, c2)+ cp := a.checkpoint(t, repo.ID, src)+ require.Equal(t, uint64(3), cp.Seq)++ // Case 7: a new checkpoint is not C_sync yet, so B catches up to 3.+ require.NoError(t, b.sy.EnsureFresh(b.ctx, b.pbRepo(localRelPath)))+ require.Equal(t, float64(1), syncCount(t, b.sy, "caught_up"))+ require.Equal(t, float64(1), syncCount(t, b.sy, "hydrated"))++ // The checkpoint becomes C_sync, and the head moves on.+ age := a.core.GCGrace - a.core.SyncTimeout - a.core.GCMargin+ a.ageCheckpoint(t, repo.ID, cp.Seq, age+time.Minute)+ a.advanceMain(t, repo.ID, src, c3)+ head, err := a.core.Meta.GetRepository(a.ctx, repo.Key)+ require.NoError(t, err)++ // Case 8: C, at 2, is below C_sync 3 and hydrates from the checkpoint.+ require.NoError(t, c.sy.EnsureFresh(c.ctx, c.pbRepo(localRelPath)))+ require.Equal(t, float64(2), syncCount(t, c.sy, "hydrated"))+ require.Zero(t, syncCount(t, c.sy, "caught_up"))+ require.Equal(t, float64(1), counter(t, c.sy, "graft_storage_sync_checkpoint_seeds_total", ""))+ c.verifySynced(t, head)++ // B, at 3, is not below C_sync 3: it catches up.+ require.NoError(t, b.sy.EnsureFresh(b.ctx, b.pbRepo(localRelPath)))+ require.Equal(t, float64(2), syncCount(t, b.sy, "caught_up"))+ require.Equal(t, float64(1), syncCount(t, b.sy, "hydrated"))+ b.verifySynced(t, head)+ })+}++// blockingBlobs is a bucket whose reads last until the end of their context.+type blockingBlobs struct {+ corestore.BlobStore+ cancelled chan error+}++func (b blockingBlobs) Get(ctx context.Context, key string) (io.ReadCloser, error) {+ <-ctx.Done()+ b.cancelled <- ctx.Err()+ return nil, ctx.Err()+}++// TestSyncerRoundBoundedBySyncTimeout: a sync round is bounded by sync_timeout,+// which cancels its bucket reads: a hydration whose download outlives it fails+// as Unavailable and its read is cancelled (GC BOUNDARY rule, enforcement).+func TestSyncerRoundBoundedBySyncTimeout(t *testing.T) {+ a := newEnv(t, envOptions{})+ src := a.newSource(t)+ c1 := a.writeCommit(t, src)+ a.register(t, a.key(localRelPath), src, c1)++ b := a.secondNode(t, sqlstore.SQLite)+ cancelled := make(chan error, 1)+ b.core.Blobs = blockingBlobs{BlobStore: b.core.Blobs, cancelled: cancelled}+ b.core.SyncTimeout = 200 * time.Millisecond++ err := b.sy.EnsureFresh(b.ctx, b.pbRepo(localRelPath))+ require.Equal(t, codes.Unavailable, status.Code(err), "%v", err)+ require.ErrorIs(t, <-cancelled, context.DeadlineExceeded, "the bucket read was cancelled by the round's deadline")++ _, ok, err := b.sy.Applied(b.ctx, b.pbRepo(localRelPath))+ require.NoError(t, err)+ require.False(t, ok, "nothing was installed")+}++// TestSyncerMissingCheckpointPack: a checkpoint whose pack is gone is+// corruption (the newest checkpoint is never collected): Internal naming the+// key, nothing installed.+func TestSyncerMissingCheckpointPack(t *testing.T) {+ forEachDialect(t, func(t *testing.T, dialect sqlstore.Dialect) {+ a := newEnv(t, envOptions{dialect: dialect})+ src := a.newSource(t)+ c1 := a.writeCommit(t, src)+ repo := a.register(t, a.key(localRelPath), src, c1)+ cp := a.checkpoint(t, repo.ID, src)+ require.NoError(t, a.core.Blobs.Delete(a.ctx, cp.PackKey))++ b := a.secondNode(t, dialect)+ err := b.sy.EnsureFresh(b.ctx, b.pbRepo(localRelPath))+ require.Equal(t, codes.Internal, status.Code(err), "%v", err)+ require.True(t, strings.Contains(err.Error(), cp.PackKey), fmt.Sprintf("the error names the pack: %v", err))+ _, ok, err := b.sy.Applied(b.ctx, b.pbRepo(localRelPath))+ require.NoError(t, err)+ require.False(t, ok)+ })+}