Skip to content

Git designed for AI agents.

Review m3: review roots before the prune and before every checkpoint

Federico Eliaf081fd75dbc3
diff --git a/doc/graft/PLAN7.md b/doc/graft/PLAN7.md
index 2309b45..1daaefd 100644
--- a/doc/graft/PLAN7.md
+++ b/doc/graft/PLAN7.md
@@ -399,10 +399,14 @@ guarded nor swept).
counted (`graft_review_roots_sweep_failures_total{trigger="close"}`), and
the backstop below deletes the reference. Not returning it is a design
decision with a backstop, not a swallowed error.
- * **Backstop, every mode.** After Gitaly's daily housekeeping of a repository
+ * **Backstop, every mode.** Before Gitaly's daily housekeeping of a repository
(`DailyOptimizationWorker`, on by default, the same job that prunes
unreferenced objects), `reviewroots.Sweeper.Housekeep` runs the backfill
- and then the full sweep of the repository's review references. The
+ and then the full sweep of the repository's review references. Before, not
+ after (integrated review m3): the prune takes unreferenced objects older
+ than its grace, so a review commit left without its reference would be
+ taken in the same run that was about to protect it; a repository whose
+ backfill fails is not optimized that day, and the error is the job's. The
backfill writes the references that open reviews lack: reviews written
before this rule, or imported from shadow-mode files. The full sweep
deletes the references no open review uses: leftovers of a failed close
@@ -416,6 +420,18 @@ guarded nor swept).
shows as a growing counter. Lost commits are logged with their review ID
and commit. Limit: an operator who disables the daily maintenance also
disables this backstop (and the scheduled object pruning it guards).
+ * **Precondition of a checkpoint, and so of garbage collection (integrated
+ review m3).** In authoritative mode the commits of a review are kept in the
+ storage core by the checkpoint closure, which is the references. A review
+ without its keep-around references whose branch is gone has its commits
+ only in changeset packs, and garbage collection deletes the packs a
+ checkpoint covers. So `checkpoint.Create` runs `Housekeep` first (step 0),
+ through the write path, before it reads the reference snapshot; a build
+ whose backfill fails records no checkpoint (`Unavailable`), and without a
+ checkpoint garbage collection deletes no changeset pack. A commit already
+ missing from the repository is logged; the checkpoint cannot hold it.
+ Test: `TestStorageCheckpointBackfillsReviewRoots` (e2e; a mutant without
+ step 0 fails it on both dialects).
* A sweep may skip references younger than `gc_grace` to make the
creation/sweep race rare. That is an efficiency measure only; correctness
comes from the invariant and the bounded retry.
diff --git a/internal/cli/gitaly/serve.go b/internal/cli/gitaly/serve.go
index 050359b..f3359a5 100644
--- a/internal/cli/gitaly/serve.go
+++ b/internal/cli/gitaly/serve.go
@@ -877,13 +877,15 @@ func run(appCtx *cli.Command, cfg config.Cfg, logger log.Logger) error {
logger,
maintenance.DailyOptimizationWorker(cfg, maintenance.OptimizerFunc(func(ctx context.Context, logger log.Logger, repo storage.Repository) error {
local := localrepo.New(logger, locator, gitCmdFactory, catfileCache, repo)
- if err := housekeepingManager.OptimizeRepository(ctx, local); err != nil {
+ // Graft: the backstop of the review roots, in every mode, before the
+ // housekeeping that prunes unreferenced objects (doc/graft/PLAN7.md,
+ // REVIEW ROOTS RULE): the commits of an open review written without
+ // its keep-around references are referenced before a prune could
+ // take them.
+ if err := housekeepReviewRoots(ctx, logger, storageCore, storageSyncer, local, repo); err != nil {
return err
}
- // Graft: the backstop of the review roots, in every mode, where the
- // housekeeping that prunes unreferenced objects runs
- // (doc/graft/PLAN7.md, REVIEW ROOTS RULE).
- return housekeepReviewRoots(ctx, logger, storageCore, storageSyncer, local, repo)
+ return housekeepingManager.OptimizeRepository(ctx, local)
})),
)
if err != nil {
@@ -925,7 +927,7 @@ func run(appCtx *cli.Command, cfg config.Cfg, logger log.Logger) error {
return waitErr
}
-// housekeepReviewRoots runs the backstop of the review roots after the daily
+// housekeepReviewRoots runs the backstop of the review roots before the daily
// housekeeping of a repository: it writes the keep-around references that
// open reviews lack and deletes those no open review uses (in authoritative
// mode after catching the repository up). Open reviews whose commits are lost
diff --git a/internal/gitaly/service/storagecore/maintenance_test.go b/internal/gitaly/service/storagecore/maintenance_test.go
index 40607dd..c1fff6f 100644
--- a/internal/gitaly/service/storagecore/maintenance_test.go
+++ b/internal/gitaly/service/storagecore/maintenance_test.go
@@ -49,7 +49,7 @@ func TestCreateCheckpoint(t *testing.T) {
t.Parallel()
forEachDialect(t, func(t *testing.T, dialect sqlstore.Dialect) {
h := setup(t, corestore.ModeAuthoritative, dialect)
- h.seed()
+ main := h.seed()
repoID := h.importRepo()
resp, err := h.client.CreateCheckpoint(h.ctx, &gitalypb.CreateCheckpointRequest{Repository: h.repo})
@@ -60,7 +60,15 @@ func TestCreateCheckpoint(t *testing.T) {
require.True(t, resp.GetCreated())
require.Equal(t, record.HeadSeq, resp.GetCheckpointSequence())
require.Positive(t, resp.GetPackBytes())
- require.Equal(t, uint32(len(h.localRefs())), resp.GetReferences(), "HEAD, the branch and the tag")
+ // The open review imported from the files has no keep-around reference:
+ // the build backfills it through the write path first (integrated review
+ // m3), so the checkpoint holds HEAD, the branch, the tag and the review
+ // root, as the database and the local copy do.
+ snapshot, err := h.core.Meta.CheckpointRefs(h.ctx, repoID, resp.GetCheckpointSequence())
+ require.NoError(t, err)
+ require.Equal(t, corestore.Target(main), snapshot[corestore.KeepAroundRef(main.String())])
+ require.Equal(t, h.localRefs(), snapshot)
+ require.Equal(t, uint32(4), resp.GetReferences())
again, err := h.client.CreateCheckpoint(h.ctx, &gitalypb.CreateCheckpointRequest{Repository: h.repo})
require.NoError(t, err)
diff --git a/internal/gitaly/service/storagecore/storagecore_test.go b/internal/gitaly/service/storagecore/storagecore_test.go
index a4d2311..3c73703 100644
--- a/internal/gitaly/service/storagecore/storagecore_test.go
+++ b/internal/gitaly/service/storagecore/storagecore_test.go
@@ -18,6 +18,7 @@ import (
"github.com/feder-cr/git-ai/internal/git/localrepo"
"github.com/feder-cr/git-ai/internal/gitaly/config"
"github.com/feder-cr/git-ai/internal/gitaly/service"
+ "github.com/feder-cr/git-ai/internal/gitaly/service/hook"
"github.com/feder-cr/git-ai/internal/graft/changelog"
"github.com/feder-cr/git-ai/internal/graft/corestore"
"github.com/feder-cr/git-ai/internal/graft/corestore/coretest"
@@ -68,9 +69,15 @@ func setup(t *testing.T, mode corestore.Mode, dialect sqlstore.Dialect, adjust .
if mode == corestore.ModeAuthoritative {
cfg.Graft.Standalone = true
}
+ // The server's own reference writes (the review roots that a checkpoint
+ // backfills) go through the reference-transaction hook and its HookService,
+ // as in production: without the hook binary git would skip it and write
+ // past the database.
+ testcfg.BuildGitalyHooks(t, cfg)
cfg.SocketPath = testserver.RunGitalyServer(t, cfg, func(srv *grpc.Server, deps *service.Dependencies) {
gitalypb.RegisterStorageCoreServiceServer(srv, NewServer(deps))
+ gitalypb.RegisterHookServiceServer(srv, hook.NewServer(deps))
}, testserver.WithGraftStorage(core))
conn, err := client.New(ctx, cfg.SocketPath)
diff --git a/internal/graft/corestore/checkpoint/checkpoint.go b/internal/graft/corestore/checkpoint/checkpoint.go
index 60b6186..ef5d29f 100644
--- a/internal/graft/corestore/checkpoint/checkpoint.go
+++ b/internal/graft/corestore/checkpoint/checkpoint.go
@@ -26,6 +26,7 @@ import (
"github.com/feder-cr/git-ai/internal/gitaly/storage"
"github.com/feder-cr/git-ai/internal/graft/corestore"
"github.com/feder-cr/git-ai/internal/graft/corestore/gitops"
+ "github.com/feder-cr/git-ai/internal/graft/corestore/reviewroots"
"github.com/feder-cr/git-ai/internal/log"
"github.com/feder-cr/git-ai/internal/structerr"
"github.com/feder-cr/git-ai/proto/go/gitalypb"
@@ -90,6 +91,26 @@ func (c *Checkpointer) Create(ctx context.Context, repo *gitalypb.Repository) (c
"repository_id": record.ID,
})
+ // Step 0: the review roots. An open review written before the REVIEW ROOTS
+ // RULE has no keep-around references: its commits would be in no checkpoint,
+ // and garbage collection could then delete the only changeset packs that
+ // hold them. The backfill writes them first (through the write path, so
+ // they are in the snapshot read below); a repository whose backfill fails
+ // gets no checkpoint. Garbage collection deletes changeset packs only behind
+ // a checkpoint, so this is its precondition too. The backfill reads the
+ // local repository, so the Freshener contract is checked before it.
+ if err := c.catchUp(ctx, repo, record.ID, record.HeadSeq); err != nil {
+ return corestore.Checkpoint{}, false, err
+ }
+ backfill, _, err := reviewroots.DefaultSweeper.Housekeep(ctx, c.core, c.fresh, c.repoFactory.Build(repo), repo)
+ if err != nil {
+ return corestore.Checkpoint{}, false, structerr.NewUnavailable("storage core: backfill the review roots before the checkpoint: %w", err)
+ }
+ for _, missing := range backfill.Missing {
+ logger.WithFields(log.Fields{"review_id": missing.ReviewID, "commit_id": missing.Commit}).
+ Warn("review roots: a commit of an open review is not in the repository; the checkpoint cannot hold it")
+ }
+
// Step 1: the reference snapshot at N, and the checkpoint at N if there is
// one.
refs, seq, err := c.core.Meta.Refs(ctx, record.ID)
@@ -120,19 +141,9 @@ func (c *Checkpointer) Create(ctx context.Context, repo *gitalypb.Repository) (c
// Step 2 (change D): catch the local repository up to a head read after
// the call, so at least N.
- if err := c.fresh.EnsureFresh(ctx, repo); err != nil {
+ if err := c.catchUp(ctx, repo, record.ID, seq); err != nil {
return corestore.Checkpoint{}, false, err
}
- wm, ok, err := c.fresh.Applied(ctx, repo)
- if err != nil {
- return corestore.Checkpoint{}, false, structerr.NewInternal("read the local watermark: %w", err)
- }
- if !ok || wm.RepoID != record.ID || wm.Seq < seq {
- // EnsureFresh succeeded, so the local repository is at a head read after
- // this call arrived: anything else breaks the Freshener contract.
- return corestore.Checkpoint{}, false, structerr.NewInternal("the local repository is not caught up after EnsureFresh: watermark %+v (present %t), repository %d at changeset %d",
- wm, ok, record.ID, seq)
- }
// Step 3. After EnsureFresh the local repository holds every object of the
// snapshot; a missing one fails the build (never a partial pack).
@@ -207,6 +218,25 @@ func (c *Checkpointer) Create(ctx context.Context, repo *gitalypb.Repository) (c
return cp, created, nil
}
+// catchUp catches the local repository up (EnsureFresh) and checks that it is
+// at changeset seq of repository id or later.
+func (c *Checkpointer) catchUp(ctx context.Context, repo *gitalypb.Repository, id int64, seq uint64) error {
+ if err := c.fresh.EnsureFresh(ctx, repo); err != nil {
+ return err
+ }
+ wm, ok, err := c.fresh.Applied(ctx, repo)
+ if err != nil {
+ return structerr.NewInternal("read the local watermark: %w", err)
+ }
+ if !ok || wm.RepoID != id || wm.Seq < seq {
+ // EnsureFresh succeeded, so the local repository is at a head read after
+ // this call arrived: anything else breaks the Freshener contract.
+ return structerr.NewInternal("the local repository is not caught up after EnsureFresh: watermark %+v (present %t), repository %d at changeset %d",
+ wm, ok, id, seq)
+ }
+ return nil
+}
+
// objectTips returns the object IDs the references point to (symbolic
// references are skipped), each validated against the repository's hash.
func objectTips(refs map[string]corestore.Target, hash git.ObjectHash) ([]git.ObjectID, error) {
diff --git a/internal/graft/e2e/storage_gc_test.go b/internal/graft/e2e/storage_gc_test.go
index a0f22cd..641bf85 100644
--- a/internal/graft/e2e/storage_gc_test.go
+++ b/internal/graft/e2e/storage_gc_test.go
@@ -127,3 +127,70 @@ func TestStorageGCAcrossNodes(t *testing.T) {
}
})
}
+
+// TestStorageCheckpointBackfillsReviewRoots: the review roots are a
+// precondition of a checkpoint, and so of garbage collection (integrated
+// review m3). An open review written before the REVIEW ROOTS RULE has no
+// keep-around references; when its branch is gone, only a changeset pack holds
+// its head commit. The checkpoint build writes the references first, so the
+// checkpoint holds the commit and the collection of the packs it covers loses
+// nothing: a node that joins afterwards still has it.
+func TestStorageCheckpointBackfillsReviewRoots(t *testing.T) {
+ forEachStorageDialect(t, func(t *testing.T, dialect sqlstore.Dialect) {
+ graft, tokens := workerGraft(1)
+ cluster := newStorageCluster(t, dialect, corestore.ModeAuthoritative, unbatched(graft))
+ cluster.adjust = func(core *corestore.Core) {
+ core.GCGrace = 2 * time.Second
+ core.GCMargin = 200 * time.Millisecond
+ }
+ a := cluster.addNode(t)
+
+ rel := a.createRepository(t).GetRelativePath()
+ onA := a.agent(t, rel, tokens[workerName(0)])
+ base, err := commitFile(a.ctx, onA, workerName(0), "main", "", "README.md", "base\n")
+ require.NoError(t, err)
+ head, err := commitFile(a.ctx, onA, workerName(0), "topic", base, "topic.txt", "topic\n")
+ require.NoError(t, err)
+ _, err = gitalypb.NewRefServiceClient(a.conn(t)).DeleteRefs(a.ctx, &gitalypb.DeleteRefsRequest{
+ Repository: a.repository(rel),
+ Refs: [][]byte{[]byte("refs/heads/topic")},
+ })
+ require.NoError(t, err)
+ require.True(t, a.dbRefs(t, rel)["refs/heads/topic"].Absent())
+
+ // A review from before the rule: recorded without its roots.
+ repoID := a.record(t, rel).ID
+ _, err = a.core.Meta.InsertReview(a.ctx, repoID, func(id string) (*gitalypb.Review, error) {
+ return &gitalypb.Review{Id: id, TargetBranch: []byte("main"), HeadCommitId: head, BaseCommitId: base, Title: []byte("before the rule")}, nil
+ }, false)
+ require.NoError(t, err)
+
+ cp, err := a.operator(t, rel).StorageCheckpoint(a.ctx)
+ require.NoError(t, err)
+ require.True(t, cp.Created)
+ snapshot, err := a.core.Meta.CheckpointRefs(a.ctx, repoID, cp.Sequence)
+ require.NoError(t, err)
+ require.Equal(t, corestore.Target(head), snapshot[corestore.KeepAroundRef(head)], "the checkpoint holds the review head")
+ require.Equal(t, corestore.Target(base), snapshot[corestore.KeepAroundRef(base)], "the checkpoint holds the review base")
+
+ covered := a.referencedPacks(t, rel)
+ require.Eventually(t, func() bool {
+ res, err := a.operator(t, rel).StorageGC(a.ctx, false)
+ require.NoError(t, err)
+ return res.Boundary == cp.Sequence && res.CoveredDeleted > 0
+ }, 60*time.Second, 200*time.Millisecond)
+ inBucket := bucketPacks(t, cluster.bucketURL)
+ for key := range covered {
+ require.False(t, inBucket[key], "covered pack %s deleted", key)
+ }
+
+ // Only the checkpoint is left to hydrate from, and it has the commit.
+ b := cluster.addNode(t)
+ onB := b.agent(t, rel, tokens[workerName(0)])
+ got, err := onB.ResolveRevision(b.ctx, "main")
+ require.NoError(t, err)
+ require.Equal(t, base, got)
+ require.Equal(t, "commit", b.git(t, rel, "cat-file", "-t", head))
+ b.requireConsistent(t, rel)
+ })
+}