Git designed for AI agents.
checkpoint: build and record checkpoints (PLAN7 WP74)
Federico Elia4ac49bcdd2ff
diff --git a/doc/graft/PLAN7.md b/doc/graft/PLAN7.mdindex 9c73b61..3fa908a 100644--- a/doc/graft/PLAN7.md+++ b/doc/graft/PLAN7.md@@ -713,14 +713,24 @@ method `func (c *Core) WriteDeadline() time.Duration` = `GCGrace − GCMargin` (// reachable from tips: `git rev-list --objects --stdin` (the tips on stdin, no// --not) piped into `git pack-objects --stdout`, hashed into a file in tmpDir.// Zero tips are ignored; no tips is an empty Pack. A tip that is not in the-// repository is an error that IsMissingObject recognizes.+// repository is an error: callers check the tips with MissingObjects first, so+// that a missing object is reported precisely and never yields a partial pack.func BuildPackFrom(ctx context.Context, repo *localrepo.Repo, tips []git.ObjectID, tmpDir string) (Pack, error)-// MissingObjects returns the tips that are not in the repository (one-// `cat-file --batch-check` run).+// MissingObjects returns the tips that are not in the repository, in the order+// given (one `cat-file --batch-check` run).func MissingObjects(ctx context.Context, repo *localrepo.Repo, tips []git.ObjectID) ([]git.ObjectID, error)++// IsReachable reports whether commit is reachable from one of tips (one+// `rev-list --stdin --max-count=1` run with the tips negated; tips that are+// blobs or trees reach nothing, annotated tags are peeled). For REVIEW ROOTS R1.+func IsReachable(ctx context.Context, repo *localrepo.Repo, commit git.ObjectID, tips []git.ObjectID) (bool, error)```+(WP74: an earlier draft had BuildPackFrom report a missing tip through an+`IsMissingObject` predicate; the build order checks with MissingObjects before+any pack is built, so no caller needs to tell that error apart.)+### Writers (WP73, change A on the write path)**One upload helper (binding).** Every path that uploads a pack and then records@@ -774,17 +784,26 @@ func New(core *corestore.Core, logger log.Logger, locator storage.Locator,// Create builds a checkpoint of repo at the current database head and records// it. created is false when a checkpoint at the head exists already.-func (c *Checkpointer) Create(ctx context.Context, repo storage.Repository) (cp corestore.Checkpoint, created bool, err error)--// FaultHook is called at "after_upload" and "after_commit" (tests only).-var FaultHook func(point string) error+func (c *Checkpointer) Create(ctx context.Context, repo *gitalypb.Repository) (cp corestore.Checkpoint, created bool, err error)```+The fault hook is an unexported field of the `Checkpointer` that tests set+(`export_test.go`), like the syncer's test hook, not a package variable, so+that parallel tests do not share it. Its points are `after_snapshot` (after+step 1: a test writes a review at `S > N` there), `after_upload` and+`after_commit`. `Create` takes `*gitalypb.Repository` because the repository+factory builds from it; both callers (the RPC handler and the maintenance+runner) have one.+Binding order:-1. Require authoritative mode (else `FailedPrecondition`, SHADOW SCOPE) and a- reftable local repository. Read `(refs, N) = Meta.Refs(repoID)` (one snapshot).- If `NewestCheckpoint` is at `N`: return it, `created = false`.+1. Require authoritative mode (else `FailedPrecondition`, SHADOW SCOPE). Read+ `(refs, N) = Meta.Refs(repoID)` (one snapshot). If `NewestCheckpoint` is at+ `N`: return it, `created = false`. (WP74: the reftable check of the draft is+ dropped. The local repository may not exist before step 3, and EnsureFresh+ replaces a copy that is not reftable, so the check could only refuse a node+ that has no copy yet. The build only reads objects, which does not depend on+ the reference backend. The object hash comes from the repository record.)2. **Roots before freshness (PM check 1).** Read the open reviews(`ListReviews(repoID, "", false, 0)`): their `head_commit_id` and`base_commit_id` (each validated as an object ID of the repository's hash;@@ -805,7 +824,8 @@ Binding order:object; a missing review root can only belong to a review written without R1(created before phase 7, or brought in by `ImportHistory` from the shadow-modefiles) and is `FailedPrecondition` naming the review IDs, with the metric;- nothing is uploaded.+ nothing is uploaded. A review commit that is not an object ID of the+ repository's hash cannot be packed and is refused the same way.5. `pack = BuildPackFrom(local, tips ∪ roots, tmpDir)`. No object tips → return`created = false` (nothing to consolidate; hydration of such a repository isalready trivial).diff --git a/internal/graft/corestore/checkpoint/checkpoint.go b/internal/graft/corestore/checkpoint/checkpoint.gonew file mode 100644index 0000000..d08adc5--- /dev/null+++ b/internal/graft/corestore/checkpoint/checkpoint.go@@ -0,0 +1,409 @@+// Package checkpoint consolidates the history of a repository in the storage+// core: a checkpoint at sequence N is one self-contained pack of every object+// reachable from the references at N and from the open reviews' commits, with+// the reference snapshot at N, so that hydration starts from it instead of+// changeset 1 and garbage collection can delete the packs it covers+// (doc/graft/PLAN7.md, "Checkpoint creation", binding).+//+// The build reads the database, catches the local repository up (EnsureFresh)+// and packs from it; it holds no reference lock and never blocks or fails a+// write (CKPT-2). A build that fails at any step records nothing; a pack it+// uploaded is an orphan that garbage collection reclaims. A build at a sequence+// that has a checkpoint already returns that checkpoint.+package checkpoint++import (+ "context"+ "errors"+ "fmt"+ "os"+ "slices"+ "time"++ "github.com/prometheus/client_golang/prometheus"++ "github.com/feder-cr/git-ai/internal/git"+ "github.com/feder-cr/git-ai/internal/git/localrepo"+ "github.com/feder-cr/git-ai/internal/gitaly/storage"+ "github.com/feder-cr/git-ai/internal/graft/corestore"+ "github.com/feder-cr/git-ai/internal/graft/corestore/gitops"+ "github.com/feder-cr/git-ai/internal/log"+ "github.com/feder-cr/git-ai/internal/structerr"+ "github.com/feder-cr/git-ai/proto/go/gitalypb"+)++// Fault points of a build, for crash injection in tests.+const (+ // PointAfterSnapshot is reached when the reference snapshot is read and the+ // open reviews are not yet: a review written here has a snapshot greater+ // than the checkpoint's sequence.+ PointAfterSnapshot = "after_snapshot"+ // PointAfterUpload is reached when the pack is uploaded and no row+ // references it yet.+ PointAfterUpload = "after_upload"+ // PointAfterCommit is reached when the checkpoint is recorded.+ PointAfterCommit = "after_commit"+)++// Checkpointer builds and records checkpoints.+type Checkpointer struct {+ core *corestore.Core+ logger log.Logger+ locator storage.Locator+ repoFactory localrepo.Factory+ fresh corestore.Freshener++ // faultHook, when set by a test, is called at the fault points; an error it+ // returns ends the build there, as a crash would. Nil in production.+ faultHook func(point string) error++ reviewObjectsMissing prometheus.Counter+}++var _ prometheus.Collector = (*Checkpointer)(nil)++// New returns the checkpointer of core. fresh catches the local repositories+// up before a build.+func New(core *corestore.Core, logger log.Logger, locator storage.Locator, repoFactory localrepo.Factory, fresh corestore.Freshener) *Checkpointer {+ return &Checkpointer{+ core: core,+ logger: logger,+ locator: locator,+ repoFactory: repoFactory,+ fresh: fresh,+ reviewObjectsMissing: prometheus.NewCounter(prometheus.CounterOpts{+ Name: "graft_storage_checkpoint_review_objects_missing_total",+ Help: "Number of checkpoint builds refused because a commit of an open review is missing from the local repository.",+ }),+ }+}++// Describe implements prometheus.Collector.+func (c *Checkpointer) Describe(descs chan<- *prometheus.Desc) {+ c.reviewObjectsMissing.Describe(descs)+}++// Collect implements prometheus.Collector.+func (c *Checkpointer) Collect(metrics chan<- prometheus.Metric) {+ c.reviewObjectsMissing.Collect(metrics)+}++// Create builds a checkpoint of repo at the current database head and records+// it. created is false, with the existing checkpoint, when a checkpoint at the+// head exists already, and false with a zero Checkpoint when the repository has+// no object to consolidate (every reference is symbolic and no review is+// open). The errors carry gRPC codes: FailedPrecondition outside authoritative+// mode, for an incomplete import, for an open review whose commits are missing+// locally (metadata review_id) and for reviews that changed during the build;+// NotFound for an untracked repository; Unavailable for the database and the+// bucket; Internal for a broken invariant.+func (c *Checkpointer) Create(ctx context.Context, repo *gitalypb.Repository) (corestore.Checkpoint, bool, error) {+ if !c.core.Authoritative() {+ return corestore.Checkpoint{}, false, structerr.NewFailedPrecondition("checkpoints are created in authoritative mode only")+ }++ key := corestore.KeyOf(repo)+ record, err := c.core.Meta.GetRepository(ctx, key)+ if err != nil {+ return corestore.Checkpoint{}, false, errRepository(err)+ }+ if record.HeadSeq == 0 {+ return corestore.Checkpoint{}, false, structerr.NewFailedPrecondition("the import of the repository into the storage core is not complete")+ }+ logger := c.logger.WithFields(log.Fields{+ "repository": key.String(),+ "repository_id": record.ID,+ })++ // Step 1: the reference snapshot at N, and the checkpoint at N if there is+ // one.+ refs, seq, err := c.core.Meta.Refs(ctx, record.ID)+ if err != nil {+ return corestore.Checkpoint{}, false, errRepository(err)+ }+ newest, ok, err := c.core.Meta.NewestCheckpoint(ctx, record.ID)+ if err != nil {+ return corestore.Checkpoint{}, false, errRepository(err)+ }+ if ok && newest.Seq == seq {+ return newest, false, nil+ }+ if err := c.fault(PointAfterSnapshot); err != nil {+ return corestore.Checkpoint{}, false, err+ }++ // The local repository may not exist before step 3 (EnsureFresh hydrates+ // it in the format of the record).+ hash, err := git.ObjectHashByFormat(record.ObjectFormat)+ if err != nil {+ return corestore.Checkpoint{}, false, structerr.NewInternal("object format of the repository in the storage core: %w", err)+ }++ // Step 2: the open reviews' commits (the roots), read after the snapshot,+ // then the head H. Every review read was written after its own check at a+ // snapshot S <= H (REVIEW ROOTS R1), so after step 3 the local repository+ // holds its commits, even those that changesets after N brought in.+ roots, err := c.reviewRoots(ctx, record.ID, hash)+ if err != nil {+ return corestore.Checkpoint{}, false, err+ }+ current, err := c.core.Meta.GetRepository(ctx, key)+ if err != nil {+ return corestore.Checkpoint{}, false, errRepository(err)+ }+ if current.ID != record.ID {+ return corestore.Checkpoint{}, false, errRemoved()+ }+ head := current.HeadSeq++ // Step 3 (change D): catch the local repository up to a head read after+ // the call, so at least H.+ if err := c.fresh.EnsureFresh(ctx, repo); err != nil {+ return corestore.Checkpoint{}, false, err+ }+ wm, ok, err := c.fresh.Applied(ctx, repo)+ if err != nil {+ return corestore.Checkpoint{}, false, structerr.NewInternal("read the local watermark: %w", err)+ }+ if !ok || wm.RepoID != record.ID || wm.Seq < head {+ // EnsureFresh succeeded, so the local repository is at a head read after+ // this call arrived: anything else breaks the Freshener contract.+ return corestore.Checkpoint{}, false, structerr.NewInternal("the local repository is not caught up after EnsureFresh: watermark %+v (present %t), repository %d at head %d",+ wm, ok, record.ID, head)+ }++ // Step 4: every object is present, or a precise error and nothing built.+ local := c.repoFactory.Build(repo)+ refTips, err := objectTips(refs, hash)+ if err != nil {+ return corestore.Checkpoint{}, false, err+ }+ tips := append(slices.Clone(refTips), roots.commits()...)+ missing, err := gitops.MissingObjects(ctx, local, tips)+ if err != nil {+ return corestore.Checkpoint{}, false, structerr.NewInternal("look up the objects: %w", err)+ }+ if err := c.checkMissing(ctx, logger, refTips, roots, missing); err != nil {+ return corestore.Checkpoint{}, false, err+ }++ // Step 5.+ if len(tips) == 0 {+ return corestore.Checkpoint{}, false, nil+ }+ tmpDir, err := c.tempDir(repo.GetStorageName())+ if err != nil {+ return corestore.Checkpoint{}, false, structerr.NewInternal("temporary directory: %w", err)+ }+ pack, err := gitops.BuildPackFrom(ctx, local, tips, tmpDir)+ if err != nil {+ return corestore.Checkpoint{}, false, structerr.NewInternal("build the checkpoint pack: %w", err)+ }+ defer func() {+ if err := pack.Remove(); err != nil {+ logger.WithError(err).Warn("storage core: removing the temporary checkpoint pack")+ }+ }()++ // Step 6: the write deadline starts at a read of the record taken right+ // before the upload (PACK OWNERSHIP RULE 4).+ readAt := time.Now()+ current, err = c.core.Meta.GetRepository(ctx, key)+ if err != nil {+ return corestore.Checkpoint{}, false, errRepository(err)+ }+ if current.ID != record.ID {+ return corestore.Checkpoint{}, false, errRemoved()+ }+ f, err := pack.Open()+ if err != nil {+ return corestore.Checkpoint{}, false, structerr.NewInternal("open the checkpoint pack: %w", err)+ }+ upload, err := c.core.UploadPack(ctx, record.ID, readAt, pack.SHA256, pack.Size, f)+ if closeErr := f.Close(); closeErr != nil && err == nil {+ err = closeErr+ }+ if err != nil {+ return corestore.Checkpoint{}, false, structerr.NewUnavailable("storage core: upload the checkpoint pack: %w", err)+ }+ if err := c.fault(PointAfterUpload); err != nil {+ return corestore.Checkpoint{}, false, err+ }++ // Step 7.+ commitCtx, cancel, err := upload.CommitContext(ctx)+ if err != nil {+ return corestore.Checkpoint{}, false, structerr.NewUnavailable("%w", err)+ }+ defer cancel()++ cp, created, err := c.core.Meta.CreateCheckpoint(commitCtx, record.ID, corestore.CheckpointInput{+ Seq: seq,+ Refs: refs,+ PackKey: upload.Key,+ PackBytes: upload.Bytes,+ Roots: roots.strings(),+ })+ switch {+ case errors.Is(err, corestore.ErrCheckpointRootsChanged):+ return corestore.Checkpoint{}, false, structerr.NewFailedPrecondition("the reviews changed during the checkpoint build, run it again: %w", err)+ case err != nil:+ return corestore.Checkpoint{}, false, errRepository(err)+ }+ if err := c.fault(PointAfterCommit); err != nil {+ return corestore.Checkpoint{}, false, err+ }++ if created {+ logger.WithFields(log.Fields{+ "sequence": cp.Seq,+ "pack_key": cp.PackKey,+ "pack_bytes": cp.PackBytes,+ }).Info("storage core: checkpoint created")+ }+ return cp, created, nil+}++// root is a commit of an open review.+type root struct {+ reviewID string+ commit git.ObjectID+}++// rootSet is the root set of a build: the head and base commits of the open+// reviews, in the order read.+type rootSet []root++func (s rootSet) commits() []git.ObjectID {+ commits := make([]git.ObjectID, 0, len(s))+ for _, r := range s {+ commits = append(commits, r.commit)+ }+ return commits+}++func (s rootSet) strings() []string {+ commits := make([]string, 0, len(s))+ for _, r := range s {+ commits = append(commits, r.commit.String())+ }+ return commits+}++// reviewRoots reads the open reviews of the repository and returns their head+// and base commits; empty values are skipped. A commit that is not an object+// ID of the repository's hash cannot be packed, so the build is refused like+// one that is missing.+func (c *Checkpointer) reviewRoots(ctx context.Context, repoID int64, hash git.ObjectHash) (rootSet, error) {+ reviews, err := c.core.Meta.ListReviews(ctx, repoID, "", false, 0)+ if err != nil {+ return nil, errRepository(err)+ }++ var roots rootSet+ var invalid []string+ for _, review := range reviews {+ for _, commit := range []string{review.GetHeadCommitId(), review.GetBaseCommitId()} {+ if commit == "" {+ continue+ }+ if err := hash.ValidateHex(commit); err != nil {+ invalid = append(invalid, review.GetId())+ continue+ }+ roots = append(roots, root{reviewID: review.GetId(), commit: git.ObjectID(commit)})+ }+ }+ if len(invalid) > 0 {+ c.reviewObjectsMissing.Inc()+ return nil, structerr.NewFailedPrecondition("open reviews %v have a commit that is not an object ID of the repository; close them to create a checkpoint", invalid).+ WithMetadata("review_id", invalid)+ }+ return roots, nil+}++// checkMissing turns the objects missing from the local repository into the+// error of the build. A missing reference tip is impossible after EnsureFresh+// (Internal). A missing review commit belongs to a review written without the+// R1 check (created before phase 7, or imported from the shadow-mode files):+// FailedPrecondition naming the reviews.+func (c *Checkpointer) checkMissing(ctx context.Context, logger log.Logger, refTips []git.ObjectID, roots rootSet, missing []git.ObjectID) error {+ if len(missing) == 0 {+ return nil+ }++ absent := make(map[git.ObjectID]struct{}, len(missing))+ for _, oid := range missing {+ absent[oid] = struct{}{}+ }+ for _, tip := range refTips {+ if _, ok := absent[tip]; ok {+ return structerr.NewInternal("object %s of a reference is missing from the local repository after EnsureFresh", tip)+ }+ }++ var reviewIDs []string+ for _, r := range roots {+ if _, ok := absent[r.commit]; ok && !slices.Contains(reviewIDs, r.reviewID) {+ reviewIDs = append(reviewIDs, r.reviewID)+ }+ }+ c.reviewObjectsMissing.Inc()+ logger.WithField("review_id", reviewIDs).WarnContext(ctx, "storage core: a checkpoint is refused: commits of open reviews are missing")+ return structerr.NewFailedPrecondition("commits of open reviews %v are missing from the repository; close them to create a checkpoint", reviewIDs).+ WithMetadata("review_id", reviewIDs)+}++// objectTips returns the object IDs the references point to (symbolic+// references are skipped), each validated against the repository's hash.+func objectTips(refs map[string]corestore.Target, hash git.ObjectHash) ([]git.ObjectID, error) {+ tips := make([]git.ObjectID, 0, len(refs))+ for name, target := range refs {+ if target.IsSymbolic() {+ continue+ }+ if err := hash.ValidateHex(string(target)); err != nil {+ return nil, structerr.NewInternal("reference %q in the storage core: %w", name, err)+ }+ tips = append(tips, git.ObjectID(target))+ }+ slices.Sort(tips)+ return slices.Compact(tips), nil+}++// tempDir returns the temporary directory of the storage, created if missing.+func (c *Checkpointer) tempDir(storageName string) (string, error) {+ dir, err := c.locator.TempDir(storageName)+ if err != nil {+ return "", err+ }+ // The server creates it when it starts, but it may have been removed since.+ if err := os.MkdirAll(dir, 0o700); err != nil {+ return "", err+ }+ return dir, nil+}++func (c *Checkpointer) fault(point string) error {+ if c.faultHook == nil {+ return nil+ }+ if err := c.faultHook(point); err != nil {+ return fmt.Errorf("fault at %s: %w", point, err)+ }+ return nil+}++// errRepository maps a database error of the build.+func errRepository(err error) error {+ if errors.Is(err, corestore.ErrRepositoryNotFound) {+ return structerr.NewNotFound("the repository is not tracked by the storage core")+ }+ return structerr.NewUnavailable("storage core: %w", err)+}++// errRemoved is the error of a build whose repository was removed (and+// possibly created again under a new ID) while it ran.+func errRemoved() error {+ return structerr.NewFailedPrecondition("the repository was removed from the storage core during the checkpoint build")+}diff --git a/internal/graft/corestore/checkpoint/checkpoint_test.go b/internal/graft/corestore/checkpoint/checkpoint_test.gonew file mode 100644index 0000000..9430c1b--- /dev/null+++ b/internal/graft/corestore/checkpoint/checkpoint_test.go@@ -0,0 +1,711 @@+package checkpoint_test++import (+ "bytes"+ "context"+ "errors"+ "fmt"+ "io"+ "testing"+ "time"++ "github.com/prometheus/client_golang/prometheus"+ "github.com/stretchr/testify/require"+ "google.golang.org/grpc/codes"+ "google.golang.org/grpc/status"++ "github.com/feder-cr/git-ai/internal/git"+ "github.com/feder-cr/git-ai/internal/git/catfile"+ "github.com/feder-cr/git-ai/internal/git/gittest"+ "github.com/feder-cr/git-ai/internal/git/localrepo"+ "github.com/feder-cr/git-ai/internal/gitaly/config"+ "github.com/feder-cr/git-ai/internal/gitaly/storage"+ "github.com/feder-cr/git-ai/internal/graft/corestore"+ "github.com/feder-cr/git-ai/internal/graft/corestore/checkpoint"+ "github.com/feder-cr/git-ai/internal/graft/corestore/coretest"+ "github.com/feder-cr/git-ai/internal/graft/corestore/gitops"+ "github.com/feder-cr/git-ai/internal/graft/corestore/sqlstore"+ "github.com/feder-cr/git-ai/internal/graft/corestore/syncer"+ "github.com/feder-cr/git-ai/internal/log"+ "github.com/feder-cr/git-ai/internal/structerr"+ "github.com/feder-cr/git-ai/internal/testhelper"+ "github.com/feder-cr/git-ai/internal/testhelper/testcfg"+ "github.com/feder-cr/git-ai/proto/go/gitalypb"+)++func TestMain(m *testing.M) {+ testhelper.Run(m)+}++// relPath is the relative path of the repository the tests checkpoint.+const relPath = "repo.git"++func forEachDialect(t *testing.T, test func(t *testing.T, dialect sqlstore.Dialect)) {+ for _, dialect := range coretest.Dialects() {+ t.Run(string(dialect), func(t *testing.T) {+ test(t, dialect)+ })+ }+}++// env is one node: a storage whose local repositories are caches, a storage+// core, its syncer and a checkpointer.+type env struct {+ ctx context.Context+ cfg config.Cfg+ logger log.Logger+ locator storage.Locator+ repoFactory localrepo.Factory+ core *corestore.Core+ sy *syncer.Syncer+ cp *checkpoint.Checkpointer+ databaseURL string+ bucketURL string+}++type envOptions struct {+ dialect sqlstore.Dialect+ mode corestore.Mode+ databaseURL string+ bucketURL string+ gcGrace time.Duration+ gcMargin time.Duration+}++func newEnv(t *testing.T, opts envOptions) *env {+ t.Helper()++ cfg := testcfg.Build(t)+ ctx := testhelper.Context(t)+ logger := testhelper.NewLogger(t)+ locator := config.NewLocator(cfg)+ gitCmdFactory := gittest.NewCommandFactory(t, cfg)+ cache := catfile.NewCache(cfg)+ t.Cleanup(cache.Stop)+ repoFactory := localrepo.NewFactory(logger, locator, gitCmdFactory, cache)++ if opts.dialect == "" {+ opts.dialect = sqlstore.SQLite+ }+ if opts.databaseURL == "" {+ opts.databaseURL = coretest.DatabaseURL(t, opts.dialect)+ }+ if opts.bucketURL == "" {+ opts.bucketURL = coretest.BucketURL(t, false)+ }+ core := coretest.Open(t, coretest.Options{+ Mode: opts.mode,+ Dialect: opts.dialect,+ DatabaseURL: opts.databaseURL,+ BucketURL: opts.bucketURL,+ GCGrace: opts.gcGrace,+ GCMargin: opts.gcMargin,+ })+ sy := syncer.New(core, logger, locator, repoFactory, gitCmdFactory)++ return &env{+ ctx: ctx,+ cfg: cfg,+ logger: logger,+ locator: locator,+ repoFactory: repoFactory,+ core: core,+ sy: sy,+ cp: checkpoint.New(core, logger, locator, repoFactory, sy),+ databaseURL: opts.databaseURL,+ bucketURL: opts.bucketURL,+ }+}++// secondNode returns another node sharing the database and the bucket of e.+func (e *env) secondNode(t *testing.T, dialect sqlstore.Dialect) *env {+ t.Helper()+ return newEnv(t, envOptions{dialect: dialect, databaseURL: e.databaseURL, bucketURL: e.bucketURL})+}++func (e *env) pbRepo() *gitalypb.Repository {+ return &gitalypb.Repository{StorageName: e.cfg.Storages[0].Name, RelativePath: relPath}+}++func (e *env) key() corestore.RepoKey {+ return corestore.KeyOf(e.pbRepo())+}++// source is the repository the test commits are written to. Its references+// stay empty, so that the pack of a changeset holds the whole history of its+// tips.+type source struct {+ repo *localrepo.Repo+ path string+}++func (e *env) newSource(t *testing.T) source {+ t.Helper()++ pb, path := gittest.CreateRepository(t, e.ctx, e.cfg, gittest.CreateRepositoryConfig{SkipCreationViaService: true})+ return source{repo: localrepo.NewTestRepo(t, e.cfg, pb), path: path}+}++func (e *env) commit(t *testing.T, src source, message string, parents ...git.ObjectID) git.ObjectID {+ t.Helper()+ return gittest.WriteCommit(t, e.cfg, src.path, gittest.WithMessage(message), gittest.WithParents(parents...))+}++// changeset uploads the pack of the tips (if any) and commits the updates as+// the next changeset of repository id.+func (e *env) changeset(t *testing.T, id int64, src source, updates []corestore.RefUpdate, tips ...git.ObjectID) corestore.CommitResult {+ t.Helper()++ pack, err := gitops.BuildPack(e.ctx, src.repo, tips, t.TempDir())+ require.NoError(t, err)+ defer func() { require.NoError(t, pack.Remove()) }()++ in := corestore.ChangesetInput{+ RepoID: id,+ Updates: updates,+ Operation: "test",+ Actor: "tester",+ NodeID: "node-a",+ }+ if !pack.Empty() {+ in.PackKey = corestore.NewPackKey(id, pack.SHA256)+ rc, err := pack.Open()+ require.NoError(t, err)+ require.NoError(t, e.core.Blobs.Put(e.ctx, in.PackKey, rc))+ require.NoError(t, rc.Close())+ in.PackBytes = pack.Size+ }++ res, err := e.core.Meta.CommitChangeset(e.ctx, in)+ require.NoError(t, err)+ require.False(t, res.NoOp)+ return res+}++// register records the repository with a first changeset that points main+// (and HEAD) at head.+func (e *env) register(t *testing.T, src source, head git.ObjectID) corestore.Repository {+ t.Helper()++ repo, _, err := e.core.Meta.RegisterRepository(e.ctx, e.key(), gittest.DefaultObjectHash.Format, corestore.InitialState{})+ require.NoError(t, err)++ e.changeset(t, repo.ID, src, []corestore.RefUpdate{+ {Name: "HEAD", Expect: corestore.ExpectValue, New: corestore.SymbolicTarget("refs/heads/main")},+ {Name: "refs/heads/main", Expect: corestore.ExpectValue, New: corestore.Target(head)},+ }, head)+ return repo+}++// setRef commits a changeset moving one reference from old to new (absent+// when empty), with the pack of new.+func (e *env) setRef(t *testing.T, id int64, src source, name string, old, new git.ObjectID) uint64 {+ t.Helper()++ var tips []git.ObjectID+ if new != "" {+ tips = append(tips, new)+ }+ return e.changeset(t, id, src, []corestore.RefUpdate{+ {Name: name, Expect: corestore.ExpectValue, Old: corestore.Target(old), New: corestore.Target(new)},+ }, tips...).Seq+}++// review records a review of head on base, open unless state says otherwise.+func (e *env) review(t *testing.T, id int64, head, base git.ObjectID, state gitalypb.Review_State) *gitalypb.Review {+ t.Helper()++ r, err := e.core.Meta.InsertReview(e.ctx, id, func(reviewID string) (*gitalypb.Review, error) {+ return &gitalypb.Review{+ Id: reviewID,+ TargetBranch: []byte("main"),+ HeadCommitId: head.String(),+ BaseCommitId: base.String(),+ State: state,+ }, nil+ })+ require.NoError(t, err)+ return r+}++func (e *env) head(t *testing.T) uint64 {+ t.Helper()++ r, err := e.core.Meta.GetRepository(e.ctx, e.key())+ require.NoError(t, err)+ return r.HeadSeq+}++func (e *env) watermark(t *testing.T) corestore.Watermark {+ t.Helper()++ wm, ok, err := e.sy.Applied(e.ctx, e.pbRepo())+ require.NoError(t, err)+ require.True(t, ok)+ return wm+}++// packKeys returns the pack objects of the repository in the bucket.+func (e *env) packKeys(t *testing.T, id int64) map[string]struct{} {+ t.Helper()++ keys := map[string]struct{}{}+ require.NoError(t, e.core.Blobs.List(e.ctx, fmt.Sprintf("repos/%d/packs/", id), func(info corestore.BlobInfo) error {+ keys[info.Key] = struct{}{}+ return nil+ }))+ return keys+}++// referenced reports whether a changeset or a checkpoint references the key.+func (e *env) referenced(t *testing.T, id int64, key string) bool {+ t.Helper()++ state, err := e.core.Meta.GCSnapshot(e.ctx, id)+ require.NoError(t, err)+ if _, ok := state.Packs[key]; ok {+ return true+ }+ for _, cp := range state.Checkpoints {+ if cp.PackKey == key {+ return true+ }+ }+ return false+}++// newKeys returns the keys of after that are not in before.+func newKeys(before, after map[string]struct{}) []string {+ var keys []string+ for key := range after {+ if _, ok := before[key]; !ok {+ keys = append(keys, key)+ }+ }+ return keys+}++// restore builds a repository from the checkpoint alone: its pack indexed into+// an empty repository and its snapshot applied. The repository must be+// connected, with exactly the snapshot as references. It returns its path.+func (e *env) restore(t *testing.T, cp corestore.Checkpoint) string {+ t.Helper()++ rc, err := e.core.Blobs.Get(e.ctx, cp.PackKey)+ require.NoError(t, err)+ data, err := io.ReadAll(rc)+ require.NoError(t, err)+ require.NoError(t, rc.Close())+ require.EqualValues(t, len(data), cp.PackBytes)++ pb, path := gittest.CreateRepository(t, e.ctx, e.cfg, gittest.CreateRepositoryConfig{SkipCreationViaService: true})+ repo := localrepo.NewTestRepo(t, e.cfg, pb)+ require.NoError(t, gitops.IndexPack(e.ctx, repo, bytes.NewReader(data)))++ refs, err := e.core.Meta.CheckpointRefs(e.ctx, cp.RepoID, cp.Seq)+ require.NoError(t, err)+ require.NoError(t, gitops.ApplyRefs(e.ctx, repo, refs))+ restored, err := gitops.ReadRefs(e.ctx, repo)+ require.NoError(t, err)+ require.Equal(t, refs, restored)++ gittest.Exec(t, e.cfg, "-C", path, "fsck", "--connectivity-only")+ return path+}++func hasObject(t *testing.T, cfg config.Cfg, path string, oid git.ObjectID) bool {+ t.Helper()+ return gittest.NewCommand(t, cfg, "-C", path, "cat-file", "-e", oid.String()).Run() == nil+}++func missingCount(t *testing.T, c *checkpoint.Checkpointer) float64 {+ t.Helper()++ reg := prometheus.NewPedanticRegistry()+ require.NoError(t, reg.Register(c))+ families, err := reg.Gather()+ require.NoError(t, err)+ for _, family := range families {+ if family.GetName() == "graft_storage_checkpoint_review_objects_missing_total" {+ return family.GetMetric()[0].GetCounter().GetValue()+ }+ }+ return 0+}++func TestCreate(t *testing.T) {+ forEachDialect(t, func(t *testing.T, dialect sqlstore.Dialect) {+ e := newEnv(t, envOptions{dialect: dialect})+ src := e.newSource(t)+ c1 := e.commit(t, src, "c1")+ repo := e.register(t, src, c1)+ c2 := e.commit(t, src, "c2", c1)+ e.setRef(t, repo.ID, src, "refs/heads/main", c1, c2)+ other := e.commit(t, src, "other", c1)+ e.setRef(t, repo.ID, src, "refs/heads/other", "", other)++ cp, created, err := e.cp.Create(e.ctx, e.pbRepo())+ require.NoError(t, err)+ require.True(t, created)+ require.Equal(t, e.head(t), cp.Seq)+ require.Equal(t, repo.ID, cp.RepoID)+ require.True(t, corestore.IsOwnedPackKey(cp.PackKey))+ keyRepo, _, ok := corestore.ParsePackKey(cp.PackKey)+ require.True(t, ok)+ require.Equal(t, repo.ID, keyRepo)++ newest, ok, err := e.core.Meta.NewestCheckpoint(e.ctx, repo.ID)+ require.NoError(t, err)+ require.True(t, ok)+ require.Equal(t, cp, newest)++ dbRefs, _, err := e.core.Meta.Refs(e.ctx, repo.ID)+ require.NoError(t, err)+ snapshot, err := e.core.Meta.CheckpointRefs(e.ctx, repo.ID, cp.Seq)+ require.NoError(t, err)+ require.Equal(t, dbRefs, snapshot)+ e.restore(t, cp)++ // A second build at the same head returns the checkpoint and uploads+ // nothing.+ before := e.packKeys(t, repo.ID)+ again, created, err := e.cp.Create(e.ctx, e.pbRepo())+ require.NoError(t, err)+ require.False(t, created)+ require.Equal(t, cp, again)+ require.Equal(t, before, e.packKeys(t, repo.ID))++ // A later head gets a new checkpoint.+ c3 := e.commit(t, src, "c3", c2)+ e.setRef(t, repo.ID, src, "refs/heads/main", c2, c3)+ next, created, err := e.cp.Create(e.ctx, e.pbRepo())+ require.NoError(t, err)+ require.True(t, created)+ require.Greater(t, next.Seq, cp.Seq)+ require.NotEqual(t, cp.PackKey, next.PackKey)+ e.restore(t, next)+ })+}++func TestCreateKeepsOpenReviewCommits(t *testing.T) {+ forEachDialect(t, func(t *testing.T, dialect sqlstore.Dialect) {+ e := newEnv(t, envOptions{dialect: dialect})+ src := e.newSource(t)+ c1 := e.commit(t, src, "c1")+ repo := e.register(t, src, c1)++ open := e.commit(t, src, "open review", c1)+ e.setRef(t, repo.ID, src, "refs/heads/open", "", open)+ e.review(t, repo.ID, open, c1, gitalypb.Review_STATE_OPEN_UNSPECIFIED)+ closed := e.commit(t, src, "closed review", c1)+ e.setRef(t, repo.ID, src, "refs/heads/closed", "", closed)+ e.review(t, repo.ID, closed, c1, gitalypb.Review_STATE_CLOSED)++ // Neither commit is reachable from a reference any more.+ e.setRef(t, repo.ID, src, "refs/heads/open", open, "")+ e.setRef(t, repo.ID, src, "refs/heads/closed", closed, "")++ cp, created, err := e.cp.Create(e.ctx, e.pbRepo())+ require.NoError(t, err)+ require.True(t, created)++ path := e.restore(t, cp)+ require.True(t, hasObject(t, e.cfg, path, open), "an open review's commit is a root of the checkpoint")+ require.False(t, hasObject(t, e.cfg, path, closed), "a closed review's commit is not (documented limit)")+ })+}++// TestCreateOnALaggingCopy is change D: the build catches the local copy up to+// the head before it packs.+func TestCreateOnALaggingCopy(t *testing.T) {+ forEachDialect(t, func(t *testing.T, dialect sqlstore.Dialect) {+ a := newEnv(t, envOptions{dialect: dialect})+ b := a.secondNode(t, dialect)+ src := a.newSource(t)+ c1 := a.commit(t, src, "c1")+ repo := a.register(t, src, c1)++ require.NoError(t, b.sy.EnsureFresh(b.ctx, b.pbRepo()))+ require.Equal(t, corestore.Watermark{RepoID: repo.ID, Seq: 1}, b.watermark(t))++ c2 := a.commit(t, src, "c2", c1)+ a.setRef(t, repo.ID, src, "refs/heads/main", c1, c2)+ c3 := a.commit(t, src, "c3", c2)+ a.setRef(t, repo.ID, src, "refs/heads/main", c2, c3)++ cp, created, err := b.cp.Create(b.ctx, b.pbRepo())+ require.NoError(t, err)+ require.True(t, created)+ require.Equal(t, uint64(3), cp.Seq)+ require.Equal(t, corestore.Watermark{RepoID: repo.ID, Seq: 3}, b.watermark(t))+ path := b.restore(t, cp)+ require.True(t, hasObject(t, b.cfg, path, c3))+ })+}++// TestCreateIncludesAReviewWrittenAfterTheSnapshot is the third review's case:+// a review written after the reference snapshot N, at S > N, on a commit that+// only the changesets after N bring and that no reference reaches any more. The+// building node is below S; the build catches it up to the head and packs the+// commit.+func TestCreateIncludesAReviewWrittenAfterTheSnapshot(t *testing.T) {+ forEachDialect(t, func(t *testing.T, dialect sqlstore.Dialect) {+ a := newEnv(t, envOptions{dialect: dialect})+ b := a.secondNode(t, dialect)+ src := a.newSource(t)+ c1 := a.commit(t, src, "c1")+ repo := a.register(t, src, c1)+ require.NoError(t, b.sy.EnsureFresh(b.ctx, b.pbRepo()))++ feature := a.commit(t, src, "feature", c1)+ checkpoint.SetFaultHook(b.cp, func(point string) error {+ if point == checkpoint.PointAfterSnapshot {+ a.setRef(t, repo.ID, src, "refs/heads/feature", "", feature)+ a.review(t, repo.ID, feature, c1, gitalypb.Review_STATE_OPEN_UNSPECIFIED)+ a.setRef(t, repo.ID, src, "refs/heads/feature", feature, "")+ }+ return nil+ })++ cp, created, err := b.cp.Create(b.ctx, b.pbRepo())+ require.NoError(t, err)+ require.True(t, created)+ require.Equal(t, uint64(1), cp.Seq, "the snapshot was read before the review")+ require.Equal(t, corestore.Watermark{RepoID: repo.ID, Seq: 3}, b.watermark(t))++ path := b.restore(t, cp)+ require.True(t, hasObject(t, b.cfg, path, feature))+ })+}++func TestCreateRefusesMissingReviewCommits(t *testing.T) {+ forEachDialect(t, func(t *testing.T, dialect sqlstore.Dialect) {+ e := newEnv(t, envOptions{dialect: dialect})+ src := e.newSource(t)+ c1 := e.commit(t, src, "c1")+ repo := e.register(t, src, c1)++ // A review written without the R1 check, on a commit the storage core+ // never received.+ pruned := e.commit(t, src, "never pushed", c1)+ r := e.review(t, repo.ID, pruned, c1, gitalypb.Review_STATE_OPEN_UNSPECIFIED)+ before := e.packKeys(t, repo.ID)++ _, _, err := e.cp.Create(e.ctx, e.pbRepo())+ require.Equal(t, codes.FailedPrecondition, status.Code(err), "%v", err)+ require.Equal(t, []string{r.GetId()}, structerr.ExtractMetadata(err)["review_id"])+ require.Equal(t, float64(1), missingCount(t, e.cp))+ require.Equal(t, before, e.packKeys(t, repo.ID), "nothing is uploaded")+ _, ok, err := e.core.Meta.NewestCheckpoint(e.ctx, repo.ID)+ require.NoError(t, err)+ require.False(t, ok)++ // Closing the review resolves it.+ stored, version, err := e.core.Meta.GetReview(e.ctx, repo.ID, r.GetId())+ require.NoError(t, err)+ stored.State = gitalypb.Review_STATE_CLOSED+ require.NoError(t, e.core.Meta.UpdateReview(e.ctx, repo.ID, r.GetId(), version, stored))+ _, created, err := e.cp.Create(e.ctx, e.pbRepo())+ require.NoError(t, err)+ require.True(t, created)+ })+}++// TestCreateRefusesReviewsChangedDuringTheBuild is REVIEW ROOTS R2.+func TestCreateRefusesReviewsChangedDuringTheBuild(t *testing.T) {+ forEachDialect(t, func(t *testing.T, dialect sqlstore.Dialect) {+ e := newEnv(t, envOptions{dialect: dialect})+ src := e.newSource(t)+ c1 := e.commit(t, src, "c1")+ repo := e.register(t, src, c1)+ c2 := e.commit(t, src, "c2", c1)+ e.setRef(t, repo.ID, src, "refs/heads/main", c1, c2)+ before := e.packKeys(t, repo.ID)++ checkpoint.SetFaultHook(e.cp, func(point string) error {+ if point == checkpoint.PointAfterUpload {+ e.review(t, repo.ID, c2, c1, gitalypb.Review_STATE_OPEN_UNSPECIFIED)+ }+ return nil+ })+ _, _, err := e.cp.Create(e.ctx, e.pbRepo())+ require.Equal(t, codes.FailedPrecondition, status.Code(err), "%v", err)+ require.ErrorIs(t, err, corestore.ErrCheckpointRootsChanged)++ _, ok, err := e.core.Meta.NewestCheckpoint(e.ctx, repo.ID)+ require.NoError(t, err)+ require.False(t, ok)+ orphans := newKeys(before, e.packKeys(t, repo.ID))+ require.Len(t, orphans, 1, "the uploaded pack is left for garbage collection")+ require.False(t, e.referenced(t, repo.ID, orphans[0]))++ // The next build includes the review.+ checkpoint.SetFaultHook(e.cp, nil)+ _, created, err := e.cp.Create(e.ctx, e.pbRepo())+ require.NoError(t, err)+ require.True(t, created)+ })+}++func TestCreateCrashPoints(t *testing.T) {+ errCrash := errors.New("crash")++ forEachDialect(t, func(t *testing.T, dialect sqlstore.Dialect) {+ for _, point := range []string{checkpoint.PointAfterSnapshot, checkpoint.PointAfterUpload, checkpoint.PointAfterCommit} {+ t.Run(point, func(t *testing.T) {+ e := newEnv(t, envOptions{dialect: dialect})+ src := e.newSource(t)+ c1 := e.commit(t, src, "c1")+ repo := e.register(t, src, c1)+ before := e.packKeys(t, repo.ID)++ checkpoint.SetFaultHook(e.cp, func(p string) error {+ if p == point {+ return errCrash+ }+ return nil+ })+ _, _, err := e.cp.Create(e.ctx, e.pbRepo())+ require.ErrorIs(t, err, errCrash)+ uploaded := newKeys(before, e.packKeys(t, repo.ID))+ newest, recorded, err := e.core.Meta.NewestCheckpoint(e.ctx, repo.ID)+ require.NoError(t, err)++ switch point {+ case checkpoint.PointAfterSnapshot:+ require.Empty(t, uploaded)+ require.False(t, recorded)+ case checkpoint.PointAfterUpload:+ require.Len(t, uploaded, 1)+ require.False(t, e.referenced(t, repo.ID, uploaded[0]), "an orphan")+ require.False(t, recorded)+ case checkpoint.PointAfterCommit:+ require.True(t, recorded)+ require.Equal(t, []string{newest.PackKey}, uploaded)+ }++ // The re-run converges.+ checkpoint.SetFaultHook(e.cp, nil)+ cp, created, err := e.cp.Create(e.ctx, e.pbRepo())+ require.NoError(t, err)+ require.Equal(t, point != checkpoint.PointAfterCommit, created)+ if !created {+ require.Equal(t, newest, cp)+ }+ e.restore(t, cp)+ })+ }+ })+}++// slowBlobs is a bucket slower than the write deadline.+type slowBlobs struct {+ corestore.BlobStore+ // late stores the object and returns only after the deadline; otherwise+ // Put waits for the end of its context and stores nothing.+ late bool+}++func (b slowBlobs) Put(ctx context.Context, key string, r io.Reader) error {+ if !b.late {+ <-ctx.Done()+ return ctx.Err()+ }+ if err := b.BlobStore.Put(context.WithoutCancel(ctx), key, r); err != nil {+ return err+ }+ <-ctx.Done()+ return nil+}++// TestCreateWriteDeadline: past the write deadline the upload is cancelled or+// the commit is not started, the build fails as a storage failure, and no row+// references the pack.+func TestCreateWriteDeadline(t *testing.T) {+ forEachDialect(t, func(t *testing.T, dialect sqlstore.Dialect) {+ for _, late := range []bool{false, true} {+ t.Run(fmt.Sprintf("late=%t", late), func(t *testing.T) {+ // The write deadline is gc_grace - gc_margin = 2s.+ e := newEnv(t, envOptions{dialect: dialect, gcGrace: 3 * time.Second, gcMargin: time.Second})+ src := e.newSource(t)+ c1 := e.commit(t, src, "c1")+ repo := e.register(t, src, c1)+ before := e.packKeys(t, repo.ID)+ e.core.Blobs = slowBlobs{BlobStore: e.core.Blobs, late: late}++ _, _, err := e.cp.Create(e.ctx, e.pbRepo())+ require.Equal(t, codes.Unavailable, status.Code(err), "%v", err)+ require.ErrorIs(t, err, corestore.ErrPackDeadline)++ _, ok, err := e.core.Meta.NewestCheckpoint(e.ctx, repo.ID)+ require.NoError(t, err)+ require.False(t, ok)+ uploaded := newKeys(before, e.packKeys(t, repo.ID))+ if late {+ require.Len(t, uploaded, 1)+ require.False(t, e.referenced(t, repo.ID, uploaded[0]))+ } else {+ require.Empty(t, uploaded)+ }+ })+ }+ })+}++func TestCreateWithoutObjects(t *testing.T) {+ forEachDialect(t, func(t *testing.T, dialect sqlstore.Dialect) {+ e := newEnv(t, envOptions{dialect: dialect})+ repo, _, err := e.core.Meta.RegisterRepository(e.ctx, e.key(), gittest.DefaultObjectHash.Format, corestore.InitialState{})+ require.NoError(t, err)+ _, err = e.core.Meta.CommitChangeset(e.ctx, corestore.ChangesetInput{+ RepoID: repo.ID,+ Updates: []corestore.RefUpdate{+ {Name: "HEAD", Expect: corestore.ExpectValue, New: corestore.SymbolicTarget("refs/heads/main")},+ },+ Operation: "test",+ Actor: "tester",+ NodeID: "node-a",+ })+ require.NoError(t, err)++ cp, created, err := e.cp.Create(e.ctx, e.pbRepo())+ require.NoError(t, err)+ require.False(t, created)+ require.Equal(t, corestore.Checkpoint{}, cp)+ })+}++func TestCreatePreconditions(t *testing.T) {+ forEachDialect(t, func(t *testing.T, dialect sqlstore.Dialect) {+ t.Run("shadow mode", func(t *testing.T) {+ e := newEnv(t, envOptions{dialect: dialect, mode: corestore.ModeShadow})+ _, _, err := e.cp.Create(e.ctx, e.pbRepo())+ require.Equal(t, codes.FailedPrecondition, status.Code(err), "%v", err)+ })++ t.Run("untracked", func(t *testing.T) {+ e := newEnv(t, envOptions{dialect: dialect})+ _, _, err := e.cp.Create(e.ctx, e.pbRepo())+ require.Equal(t, codes.NotFound, status.Code(err), "%v", err)+ })++ t.Run("incomplete import", func(t *testing.T) {+ e := newEnv(t, envOptions{dialect: dialect})+ _, _, err := e.core.Meta.RegisterRepository(e.ctx, e.key(), gittest.DefaultObjectHash.Format, corestore.InitialState{})+ require.NoError(t, err)+ _, _, err = e.cp.Create(e.ctx, e.pbRepo())+ require.Equal(t, codes.FailedPrecondition, status.Code(err), "%v", err)+ })++ // A Freshener that does not bring the local repository up to date+ // breaks its contract: the build refuses to pack from it.+ t.Run("freshener contract", func(t *testing.T) {+ e := newEnv(t, envOptions{dialect: dialect})+ src := e.newSource(t)+ e.register(t, src, e.commit(t, src, "c1"))+ cp := checkpoint.New(e.core, e.logger, e.locator, e.repoFactory, corestore.NopFreshener{})+ _, _, err := cp.Create(e.ctx, e.pbRepo())+ require.Equal(t, codes.Internal, status.Code(err), "%v", err)+ })+ })+}diff --git a/internal/graft/corestore/checkpoint/export_test.go b/internal/graft/corestore/checkpoint/export_test.gonew file mode 100644index 0000000..5a3854f--- /dev/null+++ b/internal/graft/corestore/checkpoint/export_test.go@@ -0,0 +1,4 @@+package checkpoint++// SetFaultHook sets the hook called at the fault points of c's builds.+func SetFaultHook(c *Checkpointer, hook func(point string) error) { c.faultHook = hook }