Git designed for AI agents.
corestore: one upload path with an enforced write deadline (PLAN7 WP73)
Federico Eliafd111e64e11b
diff --git a/internal/gitaly/config/config_graft_storage.go b/internal/gitaly/config/config_graft_storage.goindex 2d03f42..8e69ebb 100644--- a/internal/gitaly/config/config_graft_storage.go+++ b/internal/gitaly/config/config_graft_storage.go@@ -31,6 +31,15 @@ type GraftStorage struct {// ShadowTimeout bounds the shadow-mode work of one reference transaction// (Go duration). Empty means DefaultGraftStorageShadowTimeout.ShadowTimeout string `json:"shadow_timeout" toml:"shadow_timeout,omitempty"`+ // GCGrace is how long garbage collection waits before it deletes a pack it+ // observed unreferenced, and how old a checkpoint must be before the packs it+ // covers are deleted (Go duration). Empty means DefaultGraftStorageGCGrace.+ GCGrace string `json:"gc_grace" toml:"gc_grace,omitempty"`+ // GCMargin is the safety margin of the writers' deadline: a writer uploads a+ // pack and commits its changeset within gc_grace - gc_margin (Go duration).+ // Empty means DefaultGraftStorageGCMargin. See doc/graft/STORAGE.md for the+ // clock assumption it covers.+ GCMargin string `json:"gc_margin" toml:"gc_margin,omitempty"`}const (@@ -38,6 +47,10 @@ const (DefaultGraftStorageSyncTimeout = 10 * time.Second// DefaultGraftStorageShadowTimeout is the default GraftStorage.ShadowTimeout.DefaultGraftStorageShadowTimeout = 5 * time.Second+ // DefaultGraftStorageGCGrace is the default GraftStorage.GCGrace.+ DefaultGraftStorageGCGrace = time.Hour+ // DefaultGraftStorageGCMargin is the default GraftStorage.GCMargin.+ DefaultGraftStorageGCMargin = 5 * time.Minute)// Enabled reports whether the storage core is on.@@ -53,6 +66,40 @@ func (s GraftStorage) ShadowTimeoutOrDefault() (time.Duration, error) {return positiveDurationOrDefault("shadow_timeout", s.ShadowTimeout, DefaultGraftStorageShadowTimeout)}+// GCGraceOrDefault parses GCGrace, applying the default.+func (s GraftStorage) GCGraceOrDefault() (time.Duration, error) {+ return positiveDurationOrDefault("gc_grace", s.GCGrace, DefaultGraftStorageGCGrace)+}++// GCMarginOrDefault parses GCMargin, applying the default.+func (s GraftStorage) GCMarginOrDefault() (time.Duration, error) {+ return positiveDurationOrDefault("gc_margin", s.GCMargin, DefaultGraftStorageGCMargin)+}++// validateGC checks that the garbage collection times leave the writers a+// positive deadline, and that a sync can never meet a pack the collector+// deletes during it: gc_grace > sync_timeout + gc_margin+// (doc/graft/PLAN7.md, GC BOUNDARY rule).+func (s GraftStorage) validateGC() error {+ grace, err := s.GCGraceOrDefault()+ if err != nil {+ return err+ }+ margin, err := s.GCMarginOrDefault()+ if err != nil {+ return err+ }+ sync, err := s.SyncTimeoutOrDefault()+ if err != nil {+ return err+ }++ if grace <= sync+margin {+ return fmt.Errorf("graft storage: gc_grace %s must be greater than sync_timeout %s + gc_margin %s", grace, sync, margin)+ }+ return nil+}+func positiveDurationOrDefault(name, value string, def time.Duration) (time.Duration, error) {if value == "" {return def, nil@@ -107,5 +154,5 @@ func (s GraftStorage) validate(standalone bool) error {return err}- return nil+ return s.validateGC()}diff --git a/internal/gitaly/config/config_graft_storage_test.go b/internal/gitaly/config/config_graft_storage_test.goindex 964e9b1..04a7b8f 100644--- a/internal/gitaly/config/config_graft_storage_test.go+++ b/internal/gitaly/config/config_graft_storage_test.go@@ -34,6 +34,11 @@ func TestGraftStorageValidate(t *testing.T) {{desc: "node id with slash", storage: GraftStorage{Mode: "shadow", Database: valid.Database, Bucket: valid.Bucket, NodeID: "a/b"}, err: "node_id"},{desc: "bad sync timeout", storage: GraftStorage{Mode: "shadow", Database: valid.Database, Bucket: valid.Bucket, SyncTimeout: "0s"}, err: "sync_timeout"},{desc: "bad shadow timeout", storage: GraftStorage{Mode: "shadow", Database: valid.Database, Bucket: valid.Bucket, ShadowTimeout: "soon"}, err: "shadow_timeout"},+ {desc: "gc times", storage: GraftStorage{Mode: "shadow", Database: valid.Database, Bucket: valid.Bucket, GCGrace: "30m", GCMargin: "1m"}},+ {desc: "bad gc_grace", storage: GraftStorage{Mode: "shadow", Database: valid.Database, Bucket: valid.Bucket, GCGrace: "-1h"}, err: "gc_grace"},+ {desc: "bad gc_margin", storage: GraftStorage{Mode: "shadow", Database: valid.Database, Bucket: valid.Bucket, GCMargin: "0"}, err: "gc_margin"},+ {desc: "gc_grace not above sync_timeout + gc_margin", storage: GraftStorage{Mode: "shadow", Database: valid.Database, Bucket: valid.Bucket, GCGrace: "6m", GCMargin: "5m", SyncTimeout: "1m"}, err: "gc_grace 6m0s must be greater than sync_timeout 1m0s + gc_margin 5m0s"},+ {desc: "gc_grace just above sync_timeout + gc_margin", storage: GraftStorage{Mode: "shadow", Database: valid.Database, Bucket: valid.Bucket, GCGrace: "361s", GCMargin: "5m", SyncTimeout: "1m"}},} {t.Run(tc.desc, func(t *testing.T) {t.Parallel()@@ -62,6 +67,15 @@ func TestGraftStorageDefaults(t *testing.T) {require.NoError(t, err)require.Equal(t, DefaultGraftStorageShadowTimeout, shadow)+ grace, err := s.GCGraceOrDefault()+ require.NoError(t, err)+ require.Equal(t, DefaultGraftStorageGCGrace, grace)++ margin, err := s.GCMarginOrDefault()+ require.NoError(t, err)+ require.Equal(t, DefaultGraftStorageGCMargin, margin)+ require.NoError(t, s.validateGC(), "the defaults are consistent")+s = GraftStorage{Mode: "shadow", SyncTimeout: "250ms"}require.True(t, s.Enabled())sync, err = s.SyncTimeoutOrDefault()diff --git a/internal/gitaly/service/storagecore/import.go b/internal/gitaly/service/storagecore/import.goindex b36e087..592174e 100644--- a/internal/gitaly/service/storagecore/import.go+++ b/internal/gitaly/service/storagecore/import.go@@ -6,6 +6,7 @@ import ("maps""os""sync"+ "time""github.com/feder-cr/git-ai/internal/git""github.com/feder-cr/git-ai/internal/gitaly/storage/mode"@@ -146,14 +147,28 @@ func (s *server) ImportRepository(ctx context.Context, req *gitalypb.ImportRepos}}()- var packKey string- var packBytes int64+ // The write deadline of the pack starts at a read of the record taken right+ // before the upload, so that the build above, however long, does not consume+ // it (doc/graft/PLAN7.md, PACK OWNERSHIP RULE 4).+ var upload corestore.PackUploadif !pack.Empty() {- packKey = corestore.NewPackKey(reg.repo.ID, pack.SHA256)- if err := uploadPack(ctx, core, packKey, pack); err != nil {+ readAt := time.Now()+ if _, err := core.Meta.GetRepository(ctx, key); err != nil {+ if errors.Is(err, corestore.ErrRepositoryNotFound) {+ return nil, errRegistrationRemoved()+ }+ return nil, structerr.NewUnavailable("storage core: %w; %s", err, resumeHint)+ }++ f, err := pack.Open()+ if err != nil {+ return nil, structerr.NewInternal("open pack: %w; %s", err, resumeHint)+ }+ upload, err = core.UploadPack(ctx, reg.repo.ID, readAt, pack.SHA256, pack.Size, f)+ _ = f.Close()+ if err != nil {return nil, structerr.NewUnavailable("storage core: upload pack: %w; %s", err, resumeHint)}- packBytes = pack.Size}// Step 3.@@ -166,14 +181,20 @@ func (s *server) ImportRepository(ctx context.Context, req *gitalypb.ImportReposNew: refs[name],})}- res, err := core.Meta.CommitChangeset(ctx, corestore.ChangesetInput{+ commitCtx, cancel, err := upload.CommitContext(ctx)+ if err != nil {+ return nil, structerr.NewUnavailable("%w; %s", err, resumeHint)+ }+ defer cancel()++ res, err := core.Meta.CommitChangeset(commitCtx, corestore.ChangesetInput{RepoID: reg.repo.ID,Updates: updates,Operation: importOperation,Actor: identity.Actor(ctx, importOperation),NodeID: core.NodeID,- PackKey: packKey,- PackBytes: packBytes,+ PackKey: upload.Key,+ PackBytes: upload.Bytes,})if err != nil {var conflict *corestore.ErrRefConflict@@ -184,9 +205,7 @@ func (s *server) ImportRepository(ctx context.Context, req *gitalypb.ImportRepos// can have recorded references.return nil, errAlreadyImported()case errors.Is(err, corestore.ErrRepositoryNotFound):- // Not Aborted: Gitaly replaces the message of that code with a generic- // one, and this one tells the operator what to do.- return nil, structerr.NewFailedPrecondition("storage core: the registration of the import was removed concurrently; run ImportRepository again")+ return nil, errRegistrationRemoved()default:return nil, structerr.NewUnavailable("storage core: commit import changeset: %w; %s", err, resumeHint)}@@ -214,7 +233,7 @@ func (s *server) ImportRepository(ctx context.Context, req *gitalypb.ImportReposreturn &gitalypb.ImportRepositoryResponse{Sequence: res.Seq,References: uint32(len(refs)),- PackBytes: packBytes,+ PackBytes: upload.Bytes,ChangelogEntries: uint32(reg.changelogEntries),Reviews: uint32(reg.reviews),}, nil@@ -385,14 +404,11 @@ func acquireImport(core *corestore.Core, key corestore.RepoKey) (release func(),}, true}-// uploadPack stores the pack under key.-func uploadPack(ctx context.Context, core *corestore.Core, key string, pack gitops.Pack) error {- f, err := pack.Open()- if err != nil {- return err- }- defer func() { _ = f.Close() }()- return core.Blobs.Put(ctx, key, f)+// errRegistrationRemoved is the error of an import whose registration was+// removed while it ran. Not Aborted: Gitaly replaces the message of that code+// with a generic one, and this one tells the operator what to do.+func errRegistrationRemoved() error {+ return structerr.NewFailedPrecondition("storage core: the registration of the import was removed concurrently; run ImportRepository again")}// scanChangelog returns every entry of the file changelog of the repository,diff --git a/internal/gitaly/service/storagecore/import_deadline_test.go b/internal/gitaly/service/storagecore/import_deadline_test.gonew file mode 100644index 0000000..3a37ebe--- /dev/null+++ b/internal/gitaly/service/storagecore/import_deadline_test.go@@ -0,0 +1,141 @@+package storagecore++import (+ "context"+ "fmt"+ "io"+ "sync/atomic"+ "testing"+ "time"++ "github.com/stretchr/testify/require"+ "google.golang.org/grpc/codes"++ "github.com/feder-cr/git-ai/internal/graft/corestore"+ "github.com/feder-cr/git-ai/internal/graft/corestore/sqlstore"+ "github.com/feder-cr/git-ai/proto/go/gitalypb"+)++// The tests of this file check the write deadline of the import's pack+// (doc/graft/PLAN7.md, PACK OWNERSHIP RULE 4): an upload, or the commit of the+// changeset that references it, that would end after the deadline fails like+// a storage failure, no changeset references the pack, and the import is+// resumable as any failed import (IMPORT RULE).++const (+ slowNone int32 = iota+ slowBlock // Put waits for the end of its context and stores nothing+ slowLate // Put stores the object, then returns only after the deadline+)++// slowBlobs is a bucket that is slower than the write deadline while its mode+// is not slowNone.+type slowBlobs struct {+ corestore.BlobStore+ mode *atomic.Int32+}++func (b slowBlobs) Put(ctx context.Context, key string, r io.Reader) error {+ switch b.mode.Load() {+ case slowBlock:+ <-ctx.Done()+ return ctx.Err()+ case slowLate:+ if err := b.BlobStore.Put(ctx, key, r); err != nil {+ return err+ }+ <-ctx.Done()+ return nil+ default:+ return b.BlobStore.Put(ctx, key, r)+ }+}++// testWriteDeadline is the write deadline of these tests: long enough for a+// normal upload and commit of the test repository, short enough to wait out.+// The core is never changed while the server runs: the RPCs order the test and+// the server only through a socket, which the race detector does not see.+const testWriteDeadline = 2 * time.Second++// setupSlow is setup with a slowBlobs bucket and a write deadline of w.+func setupSlow(t *testing.T, mode corestore.Mode, dialect sqlstore.Dialect, w time.Duration) (*harness, *atomic.Int32) {+ t.Helper()++ slow := &atomic.Int32{}+ h := setup(t, mode, dialect, func(core *corestore.Core) {+ core.Blobs = slowBlobs{BlobStore: core.Blobs, mode: slow}+ core.GCMargin = w+ core.GCGrace = 2 * w+ })+ return h, slow+}++// bucketKeys lists the objects of the repository in the bucket.+func (h *harness) bucketKeys(repoID int64) []string {+ h.t.Helper()++ var keys []string+ require.NoError(h.t, h.core.Blobs.List(h.ctx, fmt.Sprintf("repos/%d/", repoID), func(info corestore.BlobInfo) error {+ keys = append(keys, info.Key)+ return nil+ }))+ return keys+}++func TestImportRepositoryUploadPastTheWriteDeadline(t *testing.T) {+ forEachCore(t, func(t *testing.T, mode corestore.Mode, dialect sqlstore.Dialect) {+ h, slow := setupSlow(t, mode, dialect, testWriteDeadline)+ h.seed()+ localBefore := h.localRefs()++ slow.Store(slowBlock)+ _, err := h.client.ImportRepository(h.ctx, &gitalypb.ImportRepositoryRequest{Repository: h.repo})+ requireResumeHint(t, err, codes.Unavailable)+ require.ErrorContains(t, err, corestore.ErrPackDeadline.Error())++ state, err := h.client.GetStorageState(h.ctx, &gitalypb.GetStorageStateRequest{Repository: h.repo})+ require.NoError(t, err)+ repoID := state.GetRepositoryId()+ h.requireIncomplete(repoID)+ require.Empty(t, h.bucketKeys(repoID), "the cancelled upload stored nothing")++ slow.Store(slowNone)+ _, err = h.client.ImportRepository(h.ctx, &gitalypb.ImportRepositoryRequest{Repository: h.repo})+ require.NoError(t, err)+ h.requireImported(repoID, localBefore)+ })+}++func TestImportRepositoryCommitPastTheWriteDeadline(t *testing.T) {+ forEachCore(t, func(t *testing.T, mode corestore.Mode, dialect sqlstore.Dialect) {+ h, slow := setupSlow(t, mode, dialect, testWriteDeadline)+ h.seed()+ localBefore := h.localRefs()++ // The upload completes, but only returns after the deadline: the changeset+ // that would reference the pack is not committed.+ slow.Store(slowLate)+ _, err := h.client.ImportRepository(h.ctx, &gitalypb.ImportRepositoryRequest{Repository: h.repo})+ requireResumeHint(t, err, codes.Unavailable)+ require.ErrorContains(t, err, corestore.ErrPackDeadline.Error())++ state, err := h.client.GetStorageState(h.ctx, &gitalypb.GetStorageStateRequest{Repository: h.repo})+ require.NoError(t, err)+ repoID := state.GetRepositoryId()+ h.requireIncomplete(repoID)+ orphans := h.bucketKeys(repoID)+ require.Len(t, orphans, 1, "the pack was stored")++ // The resumed import uploads its own pack; the late one stays an orphan+ // that no changeset references.+ slow.Store(slowNone)+ _, err = h.client.ImportRepository(h.ctx, &gitalypb.ImportRepositoryRequest{Repository: h.repo})+ require.NoError(t, err)+ h.requireImported(repoID, localBefore)++ changesets, err := h.core.Meta.ListChangesets(h.ctx, repoID, 0, 1)+ require.NoError(t, err)+ require.NotEqual(t, orphans[0], changesets[0].PackKey)+ require.ElementsMatch(t, []string{orphans[0], changesets[0].PackKey}, h.bucketKeys(repoID))+ })+}diff --git a/internal/graft/corestore/core.go b/internal/graft/corestore/core.goindex 7e36f21..8920cf3 100644--- a/internal/graft/corestore/core.go+++ b/internal/graft/corestore/core.go@@ -2,6 +2,8 @@ package corestoreimport ("context"+ "errors"+ "fmt""io""path""strings"@@ -88,6 +90,78 @@ type Core struct {// one reference transaction.SyncTimeout time.DurationShadowTimeout time.Duration+ // GCGrace is how long garbage collection waits before it deletes a pack it+ // observed unreferenced, and how old a checkpoint must be before the packs it+ // covers are deleted. GCMargin is the safety margin of the writers' deadline,+ // WriteDeadline() = GCGrace - GCMargin (doc/graft/PLAN7.md, PACK OWNERSHIP+ // RULE 4). Both are set whenever the storage core is on: garbage collection+ // runs in shadow mode too.+ GCGrace time.Duration+ GCMargin time.Duration+}++// WriteDeadline is W of doc/graft/PLAN7.md: a writer uploads a pack and+// commits the row that references it within W of a read of the repository+// record that found the repository.+func (c *Core) WriteDeadline() time.Duration { return c.GCGrace - c.GCMargin }++// PackUpload is a pack uploaded under a new owned key (NewPackKey) and the+// deadline before which a row may reference it.+type PackUpload struct {+ // Key is the owned key, empty when there was nothing to upload (the+ // transaction brings no new objects).+ Key string+ Bytes int64+ // Deadline is the time of the repository-record read that preceded the+ // upload plus WriteDeadline(). It keeps the monotonic clock reading of that+ // read, so that it is compared on the monotonic clock.+ Deadline time.Time+}++// UploadPack uploads the pack read from r (size bytes whose SHA-256 is sum)+// under a new owned key of the repository. readAt is the time of a read of the+// repository record that found the repository, taken before that read and+// before the upload; the upload must end before readAt + WriteDeadline(), else+// it is cancelled (nothing is stored) and the error wraps ErrPackDeadline. It is+// the only way the storage core uploads a pack (doc/graft/PLAN7.md, RULE 26).+func (c *Core) UploadPack(ctx context.Context, repoID int64, readAt time.Time, sum [32]byte, size int64, r io.Reader) (PackUpload, error) {+ if c.GCGrace <= 0 || c.GCMargin <= 0 || c.WriteDeadline() <= 0 {+ return PackUpload{}, fmt.Errorf("storage core: invalid write deadline: gc_grace %s, gc_margin %s", c.GCGrace, c.GCMargin)+ }++ deadline := readAt.Add(c.WriteDeadline())+ if !time.Now().Before(deadline) {+ return PackUpload{}, fmt.Errorf("storage core: upload a pack of repository %d: %w", repoID, ErrPackDeadline)+ }++ putCtx, cancel := context.WithDeadline(ctx, deadline)+ defer cancel()++ key := NewPackKey(repoID, sum)+ if err := c.Blobs.Put(putCtx, key, r); err != nil {+ if ctx.Err() == nil && errors.Is(putCtx.Err(), context.DeadlineExceeded) {+ return PackUpload{}, fmt.Errorf("storage core: upload %q: %w: %w", key, ErrPackDeadline, err)+ }+ return PackUpload{}, fmt.Errorf("storage core: upload %q: %w", key, err)+ }++ return PackUpload{Key: key, Bytes: size, Deadline: deadline}, nil+}++// CommitContext returns ctx bounded by the upload's deadline, for the commit of+// the row that references the pack. When the deadline has passed it returns+// ErrPackDeadline: the caller must not start the commit and must not reference+// the key. An upload without a key (nothing uploaded) has no deadline.+func (u PackUpload) CommitContext(ctx context.Context) (context.Context, context.CancelFunc, error) {+ if u.Key == "" {+ ctx, cancel := context.WithCancel(ctx)+ return ctx, cancel, nil+ }+ if !time.Now().Before(u.Deadline) {+ return nil, nil, fmt.Errorf("storage core: commit the changeset of pack %q: %w", u.Key, ErrPackDeadline)+ }+ ctx, cancel := context.WithDeadline(ctx, u.Deadline)+ return ctx, cancel, nil}// Enabled reports whether the storage core is on (shadow or authoritative).diff --git a/internal/graft/corestore/coretest/coretest.go b/internal/graft/corestore/coretest/coretest.goindex 2e7119b..253f6a6 100644--- a/internal/graft/corestore/coretest/coretest.go+++ b/internal/graft/corestore/coretest/coretest.go@@ -135,6 +135,11 @@ type Options struct {BucketURL string// NodeID is the node name; empty means "node-a".NodeID string+ // GCGrace and GCMargin are the garbage collection times; zero means the+ // production defaults (1h and 5m). A test of the writers' deadline sets them+ // short.+ GCGrace time.Duration+ GCMargin time.Duration}// Open opens a storage core for the test and closes it when the test ends.@@ -156,6 +161,12 @@ func Open(tb testing.TB, opts Options) *corestore.Core {if opts.NodeID == "" {opts.NodeID = "node-a"}+ if opts.GCGrace == 0 {+ opts.GCGrace = time.Hour+ }+ if opts.GCMargin == 0 {+ opts.GCMargin = 5 * time.Minute+ }ctx := context.Background()@@ -172,6 +183,8 @@ func Open(tb testing.TB, opts Options) *corestore.Core {Blobs: blobs,SyncTimeout: 10 * time.Second,ShadowTimeout: 5 * time.Second,+ GCGrace: opts.GCGrace,+ GCMargin: opts.GCMargin,}tb.Cleanup(func() { _ = core.Close() })diff --git a/internal/graft/corestore/coretest/import.go b/internal/graft/corestore/coretest/import.goindex 3939b81..3bfe69e 100644--- a/internal/graft/corestore/coretest/import.go+++ b/internal/graft/corestore/coretest/import.go@@ -2,6 +2,7 @@ package coretestimport ("testing"+ "time""github.com/feder-cr/git-ai/internal/git/localrepo""github.com/feder-cr/git-ai/internal/graft/changelog"@@ -48,12 +49,18 @@ func Import(tb testing.TB, core *corestore.Core, repo *localrepo.Repo, repoProtorequire.NoError(tb, err)defer func() { _ = pack.Remove() }()- var packKey string+ // As ImportRepository: the write deadline starts at a read of the record+ // right before the upload (doc/graft/PLAN7.md, PACK OWNERSHIP RULE 4).+ var upload corestore.PackUploadif !pack.Empty() {- packKey = corestore.NewPackKey(registered.ID, pack.SHA256)+ readAt := time.Now()+ _, err := core.Meta.GetRepository(ctx, corestore.KeyOf(repoProto))+ require.NoError(tb, err)+f, err := pack.Open()require.NoError(tb, err)- require.NoError(tb, core.Blobs.Put(ctx, packKey, f))+ upload, err = core.UploadPack(ctx, registered.ID, readAt, pack.SHA256, pack.Size, f)+ require.NoError(tb, err)require.NoError(tb, f.Close())}@@ -62,14 +69,18 @@ func Import(tb testing.TB, core *corestore.Core, repo *localrepo.Repo, repoProtoupdates = append(updates, corestore.RefUpdate{Name: name, Expect: corestore.ExpectValue, New: target})}- res, err := core.Meta.CommitChangeset(ctx, corestore.ChangesetInput{+ commitCtx, cancel, err := upload.CommitContext(ctx)+ require.NoError(tb, err)+ defer cancel()++ res, err := core.Meta.CommitChangeset(commitCtx, corestore.ChangesetInput{RepoID: registered.ID,Updates: updates,Operation: "import",Actor: "import",NodeID: core.NodeID,- PackKey: packKey,- PackBytes: pack.Size,+ PackKey: upload.Key,+ PackBytes: upload.Bytes,})require.NoError(tb, err)require.Equal(tb, uint64(1), res.Seq, "the import is changeset 1")diff --git a/internal/graft/corestore/open/open.go b/internal/graft/corestore/open/open.goindex 48dd53f..c37e14c 100644--- a/internal/graft/corestore/open/open.go+++ b/internal/graft/corestore/open/open.go@@ -34,6 +34,14 @@ func Open(ctx context.Context, cfg config.GraftStorage) (*corestore.Core, error)if err != nil {return nil, err}+ gcGrace, err := cfg.GCGraceOrDefault()+ if err != nil {+ return nil, err+ }+ gcMargin, err := cfg.GCMarginOrDefault()+ if err != nil {+ return nil, err+ }nodeID := cfg.NodeIDif nodeID == "" {@@ -60,5 +68,7 @@ func Open(ctx context.Context, cfg config.GraftStorage) (*corestore.Core, error)Blobs: blobs,SyncTimeout: syncTimeout,ShadowTimeout: shadowTimeout,+ GCGrace: gcGrace,+ GCMargin: gcMargin,}, nil}diff --git a/internal/graft/corestore/types.go b/internal/graft/corestore/types.goindex f7895c4..1ab1a21 100644--- a/internal/graft/corestore/types.go+++ b/internal/graft/corestore/types.go@@ -232,6 +232,11 @@ var (// include: the review was created or moved during the build// (doc/graft/PLAN7.md, REVIEW ROOTS RULE R2).ErrCheckpointRootsChanged = errors.New("storage core: the open reviews changed during the checkpoint build")+ // ErrPackDeadline is returned when a pack upload, or the commit of the row+ // that references it, would end after the write deadline: garbage collection+ // may then already treat the pack as an orphan (doc/graft/PLAN7.md, PACK+ // OWNERSHIP RULE 4). The write fails like any storage failure.+ ErrPackDeadline = errors.New("storage core: the pack upload is older than the write deadline")// ErrBlobCorrupt is returned by the reader of BlobStore.Get when the content// does not hash to its key.ErrBlobCorrupt = errors.New("storage core: blob content does not match its key")diff --git a/internal/graft/corestore/upload_test.go b/internal/graft/corestore/upload_test.gonew file mode 100644index 0000000..d9e080f--- /dev/null+++ b/internal/graft/corestore/upload_test.go@@ -0,0 +1,136 @@+package corestore++import (+ "context"+ "crypto/sha256"+ "errors"+ "io"+ "strings"+ "testing"+ "time"++ "github.com/stretchr/testify/require"+)++// fakeBlobs records the Puts; when block is set, Put waits for the end of its+// context instead of storing (a bucket slower than the write deadline).+type fakeBlobs struct {+ BlobStore+ block bool+ puts []string+ // deadline is the deadline of the context of the last Put.+ deadline time.Time+}++func (f *fakeBlobs) Put(ctx context.Context, key string, r io.Reader) error {+ f.puts = append(f.puts, key)+ f.deadline, _ = ctx.Deadline()+ if f.block {+ <-ctx.Done()+ return ctx.Err()+ }+ _, err := io.Copy(io.Discard, r)+ return err+}++func uploadCore(blobs BlobStore, grace, margin time.Duration) *Core {+ return &Core{Mode: ModeAuthoritative, Blobs: blobs, GCGrace: grace, GCMargin: margin}+}++func TestUploadPackUsesAnOwnedKeyAndTheWriteDeadline(t *testing.T) {+ t.Parallel()++ blobs := &fakeBlobs{}+ core := uploadCore(blobs, time.Hour, 5*time.Minute)+ sum := sha256.Sum256([]byte("pack"))+ readAt := time.Now()++ upload, err := core.UploadPack(context.Background(), 7, readAt, sum, 4, strings.NewReader("pack"))+ require.NoError(t, err)+ require.True(t, IsOwnedPackKey(upload.Key))+ id, gotSum, _ := ParsePackKey(upload.Key)+ require.Equal(t, int64(7), id)+ require.Equal(t, sum, gotSum)+ require.Equal(t, int64(4), upload.Bytes)+ require.Equal(t, readAt.Add(55*time.Minute), upload.Deadline)+ require.Equal(t, []string{upload.Key}, blobs.puts)+ require.WithinDuration(t, upload.Deadline, blobs.deadline, 0, "the Put runs under the write deadline")++ // Every upload gets its own key, also for the same bytes.+ again, err := core.UploadPack(context.Background(), 7, readAt, sum, 4, strings.NewReader("pack"))+ require.NoError(t, err)+ require.NotEqual(t, upload.Key, again.Key)++ ctx, cancel, err := upload.CommitContext(context.Background())+ require.NoError(t, err)+ defer cancel()+ deadline, ok := ctx.Deadline()+ require.True(t, ok)+ require.WithinDuration(t, upload.Deadline, deadline, 0, "the commit runs under the write deadline")+}++func TestUploadPackPastTheDeadlineStoresNothing(t *testing.T) {+ t.Parallel()++ sum := sha256.Sum256([]byte("late"))++ t.Run("deadline passed before the upload", func(t *testing.T) {+ t.Parallel()+ blobs := &fakeBlobs{}+ core := uploadCore(blobs, time.Hour, 5*time.Minute)++ _, err := core.UploadPack(context.Background(), 1, time.Now().Add(-time.Hour), sum, 4, strings.NewReader("late"))+ require.ErrorIs(t, err, ErrPackDeadline)+ require.Empty(t, blobs.puts, "nothing is uploaded")+ })++ t.Run("deadline passes during the upload", func(t *testing.T) {+ t.Parallel()+ blobs := &fakeBlobs{block: true}+ core := uploadCore(blobs, 100*time.Millisecond, 50*time.Millisecond)++ _, err := core.UploadPack(context.Background(), 1, time.Now(), sum, 4, strings.NewReader("late"))+ require.ErrorIs(t, err, ErrPackDeadline, "the Put is cancelled at the deadline")+ })++ t.Run("caller cancels", func(t *testing.T) {+ t.Parallel()+ blobs := &fakeBlobs{block: true}+ core := uploadCore(blobs, time.Hour, 5*time.Minute)+ ctx, cancel := context.WithCancel(context.Background())+ cancel()++ _, err := core.UploadPack(ctx, 1, time.Now(), sum, 4, strings.NewReader("late"))+ require.ErrorIs(t, err, context.Canceled)+ require.NotErrorIs(t, err, ErrPackDeadline, "the caller's cancellation is not a deadline")+ })+}++func TestUploadPackRequiresTheGCTimes(t *testing.T) {+ t.Parallel()++ sum := sha256.Sum256([]byte("x"))+ for _, core := range []*Core{+ uploadCore(&fakeBlobs{}, 0, 0),+ uploadCore(&fakeBlobs{}, time.Hour, 0),+ uploadCore(&fakeBlobs{}, time.Minute, time.Hour),+ } {+ _, err := core.UploadPack(context.Background(), 1, time.Now(), sum, 1, strings.NewReader("x"))+ require.ErrorContains(t, err, "invalid write deadline")+ }+}++func TestCommitContext(t *testing.T) {+ t.Parallel()++ expired := PackUpload{Key: "repos/1/packs/x.pack", Deadline: time.Now().Add(-time.Second)}+ _, _, err := expired.CommitContext(context.Background())+ require.True(t, errors.Is(err, ErrPackDeadline), "a commit past the deadline is not started")++ // A transaction without new objects uploaded nothing and has no deadline.+ ctx, cancel, err := PackUpload{}.CommitContext(context.Background())+ require.NoError(t, err)+ defer cancel()+ _, ok := ctx.Deadline()+ require.False(t, ok)+}diff --git a/internal/graft/corestore/voter/deadline_test.go b/internal/graft/corestore/voter/deadline_test.gonew file mode 100644index 0000000..36ff8ea--- /dev/null+++ b/internal/graft/corestore/voter/deadline_test.go@@ -0,0 +1,143 @@+package voter_test++import (+ "context"+ "fmt"+ "io"+ "testing"+ "time"++ "github.com/stretchr/testify/require"+ "google.golang.org/grpc/codes"+ "google.golang.org/grpc/status"++ "github.com/feder-cr/git-ai/internal/graft/corestore"+ "github.com/feder-cr/git-ai/internal/graft/corestore/sqlstore"+ "github.com/feder-cr/git-ai/internal/graft/corestore/voter"+)++// setWriteDeadline gives the fixture's core the write deadline w+// (doc/graft/PLAN7.md, PACK OWNERSHIP RULE 4: W = gc_grace - gc_margin).+func (f *fixture) setWriteDeadline(w time.Duration) {+ f.core.GCMargin = w+ f.core.GCGrace = 2 * w+}++// blockingBlobs is a bucket slower than any write deadline: every Put waits for+// the end of its context and stores nothing.+type blockingBlobs struct {+ corestore.BlobStore+}++func (blockingBlobs) Put(ctx context.Context, _ string, _ io.Reader) error {+ <-ctx.Done()+ return ctx.Err()+}++// packKeys lists the objects of the repository in the bucket.+func (f *fixture) packKeys(t *testing.T, blobs corestore.BlobStore) []string {+ t.Helper()++ var keys []string+ require.NoError(t, blobs.List(f.ctx, fmt.Sprintf("repos/%d/", f.repoID), func(info corestore.BlobInfo) error {+ keys = append(keys, info.Key)+ return nil+ }))+ return keys+}++// TestPreparedCommitsOnlyBeforeTheWriteDeadline checks that a changeset whose+// pack was uploaded is not committed once the pack's write deadline passed: the+// write fails like a storage failure and no row references the pack, which+// garbage collection may already treat as an orphan.+func TestPreparedCommitsOnlyBeforeTheWriteDeadline(t *testing.T) {+ const ref = "refs/heads/main"++ forEachDialect(t, func(t *testing.T, dialect sqlstore.Dialect) {+ for _, mode := range []corestore.Mode{corestore.ModeAuthoritative, corestore.ModeShadow} {+ t.Run(string(mode), func(t *testing.T) {+ f := newFixture(t, fixtureOptions{mode: mode, dialect: dialect})+ f.setWriteDeadline(time.Second)+ blobs := f.countPuts()+ before := f.headSeq(t)++ // The work between the upload and the commit outlasts the deadline,+ // which started before the upload.+ voter.FaultHook = func(point string) error {+ if point == "after_upload" {+ <-time.After(f.core.WriteDeadline())+ }+ return nil+ }+ t.Cleanup(func() { voter.FaultHook = nil })++ err := f.prepared(line(f.zero, f.c.String(), ref))+ if mode == corestore.ModeAuthoritative {+ require.Equal(t, codes.Unavailable, status.Code(err), "error %v", err)+ require.ErrorIs(t, err, corestore.ErrPackDeadline)+ } else {+ require.NoError(t, err, "shadow mode never fails the write")+ require.Equal(t, float64(1), metricValue(t, f.voter, "graft_storage_shadow_errors_total", nil))+ }++ require.Equal(t, 1, blobs.count(), "the pack was uploaded")+ require.Equal(t, before, f.headSeq(t), "no changeset references it")+ })+ }+ })+}++// TestPreparedUploadsAgainWhenThePreparedPackExpired checks that the pack of+// the Preparing phase is not committed when its write deadline passed before+// the Prepared phase (the transaction waited long for the reference lock): the+// Prepared phase uploads a new pack under a new key and commits that one.+func TestPreparedUploadsAgainWhenThePreparedPackExpired(t *testing.T) {+ const ref = "refs/heads/main"++ forEachDialect(t, func(t *testing.T, dialect sqlstore.Dialect) {+ f := newFixture(t, fixtureOptions{mode: corestore.ModeAuthoritative, dialect: dialect})+ f.setWriteDeadline(time.Second)+ blobs := f.countPuts()++ input := line(f.zero, f.c.String(), ref)+ f.preparing(input)+ require.Equal(t, 1, blobs.count())+ require.Equal(t, 1, f.voter.PreparedPacks())++ <-time.After(f.core.WriteDeadline())+ // A long deadline again, so that only the expiry of the prepared pack is+ // under test.+ f.setWriteDeadline(time.Hour)++ require.NoError(t, f.prepared(input))+ require.Equal(t, 2, blobs.count(), "an expired pack is never committed")+ require.NotEqual(t, blobs.puts[0], blobs.puts[1])+ require.Equal(t, blobs.puts[1], f.lastChangeset(t).PackKey)+ require.Equal(t, corestore.Target(f.c.String()), f.dbRef(t, ref))+ })+}++// TestUploadIsCancelledAtTheWriteDeadline checks that an upload slower than+// the write deadline is cancelled, stores nothing and fails the write; no+// changeset is committed.+func TestUploadIsCancelledAtTheWriteDeadline(t *testing.T) {+ const ref = "refs/heads/main"++ forEachDialect(t, func(t *testing.T, dialect sqlstore.Dialect) {+ f := newFixture(t, fixtureOptions{mode: corestore.ModeAuthoritative, dialect: dialect})+ f.setWriteDeadline(100 * time.Millisecond)+ inner := f.core.Blobs+ keysBefore := f.packKeys(t, inner)+ f.core.Blobs = blockingBlobs{BlobStore: inner}+ before := f.headSeq(t)++ input := line(f.zero, f.c.String(), ref)+ f.preparing(input)+ err := f.prepared(input)+ require.Equal(t, codes.Unavailable, status.Code(err), "error %v", err)+ require.ErrorIs(t, err, corestore.ErrPackDeadline)++ require.Equal(t, before, f.headSeq(t), "no changeset was committed")+ require.Equal(t, keysBefore, f.packKeys(t, inner), "nothing was stored")+ })+}diff --git a/internal/graft/corestore/voter/voter.go b/internal/graft/corestore/voter/voter.goindex 059e0e0..4b1e3f9 100644--- a/internal/graft/corestore/voter/voter.go+++ b/internal/graft/corestore/voter/voter.go@@ -107,25 +107,28 @@ type committedEntry struct {seq uint64}-// uploadedPack is a changeset pack in the bucket. An empty key means that the-// transaction brings no new objects.-type uploadedPack struct {- key string- bytes int64-}-// preparedPack is the outcome of the Preparing phase of a transaction.type preparedPack struct {// ok tells that pack was built and uploaded for the repository repoID.// When it is false, Preparing failed and Prepared does the work itself.ok boolrepoID int64- pack uploadedPack+ // pack is the uploaded pack, with an empty key when the transaction brings+ // no new objects.+ pack corestore.PackUpload// spent is the time Preparing took. In shadow mode it counts against the// shadow timeout, which is the budget of the whole transaction.spent time.Duration}+// usable reports whether Prepared may commit the pack of the Preparing phase:+// its write deadline has not passed (doc/graft/PLAN7.md, PACK OWNERSHIP RULE+// 4). A pack whose deadline passed may be an orphan that garbage collection+// deletes, so Prepared uploads a new one instead.+func (p preparedPack) usable(now time.Time) bool {+ return p.pack.Key == "" || now.Before(p.pack.Deadline)+}+// New returns the voter of core. applier is told about local commits (it may// be nil in shadow mode).func New(core *corestore.Core, logger log.Logger, repoFactory localrepo.Factory, locator storage.Locator, applier corestore.LocalApplier) *Voter {@@ -237,6 +240,9 @@ func (v *Voter) preparePack(ctx context.Context, payload gitcmd.HooksPayload, chreturn nil, nil}+ // The write deadline of the pack starts before this read of the record+ // (doc/graft/PLAN7.md, PACK OWNERSHIP RULE 4).+ readAt := time.Now()record, err := v.core.Meta.GetRepository(ctx, corestore.KeyOf(payload.Repo))if errors.Is(err, corestore.ErrRepositoryNotFound) {return nil, nil@@ -248,7 +254,7 @@ func (v *Voter) preparePack(ctx context.Context, payload gitcmd.HooksPayload, chreturn nil, nil}- pack, err := v.buildAndUpload(ctx, repo, payload.Repo.GetStorageName(), record.ID, updates)+ pack, err := v.buildAndUpload(ctx, repo, payload.Repo.GetStorageName(), record.ID, readAt, updates)if err != nil {return nil, err}@@ -392,6 +398,9 @@ func (v *Voter) prepare(ctx context.Context, payload gitcmd.HooksPayload, changereturn nil}+ // The write deadline of a pack uploaded here starts before this read of the+ // record (doc/graft/PLAN7.md, PACK OWNERSHIP RULE 4).+ readAt := time.Now()record, err := v.core.Meta.GetRepository(ctx, corestore.KeyOf(payload.Repo))if errors.Is(err, corestore.ErrRepositoryNotFound) {if shadow {@@ -431,14 +440,14 @@ func (v *Voter) prepare(ctx context.Context, payload gitcmd.HooksPayload, change}// The pack prepared before git took the locks is used when it was uploaded- // for this repository (not for a removed one that had the same path).- // Otherwise it is built and uploaded now, as before the LOCK AND HISTORY- // RULE.- var pack uploadedPack- if prepared != nil && prepared.ok && prepared.repoID == record.ID {+ // for this repository (not for a removed one that had the same path) and its+ // write deadline has not passed. Otherwise it is built and uploaded now, as+ // before the LOCK AND HISTORY RULE.+ var pack corestore.PackUpload+ if prepared != nil && prepared.ok && prepared.repoID == record.ID && prepared.usable(time.Now()) {pack = prepared.pack} else {- pack, err = v.buildAndUpload(ctx, repo, payload.Repo.GetStorageName(), record.ID, updates)+ pack, err = v.buildAndUpload(ctx, repo, payload.Repo.GetStorageName(), record.ID, readAt, updates)if err != nil {return v.unavailable(err)}@@ -451,8 +460,8 @@ func (v *Voter) prepare(ctx context.Context, payload gitcmd.HooksPayload, changeActor: meta.Actor,NodeID: v.core.NodeID,Force: shadow,- PackKey: pack.key,- PackBytes: pack.bytes,+ PackKey: pack.Key,+ PackBytes: pack.Bytes,}if err := v.fault(faultAfterUpload); err != nil {@@ -464,7 +473,15 @@ func (v *Voter) prepare(ctx context.Context, payload gitcmd.HooksPayload, changein.Changelog = &metaCopy}- res, err := v.core.Meta.CommitChangeset(ctx, in)+ // The changeset that references the pack commits before the pack's write+ // deadline, or not at all.+ commitCtx, cancel, err := pack.CommitContext(ctx)+ if err != nil {+ return v.unavailable(err)+ }+ defer cancel()++ res, err := v.core.Meta.CommitChangeset(commitCtx, in)var conflict *corestore.ErrRefConflictif errors.As(err, &conflict) {@@ -631,27 +648,29 @@ func newTips(updates []inputUpdate) []git.ObjectID {}// buildAndUpload builds the pack of the objects reachable from the new tips of-// updates and not from what is already in the bucket, and uploads it under the-// pack key of repoID. An empty pack is not uploaded.+// updates and not from what is already in the bucket, and uploads it under a+// new owned key of repoID with the write deadline that starts at readAt, the+// time taken before the read of the repository record (Core.UploadPack). An+// empty pack is not uploaded.//// What is already in the bucket: in authoritative mode, everything any current// local reference reaches (invariant I1, gitops.BuildPack); in shadow mode,// everything that the values the database records reach (buildPackAgainst),// because local references can hold values the storage core never recorded.-func (v *Voter) buildAndUpload(ctx context.Context, repo *localrepo.Repo, storageName string, repoID int64, updates []inputUpdate) (uploadedPack, error) {+func (v *Voter) buildAndUpload(ctx context.Context, repo *localrepo.Repo, storageName string, repoID int64, readAt time.Time, updates []inputUpdate) (corestore.PackUpload, error) {tips := newTips(updates)if len(tips) == 0 {- return uploadedPack{}, nil+ return corestore.PackUpload{}, nil}tmpDir, err := v.locator.TempDir(storageName)if err != nil {- return uploadedPack{}, err+ return corestore.PackUpload{}, err}// The server creates the temporary directory when it starts, but it may have// been cleaned up since.if err := os.MkdirAll(tmpDir, mode.Directory); err != nil {- return uploadedPack{}, fmt.Errorf("creating the temporary directory: %w", err)+ return corestore.PackUpload{}, fmt.Errorf("creating the temporary directory: %w", err)}var pack gitops.Pack@@ -661,37 +680,32 @@ func (v *Voter) buildAndUpload(ctx context.Context, repo *localrepo.Repo, storagvar known map[string]corestore.Targetknown, _, err = v.core.Meta.Refs(ctx, repoID)if err != nil {- return uploadedPack{}, fmt.Errorf("reading the database references: %w", err)+ return corestore.PackUpload{}, fmt.Errorf("reading the database references: %w", err)}pack, err = buildPackAgainst(ctx, repo, tips, known, tmpDir)}if err != nil {- return uploadedPack{}, err+ return corestore.PackUpload{}, err}defer func() { _ = pack.Remove() }()if pack.Empty() {- return uploadedPack{}, nil- }-- key := corestore.NewPackKey(repoID, pack.SHA256)- if err := v.uploadPack(ctx, key, pack); err != nil {- return uploadedPack{}, err+ return corestore.PackUpload{}, nil}- v.metrics.packBytes.Add(float64(pack.Size))- return uploadedPack{key: key, bytes: pack.Size}, nil-}--// uploadPack stores the pack under key, see BlobStore.Put.-func (v *Voter) uploadPack(ctx context.Context, key string, pack gitops.Pack) error {r, err := pack.Open()if err != nil {- return err+ return corestore.PackUpload{}, err}defer func() { _ = r.Close() }()- return v.core.Blobs.Put(ctx, key, r)+ upload, err := v.core.UploadPack(ctx, repoID, readAt, pack.SHA256, pack.Size, r)+ if err != nil {+ return corestore.PackUpload{}, err+ }+ v.metrics.packBytes.Add(float64(pack.Size))++ return upload, nil}// fault calls FaultHook and exits the process when CrashEnv selects the point.