Skip to content

Git designed for AI agents.

corestore: one upload path with an enforced write deadline (PLAN7 WP73)

Federico Eliafd111e64e11b
diff --git a/internal/gitaly/config/config_graft_storage.go b/internal/gitaly/config/config_graft_storage.go
index 2d03f42..8e69ebb 100644
--- a/internal/gitaly/config/config_graft_storage.go
+++ b/internal/gitaly/config/config_graft_storage.go
@@ -31,6 +31,15 @@ type GraftStorage struct {
// ShadowTimeout bounds the shadow-mode work of one reference transaction
// (Go duration). Empty means DefaultGraftStorageShadowTimeout.
ShadowTimeout string `json:"shadow_timeout" toml:"shadow_timeout,omitempty"`
+ // GCGrace is how long garbage collection waits before it deletes a pack it
+ // observed unreferenced, and how old a checkpoint must be before the packs it
+ // covers are deleted (Go duration). Empty means DefaultGraftStorageGCGrace.
+ GCGrace string `json:"gc_grace" toml:"gc_grace,omitempty"`
+ // GCMargin is the safety margin of the writers' deadline: a writer uploads a
+ // pack and commits its changeset within gc_grace - gc_margin (Go duration).
+ // Empty means DefaultGraftStorageGCMargin. See doc/graft/STORAGE.md for the
+ // clock assumption it covers.
+ GCMargin string `json:"gc_margin" toml:"gc_margin,omitempty"`
}
const (
@@ -38,6 +47,10 @@ const (
DefaultGraftStorageSyncTimeout = 10 * time.Second
// DefaultGraftStorageShadowTimeout is the default GraftStorage.ShadowTimeout.
DefaultGraftStorageShadowTimeout = 5 * time.Second
+ // DefaultGraftStorageGCGrace is the default GraftStorage.GCGrace.
+ DefaultGraftStorageGCGrace = time.Hour
+ // DefaultGraftStorageGCMargin is the default GraftStorage.GCMargin.
+ DefaultGraftStorageGCMargin = 5 * time.Minute
)
// Enabled reports whether the storage core is on.
@@ -53,6 +66,40 @@ func (s GraftStorage) ShadowTimeoutOrDefault() (time.Duration, error) {
return positiveDurationOrDefault("shadow_timeout", s.ShadowTimeout, DefaultGraftStorageShadowTimeout)
}
+// GCGraceOrDefault parses GCGrace, applying the default.
+func (s GraftStorage) GCGraceOrDefault() (time.Duration, error) {
+ return positiveDurationOrDefault("gc_grace", s.GCGrace, DefaultGraftStorageGCGrace)
+}
+
+// GCMarginOrDefault parses GCMargin, applying the default.
+func (s GraftStorage) GCMarginOrDefault() (time.Duration, error) {
+ return positiveDurationOrDefault("gc_margin", s.GCMargin, DefaultGraftStorageGCMargin)
+}
+
+// validateGC checks that the garbage collection times leave the writers a
+// positive deadline, and that a sync can never meet a pack the collector
+// deletes during it: gc_grace > sync_timeout + gc_margin
+// (doc/graft/PLAN7.md, GC BOUNDARY rule).
+func (s GraftStorage) validateGC() error {
+ grace, err := s.GCGraceOrDefault()
+ if err != nil {
+ return err
+ }
+ margin, err := s.GCMarginOrDefault()
+ if err != nil {
+ return err
+ }
+ sync, err := s.SyncTimeoutOrDefault()
+ if err != nil {
+ return err
+ }
+
+ if grace <= sync+margin {
+ return fmt.Errorf("graft storage: gc_grace %s must be greater than sync_timeout %s + gc_margin %s", grace, sync, margin)
+ }
+ return nil
+}
+
func positiveDurationOrDefault(name, value string, def time.Duration) (time.Duration, error) {
if value == "" {
return def, nil
@@ -107,5 +154,5 @@ func (s GraftStorage) validate(standalone bool) error {
return err
}
- return nil
+ return s.validateGC()
}
diff --git a/internal/gitaly/config/config_graft_storage_test.go b/internal/gitaly/config/config_graft_storage_test.go
index 964e9b1..04a7b8f 100644
--- a/internal/gitaly/config/config_graft_storage_test.go
+++ b/internal/gitaly/config/config_graft_storage_test.go
@@ -34,6 +34,11 @@ func TestGraftStorageValidate(t *testing.T) {
{desc: "node id with slash", storage: GraftStorage{Mode: "shadow", Database: valid.Database, Bucket: valid.Bucket, NodeID: "a/b"}, err: "node_id"},
{desc: "bad sync timeout", storage: GraftStorage{Mode: "shadow", Database: valid.Database, Bucket: valid.Bucket, SyncTimeout: "0s"}, err: "sync_timeout"},
{desc: "bad shadow timeout", storage: GraftStorage{Mode: "shadow", Database: valid.Database, Bucket: valid.Bucket, ShadowTimeout: "soon"}, err: "shadow_timeout"},
+ {desc: "gc times", storage: GraftStorage{Mode: "shadow", Database: valid.Database, Bucket: valid.Bucket, GCGrace: "30m", GCMargin: "1m"}},
+ {desc: "bad gc_grace", storage: GraftStorage{Mode: "shadow", Database: valid.Database, Bucket: valid.Bucket, GCGrace: "-1h"}, err: "gc_grace"},
+ {desc: "bad gc_margin", storage: GraftStorage{Mode: "shadow", Database: valid.Database, Bucket: valid.Bucket, GCMargin: "0"}, err: "gc_margin"},
+ {desc: "gc_grace not above sync_timeout + gc_margin", storage: GraftStorage{Mode: "shadow", Database: valid.Database, Bucket: valid.Bucket, GCGrace: "6m", GCMargin: "5m", SyncTimeout: "1m"}, err: "gc_grace 6m0s must be greater than sync_timeout 1m0s + gc_margin 5m0s"},
+ {desc: "gc_grace just above sync_timeout + gc_margin", storage: GraftStorage{Mode: "shadow", Database: valid.Database, Bucket: valid.Bucket, GCGrace: "361s", GCMargin: "5m", SyncTimeout: "1m"}},
} {
t.Run(tc.desc, func(t *testing.T) {
t.Parallel()
@@ -62,6 +67,15 @@ func TestGraftStorageDefaults(t *testing.T) {
require.NoError(t, err)
require.Equal(t, DefaultGraftStorageShadowTimeout, shadow)
+ grace, err := s.GCGraceOrDefault()
+ require.NoError(t, err)
+ require.Equal(t, DefaultGraftStorageGCGrace, grace)
+
+ margin, err := s.GCMarginOrDefault()
+ require.NoError(t, err)
+ require.Equal(t, DefaultGraftStorageGCMargin, margin)
+ require.NoError(t, s.validateGC(), "the defaults are consistent")
+
s = GraftStorage{Mode: "shadow", SyncTimeout: "250ms"}
require.True(t, s.Enabled())
sync, err = s.SyncTimeoutOrDefault()
diff --git a/internal/gitaly/service/storagecore/import.go b/internal/gitaly/service/storagecore/import.go
index b36e087..592174e 100644
--- a/internal/gitaly/service/storagecore/import.go
+++ b/internal/gitaly/service/storagecore/import.go
@@ -6,6 +6,7 @@ import (
"maps"
"os"
"sync"
+ "time"
"github.com/feder-cr/git-ai/internal/git"
"github.com/feder-cr/git-ai/internal/gitaly/storage/mode"
@@ -146,14 +147,28 @@ func (s *server) ImportRepository(ctx context.Context, req *gitalypb.ImportRepos
}
}()
- var packKey string
- var packBytes int64
+ // The write deadline of the pack starts at a read of the record taken right
+ // before the upload, so that the build above, however long, does not consume
+ // it (doc/graft/PLAN7.md, PACK OWNERSHIP RULE 4).
+ var upload corestore.PackUpload
if !pack.Empty() {
- packKey = corestore.NewPackKey(reg.repo.ID, pack.SHA256)
- if err := uploadPack(ctx, core, packKey, pack); err != nil {
+ readAt := time.Now()
+ if _, err := core.Meta.GetRepository(ctx, key); err != nil {
+ if errors.Is(err, corestore.ErrRepositoryNotFound) {
+ return nil, errRegistrationRemoved()
+ }
+ return nil, structerr.NewUnavailable("storage core: %w; %s", err, resumeHint)
+ }
+
+ f, err := pack.Open()
+ if err != nil {
+ return nil, structerr.NewInternal("open pack: %w; %s", err, resumeHint)
+ }
+ upload, err = core.UploadPack(ctx, reg.repo.ID, readAt, pack.SHA256, pack.Size, f)
+ _ = f.Close()
+ if err != nil {
return nil, structerr.NewUnavailable("storage core: upload pack: %w; %s", err, resumeHint)
}
- packBytes = pack.Size
}
// Step 3.
@@ -166,14 +181,20 @@ func (s *server) ImportRepository(ctx context.Context, req *gitalypb.ImportRepos
New: refs[name],
})
}
- res, err := core.Meta.CommitChangeset(ctx, corestore.ChangesetInput{
+ commitCtx, cancel, err := upload.CommitContext(ctx)
+ if err != nil {
+ return nil, structerr.NewUnavailable("%w; %s", err, resumeHint)
+ }
+ defer cancel()
+
+ res, err := core.Meta.CommitChangeset(commitCtx, corestore.ChangesetInput{
RepoID: reg.repo.ID,
Updates: updates,
Operation: importOperation,
Actor: identity.Actor(ctx, importOperation),
NodeID: core.NodeID,
- PackKey: packKey,
- PackBytes: packBytes,
+ PackKey: upload.Key,
+ PackBytes: upload.Bytes,
})
if err != nil {
var conflict *corestore.ErrRefConflict
@@ -184,9 +205,7 @@ func (s *server) ImportRepository(ctx context.Context, req *gitalypb.ImportRepos
// can have recorded references.
return nil, errAlreadyImported()
case errors.Is(err, corestore.ErrRepositoryNotFound):
- // Not Aborted: Gitaly replaces the message of that code with a generic
- // one, and this one tells the operator what to do.
- return nil, structerr.NewFailedPrecondition("storage core: the registration of the import was removed concurrently; run ImportRepository again")
+ return nil, errRegistrationRemoved()
default:
return nil, structerr.NewUnavailable("storage core: commit import changeset: %w; %s", err, resumeHint)
}
@@ -214,7 +233,7 @@ func (s *server) ImportRepository(ctx context.Context, req *gitalypb.ImportRepos
return &gitalypb.ImportRepositoryResponse{
Sequence: res.Seq,
References: uint32(len(refs)),
- PackBytes: packBytes,
+ PackBytes: upload.Bytes,
ChangelogEntries: uint32(reg.changelogEntries),
Reviews: uint32(reg.reviews),
}, nil
@@ -385,14 +404,11 @@ func acquireImport(core *corestore.Core, key corestore.RepoKey) (release func(),
}, true
}
-// uploadPack stores the pack under key.
-func uploadPack(ctx context.Context, core *corestore.Core, key string, pack gitops.Pack) error {
- f, err := pack.Open()
- if err != nil {
- return err
- }
- defer func() { _ = f.Close() }()
- return core.Blobs.Put(ctx, key, f)
+// errRegistrationRemoved is the error of an import whose registration was
+// removed while it ran. Not Aborted: Gitaly replaces the message of that code
+// with a generic one, and this one tells the operator what to do.
+func errRegistrationRemoved() error {
+ return structerr.NewFailedPrecondition("storage core: the registration of the import was removed concurrently; run ImportRepository again")
}
// scanChangelog returns every entry of the file changelog of the repository,
diff --git a/internal/gitaly/service/storagecore/import_deadline_test.go b/internal/gitaly/service/storagecore/import_deadline_test.go
new file mode 100644
index 0000000..3a37ebe
--- /dev/null
+++ b/internal/gitaly/service/storagecore/import_deadline_test.go
@@ -0,0 +1,141 @@
+package storagecore
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+ "google.golang.org/grpc/codes"
+
+ "github.com/feder-cr/git-ai/internal/graft/corestore"
+ "github.com/feder-cr/git-ai/internal/graft/corestore/sqlstore"
+ "github.com/feder-cr/git-ai/proto/go/gitalypb"
+)
+
+// The tests of this file check the write deadline of the import's pack
+// (doc/graft/PLAN7.md, PACK OWNERSHIP RULE 4): an upload, or the commit of the
+// changeset that references it, that would end after the deadline fails like
+// a storage failure, no changeset references the pack, and the import is
+// resumable as any failed import (IMPORT RULE).
+
+const (
+ slowNone int32 = iota
+ slowBlock // Put waits for the end of its context and stores nothing
+ slowLate // Put stores the object, then returns only after the deadline
+)
+
+// slowBlobs is a bucket that is slower than the write deadline while its mode
+// is not slowNone.
+type slowBlobs struct {
+ corestore.BlobStore
+ mode *atomic.Int32
+}
+
+func (b slowBlobs) Put(ctx context.Context, key string, r io.Reader) error {
+ switch b.mode.Load() {
+ case slowBlock:
+ <-ctx.Done()
+ return ctx.Err()
+ case slowLate:
+ if err := b.BlobStore.Put(ctx, key, r); err != nil {
+ return err
+ }
+ <-ctx.Done()
+ return nil
+ default:
+ return b.BlobStore.Put(ctx, key, r)
+ }
+}
+
+// testWriteDeadline is the write deadline of these tests: long enough for a
+// normal upload and commit of the test repository, short enough to wait out.
+// The core is never changed while the server runs: the RPCs order the test and
+// the server only through a socket, which the race detector does not see.
+const testWriteDeadline = 2 * time.Second
+
+// setupSlow is setup with a slowBlobs bucket and a write deadline of w.
+func setupSlow(t *testing.T, mode corestore.Mode, dialect sqlstore.Dialect, w time.Duration) (*harness, *atomic.Int32) {
+ t.Helper()
+
+ slow := &atomic.Int32{}
+ h := setup(t, mode, dialect, func(core *corestore.Core) {
+ core.Blobs = slowBlobs{BlobStore: core.Blobs, mode: slow}
+ core.GCMargin = w
+ core.GCGrace = 2 * w
+ })
+ return h, slow
+}
+
+// bucketKeys lists the objects of the repository in the bucket.
+func (h *harness) bucketKeys(repoID int64) []string {
+ h.t.Helper()
+
+ var keys []string
+ require.NoError(h.t, h.core.Blobs.List(h.ctx, fmt.Sprintf("repos/%d/", repoID), func(info corestore.BlobInfo) error {
+ keys = append(keys, info.Key)
+ return nil
+ }))
+ return keys
+}
+
+func TestImportRepositoryUploadPastTheWriteDeadline(t *testing.T) {
+ forEachCore(t, func(t *testing.T, mode corestore.Mode, dialect sqlstore.Dialect) {
+ h, slow := setupSlow(t, mode, dialect, testWriteDeadline)
+ h.seed()
+ localBefore := h.localRefs()
+
+ slow.Store(slowBlock)
+ _, err := h.client.ImportRepository(h.ctx, &gitalypb.ImportRepositoryRequest{Repository: h.repo})
+ requireResumeHint(t, err, codes.Unavailable)
+ require.ErrorContains(t, err, corestore.ErrPackDeadline.Error())
+
+ state, err := h.client.GetStorageState(h.ctx, &gitalypb.GetStorageStateRequest{Repository: h.repo})
+ require.NoError(t, err)
+ repoID := state.GetRepositoryId()
+ h.requireIncomplete(repoID)
+ require.Empty(t, h.bucketKeys(repoID), "the cancelled upload stored nothing")
+
+ slow.Store(slowNone)
+ _, err = h.client.ImportRepository(h.ctx, &gitalypb.ImportRepositoryRequest{Repository: h.repo})
+ require.NoError(t, err)
+ h.requireImported(repoID, localBefore)
+ })
+}
+
+func TestImportRepositoryCommitPastTheWriteDeadline(t *testing.T) {
+ forEachCore(t, func(t *testing.T, mode corestore.Mode, dialect sqlstore.Dialect) {
+ h, slow := setupSlow(t, mode, dialect, testWriteDeadline)
+ h.seed()
+ localBefore := h.localRefs()
+
+ // The upload completes, but only returns after the deadline: the changeset
+ // that would reference the pack is not committed.
+ slow.Store(slowLate)
+ _, err := h.client.ImportRepository(h.ctx, &gitalypb.ImportRepositoryRequest{Repository: h.repo})
+ requireResumeHint(t, err, codes.Unavailable)
+ require.ErrorContains(t, err, corestore.ErrPackDeadline.Error())
+
+ state, err := h.client.GetStorageState(h.ctx, &gitalypb.GetStorageStateRequest{Repository: h.repo})
+ require.NoError(t, err)
+ repoID := state.GetRepositoryId()
+ h.requireIncomplete(repoID)
+ orphans := h.bucketKeys(repoID)
+ require.Len(t, orphans, 1, "the pack was stored")
+
+ // The resumed import uploads its own pack; the late one stays an orphan
+ // that no changeset references.
+ slow.Store(slowNone)
+ _, err = h.client.ImportRepository(h.ctx, &gitalypb.ImportRepositoryRequest{Repository: h.repo})
+ require.NoError(t, err)
+ h.requireImported(repoID, localBefore)
+
+ changesets, err := h.core.Meta.ListChangesets(h.ctx, repoID, 0, 1)
+ require.NoError(t, err)
+ require.NotEqual(t, orphans[0], changesets[0].PackKey)
+ require.ElementsMatch(t, []string{orphans[0], changesets[0].PackKey}, h.bucketKeys(repoID))
+ })
+}
diff --git a/internal/graft/corestore/core.go b/internal/graft/corestore/core.go
index 7e36f21..8920cf3 100644
--- a/internal/graft/corestore/core.go
+++ b/internal/graft/corestore/core.go
@@ -2,6 +2,8 @@ package corestore
import (
"context"
+ "errors"
+ "fmt"
"io"
"path"
"strings"
@@ -88,6 +90,78 @@ type Core struct {
// one reference transaction.
SyncTimeout time.Duration
ShadowTimeout time.Duration
+ // GCGrace is how long garbage collection waits before it deletes a pack it
+ // observed unreferenced, and how old a checkpoint must be before the packs it
+ // covers are deleted. GCMargin is the safety margin of the writers' deadline,
+ // WriteDeadline() = GCGrace - GCMargin (doc/graft/PLAN7.md, PACK OWNERSHIP
+ // RULE 4). Both are set whenever the storage core is on: garbage collection
+ // runs in shadow mode too.
+ GCGrace time.Duration
+ GCMargin time.Duration
+}
+
+// WriteDeadline is W of doc/graft/PLAN7.md: a writer uploads a pack and
+// commits the row that references it within W of a read of the repository
+// record that found the repository.
+func (c *Core) WriteDeadline() time.Duration { return c.GCGrace - c.GCMargin }
+
+// PackUpload is a pack uploaded under a new owned key (NewPackKey) and the
+// deadline before which a row may reference it.
+type PackUpload struct {
+ // Key is the owned key, empty when there was nothing to upload (the
+ // transaction brings no new objects).
+ Key string
+ Bytes int64
+ // Deadline is the time of the repository-record read that preceded the
+ // upload plus WriteDeadline(). It keeps the monotonic clock reading of that
+ // read, so that it is compared on the monotonic clock.
+ Deadline time.Time
+}
+
+// UploadPack uploads the pack read from r (size bytes whose SHA-256 is sum)
+// under a new owned key of the repository. readAt is the time of a read of the
+// repository record that found the repository, taken before that read and
+// before the upload; the upload must end before readAt + WriteDeadline(), else
+// it is cancelled (nothing is stored) and the error wraps ErrPackDeadline. It is
+// the only way the storage core uploads a pack (doc/graft/PLAN7.md, RULE 26).
+func (c *Core) UploadPack(ctx context.Context, repoID int64, readAt time.Time, sum [32]byte, size int64, r io.Reader) (PackUpload, error) {
+ if c.GCGrace <= 0 || c.GCMargin <= 0 || c.WriteDeadline() <= 0 {
+ return PackUpload{}, fmt.Errorf("storage core: invalid write deadline: gc_grace %s, gc_margin %s", c.GCGrace, c.GCMargin)
+ }
+
+ deadline := readAt.Add(c.WriteDeadline())
+ if !time.Now().Before(deadline) {
+ return PackUpload{}, fmt.Errorf("storage core: upload a pack of repository %d: %w", repoID, ErrPackDeadline)
+ }
+
+ putCtx, cancel := context.WithDeadline(ctx, deadline)
+ defer cancel()
+
+ key := NewPackKey(repoID, sum)
+ if err := c.Blobs.Put(putCtx, key, r); err != nil {
+ if ctx.Err() == nil && errors.Is(putCtx.Err(), context.DeadlineExceeded) {
+ return PackUpload{}, fmt.Errorf("storage core: upload %q: %w: %w", key, ErrPackDeadline, err)
+ }
+ return PackUpload{}, fmt.Errorf("storage core: upload %q: %w", key, err)
+ }
+
+ return PackUpload{Key: key, Bytes: size, Deadline: deadline}, nil
+}
+
+// CommitContext returns ctx bounded by the upload's deadline, for the commit of
+// the row that references the pack. When the deadline has passed it returns
+// ErrPackDeadline: the caller must not start the commit and must not reference
+// the key. An upload without a key (nothing uploaded) has no deadline.
+func (u PackUpload) CommitContext(ctx context.Context) (context.Context, context.CancelFunc, error) {
+ if u.Key == "" {
+ ctx, cancel := context.WithCancel(ctx)
+ return ctx, cancel, nil
+ }
+ if !time.Now().Before(u.Deadline) {
+ return nil, nil, fmt.Errorf("storage core: commit the changeset of pack %q: %w", u.Key, ErrPackDeadline)
+ }
+ ctx, cancel := context.WithDeadline(ctx, u.Deadline)
+ return ctx, cancel, nil
}
// Enabled reports whether the storage core is on (shadow or authoritative).
diff --git a/internal/graft/corestore/coretest/coretest.go b/internal/graft/corestore/coretest/coretest.go
index 2e7119b..253f6a6 100644
--- a/internal/graft/corestore/coretest/coretest.go
+++ b/internal/graft/corestore/coretest/coretest.go
@@ -135,6 +135,11 @@ type Options struct {
BucketURL string
// NodeID is the node name; empty means "node-a".
NodeID string
+ // GCGrace and GCMargin are the garbage collection times; zero means the
+ // production defaults (1h and 5m). A test of the writers' deadline sets them
+ // short.
+ GCGrace time.Duration
+ GCMargin time.Duration
}
// Open opens a storage core for the test and closes it when the test ends.
@@ -156,6 +161,12 @@ func Open(tb testing.TB, opts Options) *corestore.Core {
if opts.NodeID == "" {
opts.NodeID = "node-a"
}
+ if opts.GCGrace == 0 {
+ opts.GCGrace = time.Hour
+ }
+ if opts.GCMargin == 0 {
+ opts.GCMargin = 5 * time.Minute
+ }
ctx := context.Background()
@@ -172,6 +183,8 @@ func Open(tb testing.TB, opts Options) *corestore.Core {
Blobs: blobs,
SyncTimeout: 10 * time.Second,
ShadowTimeout: 5 * time.Second,
+ GCGrace: opts.GCGrace,
+ GCMargin: opts.GCMargin,
}
tb.Cleanup(func() { _ = core.Close() })
diff --git a/internal/graft/corestore/coretest/import.go b/internal/graft/corestore/coretest/import.go
index 3939b81..3bfe69e 100644
--- a/internal/graft/corestore/coretest/import.go
+++ b/internal/graft/corestore/coretest/import.go
@@ -2,6 +2,7 @@ package coretest
import (
"testing"
+ "time"
"github.com/feder-cr/git-ai/internal/git/localrepo"
"github.com/feder-cr/git-ai/internal/graft/changelog"
@@ -48,12 +49,18 @@ func Import(tb testing.TB, core *corestore.Core, repo *localrepo.Repo, repoProto
require.NoError(tb, err)
defer func() { _ = pack.Remove() }()
- var packKey string
+ // As ImportRepository: the write deadline starts at a read of the record
+ // right before the upload (doc/graft/PLAN7.md, PACK OWNERSHIP RULE 4).
+ var upload corestore.PackUpload
if !pack.Empty() {
- packKey = corestore.NewPackKey(registered.ID, pack.SHA256)
+ readAt := time.Now()
+ _, err := core.Meta.GetRepository(ctx, corestore.KeyOf(repoProto))
+ require.NoError(tb, err)
+
f, err := pack.Open()
require.NoError(tb, err)
- require.NoError(tb, core.Blobs.Put(ctx, packKey, f))
+ upload, err = core.UploadPack(ctx, registered.ID, readAt, pack.SHA256, pack.Size, f)
+ require.NoError(tb, err)
require.NoError(tb, f.Close())
}
@@ -62,14 +69,18 @@ func Import(tb testing.TB, core *corestore.Core, repo *localrepo.Repo, repoProto
updates = append(updates, corestore.RefUpdate{Name: name, Expect: corestore.ExpectValue, New: target})
}
- res, err := core.Meta.CommitChangeset(ctx, corestore.ChangesetInput{
+ commitCtx, cancel, err := upload.CommitContext(ctx)
+ require.NoError(tb, err)
+ defer cancel()
+
+ res, err := core.Meta.CommitChangeset(commitCtx, corestore.ChangesetInput{
RepoID: registered.ID,
Updates: updates,
Operation: "import",
Actor: "import",
NodeID: core.NodeID,
- PackKey: packKey,
- PackBytes: pack.Size,
+ PackKey: upload.Key,
+ PackBytes: upload.Bytes,
})
require.NoError(tb, err)
require.Equal(tb, uint64(1), res.Seq, "the import is changeset 1")
diff --git a/internal/graft/corestore/open/open.go b/internal/graft/corestore/open/open.go
index 48dd53f..c37e14c 100644
--- a/internal/graft/corestore/open/open.go
+++ b/internal/graft/corestore/open/open.go
@@ -34,6 +34,14 @@ func Open(ctx context.Context, cfg config.GraftStorage) (*corestore.Core, error)
if err != nil {
return nil, err
}
+ gcGrace, err := cfg.GCGraceOrDefault()
+ if err != nil {
+ return nil, err
+ }
+ gcMargin, err := cfg.GCMarginOrDefault()
+ if err != nil {
+ return nil, err
+ }
nodeID := cfg.NodeID
if nodeID == "" {
@@ -60,5 +68,7 @@ func Open(ctx context.Context, cfg config.GraftStorage) (*corestore.Core, error)
Blobs: blobs,
SyncTimeout: syncTimeout,
ShadowTimeout: shadowTimeout,
+ GCGrace: gcGrace,
+ GCMargin: gcMargin,
}, nil
}
diff --git a/internal/graft/corestore/types.go b/internal/graft/corestore/types.go
index f7895c4..1ab1a21 100644
--- a/internal/graft/corestore/types.go
+++ b/internal/graft/corestore/types.go
@@ -232,6 +232,11 @@ var (
// include: the review was created or moved during the build
// (doc/graft/PLAN7.md, REVIEW ROOTS RULE R2).
ErrCheckpointRootsChanged = errors.New("storage core: the open reviews changed during the checkpoint build")
+ // ErrPackDeadline is returned when a pack upload, or the commit of the row
+ // that references it, would end after the write deadline: garbage collection
+ // may then already treat the pack as an orphan (doc/graft/PLAN7.md, PACK
+ // OWNERSHIP RULE 4). The write fails like any storage failure.
+ ErrPackDeadline = errors.New("storage core: the pack upload is older than the write deadline")
// ErrBlobCorrupt is returned by the reader of BlobStore.Get when the content
// does not hash to its key.
ErrBlobCorrupt = errors.New("storage core: blob content does not match its key")
diff --git a/internal/graft/corestore/upload_test.go b/internal/graft/corestore/upload_test.go
new file mode 100644
index 0000000..d9e080f
--- /dev/null
+++ b/internal/graft/corestore/upload_test.go
@@ -0,0 +1,136 @@
+package corestore
+
+import (
+ "context"
+ "crypto/sha256"
+ "errors"
+ "io"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+)
+
+// fakeBlobs records the Puts; when block is set, Put waits for the end of its
+// context instead of storing (a bucket slower than the write deadline).
+type fakeBlobs struct {
+ BlobStore
+ block bool
+ puts []string
+ // deadline is the deadline of the context of the last Put.
+ deadline time.Time
+}
+
+func (f *fakeBlobs) Put(ctx context.Context, key string, r io.Reader) error {
+ f.puts = append(f.puts, key)
+ f.deadline, _ = ctx.Deadline()
+ if f.block {
+ <-ctx.Done()
+ return ctx.Err()
+ }
+ _, err := io.Copy(io.Discard, r)
+ return err
+}
+
+func uploadCore(blobs BlobStore, grace, margin time.Duration) *Core {
+ return &Core{Mode: ModeAuthoritative, Blobs: blobs, GCGrace: grace, GCMargin: margin}
+}
+
+func TestUploadPackUsesAnOwnedKeyAndTheWriteDeadline(t *testing.T) {
+ t.Parallel()
+
+ blobs := &fakeBlobs{}
+ core := uploadCore(blobs, time.Hour, 5*time.Minute)
+ sum := sha256.Sum256([]byte("pack"))
+ readAt := time.Now()
+
+ upload, err := core.UploadPack(context.Background(), 7, readAt, sum, 4, strings.NewReader("pack"))
+ require.NoError(t, err)
+ require.True(t, IsOwnedPackKey(upload.Key))
+ id, gotSum, _ := ParsePackKey(upload.Key)
+ require.Equal(t, int64(7), id)
+ require.Equal(t, sum, gotSum)
+ require.Equal(t, int64(4), upload.Bytes)
+ require.Equal(t, readAt.Add(55*time.Minute), upload.Deadline)
+ require.Equal(t, []string{upload.Key}, blobs.puts)
+ require.WithinDuration(t, upload.Deadline, blobs.deadline, 0, "the Put runs under the write deadline")
+
+ // Every upload gets its own key, also for the same bytes.
+ again, err := core.UploadPack(context.Background(), 7, readAt, sum, 4, strings.NewReader("pack"))
+ require.NoError(t, err)
+ require.NotEqual(t, upload.Key, again.Key)
+
+ ctx, cancel, err := upload.CommitContext(context.Background())
+ require.NoError(t, err)
+ defer cancel()
+ deadline, ok := ctx.Deadline()
+ require.True(t, ok)
+ require.WithinDuration(t, upload.Deadline, deadline, 0, "the commit runs under the write deadline")
+}
+
+func TestUploadPackPastTheDeadlineStoresNothing(t *testing.T) {
+ t.Parallel()
+
+ sum := sha256.Sum256([]byte("late"))
+
+ t.Run("deadline passed before the upload", func(t *testing.T) {
+ t.Parallel()
+ blobs := &fakeBlobs{}
+ core := uploadCore(blobs, time.Hour, 5*time.Minute)
+
+ _, err := core.UploadPack(context.Background(), 1, time.Now().Add(-time.Hour), sum, 4, strings.NewReader("late"))
+ require.ErrorIs(t, err, ErrPackDeadline)
+ require.Empty(t, blobs.puts, "nothing is uploaded")
+ })
+
+ t.Run("deadline passes during the upload", func(t *testing.T) {
+ t.Parallel()
+ blobs := &fakeBlobs{block: true}
+ core := uploadCore(blobs, 100*time.Millisecond, 50*time.Millisecond)
+
+ _, err := core.UploadPack(context.Background(), 1, time.Now(), sum, 4, strings.NewReader("late"))
+ require.ErrorIs(t, err, ErrPackDeadline, "the Put is cancelled at the deadline")
+ })
+
+ t.Run("caller cancels", func(t *testing.T) {
+ t.Parallel()
+ blobs := &fakeBlobs{block: true}
+ core := uploadCore(blobs, time.Hour, 5*time.Minute)
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+
+ _, err := core.UploadPack(ctx, 1, time.Now(), sum, 4, strings.NewReader("late"))
+ require.ErrorIs(t, err, context.Canceled)
+ require.NotErrorIs(t, err, ErrPackDeadline, "the caller's cancellation is not a deadline")
+ })
+}
+
+func TestUploadPackRequiresTheGCTimes(t *testing.T) {
+ t.Parallel()
+
+ sum := sha256.Sum256([]byte("x"))
+ for _, core := range []*Core{
+ uploadCore(&fakeBlobs{}, 0, 0),
+ uploadCore(&fakeBlobs{}, time.Hour, 0),
+ uploadCore(&fakeBlobs{}, time.Minute, time.Hour),
+ } {
+ _, err := core.UploadPack(context.Background(), 1, time.Now(), sum, 1, strings.NewReader("x"))
+ require.ErrorContains(t, err, "invalid write deadline")
+ }
+}
+
+func TestCommitContext(t *testing.T) {
+ t.Parallel()
+
+ expired := PackUpload{Key: "repos/1/packs/x.pack", Deadline: time.Now().Add(-time.Second)}
+ _, _, err := expired.CommitContext(context.Background())
+ require.True(t, errors.Is(err, ErrPackDeadline), "a commit past the deadline is not started")
+
+ // A transaction without new objects uploaded nothing and has no deadline.
+ ctx, cancel, err := PackUpload{}.CommitContext(context.Background())
+ require.NoError(t, err)
+ defer cancel()
+ _, ok := ctx.Deadline()
+ require.False(t, ok)
+}
diff --git a/internal/graft/corestore/voter/deadline_test.go b/internal/graft/corestore/voter/deadline_test.go
new file mode 100644
index 0000000..36ff8ea
--- /dev/null
+++ b/internal/graft/corestore/voter/deadline_test.go
@@ -0,0 +1,143 @@
+package voter_test
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
+
+ "github.com/feder-cr/git-ai/internal/graft/corestore"
+ "github.com/feder-cr/git-ai/internal/graft/corestore/sqlstore"
+ "github.com/feder-cr/git-ai/internal/graft/corestore/voter"
+)
+
+// setWriteDeadline gives the fixture's core the write deadline w
+// (doc/graft/PLAN7.md, PACK OWNERSHIP RULE 4: W = gc_grace - gc_margin).
+func (f *fixture) setWriteDeadline(w time.Duration) {
+ f.core.GCMargin = w
+ f.core.GCGrace = 2 * w
+}
+
+// blockingBlobs is a bucket slower than any write deadline: every Put waits for
+// the end of its context and stores nothing.
+type blockingBlobs struct {
+ corestore.BlobStore
+}
+
+func (blockingBlobs) Put(ctx context.Context, _ string, _ io.Reader) error {
+ <-ctx.Done()
+ return ctx.Err()
+}
+
+// packKeys lists the objects of the repository in the bucket.
+func (f *fixture) packKeys(t *testing.T, blobs corestore.BlobStore) []string {
+ t.Helper()
+
+ var keys []string
+ require.NoError(t, blobs.List(f.ctx, fmt.Sprintf("repos/%d/", f.repoID), func(info corestore.BlobInfo) error {
+ keys = append(keys, info.Key)
+ return nil
+ }))
+ return keys
+}
+
+// TestPreparedCommitsOnlyBeforeTheWriteDeadline checks that a changeset whose
+// pack was uploaded is not committed once the pack's write deadline passed: the
+// write fails like a storage failure and no row references the pack, which
+// garbage collection may already treat as an orphan.
+func TestPreparedCommitsOnlyBeforeTheWriteDeadline(t *testing.T) {
+ const ref = "refs/heads/main"
+
+ forEachDialect(t, func(t *testing.T, dialect sqlstore.Dialect) {
+ for _, mode := range []corestore.Mode{corestore.ModeAuthoritative, corestore.ModeShadow} {
+ t.Run(string(mode), func(t *testing.T) {
+ f := newFixture(t, fixtureOptions{mode: mode, dialect: dialect})
+ f.setWriteDeadline(time.Second)
+ blobs := f.countPuts()
+ before := f.headSeq(t)
+
+ // The work between the upload and the commit outlasts the deadline,
+ // which started before the upload.
+ voter.FaultHook = func(point string) error {
+ if point == "after_upload" {
+ <-time.After(f.core.WriteDeadline())
+ }
+ return nil
+ }
+ t.Cleanup(func() { voter.FaultHook = nil })
+
+ err := f.prepared(line(f.zero, f.c.String(), ref))
+ if mode == corestore.ModeAuthoritative {
+ require.Equal(t, codes.Unavailable, status.Code(err), "error %v", err)
+ require.ErrorIs(t, err, corestore.ErrPackDeadline)
+ } else {
+ require.NoError(t, err, "shadow mode never fails the write")
+ require.Equal(t, float64(1), metricValue(t, f.voter, "graft_storage_shadow_errors_total", nil))
+ }
+
+ require.Equal(t, 1, blobs.count(), "the pack was uploaded")
+ require.Equal(t, before, f.headSeq(t), "no changeset references it")
+ })
+ }
+ })
+}
+
+// TestPreparedUploadsAgainWhenThePreparedPackExpired checks that the pack of
+// the Preparing phase is not committed when its write deadline passed before
+// the Prepared phase (the transaction waited long for the reference lock): the
+// Prepared phase uploads a new pack under a new key and commits that one.
+func TestPreparedUploadsAgainWhenThePreparedPackExpired(t *testing.T) {
+ const ref = "refs/heads/main"
+
+ forEachDialect(t, func(t *testing.T, dialect sqlstore.Dialect) {
+ f := newFixture(t, fixtureOptions{mode: corestore.ModeAuthoritative, dialect: dialect})
+ f.setWriteDeadline(time.Second)
+ blobs := f.countPuts()
+
+ input := line(f.zero, f.c.String(), ref)
+ f.preparing(input)
+ require.Equal(t, 1, blobs.count())
+ require.Equal(t, 1, f.voter.PreparedPacks())
+
+ <-time.After(f.core.WriteDeadline())
+ // A long deadline again, so that only the expiry of the prepared pack is
+ // under test.
+ f.setWriteDeadline(time.Hour)
+
+ require.NoError(t, f.prepared(input))
+ require.Equal(t, 2, blobs.count(), "an expired pack is never committed")
+ require.NotEqual(t, blobs.puts[0], blobs.puts[1])
+ require.Equal(t, blobs.puts[1], f.lastChangeset(t).PackKey)
+ require.Equal(t, corestore.Target(f.c.String()), f.dbRef(t, ref))
+ })
+}
+
+// TestUploadIsCancelledAtTheWriteDeadline checks that an upload slower than
+// the write deadline is cancelled, stores nothing and fails the write; no
+// changeset is committed.
+func TestUploadIsCancelledAtTheWriteDeadline(t *testing.T) {
+ const ref = "refs/heads/main"
+
+ forEachDialect(t, func(t *testing.T, dialect sqlstore.Dialect) {
+ f := newFixture(t, fixtureOptions{mode: corestore.ModeAuthoritative, dialect: dialect})
+ f.setWriteDeadline(100 * time.Millisecond)
+ inner := f.core.Blobs
+ keysBefore := f.packKeys(t, inner)
+ f.core.Blobs = blockingBlobs{BlobStore: inner}
+ before := f.headSeq(t)
+
+ input := line(f.zero, f.c.String(), ref)
+ f.preparing(input)
+ err := f.prepared(input)
+ require.Equal(t, codes.Unavailable, status.Code(err), "error %v", err)
+ require.ErrorIs(t, err, corestore.ErrPackDeadline)
+
+ require.Equal(t, before, f.headSeq(t), "no changeset was committed")
+ require.Equal(t, keysBefore, f.packKeys(t, inner), "nothing was stored")
+ })
+}
diff --git a/internal/graft/corestore/voter/voter.go b/internal/graft/corestore/voter/voter.go
index 059e0e0..4b1e3f9 100644
--- a/internal/graft/corestore/voter/voter.go
+++ b/internal/graft/corestore/voter/voter.go
@@ -107,25 +107,28 @@ type committedEntry struct {
seq uint64
}
-// uploadedPack is a changeset pack in the bucket. An empty key means that the
-// transaction brings no new objects.
-type uploadedPack struct {
- key string
- bytes int64
-}
-
// preparedPack is the outcome of the Preparing phase of a transaction.
type preparedPack struct {
// ok tells that pack was built and uploaded for the repository repoID.
// When it is false, Preparing failed and Prepared does the work itself.
ok bool
repoID int64
- pack uploadedPack
+ // pack is the uploaded pack, with an empty key when the transaction brings
+ // no new objects.
+ pack corestore.PackUpload
// spent is the time Preparing took. In shadow mode it counts against the
// shadow timeout, which is the budget of the whole transaction.
spent time.Duration
}
+// usable reports whether Prepared may commit the pack of the Preparing phase:
+// its write deadline has not passed (doc/graft/PLAN7.md, PACK OWNERSHIP RULE
+// 4). A pack whose deadline passed may be an orphan that garbage collection
+// deletes, so Prepared uploads a new one instead.
+func (p preparedPack) usable(now time.Time) bool {
+ return p.pack.Key == "" || now.Before(p.pack.Deadline)
+}
+
// New returns the voter of core. applier is told about local commits (it may
// be nil in shadow mode).
func New(core *corestore.Core, logger log.Logger, repoFactory localrepo.Factory, locator storage.Locator, applier corestore.LocalApplier) *Voter {
@@ -237,6 +240,9 @@ func (v *Voter) preparePack(ctx context.Context, payload gitcmd.HooksPayload, ch
return nil, nil
}
+ // The write deadline of the pack starts before this read of the record
+ // (doc/graft/PLAN7.md, PACK OWNERSHIP RULE 4).
+ readAt := time.Now()
record, err := v.core.Meta.GetRepository(ctx, corestore.KeyOf(payload.Repo))
if errors.Is(err, corestore.ErrRepositoryNotFound) {
return nil, nil
@@ -248,7 +254,7 @@ func (v *Voter) preparePack(ctx context.Context, payload gitcmd.HooksPayload, ch
return nil, nil
}
- pack, err := v.buildAndUpload(ctx, repo, payload.Repo.GetStorageName(), record.ID, updates)
+ pack, err := v.buildAndUpload(ctx, repo, payload.Repo.GetStorageName(), record.ID, readAt, updates)
if err != nil {
return nil, err
}
@@ -392,6 +398,9 @@ func (v *Voter) prepare(ctx context.Context, payload gitcmd.HooksPayload, change
return nil
}
+ // The write deadline of a pack uploaded here starts before this read of the
+ // record (doc/graft/PLAN7.md, PACK OWNERSHIP RULE 4).
+ readAt := time.Now()
record, err := v.core.Meta.GetRepository(ctx, corestore.KeyOf(payload.Repo))
if errors.Is(err, corestore.ErrRepositoryNotFound) {
if shadow {
@@ -431,14 +440,14 @@ func (v *Voter) prepare(ctx context.Context, payload gitcmd.HooksPayload, change
}
// The pack prepared before git took the locks is used when it was uploaded
- // for this repository (not for a removed one that had the same path).
- // Otherwise it is built and uploaded now, as before the LOCK AND HISTORY
- // RULE.
- var pack uploadedPack
- if prepared != nil && prepared.ok && prepared.repoID == record.ID {
+ // for this repository (not for a removed one that had the same path) and its
+ // write deadline has not passed. Otherwise it is built and uploaded now, as
+ // before the LOCK AND HISTORY RULE.
+ var pack corestore.PackUpload
+ if prepared != nil && prepared.ok && prepared.repoID == record.ID && prepared.usable(time.Now()) {
pack = prepared.pack
} else {
- pack, err = v.buildAndUpload(ctx, repo, payload.Repo.GetStorageName(), record.ID, updates)
+ pack, err = v.buildAndUpload(ctx, repo, payload.Repo.GetStorageName(), record.ID, readAt, updates)
if err != nil {
return v.unavailable(err)
}
@@ -451,8 +460,8 @@ func (v *Voter) prepare(ctx context.Context, payload gitcmd.HooksPayload, change
Actor: meta.Actor,
NodeID: v.core.NodeID,
Force: shadow,
- PackKey: pack.key,
- PackBytes: pack.bytes,
+ PackKey: pack.Key,
+ PackBytes: pack.Bytes,
}
if err := v.fault(faultAfterUpload); err != nil {
@@ -464,7 +473,15 @@ func (v *Voter) prepare(ctx context.Context, payload gitcmd.HooksPayload, change
in.Changelog = &metaCopy
}
- res, err := v.core.Meta.CommitChangeset(ctx, in)
+ // The changeset that references the pack commits before the pack's write
+ // deadline, or not at all.
+ commitCtx, cancel, err := pack.CommitContext(ctx)
+ if err != nil {
+ return v.unavailable(err)
+ }
+ defer cancel()
+
+ res, err := v.core.Meta.CommitChangeset(commitCtx, in)
var conflict *corestore.ErrRefConflict
if errors.As(err, &conflict) {
@@ -631,27 +648,29 @@ func newTips(updates []inputUpdate) []git.ObjectID {
}
// buildAndUpload builds the pack of the objects reachable from the new tips of
-// updates and not from what is already in the bucket, and uploads it under the
-// pack key of repoID. An empty pack is not uploaded.
+// updates and not from what is already in the bucket, and uploads it under a
+// new owned key of repoID with the write deadline that starts at readAt, the
+// time taken before the read of the repository record (Core.UploadPack). An
+// empty pack is not uploaded.
//
// What is already in the bucket: in authoritative mode, everything any current
// local reference reaches (invariant I1, gitops.BuildPack); in shadow mode,
// everything that the values the database records reach (buildPackAgainst),
// because local references can hold values the storage core never recorded.
-func (v *Voter) buildAndUpload(ctx context.Context, repo *localrepo.Repo, storageName string, repoID int64, updates []inputUpdate) (uploadedPack, error) {
+func (v *Voter) buildAndUpload(ctx context.Context, repo *localrepo.Repo, storageName string, repoID int64, readAt time.Time, updates []inputUpdate) (corestore.PackUpload, error) {
tips := newTips(updates)
if len(tips) == 0 {
- return uploadedPack{}, nil
+ return corestore.PackUpload{}, nil
}
tmpDir, err := v.locator.TempDir(storageName)
if err != nil {
- return uploadedPack{}, err
+ return corestore.PackUpload{}, err
}
// The server creates the temporary directory when it starts, but it may have
// been cleaned up since.
if err := os.MkdirAll(tmpDir, mode.Directory); err != nil {
- return uploadedPack{}, fmt.Errorf("creating the temporary directory: %w", err)
+ return corestore.PackUpload{}, fmt.Errorf("creating the temporary directory: %w", err)
}
var pack gitops.Pack
@@ -661,37 +680,32 @@ func (v *Voter) buildAndUpload(ctx context.Context, repo *localrepo.Repo, storag
var known map[string]corestore.Target
known, _, err = v.core.Meta.Refs(ctx, repoID)
if err != nil {
- return uploadedPack{}, fmt.Errorf("reading the database references: %w", err)
+ return corestore.PackUpload{}, fmt.Errorf("reading the database references: %w", err)
}
pack, err = buildPackAgainst(ctx, repo, tips, known, tmpDir)
}
if err != nil {
- return uploadedPack{}, err
+ return corestore.PackUpload{}, err
}
defer func() { _ = pack.Remove() }()
if pack.Empty() {
- return uploadedPack{}, nil
- }
-
- key := corestore.NewPackKey(repoID, pack.SHA256)
- if err := v.uploadPack(ctx, key, pack); err != nil {
- return uploadedPack{}, err
+ return corestore.PackUpload{}, nil
}
- v.metrics.packBytes.Add(float64(pack.Size))
- return uploadedPack{key: key, bytes: pack.Size}, nil
-}
-
-// uploadPack stores the pack under key, see BlobStore.Put.
-func (v *Voter) uploadPack(ctx context.Context, key string, pack gitops.Pack) error {
r, err := pack.Open()
if err != nil {
- return err
+ return corestore.PackUpload{}, err
}
defer func() { _ = r.Close() }()
- return v.core.Blobs.Put(ctx, key, r)
+ upload, err := v.core.UploadPack(ctx, repoID, readAt, pack.SHA256, pack.Size, r)
+ if err != nil {
+ return corestore.PackUpload{}, err
+ }
+ v.metrics.packBytes.Add(float64(pack.Size))
+
+ return upload, nil
}
// fault calls FaultHook and exits the process when CrashEnv selects the point.