Git designed for AI agents.
test(storage): split shadow mirroring from the shadow budget (PLAN7 WP73b)
Federico Eliab186f731755d
diff --git a/internal/graft/corestore/coretest/coretest.go b/internal/graft/corestore/coretest/coretest.goindex 253f6a6..93b1b95 100644--- a/internal/graft/corestore/coretest/coretest.go+++ b/internal/graft/corestore/coretest/coretest.go@@ -140,6 +140,9 @@ type Options struct {// short.GCGrace time.DurationGCMargin time.Duration+ // ShadowTimeout is the shadow budget of a transaction; zero means the+ // production default (5s).+ ShadowTimeout time.Duration}// Open opens a storage core for the test and closes it when the test ends.@@ -167,6 +170,9 @@ func Open(tb testing.TB, opts Options) *corestore.Core {if opts.GCMargin == 0 {opts.GCMargin = 5 * time.Minute}+ if opts.ShadowTimeout == 0 {+ opts.ShadowTimeout = 5 * time.Second+ }ctx := context.Background()@@ -182,7 +188,7 @@ func Open(tb testing.TB, opts Options) *corestore.Core {Meta: meta,Blobs: blobs,SyncTimeout: 10 * time.Second,- ShadowTimeout: 5 * time.Second,+ ShadowTimeout: opts.ShadowTimeout,GCGrace: opts.GCGrace,GCMargin: opts.GCMargin,}diff --git a/internal/graft/corestore/voter/shadow_budget_test.go b/internal/graft/corestore/voter/shadow_budget_test.gonew file mode 100644index 0000000..1becb19--- /dev/null+++ b/internal/graft/corestore/voter/shadow_budget_test.go@@ -0,0 +1,40 @@+package voter_test++import (+ "testing"+ "time"++ "github.com/stretchr/testify/require"++ "github.com/feder-cr/git-ai/internal/graft/corestore"+ "github.com/feder-cr/git-ai/internal/graft/corestore/sqlstore"+ "github.com/feder-cr/git-ai/internal/graft/corestore/voter"+)++// TestShadowBudgetOverrunInPrepared checks the SHADOW BUDGET requirement of+// doc/graft/PLAN7.md for a Prepared phase that runs past the shadow budget+// (TestPreparingShadowTimeoutCoversBothPhases covers a Preparing phase that+// uses it up): the write is not failed, nothing is recorded, and the failure+// is counted as a shadow error.+func TestShadowBudgetOverrunInPrepared(t *testing.T) {+ const ref = "refs/heads/main"++ forEachDialect(t, func(t *testing.T, dialect sqlstore.Dialect) {+ f := newFixture(t, fixtureOptions{mode: corestore.ModeShadow, dialect: dialect})+ f.core.ShadowTimeout = 200 * time.Millisecond+ before := f.headSeq(t)++ // The work between the upload and the commit takes the whole budget.+ voter.FaultHook = func(point string) error {+ if point == "after_upload" {+ <-time.After(f.core.ShadowTimeout)+ }+ return nil+ }+ t.Cleanup(func() { voter.FaultHook = nil })++ require.NoError(t, f.prepared(line(f.zero, f.c.String(), ref)), "shadow mode never fails the write")+ require.Equal(t, before, f.headSeq(t), "nothing is recorded")+ require.Equal(t, float64(1), metricValue(t, f.voter, "graft_storage_shadow_errors_total", nil))+ })+}diff --git a/internal/graft/e2e/storage_helpers_test.go b/internal/graft/e2e/storage_helpers_test.goindex 779b35e..a406d51 100644--- a/internal/graft/e2e/storage_helpers_test.go+++ b/internal/graft/e2e/storage_helpers_test.go@@ -10,6 +10,7 @@ import ("strings""sync/atomic""testing"+ "time""github.com/stretchr/testify/require""google.golang.org/grpc"@@ -62,6 +63,13 @@ type storageCluster struct {// logger, when set, is the logger of every node, so that a test can read// what the storage core logged.logger log.Logger+ // shadowTimeout, when set, is the shadow budget of every node instead of+ // the default.+ shadowTimeout time.Duration+ // adjust, when set, may change each node's core before its server starts+ // (wrap its database or bucket); the core is never changed afterwards, while+ // the server reads it.+ adjust func(*corestore.Core)nodes []storageNode}@@ -97,12 +105,16 @@ func (c *storageCluster) addNode(t *testing.T) storageNode {name := fmt.Sprintf("node-%c", 'a'+len(c.nodes))core := coretest.Open(t, coretest.Options{- Mode: c.mode,- Dialect: c.dialect,- DatabaseURL: c.databaseURL,- BucketURL: c.bucketURL,- NodeID: name,+ Mode: c.mode,+ Dialect: c.dialect,+ DatabaseURL: c.databaseURL,+ BucketURL: c.bucketURL,+ NodeID: name,+ ShadowTimeout: c.shadowTimeout,})+ if c.adjust != nil {+ c.adjust(core)+ }cfg := testcfg.Build(t)cfg.Auth.Token = sharedTestTokendiff --git a/internal/graft/e2e/storage_shadow_test.go b/internal/graft/e2e/storage_shadow_test.goindex ed6b93d..4c3d330 100644--- a/internal/graft/e2e/storage_shadow_test.go+++ b/internal/graft/e2e/storage_shadow_test.go@@ -1,8 +1,12 @@package e2eimport (+ "context"+ "io""strings"+ "sync/atomic""testing"+ "time""github.com/sirupsen/logrus""github.com/stretchr/testify/require"@@ -13,12 +17,147 @@ import ("github.com/feder-cr/git-ai/sdk")+// overrunBlobs is a bucket whose uploads, while on is set, last until the end+// of their context: the Preparing phase of a shadow transaction then uses up+// the whole shadow budget.+type overrunBlobs struct {+ corestore.BlobStore+ on *atomic.Bool+}++func (b overrunBlobs) Put(ctx context.Context, key string, r io.Reader) error {+ if b.on.Load() {+ <-ctx.Done()+ return ctx.Err()+ }+ return b.BlobStore.Put(ctx, key, r)+}++// overrunMeta is a database whose changeset commits, while on is set, last+// until the end of their context: the Prepared phase of a shadow transaction+// then runs past the shadow budget.+type overrunMeta struct {+ corestore.MetaStore+ on *atomic.Bool+}++func (m overrunMeta) CommitChangeset(ctx context.Context, in corestore.ChangesetInput) (corestore.CommitResult, error) {+ if m.on.Load() {+ <-ctx.Done()+ return corestore.CommitResult{}, ctx.Err()+ }+ return m.MetaStore.CommitChangeset(ctx, in)+}++// TestStorageShadowModeBudgetOverrun checks what RULE 4 guarantees for a shadow+// transaction whose storage-core work overruns shadow_timeout+// (doc/graft/PLAN7.md, SHADOW BUDGET requirement): the git write succeeds; the+// failure is logged at error level (and counted, see the voter's unit tests);+// nothing is recorded; VerifyRepository reports the reference as diverging;+// and the next write of the reference is recorded as a forced changeset and+// logged as a divergence, after which the database follows git again. Both+// phases are covered: an overrun in Preparing (the upload of the pack), which+// leaves Prepared no budget at all, and an overrun in Prepared (the database+// commit, while git holds the reference lock).+func TestStorageShadowModeBudgetOverrun(t *testing.T) {+ t.Parallel()++ for _, phase := range []string{"preparing", "prepared"} {+ t.Run(phase, func(t *testing.T) {+ forEachStorageDialect(t, func(t *testing.T, dialect sqlstore.Dialect) {+ const ref = "refs/heads/budget"+ who := workerName(0)++ graft, tokens := workerGraft(1)+ logger := testhelper.NewLogger(t)+ logs := testhelper.AddLoggerHook(logger)+ cluster := newStorageCluster(t, dialect, corestore.ModeShadow, graft)+ cluster.logger = logger++ var overrun atomic.Bool+ cluster.adjust = func(core *corestore.Core) {+ if phase == "preparing" {+ core.Blobs = overrunBlobs{BlobStore: core.Blobs, on: &overrun}+ } else {+ core.Meta = overrunMeta{MetaStore: core.Meta, on: &overrun}+ }+ }+ node := cluster.addNode(t)+ ctx := node.ctx++ rel := node.createRepository(t).GetRelativePath()+ repo := node.agent(t, rel, tokens[who])+ base, err := commitFile(ctx, repo, who, "main", "", "a.txt", "a")+ require.NoError(t, err)+ _, err = node.operator(t, rel).StorageImport(ctx)+ require.NoError(t, err)++ // A write whose shadow work overruns the budget: git commits it.+ overrun.Store(true)+ first, err := commitFile(ctx, repo, who, "budget", base, "b.txt", "b")+ overrun.Store(false)+ require.NoError(t, err, "shadow mode never fails the git write")++ require.Equal(t, corestore.Target(first), node.localRefs(t, rel)[ref])+ require.NotContains(t, node.dbRefs(t, rel), ref, "the overrun write is not recorded")+ require.True(t, hasLog(logs.AllEntries(), logrus.ErrorLevel, "shadow write failed, the git write proceeds"),+ "the overrun is logged at error level")++ verified, err := node.operator(t, rel).StorageVerify(ctx, false)+ require.NoError(t, err)+ require.Len(t, verified.Divergences, 1)+ require.Equal(t, ref, verified.Divergences[0].Reference)+ require.Equal(t, first, verified.Divergences[0].LocalTarget)+ require.Empty(t, verified.Divergences[0].DatabaseTarget)++ // The next write of the reference expects the value git has, which the+ // database does not: it is recorded as forced and counted as a+ // divergence, and the database follows git again.+ second, err := commitFile(ctx, repo, who, "budget", first, "c.txt", "c")+ require.NoError(t, err)++ rec := node.record(t, rel)+ changesets, err := node.core.Meta.ListChangesets(ctx, rec.ID, rec.HeadSeq-1, rec.HeadSeq)+ require.NoError(t, err)+ require.Len(t, changesets, 1)+ require.True(t, changesets[0].Forced, "the next write of the reference is a forced changeset")+ require.Equal(t, corestore.Target(second), changesets[0].Updates[0].New)+ require.True(t, hasLog(logs.AllEntries(), logrus.WarnLevel, "diverges from the database"))++ require.Equal(t, node.localRefs(t, rel), node.dbRefs(t, rel))+ verified, err = node.operator(t, rel).StorageVerify(ctx, true)+ require.NoError(t, err)+ require.Empty(t, verified.Divergences)+ require.True(t, verified.ConnectivityOK, verified.ConnectivityError)+ })+ })+ }+}++// hasLog reports whether an entry at level has a message containing text.+func hasLog(entries []*logrus.Entry, level logrus.Level, text string) bool {+ for _, entry := range entries {+ if entry.Level == level && strings.Contains(entry.Message, text) {+ return true+ }+ }+ return false+}+// TestStorageShadowModeBenchmarkHasNoDivergence runs the mixed landing load of// the benchmark (private changes and changes that conflict in one file, all// on one branch, batched as configured by default) against a server in shadow// mode, on a repository that had history before it was imported. Every write// is mirrored into the database without a single divergence or shadow error,// and the database ends with exactly the references git has.+//+// It tests the correctness of the mirroring, so the shadow budget is made+// non-binding: shadow mode records a transaction only if its storage-core work+// fits shadow_timeout (doc/graft/PLAN7.md, SHADOW BUDGET requirement), and+// whether the work of this load fits 5 seconds depends on how loaded the+// machine is, not on the code. The budget path itself (a write past the budget+// is counted, logged and not recorded, and the next write of its reference is+// a forced changeset) is TestStorageShadowModeBudgetOverrun.func TestStorageShadowModeBenchmarkHasNoDivergence(t *testing.T) {t.Parallel()@@ -31,6 +170,7 @@ func TestStorageShadowModeBenchmarkHasNoDivergence(t *testing.T) {logs := testhelper.AddLoggerHook(logger)cluster := newStorageCluster(t, dialect, corestore.ModeShadow, graft)cluster.logger = logger+ cluster.shadowTimeout = time.Minutenode := cluster.addNode(t)ctx := node.ctx