Skip to content

Git designed for AI agents.

test(storage): split shadow mirroring from the shadow budget (PLAN7 WP73b)

Federico Eliab186f731755d
diff --git a/internal/graft/corestore/coretest/coretest.go b/internal/graft/corestore/coretest/coretest.go
index 253f6a6..93b1b95 100644
--- a/internal/graft/corestore/coretest/coretest.go
+++ b/internal/graft/corestore/coretest/coretest.go
@@ -140,6 +140,9 @@ type Options struct {
// short.
GCGrace time.Duration
GCMargin time.Duration
+ // ShadowTimeout is the shadow budget of a transaction; zero means the
+ // production default (5s).
+ ShadowTimeout time.Duration
}
// Open opens a storage core for the test and closes it when the test ends.
@@ -167,6 +170,9 @@ func Open(tb testing.TB, opts Options) *corestore.Core {
if opts.GCMargin == 0 {
opts.GCMargin = 5 * time.Minute
}
+ if opts.ShadowTimeout == 0 {
+ opts.ShadowTimeout = 5 * time.Second
+ }
ctx := context.Background()
@@ -182,7 +188,7 @@ func Open(tb testing.TB, opts Options) *corestore.Core {
Meta: meta,
Blobs: blobs,
SyncTimeout: 10 * time.Second,
- ShadowTimeout: 5 * time.Second,
+ ShadowTimeout: opts.ShadowTimeout,
GCGrace: opts.GCGrace,
GCMargin: opts.GCMargin,
}
diff --git a/internal/graft/corestore/voter/shadow_budget_test.go b/internal/graft/corestore/voter/shadow_budget_test.go
new file mode 100644
index 0000000..1becb19
--- /dev/null
+++ b/internal/graft/corestore/voter/shadow_budget_test.go
@@ -0,0 +1,40 @@
+package voter_test
+
+import (
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/feder-cr/git-ai/internal/graft/corestore"
+ "github.com/feder-cr/git-ai/internal/graft/corestore/sqlstore"
+ "github.com/feder-cr/git-ai/internal/graft/corestore/voter"
+)
+
+// TestShadowBudgetOverrunInPrepared checks the SHADOW BUDGET requirement of
+// doc/graft/PLAN7.md for a Prepared phase that runs past the shadow budget
+// (TestPreparingShadowTimeoutCoversBothPhases covers a Preparing phase that
+// uses it up): the write is not failed, nothing is recorded, and the failure
+// is counted as a shadow error.
+func TestShadowBudgetOverrunInPrepared(t *testing.T) {
+ const ref = "refs/heads/main"
+
+ forEachDialect(t, func(t *testing.T, dialect sqlstore.Dialect) {
+ f := newFixture(t, fixtureOptions{mode: corestore.ModeShadow, dialect: dialect})
+ f.core.ShadowTimeout = 200 * time.Millisecond
+ before := f.headSeq(t)
+
+ // The work between the upload and the commit takes the whole budget.
+ voter.FaultHook = func(point string) error {
+ if point == "after_upload" {
+ <-time.After(f.core.ShadowTimeout)
+ }
+ return nil
+ }
+ t.Cleanup(func() { voter.FaultHook = nil })
+
+ require.NoError(t, f.prepared(line(f.zero, f.c.String(), ref)), "shadow mode never fails the write")
+ require.Equal(t, before, f.headSeq(t), "nothing is recorded")
+ require.Equal(t, float64(1), metricValue(t, f.voter, "graft_storage_shadow_errors_total", nil))
+ })
+}
diff --git a/internal/graft/e2e/storage_helpers_test.go b/internal/graft/e2e/storage_helpers_test.go
index 779b35e..a406d51 100644
--- a/internal/graft/e2e/storage_helpers_test.go
+++ b/internal/graft/e2e/storage_helpers_test.go
@@ -10,6 +10,7 @@ import (
"strings"
"sync/atomic"
"testing"
+ "time"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
@@ -62,6 +63,13 @@ type storageCluster struct {
// logger, when set, is the logger of every node, so that a test can read
// what the storage core logged.
logger log.Logger
+ // shadowTimeout, when set, is the shadow budget of every node instead of
+ // the default.
+ shadowTimeout time.Duration
+ // adjust, when set, may change each node's core before its server starts
+ // (wrap its database or bucket); the core is never changed afterwards, while
+ // the server reads it.
+ adjust func(*corestore.Core)
nodes []storageNode
}
@@ -97,12 +105,16 @@ func (c *storageCluster) addNode(t *testing.T) storageNode {
name := fmt.Sprintf("node-%c", 'a'+len(c.nodes))
core := coretest.Open(t, coretest.Options{
- Mode: c.mode,
- Dialect: c.dialect,
- DatabaseURL: c.databaseURL,
- BucketURL: c.bucketURL,
- NodeID: name,
+ Mode: c.mode,
+ Dialect: c.dialect,
+ DatabaseURL: c.databaseURL,
+ BucketURL: c.bucketURL,
+ NodeID: name,
+ ShadowTimeout: c.shadowTimeout,
})
+ if c.adjust != nil {
+ c.adjust(core)
+ }
cfg := testcfg.Build(t)
cfg.Auth.Token = sharedTestToken
diff --git a/internal/graft/e2e/storage_shadow_test.go b/internal/graft/e2e/storage_shadow_test.go
index ed6b93d..4c3d330 100644
--- a/internal/graft/e2e/storage_shadow_test.go
+++ b/internal/graft/e2e/storage_shadow_test.go
@@ -1,8 +1,12 @@
package e2e
import (
+ "context"
+ "io"
"strings"
+ "sync/atomic"
"testing"
+ "time"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/require"
@@ -13,12 +17,147 @@ import (
"github.com/feder-cr/git-ai/sdk"
)
+// overrunBlobs is a bucket whose uploads, while on is set, last until the end
+// of their context: the Preparing phase of a shadow transaction then uses up
+// the whole shadow budget.
+type overrunBlobs struct {
+ corestore.BlobStore
+ on *atomic.Bool
+}
+
+func (b overrunBlobs) Put(ctx context.Context, key string, r io.Reader) error {
+ if b.on.Load() {
+ <-ctx.Done()
+ return ctx.Err()
+ }
+ return b.BlobStore.Put(ctx, key, r)
+}
+
+// overrunMeta is a database whose changeset commits, while on is set, last
+// until the end of their context: the Prepared phase of a shadow transaction
+// then runs past the shadow budget.
+type overrunMeta struct {
+ corestore.MetaStore
+ on *atomic.Bool
+}
+
+func (m overrunMeta) CommitChangeset(ctx context.Context, in corestore.ChangesetInput) (corestore.CommitResult, error) {
+ if m.on.Load() {
+ <-ctx.Done()
+ return corestore.CommitResult{}, ctx.Err()
+ }
+ return m.MetaStore.CommitChangeset(ctx, in)
+}
+
+// TestStorageShadowModeBudgetOverrun checks what RULE 4 guarantees for a shadow
+// transaction whose storage-core work overruns shadow_timeout
+// (doc/graft/PLAN7.md, SHADOW BUDGET requirement): the git write succeeds; the
+// failure is logged at error level (and counted, see the voter's unit tests);
+// nothing is recorded; VerifyRepository reports the reference as diverging;
+// and the next write of the reference is recorded as a forced changeset and
+// logged as a divergence, after which the database follows git again. Both
+// phases are covered: an overrun in Preparing (the upload of the pack), which
+// leaves Prepared no budget at all, and an overrun in Prepared (the database
+// commit, while git holds the reference lock).
+func TestStorageShadowModeBudgetOverrun(t *testing.T) {
+ t.Parallel()
+
+ for _, phase := range []string{"preparing", "prepared"} {
+ t.Run(phase, func(t *testing.T) {
+ forEachStorageDialect(t, func(t *testing.T, dialect sqlstore.Dialect) {
+ const ref = "refs/heads/budget"
+ who := workerName(0)
+
+ graft, tokens := workerGraft(1)
+ logger := testhelper.NewLogger(t)
+ logs := testhelper.AddLoggerHook(logger)
+ cluster := newStorageCluster(t, dialect, corestore.ModeShadow, graft)
+ cluster.logger = logger
+
+ var overrun atomic.Bool
+ cluster.adjust = func(core *corestore.Core) {
+ if phase == "preparing" {
+ core.Blobs = overrunBlobs{BlobStore: core.Blobs, on: &overrun}
+ } else {
+ core.Meta = overrunMeta{MetaStore: core.Meta, on: &overrun}
+ }
+ }
+ node := cluster.addNode(t)
+ ctx := node.ctx
+
+ rel := node.createRepository(t).GetRelativePath()
+ repo := node.agent(t, rel, tokens[who])
+ base, err := commitFile(ctx, repo, who, "main", "", "a.txt", "a")
+ require.NoError(t, err)
+ _, err = node.operator(t, rel).StorageImport(ctx)
+ require.NoError(t, err)
+
+ // A write whose shadow work overruns the budget: git commits it.
+ overrun.Store(true)
+ first, err := commitFile(ctx, repo, who, "budget", base, "b.txt", "b")
+ overrun.Store(false)
+ require.NoError(t, err, "shadow mode never fails the git write")
+
+ require.Equal(t, corestore.Target(first), node.localRefs(t, rel)[ref])
+ require.NotContains(t, node.dbRefs(t, rel), ref, "the overrun write is not recorded")
+ require.True(t, hasLog(logs.AllEntries(), logrus.ErrorLevel, "shadow write failed, the git write proceeds"),
+ "the overrun is logged at error level")
+
+ verified, err := node.operator(t, rel).StorageVerify(ctx, false)
+ require.NoError(t, err)
+ require.Len(t, verified.Divergences, 1)
+ require.Equal(t, ref, verified.Divergences[0].Reference)
+ require.Equal(t, first, verified.Divergences[0].LocalTarget)
+ require.Empty(t, verified.Divergences[0].DatabaseTarget)
+
+ // The next write of the reference expects the value git has, which the
+ // database does not: it is recorded as forced and counted as a
+ // divergence, and the database follows git again.
+ second, err := commitFile(ctx, repo, who, "budget", first, "c.txt", "c")
+ require.NoError(t, err)
+
+ rec := node.record(t, rel)
+ changesets, err := node.core.Meta.ListChangesets(ctx, rec.ID, rec.HeadSeq-1, rec.HeadSeq)
+ require.NoError(t, err)
+ require.Len(t, changesets, 1)
+ require.True(t, changesets[0].Forced, "the next write of the reference is a forced changeset")
+ require.Equal(t, corestore.Target(second), changesets[0].Updates[0].New)
+ require.True(t, hasLog(logs.AllEntries(), logrus.WarnLevel, "diverges from the database"))
+
+ require.Equal(t, node.localRefs(t, rel), node.dbRefs(t, rel))
+ verified, err = node.operator(t, rel).StorageVerify(ctx, true)
+ require.NoError(t, err)
+ require.Empty(t, verified.Divergences)
+ require.True(t, verified.ConnectivityOK, verified.ConnectivityError)
+ })
+ })
+ }
+}
+
+// hasLog reports whether an entry at level has a message containing text.
+func hasLog(entries []*logrus.Entry, level logrus.Level, text string) bool {
+ for _, entry := range entries {
+ if entry.Level == level && strings.Contains(entry.Message, text) {
+ return true
+ }
+ }
+ return false
+}
+
// TestStorageShadowModeBenchmarkHasNoDivergence runs the mixed landing load of
// the benchmark (private changes and changes that conflict in one file, all
// on one branch, batched as configured by default) against a server in shadow
// mode, on a repository that had history before it was imported. Every write
// is mirrored into the database without a single divergence or shadow error,
// and the database ends with exactly the references git has.
+//
+// It tests the correctness of the mirroring, so the shadow budget is made
+// non-binding: shadow mode records a transaction only if its storage-core work
+// fits shadow_timeout (doc/graft/PLAN7.md, SHADOW BUDGET requirement), and
+// whether the work of this load fits 5 seconds depends on how loaded the
+// machine is, not on the code. The budget path itself (a write past the budget
+// is counted, logged and not recorded, and the next write of its reference is
+// a forced changeset) is TestStorageShadowModeBudgetOverrun.
func TestStorageShadowModeBenchmarkHasNoDivergence(t *testing.T) {
t.Parallel()
@@ -31,6 +170,7 @@ func TestStorageShadowModeBenchmarkHasNoDivergence(t *testing.T) {
logs := testhelper.AddLoggerHook(logger)
cluster := newStorageCluster(t, dialect, corestore.ModeShadow, graft)
cluster.logger = logger
+ cluster.shadowTimeout = time.Minute
node := cluster.addNode(t)
ctx := node.ctx