Skip to content

Git designed for AI agents.

Review m5, m7, m9: config error, covered checkpoints, a deterministic lease test

Federico Elia86f8316e9078
diff --git a/internal/gitaly/config/config_graft_storage.go b/internal/gitaly/config/config_graft_storage.go
index 3db9f67..47aa398 100644
--- a/internal/gitaly/config/config_graft_storage.go
+++ b/internal/gitaly/config/config_graft_storage.go
@@ -216,7 +216,9 @@ func (s GraftStorage) validateGC() error {
}
if grace <= sync+margin {
- return fmt.Errorf("graft storage: gc_grace %s must be greater than sync_timeout %s + gc_margin %s", grace, sync, margin)
+ return fmt.Errorf("graft storage: gc_grace %s must be greater than sync_timeout %s + gc_margin %s: "+
+ "raise gc_grace or lower sync_timeout (a rule since v0.3.0: a v0.2 configuration with sync_timeout of %s or more must set gc_grace)",
+ grace, sync, margin, DefaultGraftStorageGCGrace-DefaultGraftStorageGCMargin)
}
return nil
}
diff --git a/internal/gitaly/config/config_graft_storage_test.go b/internal/gitaly/config/config_graft_storage_test.go
index ba1973d..571e7b3 100644
--- a/internal/gitaly/config/config_graft_storage_test.go
+++ b/internal/gitaly/config/config_graft_storage_test.go
@@ -38,7 +38,8 @@ func TestGraftStorageValidate(t *testing.T) {
{desc: "gc times", storage: GraftStorage{Mode: "shadow", Database: valid.Database, Bucket: valid.Bucket, GCGrace: "30m", GCMargin: "1m"}},
{desc: "bad gc_grace", storage: GraftStorage{Mode: "shadow", Database: valid.Database, Bucket: valid.Bucket, GCGrace: "-1h"}, err: "gc_grace"},
{desc: "bad gc_margin", storage: GraftStorage{Mode: "shadow", Database: valid.Database, Bucket: valid.Bucket, GCMargin: "0"}, err: "gc_margin"},
- {desc: "gc_grace not above sync_timeout + gc_margin", storage: GraftStorage{Mode: "shadow", Database: valid.Database, Bucket: valid.Bucket, GCGrace: "6m", GCMargin: "5m", SyncTimeout: "1m"}, err: "gc_grace 6m0s must be greater than sync_timeout 1m0s + gc_margin 5m0s"},
+ {desc: "gc_grace not above sync_timeout + gc_margin", storage: GraftStorage{Mode: "shadow", Database: valid.Database, Bucket: valid.Bucket, GCGrace: "6m", GCMargin: "5m", SyncTimeout: "1m"}, err: "gc_grace 6m0s must be greater than sync_timeout 1m0s + gc_margin 5m0s: raise gc_grace or lower sync_timeout"},
+ {desc: "a v0.2 configuration with a long sync_timeout", storage: GraftStorage{Mode: "shadow", Database: valid.Database, Bucket: valid.Bucket, SyncTimeout: "1h"}, err: "a v0.2 configuration with sync_timeout of 55m0s or more must set gc_grace"},
{desc: "gc_grace just above sync_timeout + gc_margin", storage: GraftStorage{Mode: "shadow", Database: valid.Database, Bucket: valid.Bucket, GCGrace: "361s", GCMargin: "5m", SyncTimeout: "1m"}},
{desc: "checkpoints and gc auto in authoritative mode", storage: GraftStorage{Mode: "authoritative", Database: valid.Database, Bucket: valid.Bucket, Checkpoints: "auto", GC: "auto", MaintenanceInterval: "15m", LeaseTTL: "2m", CheckpointChangesets: ptr[uint64](0), CheckpointPackBytes: ptr[int64](0)}, standalone: true},
{desc: "checkpoints and gc off", storage: GraftStorage{Mode: "shadow", Database: valid.Database, Bucket: valid.Bucket, Checkpoints: "off", GC: "off"}},
diff --git a/internal/graft/corestore/maintenance/maintenance_test.go b/internal/graft/corestore/maintenance/maintenance_test.go
index 24b2cbd..2ec7a3e 100644
--- a/internal/graft/corestore/maintenance/maintenance_test.go
+++ b/internal/graft/corestore/maintenance/maintenance_test.go
@@ -24,6 +24,7 @@ import (
"github.com/feder-cr/git-ai/internal/graft/corestore/gc"
"github.com/feder-cr/git-ai/internal/graft/corestore/gitops"
"github.com/feder-cr/git-ai/internal/graft/corestore/maintenance"
+ "github.com/feder-cr/git-ai/internal/graft/corestore/sqlstore"
"github.com/feder-cr/git-ai/internal/graft/corestore/syncer"
"github.com/feder-cr/git-ai/internal/testhelper"
"github.com/feder-cr/git-ai/internal/testhelper/testcfg"
@@ -225,6 +226,37 @@ func TestRunOnceDropsSupersededCheckpointsWithGCOff(t *testing.T) {
require.Equal(t, before+1, e.metric("graft_storage_maintenance_skipped_total", "nothing_due"))
}
+// leaseExpiry returns the expiry of the repository's maintenance lease, which
+// holder must hold.
+func (e *env) leaseExpiry(holder string) time.Time {
+ e.t.Helper()
+ store := e.core.Meta.(*sqlstore.Store)
+ query := "SELECT holder, expires_at FROM graft_gc_leases WHERE repo_id = ?"
+ if store.Dialect() == sqlstore.Postgres {
+ query = "SELECT holder, expires_at FROM graft_gc_leases WHERE repo_id = $1"
+ }
+ var (
+ got string
+ raw any
+ )
+ require.NoError(e.t, store.DB().QueryRowContext(testhelper.Context(e.t), query, e.repo.ID).Scan(&got, &raw))
+ require.Equal(e.t, holder, got)
+ switch v := raw.(type) {
+ case time.Time:
+ return v
+ case string:
+ expiry, err := time.Parse(time.RFC3339Nano, v)
+ require.NoError(e.t, err)
+ return expiry
+ case []byte:
+ expiry, err := time.Parse(time.RFC3339Nano, string(v))
+ require.NoError(e.t, err)
+ return expiry
+ }
+ require.FailNow(e.t, "unexpected expiry type", "%T", raw)
+ return time.Time{}
+}
+
// checkpoints returns the checkpoints of the repository.
func (e *env) checkpoints() []corestore.Checkpoint {
e.t.Helper()
@@ -434,33 +466,31 @@ func TestRunOnceStopsWhenTheLeaseIsLost(t *testing.T) {
}
// TestHoldLeaseRenewsUntilReleased verifies the lease the background
-// maintenance and the CollectGarbage request hold while they work: no other
-// holder takes it for several times its time to live, and it is free once
-// released.
+// maintenance and the CollectGarbage request hold while they work: the renewal
+// moves its expiry forward, the holder keeps it, and it is free once released.
+// The test waits for the renewal itself (the expiry in the database moving past
+// the one the acquisition set), not for a span of time, so it does not depend
+// on how fast the database writes on the host.
func TestHoldLeaseRenewsUntilReleased(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
e := newEnv(t, coretest.Options{}, config3(0, 0))
- // The renewal runs every third of the time to live, so a renewal delayed by
- // up to two thirds of it (2 s) keeps the lease. Every renewal and every
- // probe below is a database write transaction (SQLite: the write admission
- // and an fsync), whose latency on a loaded host is what a shorter time to
- // live would measure instead of the renewal. The probes are spaced for the
- // same reason.
- const ttl = 3 * time.Second
+ const ttl = 900 * time.Millisecond
const holder = "node-a/1/request-1"
acquired, err := e.core.Meta.AcquireMaintenanceLease(ctx, e.repo.ID, holder, ttl)
require.NoError(t, err)
require.True(t, acquired)
+ first := e.leaseExpiry(holder)
var lost atomic.Bool
leaseCtx, release := maintenance.HoldLease(ctx, e.core, testhelper.NewLogger(t), e.repo.ID, holder, ttl, func() { lost.Store(true) })
- require.Never(t, func() bool {
- acquired, err := e.core.Meta.AcquireMaintenanceLease(ctx, e.repo.ID, "node-b/1", ttl)
- require.NoError(t, err)
- return acquired
- }, 5*ttl/2, ttl/6, "the lease is renewed")
+ for renewals := 0; renewals < 2; renewals++ {
+ previous := e.leaseExpiry(holder)
+ require.Eventually(t, func() bool { return e.leaseExpiry(holder).After(previous) },
+ 30*time.Second, 10*time.Millisecond, "renewal %d moves the expiry forward", renewals+1)
+ }
+ require.True(t, e.leaseExpiry(holder).After(first))
require.NoError(t, leaseCtx.Err())
require.False(t, lost.Load())
diff --git a/internal/graft/corestore/metastore.go b/internal/graft/corestore/metastore.go
index 23f3ad0..691a4a7 100644
--- a/internal/graft/corestore/metastore.go
+++ b/internal/graft/corestore/metastore.go
@@ -74,8 +74,8 @@ type InitialState struct {
// CheckpointInput is the state MetaStore.CreateCheckpoint records.
type CheckpointInput struct {
- // Seq is the changeset sequence of the reference snapshot: 1..head sequence,
- // greater than the newest checkpoint's.
+ // Seq is the changeset sequence of the reference snapshot: 1..head sequence.
+ // A newer checkpoint covers it (CreateCheckpoint returns that one).
Seq uint64
// Refs is the reference snapshot at Seq (every existing reference, HEAD
// included); it holds no absent value and is not empty.
@@ -181,13 +181,14 @@ type MetaStore interface {
UpdateReview(ctx context.Context, repoID int64, id string, version int64, r *gitalypb.Review, requireRoots bool) error
// CreateCheckpoint records a checkpoint and its reference snapshot in one
- // transaction, with the database time as its creation time. Its first
- // statement locks the repository row (doc/graft/PLAN7.md, LOCK ORDER). It
- // requires Seq <= head sequence, Seq greater than the newest checkpoint's,
- // an owned pack key of the repository and a valid snapshot, else an error and
- // nothing is written. A checkpoint at Seq that exists already is returned
- // unchanged with created false (its pack is kept; the caller's becomes an
- // orphan). ErrRepositoryNotFound for an unknown repository.
+ // transaction, with the database time as its creation time. It does not lock
+ // the repository row (doc/graft/PLAN7.md, LOCK ORDER). It requires Seq <=
+ // head sequence, an owned pack key of the repository, a valid snapshot and
+ // the pack's write deadline not reached on the database clock, else an error
+ // and nothing is written. A checkpoint that covers the request (one at Seq,
+ // or a newer one) is returned unchanged with created false: its pack is
+ // kept, and the caller's becomes an orphan. ErrRepositoryNotFound for an
+ // unknown repository.
CreateCheckpoint(ctx context.Context, repoID int64, in CheckpointInput) (cp Checkpoint, created bool, err error)
// NewestCheckpoint returns the checkpoint with the greatest sequence; ok is
// false when the repository has none.
diff --git a/internal/graft/corestore/sqlstore/checkpoint.go b/internal/graft/corestore/sqlstore/checkpoint.go
index 108f5cc..99b90ed 100644
--- a/internal/graft/corestore/sqlstore/checkpoint.go
+++ b/internal/graft/corestore/sqlstore/checkpoint.go
@@ -62,7 +62,11 @@ func (s *Store) CreateCheckpoint(ctx context.Context, repoID int64, in corestore
return err
}
if ok && newest.Seq > in.Seq {
- return fmt.Errorf("storage core: checkpoint at changeset %d is older than the newest checkpoint at %d of repository %d", in.Seq, newest.Seq, repoID)
+ // A newer checkpoint (the background maintenance's, another node's)
+ // covers this one: it is returned, and this build's pack is an orphan
+ // that garbage collection reclaims.
+ cp = newest
+ return nil
}
if err := s.checkPackDeadline(ctx, tx, in.PackDeadline); err != nil {
diff --git a/internal/graft/corestore/sqlstore/deadline_test.go b/internal/graft/corestore/sqlstore/deadline_test.go
index 5144d86..9b5c9e4 100644
--- a/internal/graft/corestore/sqlstore/deadline_test.go
+++ b/internal/graft/corestore/sqlstore/deadline_test.go
@@ -2,6 +2,7 @@ package sqlstore_test
import (
"context"
+ "fmt"
"testing"
"time"
@@ -11,6 +12,52 @@ import (
"github.com/feder-cr/git-ai/internal/graft/corestore/sqlstore"
)
+// TestConcurrentCheckpointsOfOneSequence verifies that builds of the same
+// checkpoint racing each other (a CreateCheckpoint request and the background
+// maintenance, two nodes) all succeed: one records it, the others get it back
+// (integrated review m7). The record does not lock the repository row, so on
+// Postgres they meet at the primary key.
+func TestConcurrentCheckpointsOfOneSequence(t *testing.T) {
+ forEachDialect(t, func(t *testing.T, ctx context.Context, s *sqlstore.Store) {
+ repo, _ := headRepo(t, ctx, s, "racing checkpoints", 1)
+ const builds = 6
+
+ type result struct {
+ cp corestore.Checkpoint
+ created bool
+ err error
+ }
+ start := make(chan struct{})
+ results := make(chan result, builds)
+ for i := 0; i < builds; i++ {
+ go func() {
+ <-start
+ cp, created, err := s.CreateCheckpoint(ctx, repo.ID, corestore.CheckpointInput{
+ Seq: 1, Refs: map[string]corestore.Target{"refs/heads/main": oid(1)},
+ PackKey: ownedKey(repo.ID, fmt.Sprint("build ", i)), PackBytes: 1, PackDeadline: time.Now().Add(time.Hour),
+ })
+ results <- result{cp: cp, created: created, err: err}
+ }()
+ }
+ close(start)
+
+ var recorded []corestore.Checkpoint
+ created := 0
+ for i := 0; i < builds; i++ {
+ r := <-results
+ require.NoError(t, r.err)
+ if r.created {
+ created++
+ }
+ recorded = append(recorded, r.cp)
+ }
+ require.Equal(t, 1, created, "one build records the checkpoint")
+ for _, cp := range recorded {
+ require.Equal(t, recorded[0], cp, "every build gets the recorded checkpoint")
+ }
+ })
+}
+
// TestCommitChecksThePackDeadlineOnTheDatabaseClock verifies the write
// deadline of a pack in the commit transaction (integrated review m6). A
// writer's monotonic clock stops while its host is suspended or its VM paused,
diff --git a/internal/graft/corestore/sqlstore/gc_test.go b/internal/graft/corestore/sqlstore/gc_test.go
index 050c055..b5dafec 100644
--- a/internal/graft/corestore/sqlstore/gc_test.go
+++ b/internal/graft/corestore/sqlstore/gc_test.go
@@ -197,16 +197,24 @@ func TestCheckpointLifecycle(t *testing.T) {
require.False(t, created)
require.Equal(t, cp, again)
+ // An older build is covered by the newest checkpoint, which it gets back
+ // (a request that lost the race against the background maintenance).
+ older, created, err := s.CreateCheckpoint(ctx, repo.ID, corestore.CheckpointInput{
+ Seq: 1, Refs: refs, PackKey: ownedKey(repo.ID, "older build"), PackDeadline: time.Now().Add(time.Hour),
+ })
+ require.NoError(t, err)
+ require.False(t, created)
+ require.Equal(t, cp, older)
+
for name, bad := range map[string]corestore.CheckpointInput{
- "sequence 0": {Seq: 0, Refs: refs, PackKey: ownedKey(repo.ID, "x"), PackDeadline: time.Now().Add(time.Hour)},
- "beyond the head": {Seq: 4, Refs: refs, PackKey: ownedKey(repo.ID, "x"), PackDeadline: time.Now().Add(time.Hour)},
- "older than the newest": {Seq: 1, Refs: refs, PackKey: ownedKey(repo.ID, "x"), PackDeadline: time.Now().Add(time.Hour)},
- "legacy pack key": {Seq: 3, Refs: refs, PackKey: legacyKey(repo.ID, "x"), PackDeadline: time.Now().Add(time.Hour)},
- "another repository": {Seq: 3, Refs: refs, PackKey: ownedKey(repo.ID+1000, "x"), PackDeadline: time.Now().Add(time.Hour)},
- "no pack key": {Seq: 3, Refs: refs},
- "no references": {Seq: 3, PackKey: ownedKey(repo.ID, "x"), PackDeadline: time.Now().Add(time.Hour)},
- "absent target": {Seq: 3, Refs: map[string]corestore.Target{"refs/heads/main": ""}, PackKey: ownedKey(repo.ID, "x"), PackDeadline: time.Now().Add(time.Hour)},
- "empty name": {Seq: 3, Refs: map[string]corestore.Target{"": oid(1)}, PackKey: ownedKey(repo.ID, "x"), PackDeadline: time.Now().Add(time.Hour)},
+ "sequence 0": {Seq: 0, Refs: refs, PackKey: ownedKey(repo.ID, "x"), PackDeadline: time.Now().Add(time.Hour)},
+ "beyond the head": {Seq: 4, Refs: refs, PackKey: ownedKey(repo.ID, "x"), PackDeadline: time.Now().Add(time.Hour)},
+ "legacy pack key": {Seq: 3, Refs: refs, PackKey: legacyKey(repo.ID, "x"), PackDeadline: time.Now().Add(time.Hour)},
+ "another repository": {Seq: 3, Refs: refs, PackKey: ownedKey(repo.ID+1000, "x"), PackDeadline: time.Now().Add(time.Hour)},
+ "no pack key": {Seq: 3, Refs: refs},
+ "no references": {Seq: 3, PackKey: ownedKey(repo.ID, "x"), PackDeadline: time.Now().Add(time.Hour)},
+ "absent target": {Seq: 3, Refs: map[string]corestore.Target{"refs/heads/main": ""}, PackKey: ownedKey(repo.ID, "x"), PackDeadline: time.Now().Add(time.Hour)},
+ "empty name": {Seq: 3, Refs: map[string]corestore.Target{"": oid(1)}, PackKey: ownedKey(repo.ID, "x"), PackDeadline: time.Now().Add(time.Hour)},
} {
_, _, err := s.CreateCheckpoint(ctx, repo.ID, bad)
require.Error(t, err, name)