Git designed for AI agents.
WP79 (1/2): milestone 3 end to end, and the docs
Federico Elia6306b829f8c3
diff --git a/doc/graft/PLAN7.md b/doc/graft/PLAN7.mdindex ca96608..c376650 100644--- a/doc/graft/PLAN7.md+++ b/doc/graft/PLAN7.md@@ -1773,6 +1773,30 @@ every entry point returns `FailedPrecondition`.| WP78 | proto, handlers, SDK, CLI | WP74, WP75 || WP79 | e2e, STORAGE.md and CONFIGURATION.md, benchmark | all |+**As implemented (WP79, e2e and docs; the benchmark waits for its window).**+* `storage_gc_test.go`, in-process servers sharing the database and the+ bucket, on SQLite and Postgres:+ * a repository written on two nodes is checkpointed on node A;+ * node B collects every covered changeset pack once the checkpoint is+ gc_grace old;+ * a node that joins after the collection hydrates from the checkpoint and+ serves every write;+ * writes go on and every node serves them;+ * an orphan pack and a removed repository are collected.+* `storage_localrefs_test.go`, the real `gitaly` binary:+ * the daily housekeeping, scheduled a minute ahead, backfills the+ keep-around references of a review written without them (the+ authoritative `Housekeep` path end to end);+ * the background maintenance checkpoints;+ * the deferred durability scenarios.+* STORAGE.md:+ * checkpoints and garbage collection for operators;+ * the clock assumption of `gc_margin`, precisely: a rate drift over+ `gc_grace`, with a forward jump of the database clock counting;+ * the configuration;+ * the milestone's e2e.+* CONFIGURATION.md: every key.+**Opus writes** everything that deletes data, is concurrent, is a compare-and-swapor is crash recovery: WP71–WP77 entirely, the handlers of WP78, and the race, crashand e2e tests. **Haiku drafts** from an exact spec, monitored and checked by Opus:diff --git a/doc/graft/STORAGE.md b/doc/graft/STORAGE.mdindex 7f2950d..abc919f 100644--- a/doc/graft/STORAGE.md+++ b/doc/graft/STORAGE.md@@ -322,6 +322,28 @@ authoritative mode.### Checkpoints and garbage collection+How it stays safe:+- Every write uploads its pack under a new key of its own, and commits the+ database row that references it within `gc_grace - gc_margin` of a database+ read that found the repository. A write that misses that deadline fails, and+ its pack is an orphan.+- Garbage collection deletes a pack only when:+ - no row references it, and the collector itself observed that at least+ `gc_grace` earlier;+ - or it is covered by a checkpoint created at least `gc_grace` ago.+- Every "how old" decision uses the database clock: Postgres's `now()`; with+ SQLite, the host clock, since SQLite serves one host. The clocks of the+ nodes never decide.+- A writer measures its deadline on its own monotonic clock, so the steps of+ a node's wall clock do not matter. The assumption is that, over `gc_grace`,+ the writer's clock runs at most `gc_margin` slower than the database's, a+ forward jump of the database clock included. With the defaults, 5 minutes+ over an hour. Do not step the database host's clock forward by more than+ `gc_margin`.+- `gc_grace` must also be greater than `sync_timeout + gc_margin`, so that a+ sync that started from an older checkpoint never meets a pack deleted under+ it.+```shgraft storage checkpoint -address unix:/var/opt/graft/gitaly.sock -repo @hashed/ab/cd/x.git [--json]graft storage gc -address unix:/var/opt/graft/gitaly.sock -repo @hashed/ab/cd/x.git [--dry-run] [--json]@@ -532,6 +554,18 @@ or re-import the repository from scratch (seepath: the repository it named cannot be identified. Check the storage, thenremove the marker; the repository's next request hydrates it if it is stale.+Milestone 3 is proved by `internal/graft/e2e/storage_gc_test.go` and+`storage_localrefs_test.go`, on SQLite and Postgres:+- a repository written on two nodes is checkpointed on one; the other collects+ the covered packs once the checkpoint is `gc_grace` old;+- a node that joins afterwards hydrates from the checkpoint and serves every+ write, and writes go on after the collection;+- orphan packs and removed repositories are collected;+- with the real `gitaly` binary: the deferred reference durability across+ process kills, a simulated power loss and a graceful upgrade; the background+ maintenance; and the daily housekeeping that backfills the keep-around+ references of a review from before the REVIEW ROOTS RULE.+These behaviours are proved by the end-to-end tests`internal/graft/e2e/storage_*_test.go`, with several real servers sharing onedatabase and one bucket, on SQLite and Postgres: writes on one node and readsdiff --git a/internal/graft/e2e/storage_gc_test.go b/internal/graft/e2e/storage_gc_test.gonew file mode 100644index 0000000..a0f22cd--- /dev/null+++ b/internal/graft/e2e/storage_gc_test.go@@ -0,0 +1,129 @@+package e2e++import (+ "bytes"+ "crypto/sha256"+ "fmt"+ "testing"+ "time"++ "github.com/stretchr/testify/require"++ "github.com/feder-cr/git-ai/internal/graft/corestore"+ "github.com/feder-cr/git-ai/internal/graft/corestore/sqlstore"+ "github.com/feder-cr/git-ai/proto/go/gitalypb"+ "github.com/feder-cr/git-ai/sdk"+)++// TestStorageGCAcrossNodes runs the compaction of milestone 3 on a cluster of+// real servers sharing one database and one bucket (doc/graft/PLAN7.md): node+// A checkpoints a repository written on two nodes, node B collects the packs+// the checkpoint covers once it is gc_grace old, and a node that joins+// afterwards hydrates from the checkpoint and serves every write, before and+// after the collection. Orphan packs and removed repositories are collected+// too.+func TestStorageGCAcrossNodes(t *testing.T) {+ forEachStorageDialect(t, func(t *testing.T, dialect sqlstore.Dialect) {+ graft, tokens := workerGraft(1)+ cluster := newStorageCluster(t, dialect, corestore.ModeAuthoritative, unbatched(graft))+ // A short grace, so that the test sees packs collected. The writers'+ // deadline (grace - margin) stays long enough for an upload under load.+ cluster.adjust = func(core *corestore.Core) {+ core.GCGrace = 2 * time.Second+ core.GCMargin = 200 * time.Millisecond+ }+ a := cluster.addNode(t)+ b := cluster.addNode(t)++ rel := a.createRepository(t).GetRelativePath()+ onA := a.agent(t, rel, tokens[workerName(0)])+ onB := b.agent(t, rel, tokens[workerName(0)])++ written := map[string]string{}+ base, err := commitFile(a.ctx, onA, workerName(0), "main", "", "README.md", "base\n")+ require.NoError(t, err)+ written["main"] = base+ for i, repo := range []*sdk.Repo{onA, onB, onA, onB} {+ branch := fmt.Sprintf("topic/%d", i)+ commit, err := commitFile(a.ctx, repo, workerName(0), branch, base, branch+".txt", branch+"\n")+ require.NoError(t, err)+ written[branch] = commit+ }+ before := a.referencedPacks(t, rel)+ require.GreaterOrEqual(t, len(before), len(written), "one pack per write")++ cp, err := a.operator(t, rel).StorageCheckpoint(a.ctx)+ require.NoError(t, err)+ require.True(t, cp.Created)+ require.Equal(t, a.record(t, rel).HeadSeq, cp.Sequence)++ // Node B collects once the checkpoint is gc_grace old.+ var collected *sdk.StorageGC+ require.Eventually(t, func() bool {+ res, err := b.operator(t, rel).StorageGC(b.ctx, false)+ require.NoError(t, err)+ collected = res+ return res.Boundary == cp.Sequence && res.CoveredDeleted > 0+ }, 60*time.Second, 200*time.Millisecond)+ require.Equal(t, uint32(len(before)), collected.CoveredDeleted, "every changeset pack is covered by the checkpoint")+ inBucket := bucketPacks(t, cluster.bucketURL)+ for key := range before {+ require.False(t, inBucket[key], "covered pack %s deleted", key)+ }++ // A node that joins now has only the checkpoint to hydrate from.+ c := cluster.addNode(t)+ onC := c.agent(t, rel, tokens[workerName(0)])+ for branch, commit := range written {+ got, err := onC.ResolveRevision(c.ctx, branch)+ require.NoError(t, err, "branch %s", branch)+ require.Equal(t, commit, got, "branch %s", branch)+ }+ content, err := onC.ReadFile(c.ctx, "topic/3", "topic/3.txt")+ require.NoError(t, err)+ require.Equal(t, "topic/3\n", string(content))+ c.requireConsistent(t, rel)++ // Writes go on after the collection, and every node serves them.+ after, err := commitFile(b.ctx, onB, workerName(0), "after-gc", base, "after.txt", "after the collection\n")+ require.NoError(t, err)+ for _, node := range []struct {+ node storageNode+ repo *sdk.Repo+ }{{a, onA}, {b, onB}, {c, onC}} {+ got, err := node.repo.ResolveRevision(node.node.ctx, "after-gc")+ require.NoError(t, err, node.node.name)+ require.Equal(t, after, got, node.node.name)+ node.node.requireConsistent(t, rel)+ }++ // An orphan (the pack of a write that died before its database commit)+ // is deleted gc_grace after a collection observed it.+ orphan := []byte("the pack of a write that died")+ orphanKey := corestore.NewPackKey(a.record(t, rel).ID, sha256.Sum256(orphan))+ require.NoError(t, a.core.Blobs.Put(a.ctx, orphanKey, bytes.NewReader(orphan)))+ require.Eventually(t, func() bool {+ res, err := a.operator(t, rel).StorageGC(a.ctx, false)+ require.NoError(t, err)+ return res.OrphansDeleted == 1+ }, 60*time.Second, 200*time.Millisecond)+ require.False(t, bucketPacks(t, cluster.bucketURL)[orphanKey])++ // A removed repository's objects go once its tombstone is gc_grace old.+ repoID := a.record(t, rel).ID+ _, err = gitalypb.NewRepositoryServiceClient(a.conn(t)).RemoveRepository(a.setupCtx, &gitalypb.RemoveRepositoryRequest{Repository: a.repository(rel)})+ require.NoError(t, err)+ client, err := sdk.Dial(a.ctx, a.cfg.SocketPath, sdk.WithStorage(a.cfg.Storages[0].Name), sdk.WithToken(sharedTestToken))+ require.NoError(t, err)+ t.Cleanup(func() { _ = client.Close() })+ require.Eventually(t, func() bool {+ res, err := client.StorageGCRemoved(a.ctx, "", false)+ require.NoError(t, err)+ return res.Repositories == 1+ }, 60*time.Second, 200*time.Millisecond)+ prefix := fmt.Sprintf("repos/%d/", repoID)+ for key := range bucketPacks(t, cluster.bucketURL) {+ require.NotContains(t, key, prefix, "an object of the removed repository is left")+ }+ })+}diff --git a/internal/graft/e2e/storage_localrefs_test.go b/internal/graft/e2e/storage_localrefs_test.goindex f682252..cfe84b8 100644--- a/internal/graft/e2e/storage_localrefs_test.go+++ b/internal/graft/e2e/storage_localrefs_test.go@@ -25,11 +25,14 @@ import ("github.com/feder-cr/git-ai/internal/featureflag""github.com/feder-cr/git-ai/internal/git/gittest""github.com/feder-cr/git-ai/internal/gitaly/config"+ "github.com/feder-cr/git-ai/internal/graft/corestore""github.com/feder-cr/git-ai/internal/graft/corestore/coretest""github.com/feder-cr/git-ai/internal/graft/corestore/localrefs""github.com/feder-cr/git-ai/internal/graft/corestore/sqlstore"+ "github.com/feder-cr/git-ai/internal/helper/duration""github.com/feder-cr/git-ai/internal/testhelper""github.com/feder-cr/git-ai/internal/testhelper/testcfg"+ "github.com/feder-cr/git-ai/proto/go/gitalypb""github.com/feder-cr/git-ai/sdk")@@ -186,6 +189,13 @@ func (n *realNode) log(t *testing.T) string {func (n *realNode) storagePath() string { return n.cfg.Storages[0].Path }+// git runs git in the node's local copy of the repository, reading the disk+// directly.+func (n *realNode) git(t *testing.T, relPath string, args ...string) string {+ t.Helper()+ return strings.TrimSpace(string(gittest.Exec(t, n.cfg, append([]string{"-C", n.repoPath(relPath)}, args...)...)))+}+// freeAddr returns a local TCP address that was free a moment ago.func freeAddr(t *testing.T) string {t.Helper()@@ -489,6 +499,66 @@ func TestStorageBackgroundMaintenanceCheckpoints(t *testing.T) {n.requireServed(t, rel, written)}+// TestStorageHousekeepingBackfillsReviewRoots runs the daily housekeeping of+// the real server in authoritative mode (doc/graft/PLAN7.md, REVIEW ROOTS RULE,+// the backstop of the daily worker): a review written without the keep-around+// references of its commits (a review from before the rule) gets them, in the+// database and on the node, through the ordinary write path.+func TestStorageHousekeepingBackfillsReviewRoots(t *testing.T) {+ t.Parallel()++ // The daily worker starts at a minute of the clock: the next one that the+ // server is sure to see as future.+ start := time.Now().Add(time.Minute)+ if start.Second() > 40 {+ start = start.Add(time.Minute)+ }+ n := newRealNode(t)+ n.cfg.DailyMaintenance = config.DailyJob{+ Hour: uint(start.Hour()),+ Minute: uint(start.Minute()),+ Duration: duration.Duration(5 * time.Minute),+ Storages: []string{n.cfg.Storages[0].Name},+ }+ n.cfgPath = testcfg.WriteTemporaryGitalyConfigFile(t, n.cfg)+ n.start(t)++ rel := n.createRepository(t)+ onA := n.agent(t, rel)+ base, err := commitFile(n.ctx, onA, workerName(0), "main", "", "README.md", "base\n")+ require.NoError(t, err)+ head, err := commitFile(n.ctx, onA, workerName(0), "topic", base, "topic.txt", "topic\n")+ require.NoError(t, err)+ state, err := n.operator(t, rel).StorageState(n.ctx)+ require.NoError(t, err)++ // A review from before the rule: recorded without its roots.+ meta, err := sqlstore.Open(n.ctx, n.cfg.Graft.Storage.Database)+ require.NoError(t, err)+ t.Cleanup(func() { _ = meta.Close() })+ _, err = meta.InsertReview(n.ctx, state.RepositoryID, func(id string) (*gitalypb.Review, error) {+ return &gitalypb.Review{Id: id, TargetBranch: []byte("main"), HeadCommitId: head, BaseCommitId: base, Title: []byte("before the rule")}, nil+ }, false)+ require.NoError(t, err)++ roots := []string{corestore.KeepAroundRef(head), corestore.KeepAroundRef(base)}+ require.Eventually(t, func() bool {+ refs, _, err := meta.Refs(n.ctx, state.RepositoryID)+ require.NoError(t, err)+ for _, root := range roots {+ if refs[root].Absent() {+ return false+ }+ }+ return true+ }, 4*time.Minute, time.Second, "the daily housekeeping backfills the roots")+ require.GreaterOrEqual(t, n.counter(t, "graft_review_roots_backfilled_total"), float64(len(roots)))+ for _, root := range roots {+ require.NotEmpty(t, n.git(t, rel, "rev-parse", "--verify", root), "the node has %s", root)+ }+ n.requireServed(t, rel, map[string]string{"main": base, "topic": head})+}+// TestStorageLocalReferencesStackDepthIsBounded writes many references// without fsync from concurrent writers and checks that git's compaction keeps// the stack shallow: the durable step only makes it durable, compaction stays