Git designed for AI agents.
PLAN11 follow-ups: the scan fallback tested, compaction failures counted, the due size
Federico Elia10bf1f9abb46
diff --git a/doc/graft/LAND-BATCHING.md b/doc/graft/LAND-BATCHING.mdindex 5b2d4e6..d5dd6bc 100644--- a/doc/graft/LAND-BATCHING.md+++ b/doc/graft/LAND-BATCHING.md@@ -415,7 +415,9 @@ The batch executor (`internal/gitaly/service/operations`) defines:The hooks (`internal/gitaly/hook`) define `graft_land_intents_seconds`(histogram): the land intents of one prepared reference transaction in the filechangelog, the wait for their lock file, one write and one fsync, under the-references' locks ([PLAN11](PLAN11.md), 4.2).+references' locks ([PLAN11](PLAN11.md), 4.2), and+`graft_land_intents_compaction_failures_total` (counter): compactions of the+land intents that failed, which leave the intents as they were.The server registers the lanes of the process (`landlane.Shared`) and thesemetrics with the Prometheus registry, so they are exported on `/metrics`. Thediff --git a/doc/graft/PLAN11.md b/doc/graft/PLAN11.mdindex f6cb1ac..7b5d60e 100644--- a/doc/graft/PLAN11.md+++ b/doc/graft/PLAN11.md@@ -165,8 +165,10 @@ of a land as intents in the prepared phase, before git may commit:64 KiB on, when at least half of its intents may go: the rest is written to a new file,fsync'd and renamed over the old one, with the directory fsync'd. The lookups compact,and so does the append: once the file doubled since the last compaction, the hook- starts one after the update, outside the references' locks, so that the intents of- lands that never look up (fast-forwards only) go too. The intents of updates that were+ starts one in the prepared phase, in a goroutine it does not wait for, which takes none+ of the references' locks, only the intents' lock file, so that the intents of lands+ that never look up (fast-forwards only) go too. A failed compaction is counted+ (`graft_land_intents_compaction_failures_total`) and logged. The intents of updates that werenever committed stay; there is one line, about 200 bytes, per such update.* The cost is one write and one fsync per land update in the prepared phase, under thereferences' locks, serialized per repository by the intents' lock file, not shareddiff --git a/internal/gitaly/hook/changeset_order.go b/internal/gitaly/hook/changeset_order.goindex 9a26467..3025ed8 100644--- a/internal/gitaly/hook/changeset_order.go+++ b/internal/gitaly/hook/changeset_order.go@@ -33,6 +33,11 @@ var (Buckets: []float64{0.0005, 0.001, 0.002, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1},})+ landIntentsCompactionFailuresTotal = promauto.NewCounter(prometheus.CounterOpts{+ Name: "graft_land_intents_compaction_failures_total",+ Help: "Compactions of the land intents that a prepared reference transaction started and that failed; the intents stay as they were.",+ })+changelogAppendSeconds = promauto.NewHistogram(prometheus.HistogramOpts{Name: "graft_changelog_append_seconds",Help: "Time of one write of the changeset log, with its fsync, for a group of reference transactions.",diff --git a/internal/gitaly/hook/manager.go b/internal/gitaly/hook/manager.goindex 28065e8..de80d7b 100644--- a/internal/gitaly/hook/manager.go+++ b/internal/gitaly/hook/manager.go@@ -87,8 +87,9 @@ type GitLabHookManager struct {observer *eager.Observer// storageVoter commits reference transactions to the Graft storage core, nil when it is off.storageVoter StorageVoter- // startLandIntentsCompaction runs a compaction of land intents that a prepared transaction- // asked for, after the hook. Nil runs it in a goroutine; tests set it.+ // startLandIntentsCompaction starts a compaction of land intents that a prepared+ // transaction asked for, without the hook waiting for it. Nil runs it in a goroutine;+ // tests set it.startLandIntentsCompaction func(compact func())}diff --git a/internal/gitaly/hook/referencetransaction.go b/internal/gitaly/hook/referencetransaction.goindex f163a14..29a929c 100644--- a/internal/gitaly/hook/referencetransaction.go+++ b/internal/gitaly/hook/referencetransaction.go@@ -234,11 +234,15 @@ func (m *GitLabHookManager) recordLandIntents(ctx context.Context, payload gitcmreturn err}- // The compaction reads the changelog and rewrites the intents: it runs after the hook, not- // under the locks of the references, and its failure leaves the intents as they are.+ // The compaction reads the changelog and rewrites the intents. It starts in the prepared+ // phase, in a goroutine that the hook does not wait for, so it runs while git still holds+ // the references' locks and after: it takes none of them, only the intents' lock file, which+ // the intents of other transactions of the repository wait for meanwhile. Its failure leaves+ // the intents as they are; it is counted and logged.if compact {run := func() {if err := changelog.CompactLandIntents(repoPath); err != nil {+ landIntentsCompactionFailuresTotal.Inc()m.logger.WithError(err).WithField("repository", repoPath).Error("graft: compacting the land intents failed")}}diff --git a/internal/gitaly/hook/referencetransaction_intents_test.go b/internal/gitaly/hook/referencetransaction_intents_test.goindex 7838acf..cfe9eb7 100644--- a/internal/gitaly/hook/referencetransaction_intents_test.go+++ b/internal/gitaly/hook/referencetransaction_intents_test.go@@ -9,6 +9,7 @@ import ("strings""testing"+ "github.com/prometheus/client_golang/prometheus/testutil""github.com/stretchr/testify/require""github.com/feder-cr/git-ai/internal/git/gitcmd"@@ -118,6 +119,30 @@ func TestReferenceTransactionHook_recordsLandIntents(t *testing.T) {require.Equal(t, []changelog.LandRecord{{Landed: land.Landed}}, records2, "the intent of the update stays")})+ t.Run("a failed compaction is counted", func(t *testing.T) {+ t.Parallel()++ // The intents file asks for a compaction, which fails: the land index cannot be opened, a+ // directory is in its place.+ var lines strings.Builder+ for i := 0; i < 600; i++ {+ line, err := json.Marshal(map[string]any{"ref": []byte("refs/heads/main"), "original": fmt.Sprintf("%040x", i+1), "landed": fmt.Sprintf("%040x", i+1), "invocation": fmt.Sprint("inv-", i)})+ require.NoError(t, err)+ lines.Write(line)+ lines.WriteByte('\n')+ }++ failures := testutil.ToFloat64(landIntentsCompactionFailuresTotal)+ _, err := prepare(t, testcfg.Build(t), "refs/heads/main", func(repoPath string) {+ require.NoError(t, os.MkdirAll(filepath.Join(repoPath, "graft", "changelog.lands"), 0o755))+ require.NoError(t, os.WriteFile(filepath.Join(repoPath, "graft", "changelog.intents"), []byte(lines.String()), 0o644))+ _, err := changelog.Append(repoPath, changelog.Meta{Operation: "push"}, []changelog.Update{{Reference: "refs/heads/other", OldOID: gittest.DefaultObjectHash.ZeroOID.String(), NewOID: strings.Repeat("c", 40)}})+ require.NoError(t, err)+ }, func(compact func()) { compact() })+ require.NoError(t, err, "the update does not wait for its compaction, nor fail with it")+ require.GreaterOrEqual(t, testutil.ToFloat64(landIntentsCompactionFailuresTotal), failures+1)+ })+t.Run("a failure to record the intents aborts the update", func(t *testing.T) {t.Parallel()diff --git a/internal/graft/changelog/lands_intents.go b/internal/graft/changelog/lands_intents.goindex 656b51d..cf33fef 100644--- a/internal/graft/changelog/lands_intents.go+++ b/internal/graft/changelog/lands_intents.go@@ -61,8 +61,8 @@ var (// durable, see RecordLandIntents. A file created again at the same path, as with a// repository created again, is another file.intentDirsSynced sync.Map- // intentsCompactionSize holds, by path, the size of the intents file when RecordLandIntents- // last asked for a compaction, see compactionDue.+ // intentsCompactionSize holds, by path, the size of the intents file that the last+ // CompactLandIntents of this process left, see compactionDue.intentsCompactionSize sync.Map)@@ -147,20 +147,17 @@ func RecordLandIntents(repoPath, reference, invocation string, lands []Land) (co}// compactionDue reports whether an intents file of the given size is due for a compaction: from-// compactIntentsMinSize on, each time it doubled since the last time it was due, or since the last-// CompactLandIntents left it, in this process.+// compactIntentsMinSize on, once it doubled since the last CompactLandIntents of this process left+// it. Only a compaction that runs moves that size: a request that found one running does not, so+// the next append asks again.func compactionDue(path string, size int64) bool {if size < compactIntentsMinSize {return false}last, _ := intentsCompactionSize.Load(path)- if lastSize, ok := last.(int64); ok && size < 2*lastSize {- return false- }-- intentsCompactionSize.Store(path, size)- return true+ lastSize, ok := last.(int64)+ return !ok || size >= 2*lastSize}// compactionsRunning holds the repositories whose intents CompactLandIntents compacts right now.diff --git a/internal/graft/changelog/lands_intents_test.go b/internal/graft/changelog/lands_intents_test.goindex 8cdbfa3..3278ca0 100644--- a/internal/graft/changelog/lands_intents_test.go+++ b/internal/graft/changelog/lands_intents_test.go@@ -325,6 +325,82 @@ func TestLandIntentsCompaction(t *testing.T) {})}+func TestLandIntentsCompactionDue(t *testing.T) {+ t.Parallel()++ // A compaction that was asked for while another one ran does not move the size from which+ // the next one is due: the next append asks again.+ repoPath := t.TempDir()+ path := intentsPath(repoPath)+ require.False(t, compactionDue(path, compactIntentsMinSize-1))+ require.True(t, compactionDue(path, compactIntentsMinSize))+ require.True(t, compactionDue(path, compactIntentsMinSize), "asking does not move the size")++ compactionsRunning.Store(repoPath, struct{}{})+ require.NoError(t, CompactLandIntents(repoPath))+ compactionsRunning.Delete(repoPath)+ require.True(t, compactionDue(path, compactIntentsMinSize), "a compaction that did not run does not move it")++ recordIntent(t, repoPath, "refs/heads/main", landOID(1), landOID(2))+ require.NoError(t, CompactLandIntents(repoPath))+ size := fileSize(t, path)+ require.False(t, compactionDue(path, max(compactIntentsMinSize, 2*size)-1), "a compaction that ran moves it")+ require.True(t, compactionDue(path, max(compactIntentsMinSize, 2*size)))+}++func TestLandsIndexScanFallsBackFromAWrongOffset(t *testing.T) {+ t.Parallel()++ // The index in memory remembers the offset of its mark's line; an offset whose line does not+ // hold the mark's entry (another entry, the middle of a line, past the end) is not used: the+ // lookup reads from the index point instead, and answers as it would.+ for _, tc := range []struct {+ desc string+ offset func(repoPath string) int64+ }{+ {desc: "the line of another entry", offset: func(string) int64 { return 0 }},+ {desc: "the middle of a line", offset: func(string) int64 { return 7 }},+ {desc: "past the end of the log", offset: func(repoPath string) int64 { return 1 << 30 }},+ } {+ t.Run(tc.desc, func(t *testing.T) {+ t.Parallel()++ repoPath := t.TempDir()+ first := appendLand(t, repoPath, "refs/heads/main", landOID(1), landOID(2))+ second := appendLand(t, repoPath, "refs/heads/main", landOID(3), landOID(4))+ _, found := lookup(t, repoPath, "refs/heads/main", landOID(1))+ require.True(t, found)++ index, release := acquireLandsIndex(landsPath(repoPath))+ index.mu.Lock()+ require.NotEqual(t, int64(-1), index.markOffset, "the warm index knows its mark's line")+ index.markOffset = tc.offset(repoPath)+ index.mu.Unlock()+ release()++ before, err := os.ReadFile(landsPath(repoPath))+ require.NoError(t, err)++ third := appendLand(t, repoPath, "refs/heads/main", landOID(5), landOID(6))+ for original, want := range map[string]LandRecord{+ landOID(1): {Sequence: first, Landed: landOID(2)},+ landOID(3): {Sequence: second, Landed: landOID(4)},+ landOID(5): {Sequence: third, Landed: landOID(6)},+ } {+ record, found := lookup(t, repoPath, "refs/heads/main", original)+ require.True(t, found, original)+ require.Equal(t, want, record)+ }++ // The index caught up from the index point; it was not rebuilt.+ after, err := os.ReadFile(landsPath(repoPath))+ require.NoError(t, err)+ require.True(t, bytes.HasPrefix(after, before), "the index file was extended, not written again")+ require.Greater(t, len(after), len(before))+ })+ }+}+func TestLandsIndexCompaction(t *testing.T) {t.Parallel()