Git designed for AI agents.
housekeeping: remove stale reftable table locks; test contention with both gits
Federico Elia0baff276e3e2
diff --git a/internal/git/housekeeping/clean_stale_data.go b/internal/git/housekeeping/clean_stale_data.goindex 22b38bc..972b1ff 100644--- a/internal/git/housekeeping/clean_stale_data.go+++ b/internal/git/housekeeping/clean_stale_data.go@@ -491,24 +491,46 @@ func FindStaleReferenceLocks(gracePeriod time.Duration) FindStaleFileFunc {}}-// FindStaleReftableLock provides a function which scans the reftable folder-// for stale a reftable lock and deletes it.+// FindStaleReftableLock provides a function which scans the reftable folder for the locks git's+// reftable stack takes that are older than ReftableLockfileGracePeriod, left by a git process that+// died: the lock of tables.list, and the lock of a table being compacted, <table>.ref.lock+// (flock_acquire in reftable/stack.c). A stale table lock keeps every later compaction from+// compacting that table, so git-pack-refs(1) fails on each housekeeping until it is gone.func FindStaleReftableLock(_ context.Context, repoPath string) ([]string, error) {- lockPath := filepath.Join(repoPath, "reftable", "tables.list.lock")- stat, err := os.Stat(lockPath)+ reftablePath := filepath.Join(repoPath, "reftable")+ entries, err := os.ReadDir(reftablePath)if err != nil {if errors.Is(err, fs.ErrNotExist) {return nil, nil}- return nil, fmt.Errorf("statting reftable lock: %w", err)+ return nil, fmt.Errorf("reading reftable directory: %w", err)}- if time.Since(stat.ModTime()) < ReftableLockfileGracePeriod {- return nil, nil+ var staleLocks []string+ for _, entry := range entries {+ name := entry.Name()+ if name != "tables.list.lock" && !strings.HasSuffix(name, ".ref.lock") {+ continue+ }++ info, err := entry.Info()+ if err != nil {+ if errors.Is(err, fs.ErrNotExist) {+ continue+ }++ return nil, fmt.Errorf("statting reftable lock: %w", err)+ }++ if time.Since(info.ModTime()) < ReftableLockfileGracePeriod {+ continue+ }++ staleLocks = append(staleLocks, filepath.Join(reftablePath, name))}- return []string{lockPath}, nil+ return staleLocks, nil}func removeUnnecessaryConfig(ctx context.Context, repository *localrepo.Repo, txManager transaction.Manager) (int, error) {diff --git a/internal/git/housekeeping/manager/commit_graph_test.go b/internal/git/housekeeping/manager/commit_graph_test.goindex 683f030..c9e12b7 100644--- a/internal/git/housekeeping/manager/commit_graph_test.go+++ b/internal/git/housekeeping/manager/commit_graph_test.go@@ -128,11 +128,16 @@ func TestWriteCommitGraphIfIdle(t *testing.T) {// TestWriteCommitGraphIfIdle_ConcurrentWithOptimizeRepository runs background commit-graph writes// while OptimizeRepository writes the commit-graph of the same repository. Without the shared-// guard, one of the two failed on the commit-graph chain lock in most rounds.+// guard, one of the two failed on the commit-graph chain lock in most rounds. On reftable the+// commits race OptimizeRepository's git-pack-refs(1) too, whose contention errors differ between the+// bundled gits: the test runs with both.func TestWriteCommitGraphIfIdle_ConcurrentWithOptimizeRepository(t *testing.T) {t.Parallel()- ctx := testhelper.Context(t)+ forEachBundledGit(t, testWriteCommitGraphIfIdleConcurrentWithOptimizeRepository)+}++func testWriteCommitGraphIfIdleConcurrentWithOptimizeRepository(t *testing.T, ctx context.Context) {cfg := testcfg.Build(t)repoProto, repoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{diff --git a/internal/git/housekeeping/manager/optimize_repository.go b/internal/git/housekeeping/manager/optimize_repository.goindex 22109e3..1ac2368 100644--- a/internal/git/housekeeping/manager/optimize_repository.go+++ b/internal/git/housekeeping/manager/optimize_repository.go@@ -571,20 +571,28 @@ func (m *RepositoryManager) packRefsIfNeeded(ctx context.Context, repo *localrepvar errReftableCompactionContended = errors.New("reftable compaction contended")// Exit codes and stderr of git-pack-refs(1) on a reftable repository when the stack is contended.-// They are git internals, read from git 2.56.0-rc1 (3bbf434) and 2.55.0-rc1 (1ea786d), the bundled-// builds, and pinned by TestPackRefsIfNeeded_reftableContention against the real git:+// They are git internals, read from the bundled builds, git 2.56.0-rc1 (3bbf434, "master") and+// 2.55.0-rc1 (1ea786d, "prev"). TestPackRefsIfNeeded_reftableContention pins 251 and 255 "data is+// locked" against both; 244 and 255 "data concurrently modified" need a writer racing pack-refs,+// which TestWriteCommitGraphIfIdle_ConcurrentWithOptimizeRepository runs with both, without+// guaranteeing the race in a given run://-// - reftable_be_optimize (refs/reftable-backend.c:1699) returns what reftable_stack_clean-// (reftable/stack.c:1845) returns as is, and git exits with it modulo 256 and an empty stderr:-// REFTABLE_LOCK_ERROR (-5) when tables.list.lock is held, REFTABLE_OUTDATED_ERROR (-12) when-// the stack changed under it (reftable/reftable-error.h).+// - reftable_be_optimize (refs/reftable-backend.c:1699 in master, :1656 in prev) returns what+// reftable_stack_clean (reftable/stack.c:1845 in master, :1807 in prev) returns as is, and git+// exits with it modulo 256 and an empty stderr. reftable_stack_clean takes tables.list.lock+// without waiting (no write options, so reftable.lockTimeout does not apply) and fails with+// REFTABLE_LOCK_ERROR (-5, exit 251) when a writer holds it. Only prev also fails with+// REFTABLE_OUTDATED_ERROR (-12, exit 244) when a writer changed the stack since pack-refs read+// it (reftable_stack_init_addition); master reloads the stack there instead. Both codes are in+// reftable/reftable-error.h.// - When the compaction itself fails, reftable_be_optimize reports it with error(), exiting 255,// and "unable to compact stack: <reftable_error_str>", whose texts for these two errors come// from reftable/error.c.//// git's own auto-compaction treats both errors as benign (reftable/stack.c:833). A lock left by a-// crashed git is no reason to retry here either: CleanStaleData removes tables.list.lock once it is-// older than housekeeping.ReftableLockfileGracePeriod, before the next pack-refs runs.+// git process that died is not contention that passes: CleanStaleData removes the locks of+// tables.list and of single tables once they are older than+// housekeeping.ReftableLockfileGracePeriod, before pack-refs runs.const (reftableLockErrorExitCode = 256 - 5reftableOutdatedErrorExitCode = 256 - 12diff --git a/internal/git/housekeeping/manager/optimize_repository_test.go b/internal/git/housekeeping/manager/optimize_repository_test.goindex fd6a05c..75f18b9 100644--- a/internal/git/housekeeping/manager/optimize_repository_test.go+++ b/internal/git/housekeeping/manager/optimize_repository_test.go@@ -1680,17 +1680,35 @@ func TestRepositoryManager_CleanStaleData_reftable(t *testing.T) {for _, tc := range []struct {desc string+ lock stringage time.Durationexpected boolexpectedMetrics cleanStaleDataMetrics}{{desc: "fresh lock",+ lock: "tables.list.lock",age: 0,expected: true,},{desc: "stale lock",+ lock: "tables.list.lock",+ age: housekeeping.ReftableLockfileGracePeriod + time.Minute,+ expected: false,+ expectedMetrics: cleanStaleDataMetrics{+ reftablelocks: 1,+ },+ },+ {+ desc: "fresh table lock",+ lock: "0x000000000001-0x000000000002-0123abcd.ref.lock",+ age: 0,+ expected: true,+ },+ {+ desc: "stale table lock",+ lock: "0x000000000001-0x000000000002-0123abcd.ref.lock",age: housekeeping.ReftableLockfileGracePeriod + time.Minute,expected: false,expectedMetrics: cleanStaleDataMetrics{@@ -1709,7 +1727,7 @@ func TestRepositoryManager_CleanStaleData_reftable(t *testing.T) {})repo := localrepo.NewTestRepo(t, cfg, repoProto)- path := filepath.Join(repoPath, "reftable", "tables.list.lock")+ path := filepath.Join(repoPath, "reftable", tc.lock)require.NoError(t, os.WriteFile(path, []byte{}, mode.File))filetime := time.Now().Add(-tc.age)diff --git a/internal/git/housekeeping/manager/pack_refs_contention_test.go b/internal/git/housekeeping/manager/pack_refs_contention_test.goindex a7460af..14db4c6 100644--- a/internal/git/housekeeping/manager/pack_refs_contention_test.go+++ b/internal/git/housekeeping/manager/pack_refs_contention_test.go@@ -1,14 +1,16 @@package managerimport (- "bytes""context""errors""fmt""os""path/filepath"+ "strings""testing"+ "time"+ "github.com/feder-cr/git-ai/internal/featureflag""github.com/feder-cr/git-ai/internal/git""github.com/feder-cr/git-ai/internal/git/gittest""github.com/feder-cr/git-ai/internal/git/housekeeping"@@ -17,6 +19,7 @@ import (gitalycfgprom "github.com/feder-cr/git-ai/internal/gitaly/config/prometheus""github.com/feder-cr/git-ai/internal/testhelper""github.com/feder-cr/git-ai/internal/testhelper/testcfg"+ "github.com/prometheus/client_golang/prometheus/testutil""github.com/sirupsen/logrus""github.com/stretchr/testify/require")@@ -44,9 +47,34 @@ func setupPackRefsRepository(t *testing.T, ctx context.Context, backend git.Refefunc alwaysPackReferences(context.Context) bool { return true }-// TestPackRefsIfNeeded_reftableContention runs the real git-pack-refs(1) while reftable's-// tables.list.lock is held, as a concurrent writer holds it, and pins the exit codes and stderr that-// isReftableContention classifies: a git that changes them fails here.+// forEachBundledGit runs test once with each bundled git, master and prev, selected by the feature+// flag in the context, instead of the one testhelper.Context picks at random: the reftable+// contention errors differ between them.+func forEachBundledGit(t *testing.T, test func(t *testing.T, ctx context.Context)) {+ t.Helper()++ gitCmdFactory := gittest.NewCommandFactory(t, testcfg.Build(t))+ versions := map[bool]string{}+ for _, master := range []bool{true, false} {+ version, err := gitCmdFactory.GitVersion(featureflag.ContextWithFeatureFlag(testhelper.Context(t), featureflag.GitMaster, master))+ require.NoError(t, err)+ versions[master] = version+ }+ require.NotEqual(t, versions[true], versions[false], "the feature flag does not select another git")++ for _, master := range []bool{true, false} {+ t.Run(testhelper.DependingOn(master, "git-master", "git-prev"), func(t *testing.T) {+ t.Parallel()+ t.Logf("git %s", versions[master])++ test(t, featureflag.ContextWithFeatureFlag(testhelper.Context(t), featureflag.GitMaster, master))+ })+ }+}++// TestPackRefsIfNeeded_reftableContention runs the real git-pack-refs(1) of both bundled gits while+// reftable's tables.list.lock is held, as a concurrent writer holds it, and pins the exit codes and+// stderr that isReftableContention classifies: a git that changes them fails here.func TestPackRefsIfNeeded_reftableContention(t *testing.T) {t.Parallel()@@ -71,26 +99,102 @@ func TestPackRefsIfNeeded_reftableContention(t *testing.T) {t.Run(tc.desc, func(t *testing.T) {t.Parallel()- ctx := testhelper.Context(t)- repo, repoPath := setupPackRefsRepository(t, ctx, git.ReferenceBackendReftables, tc.tables)- manager := New(gitalycfgprom.Config{}, testhelper.NewLogger(t), nil, nil)- strategy := mockOptimizationStrategy{shouldRepackReferences: alwaysPackReferences}+ forEachBundledGit(t, func(t *testing.T, ctx context.Context) {+ repo, repoPath := setupPackRefsRepository(t, ctx, git.ReferenceBackendReftables, tc.tables)+ manager := New(gitalycfgprom.Config{}, testhelper.NewLogger(t), nil, nil)+ strategy := mockOptimizationStrategy{shouldRepackReferences: alwaysPackReferences}- lockPath := filepath.Join(repoPath, "reftable", "tables.list.lock")- require.NoError(t, os.WriteFile(lockPath, nil, 0o644))+ lockPath := filepath.Join(repoPath, "reftable", "tables.list.lock")+ require.NoError(t, os.WriteFile(lockPath, nil, 0o644))- didPackRefs, err := manager.packRefsIfNeeded(ctx, repo, strategy)- require.ErrorIs(t, err, errReftableCompactionContended)- require.EqualError(t, err, tc.expectedErr)- require.False(t, didPackRefs)+ didPackRefs, err := manager.packRefsIfNeeded(ctx, repo, strategy)+ require.ErrorIs(t, err, errReftableCompactionContended)+ require.EqualError(t, err, tc.expectedErr)+ require.False(t, didPackRefs)++ // Once the writer is gone, the stack is compacted into a single table.+ require.NoError(t, os.Remove(lockPath))+ didPackRefs, err = manager.packRefsIfNeeded(ctx, repo, strategy)+ require.NoError(t, err)+ require.True(t, didPackRefs)+ require.Len(t, reftableTables(t, repoPath), 1)+ })+ })+ }+}++// reftableTables returns the tables of the reftable stack of the repository at repoPath.+func reftableTables(t *testing.T, repoPath string) []string {+ t.Helper()- // Once the writer is gone, the stack is compacted into a single table.- require.NoError(t, os.Remove(lockPath))- didPackRefs, err = manager.packRefsIfNeeded(ctx, repo, strategy)- require.NoError(t, err)- require.True(t, didPackRefs)- tablesList := testhelper.MustReadFile(t, filepath.Join(repoPath, "reftable", "tables.list"))- require.Equal(t, 1, bytes.Count(tablesList, []byte("\n")))+ return strings.Fields(string(testhelper.MustReadFile(t, filepath.Join(repoPath, "reftable", "tables.list"))))+}++// TestOptimizeRepository_staleReftableTableLock leaves the lock of a table being compacted behind, as+// a git killed during the compaction does, and checks that housekeeping recovers: once the lock is+// older than the grace period, CleanStaleData removes it and the stack is compacted again. While it+// is fresh, it is a concurrent compaction: the lock stays and the reference packing is skipped.+func TestOptimizeRepository_staleReftableTableLock(t *testing.T) {+ t.Parallel()++ for _, tc := range []struct {+ desc string+ age time.Duration+ expectedLock bool+ expectedPackedRefs map[string]float64+ expectedTables int+ }{+ {+ desc: "fresh lock",+ age: 0,+ expectedLock: true,+ expectedPackedRefs: map[string]float64{"success": 1, "skipped": 1},+ expectedTables: 2,+ },+ {+ desc: "stale lock",+ age: housekeeping.ReftableLockfileGracePeriod + time.Minute,+ expectedLock: false,+ expectedPackedRefs: map[string]float64{"success": 2, "skipped": 0},+ expectedTables: 1,+ },+ } {+ t.Run(tc.desc, func(t *testing.T) {+ t.Parallel()++ forEachBundledGit(t, func(t *testing.T, ctx context.Context) {+ repo, repoPath := setupPackRefsRepository(t, ctx, git.ReferenceBackendReftables, 10)++ // The lock of the oldest table: a compaction locks the tables from the newest+ // down, so git compacts the newer ones around it, and the oldest table and the+ // compacted one stay apart for as long as the lock is there.+ lockPath := filepath.Join(repoPath, "reftable", reftableTables(t, repoPath)[0]+".lock")+ require.NoError(t, os.WriteFile(lockPath, nil, 0o644))+ lockTime := time.Now().Add(-tc.age)+ require.NoError(t, os.Chtimes(lockPath, lockTime, lockTime))++ manager := New(gitalycfgprom.Config{}, testhelper.NewLogger(t), nil, nil)+ optimize := func() {+ require.NoError(t, manager.OptimizeRepository(ctx, repo, WithOptimizationStrategyConstructor(+ func(stats.RepositoryInfo) housekeeping.OptimizationStrategy {+ return mockOptimizationStrategy{shouldRepackReferences: alwaysPackReferences}+ },+ )))+ }++ // The first run compacts the tables newer than a held lock; the second finds only+ // the locked table and the compacted one, and cannot compact them.+ optimize()+ optimize()++ _, err := os.Stat(lockPath)+ require.Equal(t, tc.expectedLock, err == nil, "lock exists: %v", err)+ require.Len(t, reftableTables(t, repoPath), tc.expectedTables)+ for status, count := range tc.expectedPackedRefs {+ require.Equal(t, count, testutil.ToFloat64(+ manager.metrics.TasksTotal.WithLabelValues("packed_refs", status)), status)+ }+ })})}}