Skip to content

Git designed for AI agents.

housekeeping: remove stale reftable table locks; test contention with both gits

Federico Elia0baff276e3e2
diff --git a/internal/git/housekeeping/clean_stale_data.go b/internal/git/housekeeping/clean_stale_data.go
index 22b38bc..972b1ff 100644
--- a/internal/git/housekeeping/clean_stale_data.go
+++ b/internal/git/housekeeping/clean_stale_data.go
@@ -491,24 +491,46 @@ func FindStaleReferenceLocks(gracePeriod time.Duration) FindStaleFileFunc {
}
}
-// FindStaleReftableLock provides a function which scans the reftable folder
-// for stale a reftable lock and deletes it.
+// FindStaleReftableLock provides a function which scans the reftable folder for the locks git's
+// reftable stack takes that are older than ReftableLockfileGracePeriod, left by a git process that
+// died: the lock of tables.list, and the lock of a table being compacted, <table>.ref.lock
+// (flock_acquire in reftable/stack.c). A stale table lock keeps every later compaction from
+// compacting that table, so git-pack-refs(1) fails on each housekeeping until it is gone.
func FindStaleReftableLock(_ context.Context, repoPath string) ([]string, error) {
- lockPath := filepath.Join(repoPath, "reftable", "tables.list.lock")
- stat, err := os.Stat(lockPath)
+ reftablePath := filepath.Join(repoPath, "reftable")
+ entries, err := os.ReadDir(reftablePath)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
return nil, nil
}
- return nil, fmt.Errorf("statting reftable lock: %w", err)
+ return nil, fmt.Errorf("reading reftable directory: %w", err)
}
- if time.Since(stat.ModTime()) < ReftableLockfileGracePeriod {
- return nil, nil
+ var staleLocks []string
+ for _, entry := range entries {
+ name := entry.Name()
+ if name != "tables.list.lock" && !strings.HasSuffix(name, ".ref.lock") {
+ continue
+ }
+
+ info, err := entry.Info()
+ if err != nil {
+ if errors.Is(err, fs.ErrNotExist) {
+ continue
+ }
+
+ return nil, fmt.Errorf("statting reftable lock: %w", err)
+ }
+
+ if time.Since(info.ModTime()) < ReftableLockfileGracePeriod {
+ continue
+ }
+
+ staleLocks = append(staleLocks, filepath.Join(reftablePath, name))
}
- return []string{lockPath}, nil
+ return staleLocks, nil
}
func removeUnnecessaryConfig(ctx context.Context, repository *localrepo.Repo, txManager transaction.Manager) (int, error) {
diff --git a/internal/git/housekeeping/manager/commit_graph_test.go b/internal/git/housekeeping/manager/commit_graph_test.go
index 683f030..c9e12b7 100644
--- a/internal/git/housekeeping/manager/commit_graph_test.go
+++ b/internal/git/housekeeping/manager/commit_graph_test.go
@@ -128,11 +128,16 @@ func TestWriteCommitGraphIfIdle(t *testing.T) {
// TestWriteCommitGraphIfIdle_ConcurrentWithOptimizeRepository runs background commit-graph writes
// while OptimizeRepository writes the commit-graph of the same repository. Without the shared
-// guard, one of the two failed on the commit-graph chain lock in most rounds.
+// guard, one of the two failed on the commit-graph chain lock in most rounds. On reftable the
+// commits race OptimizeRepository's git-pack-refs(1) too, whose contention errors differ between the
+// bundled gits: the test runs with both.
func TestWriteCommitGraphIfIdle_ConcurrentWithOptimizeRepository(t *testing.T) {
t.Parallel()
- ctx := testhelper.Context(t)
+ forEachBundledGit(t, testWriteCommitGraphIfIdleConcurrentWithOptimizeRepository)
+}
+
+func testWriteCommitGraphIfIdleConcurrentWithOptimizeRepository(t *testing.T, ctx context.Context) {
cfg := testcfg.Build(t)
repoProto, repoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{
diff --git a/internal/git/housekeeping/manager/optimize_repository.go b/internal/git/housekeeping/manager/optimize_repository.go
index 22109e3..1ac2368 100644
--- a/internal/git/housekeeping/manager/optimize_repository.go
+++ b/internal/git/housekeeping/manager/optimize_repository.go
@@ -571,20 +571,28 @@ func (m *RepositoryManager) packRefsIfNeeded(ctx context.Context, repo *localrep
var errReftableCompactionContended = errors.New("reftable compaction contended")
// Exit codes and stderr of git-pack-refs(1) on a reftable repository when the stack is contended.
-// They are git internals, read from git 2.56.0-rc1 (3bbf434) and 2.55.0-rc1 (1ea786d), the bundled
-// builds, and pinned by TestPackRefsIfNeeded_reftableContention against the real git:
+// They are git internals, read from the bundled builds, git 2.56.0-rc1 (3bbf434, "master") and
+// 2.55.0-rc1 (1ea786d, "prev"). TestPackRefsIfNeeded_reftableContention pins 251 and 255 "data is
+// locked" against both; 244 and 255 "data concurrently modified" need a writer racing pack-refs,
+// which TestWriteCommitGraphIfIdle_ConcurrentWithOptimizeRepository runs with both, without
+// guaranteeing the race in a given run:
//
-// - reftable_be_optimize (refs/reftable-backend.c:1699) returns what reftable_stack_clean
-// (reftable/stack.c:1845) returns as is, and git exits with it modulo 256 and an empty stderr:
-// REFTABLE_LOCK_ERROR (-5) when tables.list.lock is held, REFTABLE_OUTDATED_ERROR (-12) when
-// the stack changed under it (reftable/reftable-error.h).
+// - reftable_be_optimize (refs/reftable-backend.c:1699 in master, :1656 in prev) returns what
+// reftable_stack_clean (reftable/stack.c:1845 in master, :1807 in prev) returns as is, and git
+// exits with it modulo 256 and an empty stderr. reftable_stack_clean takes tables.list.lock
+// without waiting (no write options, so reftable.lockTimeout does not apply) and fails with
+// REFTABLE_LOCK_ERROR (-5, exit 251) when a writer holds it. Only prev also fails with
+// REFTABLE_OUTDATED_ERROR (-12, exit 244) when a writer changed the stack since pack-refs read
+// it (reftable_stack_init_addition); master reloads the stack there instead. Both codes are in
+// reftable/reftable-error.h.
// - When the compaction itself fails, reftable_be_optimize reports it with error(), exiting 255,
// and "unable to compact stack: <reftable_error_str>", whose texts for these two errors come
// from reftable/error.c.
//
// git's own auto-compaction treats both errors as benign (reftable/stack.c:833). A lock left by a
-// crashed git is no reason to retry here either: CleanStaleData removes tables.list.lock once it is
-// older than housekeeping.ReftableLockfileGracePeriod, before the next pack-refs runs.
+// git process that died is not contention that passes: CleanStaleData removes the locks of
+// tables.list and of single tables once they are older than
+// housekeeping.ReftableLockfileGracePeriod, before pack-refs runs.
const (
reftableLockErrorExitCode = 256 - 5
reftableOutdatedErrorExitCode = 256 - 12
diff --git a/internal/git/housekeeping/manager/optimize_repository_test.go b/internal/git/housekeeping/manager/optimize_repository_test.go
index fd6a05c..75f18b9 100644
--- a/internal/git/housekeeping/manager/optimize_repository_test.go
+++ b/internal/git/housekeeping/manager/optimize_repository_test.go
@@ -1680,17 +1680,35 @@ func TestRepositoryManager_CleanStaleData_reftable(t *testing.T) {
for _, tc := range []struct {
desc string
+ lock string
age time.Duration
expected bool
expectedMetrics cleanStaleDataMetrics
}{
{
desc: "fresh lock",
+ lock: "tables.list.lock",
age: 0,
expected: true,
},
{
desc: "stale lock",
+ lock: "tables.list.lock",
+ age: housekeeping.ReftableLockfileGracePeriod + time.Minute,
+ expected: false,
+ expectedMetrics: cleanStaleDataMetrics{
+ reftablelocks: 1,
+ },
+ },
+ {
+ desc: "fresh table lock",
+ lock: "0x000000000001-0x000000000002-0123abcd.ref.lock",
+ age: 0,
+ expected: true,
+ },
+ {
+ desc: "stale table lock",
+ lock: "0x000000000001-0x000000000002-0123abcd.ref.lock",
age: housekeeping.ReftableLockfileGracePeriod + time.Minute,
expected: false,
expectedMetrics: cleanStaleDataMetrics{
@@ -1709,7 +1727,7 @@ func TestRepositoryManager_CleanStaleData_reftable(t *testing.T) {
})
repo := localrepo.NewTestRepo(t, cfg, repoProto)
- path := filepath.Join(repoPath, "reftable", "tables.list.lock")
+ path := filepath.Join(repoPath, "reftable", tc.lock)
require.NoError(t, os.WriteFile(path, []byte{}, mode.File))
filetime := time.Now().Add(-tc.age)
diff --git a/internal/git/housekeeping/manager/pack_refs_contention_test.go b/internal/git/housekeeping/manager/pack_refs_contention_test.go
index a7460af..14db4c6 100644
--- a/internal/git/housekeeping/manager/pack_refs_contention_test.go
+++ b/internal/git/housekeeping/manager/pack_refs_contention_test.go
@@ -1,14 +1,16 @@
package manager
import (
- "bytes"
"context"
"errors"
"fmt"
"os"
"path/filepath"
+ "strings"
"testing"
+ "time"
+ "github.com/feder-cr/git-ai/internal/featureflag"
"github.com/feder-cr/git-ai/internal/git"
"github.com/feder-cr/git-ai/internal/git/gittest"
"github.com/feder-cr/git-ai/internal/git/housekeeping"
@@ -17,6 +19,7 @@ import (
gitalycfgprom "github.com/feder-cr/git-ai/internal/gitaly/config/prometheus"
"github.com/feder-cr/git-ai/internal/testhelper"
"github.com/feder-cr/git-ai/internal/testhelper/testcfg"
+ "github.com/prometheus/client_golang/prometheus/testutil"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/require"
)
@@ -44,9 +47,34 @@ func setupPackRefsRepository(t *testing.T, ctx context.Context, backend git.Refe
func alwaysPackReferences(context.Context) bool { return true }
-// TestPackRefsIfNeeded_reftableContention runs the real git-pack-refs(1) while reftable's
-// tables.list.lock is held, as a concurrent writer holds it, and pins the exit codes and stderr that
-// isReftableContention classifies: a git that changes them fails here.
+// forEachBundledGit runs test once with each bundled git, master and prev, selected by the feature
+// flag in the context, instead of the one testhelper.Context picks at random: the reftable
+// contention errors differ between them.
+func forEachBundledGit(t *testing.T, test func(t *testing.T, ctx context.Context)) {
+ t.Helper()
+
+ gitCmdFactory := gittest.NewCommandFactory(t, testcfg.Build(t))
+ versions := map[bool]string{}
+ for _, master := range []bool{true, false} {
+ version, err := gitCmdFactory.GitVersion(featureflag.ContextWithFeatureFlag(testhelper.Context(t), featureflag.GitMaster, master))
+ require.NoError(t, err)
+ versions[master] = version
+ }
+ require.NotEqual(t, versions[true], versions[false], "the feature flag does not select another git")
+
+ for _, master := range []bool{true, false} {
+ t.Run(testhelper.DependingOn(master, "git-master", "git-prev"), func(t *testing.T) {
+ t.Parallel()
+ t.Logf("git %s", versions[master])
+
+ test(t, featureflag.ContextWithFeatureFlag(testhelper.Context(t), featureflag.GitMaster, master))
+ })
+ }
+}
+
+// TestPackRefsIfNeeded_reftableContention runs the real git-pack-refs(1) of both bundled gits while
+// reftable's tables.list.lock is held, as a concurrent writer holds it, and pins the exit codes and
+// stderr that isReftableContention classifies: a git that changes them fails here.
func TestPackRefsIfNeeded_reftableContention(t *testing.T) {
t.Parallel()
@@ -71,26 +99,102 @@ func TestPackRefsIfNeeded_reftableContention(t *testing.T) {
t.Run(tc.desc, func(t *testing.T) {
t.Parallel()
- ctx := testhelper.Context(t)
- repo, repoPath := setupPackRefsRepository(t, ctx, git.ReferenceBackendReftables, tc.tables)
- manager := New(gitalycfgprom.Config{}, testhelper.NewLogger(t), nil, nil)
- strategy := mockOptimizationStrategy{shouldRepackReferences: alwaysPackReferences}
+ forEachBundledGit(t, func(t *testing.T, ctx context.Context) {
+ repo, repoPath := setupPackRefsRepository(t, ctx, git.ReferenceBackendReftables, tc.tables)
+ manager := New(gitalycfgprom.Config{}, testhelper.NewLogger(t), nil, nil)
+ strategy := mockOptimizationStrategy{shouldRepackReferences: alwaysPackReferences}
- lockPath := filepath.Join(repoPath, "reftable", "tables.list.lock")
- require.NoError(t, os.WriteFile(lockPath, nil, 0o644))
+ lockPath := filepath.Join(repoPath, "reftable", "tables.list.lock")
+ require.NoError(t, os.WriteFile(lockPath, nil, 0o644))
- didPackRefs, err := manager.packRefsIfNeeded(ctx, repo, strategy)
- require.ErrorIs(t, err, errReftableCompactionContended)
- require.EqualError(t, err, tc.expectedErr)
- require.False(t, didPackRefs)
+ didPackRefs, err := manager.packRefsIfNeeded(ctx, repo, strategy)
+ require.ErrorIs(t, err, errReftableCompactionContended)
+ require.EqualError(t, err, tc.expectedErr)
+ require.False(t, didPackRefs)
+
+ // Once the writer is gone, the stack is compacted into a single table.
+ require.NoError(t, os.Remove(lockPath))
+ didPackRefs, err = manager.packRefsIfNeeded(ctx, repo, strategy)
+ require.NoError(t, err)
+ require.True(t, didPackRefs)
+ require.Len(t, reftableTables(t, repoPath), 1)
+ })
+ })
+ }
+}
+
+// reftableTables returns the tables of the reftable stack of the repository at repoPath.
+func reftableTables(t *testing.T, repoPath string) []string {
+ t.Helper()
- // Once the writer is gone, the stack is compacted into a single table.
- require.NoError(t, os.Remove(lockPath))
- didPackRefs, err = manager.packRefsIfNeeded(ctx, repo, strategy)
- require.NoError(t, err)
- require.True(t, didPackRefs)
- tablesList := testhelper.MustReadFile(t, filepath.Join(repoPath, "reftable", "tables.list"))
- require.Equal(t, 1, bytes.Count(tablesList, []byte("\n")))
+ return strings.Fields(string(testhelper.MustReadFile(t, filepath.Join(repoPath, "reftable", "tables.list"))))
+}
+
+// TestOptimizeRepository_staleReftableTableLock leaves the lock of a table being compacted behind, as
+// a git killed during the compaction does, and checks that housekeeping recovers: once the lock is
+// older than the grace period, CleanStaleData removes it and the stack is compacted again. While it
+// is fresh, it is a concurrent compaction: the lock stays and the reference packing is skipped.
+func TestOptimizeRepository_staleReftableTableLock(t *testing.T) {
+ t.Parallel()
+
+ for _, tc := range []struct {
+ desc string
+ age time.Duration
+ expectedLock bool
+ expectedPackedRefs map[string]float64
+ expectedTables int
+ }{
+ {
+ desc: "fresh lock",
+ age: 0,
+ expectedLock: true,
+ expectedPackedRefs: map[string]float64{"success": 1, "skipped": 1},
+ expectedTables: 2,
+ },
+ {
+ desc: "stale lock",
+ age: housekeeping.ReftableLockfileGracePeriod + time.Minute,
+ expectedLock: false,
+ expectedPackedRefs: map[string]float64{"success": 2, "skipped": 0},
+ expectedTables: 1,
+ },
+ } {
+ t.Run(tc.desc, func(t *testing.T) {
+ t.Parallel()
+
+ forEachBundledGit(t, func(t *testing.T, ctx context.Context) {
+ repo, repoPath := setupPackRefsRepository(t, ctx, git.ReferenceBackendReftables, 10)
+
+ // The lock of the oldest table: a compaction locks the tables from the newest
+ // down, so git compacts the newer ones around it, and the oldest table and the
+ // compacted one stay apart for as long as the lock is there.
+ lockPath := filepath.Join(repoPath, "reftable", reftableTables(t, repoPath)[0]+".lock")
+ require.NoError(t, os.WriteFile(lockPath, nil, 0o644))
+ lockTime := time.Now().Add(-tc.age)
+ require.NoError(t, os.Chtimes(lockPath, lockTime, lockTime))
+
+ manager := New(gitalycfgprom.Config{}, testhelper.NewLogger(t), nil, nil)
+ optimize := func() {
+ require.NoError(t, manager.OptimizeRepository(ctx, repo, WithOptimizationStrategyConstructor(
+ func(stats.RepositoryInfo) housekeeping.OptimizationStrategy {
+ return mockOptimizationStrategy{shouldRepackReferences: alwaysPackReferences}
+ },
+ )))
+ }
+
+ // The first run compacts the tables newer than a held lock; the second finds only
+ // the locked table and the compacted one, and cannot compact them.
+ optimize()
+ optimize()
+
+ _, err := os.Stat(lockPath)
+ require.Equal(t, tc.expectedLock, err == nil, "lock exists: %v", err)
+ require.Len(t, reftableTables(t, repoPath), tc.expectedTables)
+ for status, count := range tc.expectedPackedRefs {
+ require.Equal(t, count, testutil.ToFloat64(
+ manager.metrics.TasksTotal.WithLabelValues("packed_refs", status)), status)
+ }
+ })
})
}
}