Git designed for AI agents.
housekeeping: skip reference packing when the reftable stack is contended
Federico Elia243aa4402c2c
diff --git a/internal/git/housekeeping/manager/optimize_repository.go b/internal/git/housekeeping/manager/optimize_repository.goindex a95440b..22109e3 100644--- a/internal/git/housekeeping/manager/optimize_repository.go+++ b/internal/git/housekeeping/manager/optimize_repository.go@@ -8,6 +8,7 @@ import ("os""time"+ "github.com/feder-cr/git-ai/internal/command""github.com/feder-cr/git-ai/internal/git""github.com/feder-cr/git-ai/internal/git/gitcmd""github.com/feder-cr/git-ai/internal/git/housekeeping"@@ -353,10 +354,16 @@ func (m *RepositoryManager) optimizeRepository(finishTimer = m.metrics.ReportTaskLatency("pack-refs", "apply")didPackRefs, err := m.packRefsIfNeeded(ctx, repo, strategy)- if err != nil {+ switch {+ case errors.Is(err, errReftableCompactionContended):+ // Benign: the stack stays consistent and the next housekeeping compacts or cleans what+ // this one could not, as git's own auto-compaction assumes. The rest still runs.+ optimizations["packed_refs"] = "skipped"+ m.logger.WithError(err).WarnContext(ctx, "skipped packing references: reftable stack locked or modified concurrently")+ case err != nil:optimizations["packed_refs"] = "failure"return fmt.Errorf("could not pack refs: %w", err)- } else if didPackRefs {+ case didPackRefs:optimizations["packed_refs"] = "success"}packRefsTime = finishTimer()@@ -549,12 +556,63 @@ func (m *RepositoryManager) packRefsIfNeeded(ctx context.Context, repo *localrepgitcmd.Flag{Name: "--all"},},}, gitcmd.WithStderr(&stderr)); err != nil {- return false, fmt.Errorf("packing refs: %w, stderr: %q", err, stderr.String())+ err = fmt.Errorf("packing refs: %w, stderr: %q", err, stderr.String())+ if backend.Name == git.ReferenceBackendReftables.Name && isReftableContention(err, stderr.String()) {+ return false, fmt.Errorf("%w: %w", errReftableCompactionContended, err)+ }+ return false, err}return true, nil}+// errReftableCompactionContended is returned by packRefsIfNeeded when git-pack-refs(1) failed on a+// reftable repository only because a concurrent writer held the stack's lock or changed the stack.+var errReftableCompactionContended = errors.New("reftable compaction contended")++// Exit codes and stderr of git-pack-refs(1) on a reftable repository when the stack is contended.+// They are git internals, read from git 2.56.0-rc1 (3bbf434) and 2.55.0-rc1 (1ea786d), the bundled+// builds, and pinned by TestPackRefsIfNeeded_reftableContention against the real git:+//+// - reftable_be_optimize (refs/reftable-backend.c:1699) returns what reftable_stack_clean+// (reftable/stack.c:1845) returns as is, and git exits with it modulo 256 and an empty stderr:+// REFTABLE_LOCK_ERROR (-5) when tables.list.lock is held, REFTABLE_OUTDATED_ERROR (-12) when+// the stack changed under it (reftable/reftable-error.h).+// - When the compaction itself fails, reftable_be_optimize reports it with error(), exiting 255,+// and "unable to compact stack: <reftable_error_str>", whose texts for these two errors come+// from reftable/error.c.+//+// git's own auto-compaction treats both errors as benign (reftable/stack.c:833). A lock left by a+// crashed git is no reason to retry here either: CleanStaleData removes tables.list.lock once it is+// older than housekeeping.ReftableLockfileGracePeriod, before the next pack-refs runs.+const (+ reftableLockErrorExitCode = 256 - 5+ reftableOutdatedErrorExitCode = 256 - 12+ gitErrorExitCode = 255++ reftableCompactLockedStderr = "error: unable to compact stack: data is locked\n"+ reftableCompactOutdatedStderr = "error: unable to compact stack: data concurrently modified\n"+)++// isReftableContention reports whether git-pack-refs(1) failed with err and stderr on a reftable+// repository only because the stack was locked or modified concurrently. Any other failure is a+// real error.+func isReftableContention(err error, stderr string) bool {+ exitCode, ok := command.ExitStatus(err)+ if !ok {+ return false+ }++ switch exitCode {+ case reftableLockErrorExitCode, reftableOutdatedErrorExitCode:+ return stderr == ""+ case gitErrorExitCode:+ return stderr == reftableCompactLockedStderr || stderr == reftableCompactOutdatedStderr+ default:+ return false+ }+}+// CleanStaleData removes any stale data in the repository as per the provided configuration.func (m *RepositoryManager) CleanStaleData(ctx context.Context, repo *localrepo.Repo, cfg housekeeping.CleanStaleDataConfig) error {span, ctx := tracing.StartSpanIfHasParent(ctx, "housekeeping.CleanStaleData", nil)diff --git a/internal/git/housekeeping/manager/pack_refs_contention_test.go b/internal/git/housekeeping/manager/pack_refs_contention_test.gonew file mode 100644index 0000000..a7460af--- /dev/null+++ b/internal/git/housekeeping/manager/pack_refs_contention_test.go@@ -0,0 +1,227 @@+package manager++import (+ "bytes"+ "context"+ "errors"+ "fmt"+ "os"+ "path/filepath"+ "testing"++ "github.com/feder-cr/git-ai/internal/git"+ "github.com/feder-cr/git-ai/internal/git/gittest"+ "github.com/feder-cr/git-ai/internal/git/housekeeping"+ "github.com/feder-cr/git-ai/internal/git/localrepo"+ "github.com/feder-cr/git-ai/internal/git/stats"+ gitalycfgprom "github.com/feder-cr/git-ai/internal/gitaly/config/prometheus"+ "github.com/feder-cr/git-ai/internal/testhelper"+ "github.com/feder-cr/git-ai/internal/testhelper/testcfg"+ "github.com/sirupsen/logrus"+ "github.com/stretchr/testify/require"+)++// setupPackRefsRepository creates a repository with the given reference backend and tables branches,+// so that git-pack-refs(1) --auto has work to do: on reftable each branch is a table of its own that+// auto-compaction left alone, on files a loose reference (files packs from 16 of them).+func setupPackRefsRepository(t *testing.T, ctx context.Context, backend git.ReferenceBackend, tables int) (*localrepo.Repo, string) {+ t.Helper()++ cfg := testcfg.Build(t)+ repoProto, repoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{+ SkipCreationViaService: true,+ ReferenceBackend: backend,+ })+ commitID := gittest.WriteCommit(t, cfg, repoPath, gittest.WithBranch("main"))+ for i := 1; i < tables; i++ {+ gittest.ExecOpts(t, cfg, gittest.ExecConfig{+ Env: []string{"GIT_TEST_REFTABLE_AUTOCOMPACTION=false"},+ }, "-C", repoPath, "update-ref", fmt.Sprintf("refs/heads/branch-%d", i), commitID.String())+ }++ return localrepo.NewTestRepo(t, cfg, repoProto), repoPath+}++func alwaysPackReferences(context.Context) bool { return true }++// TestPackRefsIfNeeded_reftableContention runs the real git-pack-refs(1) while reftable's+// tables.list.lock is held, as a concurrent writer holds it, and pins the exit codes and stderr that+// isReftableContention classifies: a git that changes them fails here.+func TestPackRefsIfNeeded_reftableContention(t *testing.T) {+ t.Parallel()++ for _, tc := range []struct {+ desc string+ tables int+ expectedErr string+ }{+ {+ // Nothing to compact: reftable_stack_clean fails to take the lock.+ desc: "lock held, nothing to compact",+ tables: 1,+ expectedErr: `reftable compaction contended: packing refs: exit status 251, stderr: ""`,+ },+ {+ // The compaction fails to take the lock.+ desc: "lock held, stack to compact",+ tables: 10,+ expectedErr: `reftable compaction contended: packing refs: exit status 255, stderr: "error: unable to compact stack: data is locked\n"`,+ },+ } {+ t.Run(tc.desc, func(t *testing.T) {+ t.Parallel()++ ctx := testhelper.Context(t)+ repo, repoPath := setupPackRefsRepository(t, ctx, git.ReferenceBackendReftables, tc.tables)+ manager := New(gitalycfgprom.Config{}, testhelper.NewLogger(t), nil, nil)+ strategy := mockOptimizationStrategy{shouldRepackReferences: alwaysPackReferences}++ lockPath := filepath.Join(repoPath, "reftable", "tables.list.lock")+ require.NoError(t, os.WriteFile(lockPath, nil, 0o644))++ didPackRefs, err := manager.packRefsIfNeeded(ctx, repo, strategy)+ require.ErrorIs(t, err, errReftableCompactionContended)+ require.EqualError(t, err, tc.expectedErr)+ require.False(t, didPackRefs)++ // Once the writer is gone, the stack is compacted into a single table.+ require.NoError(t, os.Remove(lockPath))+ didPackRefs, err = manager.packRefsIfNeeded(ctx, repo, strategy)+ require.NoError(t, err)+ require.True(t, didPackRefs)+ tablesList := testhelper.MustReadFile(t, filepath.Join(repoPath, "reftable", "tables.list"))+ require.Equal(t, 1, bytes.Count(tablesList, []byte("\n")))+ })+ }+}++// TestPackRefsIfNeeded_failuresStayFatal checks that git-pack-refs(1) failures other than reftable+// contention are returned as they are: a broken reftable stack, and a held lock on the files backend,+// whose behaviour does not change.+func TestPackRefsIfNeeded_failuresStayFatal(t *testing.T) {+ t.Parallel()++ for _, tc := range []struct {+ desc string+ backend git.ReferenceBackend+ setup func(t *testing.T, repoPath string)+ expectedErr string+ }{+ {+ desc: "reftable stack naming a missing table",+ backend: git.ReferenceBackendReftables,+ setup: func(t *testing.T, repoPath string) {+ tablesList, err := os.OpenFile(filepath.Join(repoPath, "reftable", "tables.list"), os.O_APPEND|os.O_WRONLY, 0)+ require.NoError(t, err)+ _, err = tablesList.WriteString("0x000000000009-0x000000000009-deadbeef.ref\n")+ require.NoError(t, err)+ require.NoError(t, tablesList.Close())+ },+ expectedErr: `packing refs: exit status 252, stderr: ""`,+ },+ {+ desc: "files packed-refs.lock held",+ backend: git.ReferenceBackendFiles,+ setup: func(t *testing.T, repoPath string) {+ require.NoError(t, os.WriteFile(filepath.Join(repoPath, "packed-refs.lock"), nil, 0o644))+ },+ expectedErr: "packing refs: exit status 128",+ },+ } {+ t.Run(tc.desc, func(t *testing.T) {+ t.Parallel()++ ctx := testhelper.Context(t)+ repo, repoPath := setupPackRefsRepository(t, ctx, tc.backend, 32)+ tc.setup(t, repoPath)+ manager := New(gitalycfgprom.Config{}, testhelper.NewLogger(t), nil, nil)++ didPackRefs, err := manager.packRefsIfNeeded(ctx, repo, mockOptimizationStrategy{+ shouldRepackReferences: alwaysPackReferences,+ })+ require.False(t, errors.Is(err, errReftableCompactionContended), "classified as contention: %v", err)+ require.ErrorContains(t, err, tc.expectedErr)+ require.False(t, didPackRefs)+ })+ }+}++// TestOptimizeRepository_packRefsContention checks how OptimizeRepository reports git-pack-refs(1)+// failures: reftable contention is skipped, logged as a warning and counted with the "skipped"+// status while the commit-graph is still written; on the files backend a held lock still fails the+// housekeeping.+func TestOptimizeRepository_packRefsContention(t *testing.T) {+ t.Parallel()++ for _, tc := range []struct {+ desc string+ backend git.ReferenceBackend+ lockFile string+ expectedErr string+ expectedMetrics string+ expectedWarning bool+ }{+ {+ desc: "reftable",+ backend: git.ReferenceBackendReftables,+ lockFile: filepath.Join("reftable", "tables.list.lock"),+ expectedErr: "",+ expectedMetrics: `gitaly_housekeeping_tasks_total{housekeeping_task="packed_refs", status="skipped"} 1+gitaly_housekeeping_tasks_total{housekeeping_task="total", status="success"} 1+gitaly_housekeeping_tasks_total{housekeeping_task="written_commit_graph_incremental", status="success"} 1+`,+ expectedWarning: true,+ },+ {+ desc: "files",+ backend: git.ReferenceBackendFiles,+ lockFile: "packed-refs.lock",+ expectedErr: "could not pack refs: packing refs: exit status 128",+ expectedMetrics: `gitaly_housekeeping_tasks_total{housekeeping_task="packed_refs", status="failure"} 1+gitaly_housekeeping_tasks_total{housekeeping_task="total", status="failure"} 1+`,+ },+ } {+ t.Run(tc.desc, func(t *testing.T) {+ t.Parallel()++ ctx := testhelper.Context(t)+ repo, repoPath := setupPackRefsRepository(t, ctx, tc.backend, 32)+ require.NoError(t, os.WriteFile(filepath.Join(repoPath, tc.lockFile), nil, 0o644))++ logger := testhelper.NewLogger(t)+ hook := testhelper.AddLoggerHook(logger)+ manager := New(gitalycfgprom.Config{}, logger, nil, nil)++ err := manager.OptimizeRepository(ctx, repo, WithOptimizationStrategyConstructor(+ func(stats.RepositoryInfo) housekeeping.OptimizationStrategy {+ return mockOptimizationStrategy{+ shouldRepackReferences: alwaysPackReferences,+ shouldWriteCommitGraph: true,+ }+ },+ ))+ if tc.expectedErr == "" {+ require.NoError(t, err)+ } else {+ require.ErrorContains(t, err, tc.expectedErr)+ }++ testhelper.RequirePromMetrics(t, manager.metrics.TasksTotal,+ "# HELP gitaly_housekeeping_tasks_total Total number of housekeeping tasks performed in the repository\n"++ "# TYPE gitaly_housekeeping_tasks_total counter\n"+tc.expectedMetrics)++ var warnings []string+ for _, entry := range hook.AllEntries() {+ if entry.Level <= logrus.WarnLevel {+ warnings = append(warnings, entry.Message)+ }+ }+ if tc.expectedWarning {+ require.Equal(t, []string{"skipped packing references: reftable stack locked or modified concurrently"}, warnings)+ } else {+ require.Empty(t, warnings)+ }+ })+ }+}