Skip to content

Git designed for AI agents.

housekeeping: skip reference packing when the reftable stack is contended

Federico Elia243aa4402c2c
diff --git a/internal/git/housekeeping/manager/optimize_repository.go b/internal/git/housekeeping/manager/optimize_repository.go
index a95440b..22109e3 100644
--- a/internal/git/housekeeping/manager/optimize_repository.go
+++ b/internal/git/housekeeping/manager/optimize_repository.go
@@ -8,6 +8,7 @@ import (
"os"
"time"
+ "github.com/feder-cr/git-ai/internal/command"
"github.com/feder-cr/git-ai/internal/git"
"github.com/feder-cr/git-ai/internal/git/gitcmd"
"github.com/feder-cr/git-ai/internal/git/housekeeping"
@@ -353,10 +354,16 @@ func (m *RepositoryManager) optimizeRepository(
finishTimer = m.metrics.ReportTaskLatency("pack-refs", "apply")
didPackRefs, err := m.packRefsIfNeeded(ctx, repo, strategy)
- if err != nil {
+ switch {
+ case errors.Is(err, errReftableCompactionContended):
+ // Benign: the stack stays consistent and the next housekeeping compacts or cleans what
+ // this one could not, as git's own auto-compaction assumes. The rest still runs.
+ optimizations["packed_refs"] = "skipped"
+ m.logger.WithError(err).WarnContext(ctx, "skipped packing references: reftable stack locked or modified concurrently")
+ case err != nil:
optimizations["packed_refs"] = "failure"
return fmt.Errorf("could not pack refs: %w", err)
- } else if didPackRefs {
+ case didPackRefs:
optimizations["packed_refs"] = "success"
}
packRefsTime = finishTimer()
@@ -549,12 +556,63 @@ func (m *RepositoryManager) packRefsIfNeeded(ctx context.Context, repo *localrep
gitcmd.Flag{Name: "--all"},
},
}, gitcmd.WithStderr(&stderr)); err != nil {
- return false, fmt.Errorf("packing refs: %w, stderr: %q", err, stderr.String())
+ err = fmt.Errorf("packing refs: %w, stderr: %q", err, stderr.String())
+ if backend.Name == git.ReferenceBackendReftables.Name && isReftableContention(err, stderr.String()) {
+ return false, fmt.Errorf("%w: %w", errReftableCompactionContended, err)
+ }
+ return false, err
}
return true, nil
}
+// errReftableCompactionContended is returned by packRefsIfNeeded when git-pack-refs(1) failed on a
+// reftable repository only because a concurrent writer held the stack's lock or changed the stack.
+var errReftableCompactionContended = errors.New("reftable compaction contended")
+
+// Exit codes and stderr of git-pack-refs(1) on a reftable repository when the stack is contended.
+// They are git internals, read from git 2.56.0-rc1 (3bbf434) and 2.55.0-rc1 (1ea786d), the bundled
+// builds, and pinned by TestPackRefsIfNeeded_reftableContention against the real git:
+//
+// - reftable_be_optimize (refs/reftable-backend.c:1699) returns what reftable_stack_clean
+// (reftable/stack.c:1845) returns as is, and git exits with it modulo 256 and an empty stderr:
+// REFTABLE_LOCK_ERROR (-5) when tables.list.lock is held, REFTABLE_OUTDATED_ERROR (-12) when
+// the stack changed under it (reftable/reftable-error.h).
+// - When the compaction itself fails, reftable_be_optimize reports it with error(), exiting 255,
+// and "unable to compact stack: <reftable_error_str>", whose texts for these two errors come
+// from reftable/error.c.
+//
+// git's own auto-compaction treats both errors as benign (reftable/stack.c:833). A lock left by a
+// crashed git is no reason to retry here either: CleanStaleData removes tables.list.lock once it is
+// older than housekeeping.ReftableLockfileGracePeriod, before the next pack-refs runs.
+const (
+ reftableLockErrorExitCode = 256 - 5
+ reftableOutdatedErrorExitCode = 256 - 12
+ gitErrorExitCode = 255
+
+ reftableCompactLockedStderr = "error: unable to compact stack: data is locked\n"
+ reftableCompactOutdatedStderr = "error: unable to compact stack: data concurrently modified\n"
+)
+
+// isReftableContention reports whether git-pack-refs(1) failed with err and stderr on a reftable
+// repository only because the stack was locked or modified concurrently. Any other failure is a
+// real error.
+func isReftableContention(err error, stderr string) bool {
+ exitCode, ok := command.ExitStatus(err)
+ if !ok {
+ return false
+ }
+
+ switch exitCode {
+ case reftableLockErrorExitCode, reftableOutdatedErrorExitCode:
+ return stderr == ""
+ case gitErrorExitCode:
+ return stderr == reftableCompactLockedStderr || stderr == reftableCompactOutdatedStderr
+ default:
+ return false
+ }
+}
+
// CleanStaleData removes any stale data in the repository as per the provided configuration.
func (m *RepositoryManager) CleanStaleData(ctx context.Context, repo *localrepo.Repo, cfg housekeeping.CleanStaleDataConfig) error {
span, ctx := tracing.StartSpanIfHasParent(ctx, "housekeeping.CleanStaleData", nil)
diff --git a/internal/git/housekeeping/manager/pack_refs_contention_test.go b/internal/git/housekeeping/manager/pack_refs_contention_test.go
new file mode 100644
index 0000000..a7460af
--- /dev/null
+++ b/internal/git/housekeeping/manager/pack_refs_contention_test.go
@@ -0,0 +1,227 @@
+package manager
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/feder-cr/git-ai/internal/git"
+ "github.com/feder-cr/git-ai/internal/git/gittest"
+ "github.com/feder-cr/git-ai/internal/git/housekeeping"
+ "github.com/feder-cr/git-ai/internal/git/localrepo"
+ "github.com/feder-cr/git-ai/internal/git/stats"
+ gitalycfgprom "github.com/feder-cr/git-ai/internal/gitaly/config/prometheus"
+ "github.com/feder-cr/git-ai/internal/testhelper"
+ "github.com/feder-cr/git-ai/internal/testhelper/testcfg"
+ "github.com/sirupsen/logrus"
+ "github.com/stretchr/testify/require"
+)
+
+// setupPackRefsRepository creates a repository with the given reference backend and tables branches,
+// so that git-pack-refs(1) --auto has work to do: on reftable each branch is a table of its own that
+// auto-compaction left alone, on files a loose reference (files packs from 16 of them).
+func setupPackRefsRepository(t *testing.T, ctx context.Context, backend git.ReferenceBackend, tables int) (*localrepo.Repo, string) {
+ t.Helper()
+
+ cfg := testcfg.Build(t)
+ repoProto, repoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{
+ SkipCreationViaService: true,
+ ReferenceBackend: backend,
+ })
+ commitID := gittest.WriteCommit(t, cfg, repoPath, gittest.WithBranch("main"))
+ for i := 1; i < tables; i++ {
+ gittest.ExecOpts(t, cfg, gittest.ExecConfig{
+ Env: []string{"GIT_TEST_REFTABLE_AUTOCOMPACTION=false"},
+ }, "-C", repoPath, "update-ref", fmt.Sprintf("refs/heads/branch-%d", i), commitID.String())
+ }
+
+ return localrepo.NewTestRepo(t, cfg, repoProto), repoPath
+}
+
+func alwaysPackReferences(context.Context) bool { return true }
+
+// TestPackRefsIfNeeded_reftableContention runs the real git-pack-refs(1) while reftable's
+// tables.list.lock is held, as a concurrent writer holds it, and pins the exit codes and stderr that
+// isReftableContention classifies: a git that changes them fails here.
+func TestPackRefsIfNeeded_reftableContention(t *testing.T) {
+ t.Parallel()
+
+ for _, tc := range []struct {
+ desc string
+ tables int
+ expectedErr string
+ }{
+ {
+ // Nothing to compact: reftable_stack_clean fails to take the lock.
+ desc: "lock held, nothing to compact",
+ tables: 1,
+ expectedErr: `reftable compaction contended: packing refs: exit status 251, stderr: ""`,
+ },
+ {
+ // The compaction fails to take the lock.
+ desc: "lock held, stack to compact",
+ tables: 10,
+ expectedErr: `reftable compaction contended: packing refs: exit status 255, stderr: "error: unable to compact stack: data is locked\n"`,
+ },
+ } {
+ t.Run(tc.desc, func(t *testing.T) {
+ t.Parallel()
+
+ ctx := testhelper.Context(t)
+ repo, repoPath := setupPackRefsRepository(t, ctx, git.ReferenceBackendReftables, tc.tables)
+ manager := New(gitalycfgprom.Config{}, testhelper.NewLogger(t), nil, nil)
+ strategy := mockOptimizationStrategy{shouldRepackReferences: alwaysPackReferences}
+
+ lockPath := filepath.Join(repoPath, "reftable", "tables.list.lock")
+ require.NoError(t, os.WriteFile(lockPath, nil, 0o644))
+
+ didPackRefs, err := manager.packRefsIfNeeded(ctx, repo, strategy)
+ require.ErrorIs(t, err, errReftableCompactionContended)
+ require.EqualError(t, err, tc.expectedErr)
+ require.False(t, didPackRefs)
+
+ // Once the writer is gone, the stack is compacted into a single table.
+ require.NoError(t, os.Remove(lockPath))
+ didPackRefs, err = manager.packRefsIfNeeded(ctx, repo, strategy)
+ require.NoError(t, err)
+ require.True(t, didPackRefs)
+ tablesList := testhelper.MustReadFile(t, filepath.Join(repoPath, "reftable", "tables.list"))
+ require.Equal(t, 1, bytes.Count(tablesList, []byte("\n")))
+ })
+ }
+}
+
+// TestPackRefsIfNeeded_failuresStayFatal checks that git-pack-refs(1) failures other than reftable
+// contention are returned as they are: a broken reftable stack, and a held lock on the files backend,
+// whose behaviour does not change.
+func TestPackRefsIfNeeded_failuresStayFatal(t *testing.T) {
+ t.Parallel()
+
+ for _, tc := range []struct {
+ desc string
+ backend git.ReferenceBackend
+ setup func(t *testing.T, repoPath string)
+ expectedErr string
+ }{
+ {
+ desc: "reftable stack naming a missing table",
+ backend: git.ReferenceBackendReftables,
+ setup: func(t *testing.T, repoPath string) {
+ tablesList, err := os.OpenFile(filepath.Join(repoPath, "reftable", "tables.list"), os.O_APPEND|os.O_WRONLY, 0)
+ require.NoError(t, err)
+ _, err = tablesList.WriteString("0x000000000009-0x000000000009-deadbeef.ref\n")
+ require.NoError(t, err)
+ require.NoError(t, tablesList.Close())
+ },
+ expectedErr: `packing refs: exit status 252, stderr: ""`,
+ },
+ {
+ desc: "files packed-refs.lock held",
+ backend: git.ReferenceBackendFiles,
+ setup: func(t *testing.T, repoPath string) {
+ require.NoError(t, os.WriteFile(filepath.Join(repoPath, "packed-refs.lock"), nil, 0o644))
+ },
+ expectedErr: "packing refs: exit status 128",
+ },
+ } {
+ t.Run(tc.desc, func(t *testing.T) {
+ t.Parallel()
+
+ ctx := testhelper.Context(t)
+ repo, repoPath := setupPackRefsRepository(t, ctx, tc.backend, 32)
+ tc.setup(t, repoPath)
+ manager := New(gitalycfgprom.Config{}, testhelper.NewLogger(t), nil, nil)
+
+ didPackRefs, err := manager.packRefsIfNeeded(ctx, repo, mockOptimizationStrategy{
+ shouldRepackReferences: alwaysPackReferences,
+ })
+ require.False(t, errors.Is(err, errReftableCompactionContended), "classified as contention: %v", err)
+ require.ErrorContains(t, err, tc.expectedErr)
+ require.False(t, didPackRefs)
+ })
+ }
+}
+
+// TestOptimizeRepository_packRefsContention checks how OptimizeRepository reports git-pack-refs(1)
+// failures: reftable contention is skipped, logged as a warning and counted with the "skipped"
+// status while the commit-graph is still written; on the files backend a held lock still fails the
+// housekeeping.
+func TestOptimizeRepository_packRefsContention(t *testing.T) {
+ t.Parallel()
+
+ for _, tc := range []struct {
+ desc string
+ backend git.ReferenceBackend
+ lockFile string
+ expectedErr string
+ expectedMetrics string
+ expectedWarning bool
+ }{
+ {
+ desc: "reftable",
+ backend: git.ReferenceBackendReftables,
+ lockFile: filepath.Join("reftable", "tables.list.lock"),
+ expectedErr: "",
+ expectedMetrics: `gitaly_housekeeping_tasks_total{housekeeping_task="packed_refs", status="skipped"} 1
+gitaly_housekeeping_tasks_total{housekeeping_task="total", status="success"} 1
+gitaly_housekeeping_tasks_total{housekeeping_task="written_commit_graph_incremental", status="success"} 1
+`,
+ expectedWarning: true,
+ },
+ {
+ desc: "files",
+ backend: git.ReferenceBackendFiles,
+ lockFile: "packed-refs.lock",
+ expectedErr: "could not pack refs: packing refs: exit status 128",
+ expectedMetrics: `gitaly_housekeeping_tasks_total{housekeeping_task="packed_refs", status="failure"} 1
+gitaly_housekeeping_tasks_total{housekeeping_task="total", status="failure"} 1
+`,
+ },
+ } {
+ t.Run(tc.desc, func(t *testing.T) {
+ t.Parallel()
+
+ ctx := testhelper.Context(t)
+ repo, repoPath := setupPackRefsRepository(t, ctx, tc.backend, 32)
+ require.NoError(t, os.WriteFile(filepath.Join(repoPath, tc.lockFile), nil, 0o644))
+
+ logger := testhelper.NewLogger(t)
+ hook := testhelper.AddLoggerHook(logger)
+ manager := New(gitalycfgprom.Config{}, logger, nil, nil)
+
+ err := manager.OptimizeRepository(ctx, repo, WithOptimizationStrategyConstructor(
+ func(stats.RepositoryInfo) housekeeping.OptimizationStrategy {
+ return mockOptimizationStrategy{
+ shouldRepackReferences: alwaysPackReferences,
+ shouldWriteCommitGraph: true,
+ }
+ },
+ ))
+ if tc.expectedErr == "" {
+ require.NoError(t, err)
+ } else {
+ require.ErrorContains(t, err, tc.expectedErr)
+ }
+
+ testhelper.RequirePromMetrics(t, manager.metrics.TasksTotal,
+ "# HELP gitaly_housekeeping_tasks_total Total number of housekeeping tasks performed in the repository\n"+
+ "# TYPE gitaly_housekeeping_tasks_total counter\n"+tc.expectedMetrics)
+
+ var warnings []string
+ for _, entry := range hook.AllEntries() {
+ if entry.Level <= logrus.WarnLevel {
+ warnings = append(warnings, entry.Message)
+ }
+ }
+ if tc.expectedWarning {
+ require.Equal(t, []string{"skipped packing references: reftable stack locked or modified concurrently"}, warnings)
+ } else {
+ require.Empty(t, warnings)
+ }
+ })
+ }
+}