Git designed for AI agents.
package housekeeping
import (
"context"
"errors"
"fmt"
"os/exec"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/feder-cr/git-ai/internal/git"
"github.com/feder-cr/git-ai/internal/git/gitcmd"
"github.com/feder-cr/git-ai/internal/git/housekeeping/config"
"github.com/feder-cr/git-ai/internal/git/localrepo"
"github.com/feder-cr/git-ai/internal/git/stats"
"github.com/feder-cr/git-ai/internal/gitaly/storage"
"github.com/feder-cr/git-ai/internal/structerr"
)
const (
// LooseObjectLimit is the limit of loose objects we accept both when doing incremental
// repacks and when pruning objects.
LooseObjectLimit = 1024
)
// ValidateRepacking validates the input repacking config. This function any validating error and if the configuration
// is for full repack.
func ValidateRepacking(cfg config.RepackObjectsConfig) (bool, error) {
var isFullRepack bool
switch cfg.Strategy {
case config.RepackObjectsStrategyIncrementalWithUnreachable:
isFullRepack = false
if cfg.WriteBitmap {
return false, structerr.NewInvalidArgument("cannot write packfile bitmap for an incremental repack")
}
if cfg.WriteMultiPackIndex {
return false, structerr.NewInvalidArgument("cannot write multi-pack index for an incremental repack")
}
case config.RepackObjectsStrategyGeometric:
isFullRepack = false
case config.RepackObjectsStrategyFullWithCruft, config.RepackObjectsStrategyFullWithUnreachable:
isFullRepack = true
default:
return false, structerr.NewInvalidArgument("invalid strategy: %q", cfg.Strategy)
}
if !isFullRepack && !cfg.WriteMultiPackIndex && cfg.WriteBitmap {
return false, structerr.NewInvalidArgument("cannot write packfile bitmap for an incremental repack")
}
if cfg.Strategy != config.RepackObjectsStrategyFullWithCruft && !cfg.CruftExpireBefore.IsZero() {
return isFullRepack, structerr.NewInvalidArgument("cannot expire cruft objects when not writing cruft packs")
}
return isFullRepack, nil
}
// RepackObjects repacks objects in the given repository and updates the commit-graph. The way
// objects are repacked is determined via the config.RepackObjectsConfig.
func RepackObjects(ctx context.Context, repo *localrepo.Repo, cfg config.RepackObjectsConfig) error {
repoPath, err := repo.Path(ctx)
if err != nil {
return err
}
isFullRepack, err := ValidateRepacking(cfg)
if err != nil {
return err
}
if isFullRepack {
// When we have performed a full repack we're updating the "full-repack-timestamp"
// file. This is done so that we can tell when we have last performed a full repack
// in a repository. This information can be used by our heuristics to effectively
// rate-limit the frequency of full repacks.
//
// Note that we write the file _before_ actually writing the new pack, which means
// that even if the full repack fails, we would still pretend to have done it. This
// is done intentionally, as the likelihood for huge repositories to fail during a
// full repack is comparatively high. So if we didn't update the timestamp in case
// of a failure we'd potentially busy-spin trying to do a full repack.
if err := stats.UpdateFullRepackTimestamp(repoPath, time.Now()); err != nil {
return fmt.Errorf("updating full-repack timestamp: %w", err)
}
}
switch cfg.Strategy {
case config.RepackObjectsStrategyIncrementalWithUnreachable:
if err := PerformIncrementalRepackingWithUnreachable(ctx, repo); err != nil {
return fmt.Errorf("perform incremental repacking with unreachable: %w", err)
}
// The `-d` switch of git-repack(1) handles deletion of objects that have just been
// packed into a new packfile. As we pack objects ourselves, we have to manually
// ensure that packed loose objects are deleted.
var stderr strings.Builder
if err := repo.ExecAndWait(ctx,
gitcmd.Command{
Name: "prune-packed",
Flags: []gitcmd.Option{
// We don't care about any kind of progress meter.
gitcmd.Flag{Name: "--quiet"},
},
},
gitcmd.WithStderr(&stderr),
); err != nil {
var exitErr *exec.ExitError
if errors.As(err, &exitErr) {
return structerr.New("prune-packed failed with error code %d", exitErr.ExitCode()).WithMetadata("stderr", stderr.String())
}
return fmt.Errorf("prune-packed failed: %w", err)
}
return nil
case config.RepackObjectsStrategyFullWithCruft:
options := []gitcmd.Option{
gitcmd.Flag{Name: "--cruft"},
gitcmd.Flag{Name: "--pack-kept-objects"},
gitcmd.Flag{Name: "-l"},
gitcmd.Flag{Name: "-d"},
}
if !cfg.CruftExpireBefore.IsZero() {
options = append(options, gitcmd.ValueFlag{
Name: "--cruft-expiration",
Value: git.FormatTime(cfg.CruftExpireBefore),
})
}
return PerformRepack(ctx, repo, cfg, options...)
case config.RepackObjectsStrategyFullWithUnreachable:
return PerformFullRepackingWithUnreachable(ctx, repo, cfg)
case config.RepackObjectsStrategyGeometric:
return PerformGeometricRepacking(ctx, repo, cfg)
}
return nil
}
// PerformIncrementalRepackingWithUnreachable performs an incremental repacking task using git-repack(1) command.
// It packs all loose objects into a pack. This will omit packing objects part of alternates. It does not prune the
// loose objects that were packed.
func PerformIncrementalRepackingWithUnreachable(ctx context.Context, repo *localrepo.Repo) error {
repoPath, err := repo.Path(ctx)
if err != nil {
return fmt.Errorf("repo path: %w", err)
}
var stderr strings.Builder
// Pack all loose objects into a new packfile, regardless of their reachability.
// There is no git-repack(1) mode that would allow us to do this, so we have to
// instead do it ourselves.
if err := repo.ExecAndWait(ctx,
gitcmd.Command{
Name: "pack-objects",
Flags: []gitcmd.Option{
// We ask git-pack-objects(1) to pack loose unreachable
// objects. This implies `--revs`, but as we don't supply
// any revisions via stdin all objects will be considered
// unreachable. The effect is that we simply pack all loose
// objects into a new packfile, regardless of whether they
// are reachable or not.
gitcmd.Flag{Name: "--pack-loose-unreachable"},
// Skip any objects which are part of an alternative object
// directory.
gitcmd.Flag{Name: "--local"},
// Only pack objects which are not yet part of a different,
// local pack.
gitcmd.Flag{Name: "--incremental"},
// Only create the packfile if it would contain at least one
// object.
gitcmd.Flag{Name: "--non-empty"},
// We don't care about any kind of progress meter.
gitcmd.Flag{Name: "--quiet"},
},
Args: []string{
// We need to tell git-pack-objects(1) where to write the
// new packfile and what prefix it should have. We of course
// want to write it into the main object directory and have
// the same "pack-" prefix like normal packfiles would.
filepath.Join(repoPath, "objects", "pack", "pack"),
},
},
// Note: we explicitly do not pass `GetRepackGitConfig()` here as none of
// its options apply to this kind of repack: we have no delta islands given
// that we do not walk the revision graph, and we won't ever write bitmaps.
gitcmd.WithStderr(&stderr),
); err != nil {
var exitErr *exec.ExitError
if errors.As(err, &exitErr) {
return structerr.New("pack-objects failed with error code %d", exitErr.ExitCode()).WithMetadata("stderr", stderr.String())
}
return fmt.Errorf("pack-objects failed: %w", err)
}
return nil
}
// PerformFullRepackingWithUnreachable performs a full repacking task using git-repack(1) command. This will omit packing objects part of alternates.
func PerformFullRepackingWithUnreachable(ctx context.Context, repo *localrepo.Repo, cfg config.RepackObjectsConfig) error {
return PerformRepack(ctx, repo, cfg,
// Do a full repack.
gitcmd.Flag{Name: "-a"},
// Don't include objects part of alternate.
gitcmd.Flag{Name: "-l"},
// Delete loose objects made redundant by this repack.
gitcmd.Flag{Name: "-d"},
// Keep unreachable objects part of the old packs in the new pack.
gitcmd.Flag{Name: "--keep-unreachable"},
)
}
// PerformGeometricRepacking performs geometric repacking task using git-repack(1) command. It allows us to merge
// multiple packfiles without having to rewrite all packfiles into one. This new "geometric" strategy tries to ensure
// that existing packfiles in the repository form a geometric sequence where each successive packfile contains at least
// n times as many objects as the preceding packfile. If the sequence isn't maintained, Git will determine a slice of
// packfiles that it must repack to maintain the sequence again. With this process, we can limit the number of packfiles
// that exist in the repository without having to repack all objects into a single packfile regularly.
// This repacking does not take reachability into account.
// For more information, https://about.gitlab.com/blog/2023/11/02/rearchitecting-git-object-database-mainentance-for-scale/#geometric-repacking
func PerformGeometricRepacking(ctx context.Context, repo *localrepo.Repo, cfg config.RepackObjectsConfig) error {
return PerformRepack(ctx, repo, cfg,
// We use a geometric factor `r`, which means that every successively larger
// packfile must have at least `r` times the number of objects.
//
// This factor ultimately determines how many packfiles there can be at a
// maximum in a repository for a given number of objects. The maximum number
// of objects with `n` packfiles and a factor `r` is `(1 - r^n) / (1 - r)`.
// E.g. with a factor of 4 and 10 packfiles, we can have at most 349,525
// objects, with 16 packfiles we can have 1,431,655,765 objects. Contrary to
// that, having a factor of 2 will translate to 1023 objects at 10 packfiles
// and 65535 objects at 16 packfiles at a maximum.
//
// So what we're effectively choosing here is how often we need to repack
// larger parts of the repository. The higher the factor the more we'll have
// to repack as the packfiles will be larger. On the other hand, having a
// smaller factor means we'll have to repack less objects as the slices we
// need to repack will have less objects.
//
// The end result is a hybrid approach between incremental repacks and full
// repacks: we won't typically repack the full repository, but only a subset
// of packfiles.
//
// For now, we choose a geometric factor of two. Large repositories nowadays
// typically have a few million objects, which would boil down to having at
// most 32 packfiles in the repository. This number is not scientifically
// chosen though any may be changed at a later point in time.
gitcmd.ValueFlag{Name: "--geometric", Value: "2"},
// Make sure to delete loose objects and packfiles that are made obsolete
// by the new packfile.
gitcmd.Flag{Name: "-d"},
// Don't include objects part of an alternate.
gitcmd.Flag{Name: "-l"},
)
}
// PerformRepack performs `git-repack(1)` command on a repository with some pre-built configs.
func PerformRepack(ctx context.Context, repo *localrepo.Repo, cfg config.RepackObjectsConfig, opts ...gitcmd.Option) error {
if cfg.WriteMultiPackIndex {
opts = append(opts, gitcmd.Flag{Name: "--write-midx"})
}
var stderr strings.Builder
if err := repo.ExecAndWait(ctx,
gitcmd.Command{
Name: "repack",
Flags: opts,
},
gitcmd.WithConfig(GetRepackGitConfig(ctx, repo, cfg.WriteBitmap)...),
gitcmd.WithStderr(&stderr),
); err != nil {
var exitErr *exec.ExitError
if errors.As(err, &exitErr) {
// We do not embed the `exec.ExitError` directly as it wouldn't typically
// contain any useful information anyway except for its error code. So we
// instead only expose what matters and attach stderr to the error metadata.
return structerr.New("repack failed with error code %d", exitErr.ExitCode()).WithMetadata("stderr", stderr.String())
}
return fmt.Errorf("repack failed: %w", err)
}
return nil
}
// GetRepackGitConfig returns configuration suitable for Git commands which write new packfiles.
func GetRepackGitConfig(ctx context.Context, repo storage.Repository, bitmap bool) []gitcmd.ConfigPair {
config := []gitcmd.ConfigPair{
{Key: "repack.useDeltaIslands", Value: "true"},
{Key: "repack.writeBitmaps", Value: strconv.FormatBool(bitmap)},
{Key: "pack.writeBitmapLookupTable", Value: "true"},
}
if storage.IsPoolRepository(repo) {
config = append(config,
gitcmd.ConfigPair{Key: "pack.island", Value: git.ObjectPoolRefNamespace + "/he(a)ds"},
gitcmd.ConfigPair{Key: "pack.island", Value: git.ObjectPoolRefNamespace + "/t(a)gs"},
gitcmd.ConfigPair{Key: "pack.islandCore", Value: "a"},
)
} else {
config = append(config,
gitcmd.ConfigPair{Key: "pack.island", Value: "r(e)fs/heads"},
gitcmd.ConfigPair{Key: "pack.island", Value: "r(e)fs/tags"},
gitcmd.ConfigPair{Key: "pack.islandCore", Value: "e"},
)
}
return config
}
// PruneObjectsConfig determines which objects should be pruned in PruneObjects.
type PruneObjectsConfig struct {
// ExpireBefore controls the grace period after which unreachable objects shall be pruned.
// An unreachable object must be older than the given date in order to be considered for
// deletion.
ExpireBefore time.Time
}
// PruneObjects prunes loose objects from the repository that are already packed or which are
// unreachable and older than the configured expiry date.
func PruneObjects(ctx context.Context, repo *localrepo.Repo, cfg PruneObjectsConfig) error {
if err := repo.ExecAndWait(ctx, gitcmd.Command{
Name: "prune",
Flags: []gitcmd.Option{
// By default, this prunes all unreachable objects regardless of when they
// have last been accessed. This opens us up for races when there are
// concurrent commands which are just at the point of writing objects into
// the repository, but which haven't yet updated any references to make them
// reachable.
//
// To avoid this race, we use a grace window that can be specified by the
// caller so that we only delete objects that are older than this grace
// window.
gitcmd.ValueFlag{Name: "--expire", Value: git.FormatTime(cfg.ExpireBefore)},
},
}); err != nil {
return fmt.Errorf("executing prune: %w", err)
}
return nil
}