Git designed for AI agents.
package manager
import (
"context"
"fmt"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
"github.com/feder-cr/git-ai/internal/git"
"github.com/feder-cr/git-ai/internal/git/catfile"
"github.com/feder-cr/git-ai/internal/git/gittest"
"github.com/feder-cr/git-ai/internal/git/housekeeping"
housekeepingcfg "github.com/feder-cr/git-ai/internal/git/housekeeping/config"
"github.com/feder-cr/git-ai/internal/git/localrepo"
gitalycfg "github.com/feder-cr/git-ai/internal/gitaly/config"
"github.com/feder-cr/git-ai/internal/gitaly/storage"
"github.com/feder-cr/git-ai/internal/gitaly/storage/keyvalue"
"github.com/feder-cr/git-ai/internal/gitaly/storage/keyvalue/databasemgr"
"github.com/feder-cr/git-ai/internal/gitaly/storage/mode"
nodeimpl "github.com/feder-cr/git-ai/internal/gitaly/storage/node"
"github.com/feder-cr/git-ai/internal/gitaly/storage/storagemgr"
"github.com/feder-cr/git-ai/internal/gitaly/storage/storagemgr/partition"
"github.com/feder-cr/git-ai/internal/helper"
"github.com/feder-cr/git-ai/internal/helper/text"
"github.com/feder-cr/git-ai/internal/offloading"
"github.com/feder-cr/git-ai/internal/testhelper"
"github.com/feder-cr/git-ai/internal/testhelper/testcfg"
"github.com/stretchr/testify/require"
"gocloud.dev/blob"
)
func TestOffloadRepository_HappyPath(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
cfg := testcfg.Build(t)
localBucketDir := testhelper.TempDir(t)
cacheRoot := testhelper.TempDir(t)
sinkURL := fmt.Sprintf("file://%s", localBucketDir)
sink, bucket := getOffloadingStorageSink(t, ctx, sinkURL)
t.Cleanup(func() { _ = bucket.Close() })
cfg.Offloading = gitalycfg.Offloading{
Enabled: true,
CacheRoot: cacheRoot,
GoCloudURL: sinkURL,
}
// Setup repo, node and housekeeping manager.
repoSetup := setupRepoForOffloading(t, ctx, cfg)
repo := repoSetup.repo
blobs := repoSetup.blobs
commits := repoSetup.commits
trees := repoSetup.trees
repoPath := repoSetup.repoPath
node := setupNodeForTransaction(t, ctx, cfg, sink)
defer node.Close()
housekeepingManager := New(cfg.Prometheus, testhelper.SharedLogger(t), nil, node)
// Execute offloading in repository.
offloadingCfg := housekeepingcfg.OffloadingConfig{
CacheRoot: cacheRoot,
SinkBaseURL: cfg.Offloading.GoCloudURL,
}
require.NoError(t, housekeepingManager.OffloadRepository(ctx, repo, offloadingCfg))
applyWAL(t, ctx, repo, node)
// First, verify blobs are properly offloaded and no longer present in the repository
assertOffloadedRepoObjects(t, false, "", blobs, trees, commits, repoPath, cfg)
// Then, verify that downloading offloaded files to the cache resolves missing objects, which confirms:
// 1. Offloaded files contain the expected blob content
// 2. After injecting the cache path into GIT_ALTERNATE_OBJECT_DIRECTORIES, the downloaded files are
// recognized as part of the repository
cachePathObjectsDir := filepath.Join(cfg.Offloading.CacheRoot, repo.GetRelativePath(), "objects")
cachePathPackDir := filepath.Join(cachePathObjectsDir, "pack")
downloadFilesToCache(t, ctx, repoPath, cachePathPackDir, sink, repo, cfg)
assertOffloadedRepoObjects(t, true, cachePathObjectsDir, blobs, trees, commits, repoPath, cfg)
}
func TestRehydrateRepository_HappyPath(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
cfg := testcfg.Build(t)
localBucketDir := testhelper.TempDir(t)
cacheRoot := testhelper.TempDir(t)
sinkURL := fmt.Sprintf("file://%s", localBucketDir)
sink, bucket := getOffloadingStorageSink(t, ctx, sinkURL)
t.Cleanup(func() { _ = bucket.Close() })
cfg.Offloading = gitalycfg.Offloading{
Enabled: true,
CacheRoot: cacheRoot,
GoCloudURL: sinkURL,
}
// Setup repo, node and housekeeping manager.
repoSetup := setupRepoForOffloading(t, ctx, cfg)
repo := repoSetup.repo
blobs := repoSetup.blobs
commits := repoSetup.commits
trees := repoSetup.trees
repoPath := repoSetup.repoPath
node := setupNodeForTransaction(t, ctx, cfg, sink)
defer node.Close()
housekeepingManager := New(cfg.Prometheus, testhelper.SharedLogger(t), nil, node)
// Execute offloading in repository.
offloadingCfg := housekeepingcfg.OffloadingConfig{
CacheRoot: cacheRoot,
SinkBaseURL: cfg.Offloading.GoCloudURL,
}
require.NoError(t, housekeepingManager.OffloadRepository(ctx, repo, offloadingCfg))
applyWAL(t, ctx, repo, node)
// First, verify blobs are properly offloaded and no longer present in the repository
assertOffloadedRepoObjects(t, false, "", blobs, trees, commits, repoPath, cfg)
isOffloaded, offloadURL, err := repo.IsOffloaded(ctx)
require.True(t, isOffloaded, "Repository should be offloaded")
require.NoError(t, err)
// Extract the prefix by removing the storage URL path from the offload URL path
parsedURL, _ := url.Parse(offloadURL)
storageURL, _ := url.Parse(cfg.Offloading.GoCloudURL)
prefix := strings.TrimPrefix(parsedURL.Path, storageURL.Path+"/")
require.NoError(t, housekeepingManager.RehydrateRepository(ctx, repo, prefix))
applyWAL(t, ctx, repo, node)
expectedObjects := make([]git.ObjectID, 0, len(blobs)+len(trees)+len(commits))
expectedObjects = append(expectedObjects, commits...)
expectedObjects = append(expectedObjects, trees...)
expectedObjects = append(expectedObjects, blobs...)
expectedObjectHashes := make([]string, 0, len(blobs)+len(trees)+len(commits))
for _, obj := range expectedObjects {
expectedObjectHashes = append(expectedObjectHashes, string(obj))
}
actualObjectHashes := make([]string, 0, len(trees)+len(commits)+len(blobs))
output := gittest.Exec(t, cfg, "-C", repoPath, "rev-list", "--objects", "--all", "--missing=print", "--no-object-names")
actualObjectHashes = append(actualObjectHashes, strings.Split(text.ChompBytes(output), "\n")...)
require.ElementsMatch(t, expectedObjectHashes, actualObjectHashes)
}
type offloadingRepoSetup struct {
repoPath string
repo *localrepo.Repo
blobs, trees, commits []git.ObjectID
refs map[git.ReferenceName]git.ObjectID
}
func setupRepoForOffloading(t *testing.T, ctx context.Context, cfg gitalycfg.Cfg) offloadingRepoSetup {
repoProto, repoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{
SkipCreationViaService: true,
RelativePath: gittest.NewRepositoryName(t),
})
blobs := make([]git.ObjectID, 4)
for i := range len(blobs) {
blobs[i] = gittest.WriteBlob(t, cfg, repoPath, []byte(strconv.Itoa(i)))
}
subsubTree := gittest.WriteTree(t, cfg, repoPath, []gittest.TreeEntry{
{Path: "subsubfile", Mode: "100644", OID: blobs[0]},
})
firstCommit := gittest.WriteCommit(t, cfg, repoPath, gittest.WithTree(subsubTree), gittest.WithParents())
subTree := gittest.WriteTree(t, cfg, repoPath, []gittest.TreeEntry{
{Path: "subfile", Mode: "100644", OID: blobs[1]},
{Path: "subsubdir", Mode: "040000", OID: subsubTree},
})
secondCommit := gittest.WriteCommit(t, cfg, repoPath, gittest.WithTree(subTree), gittest.WithParents(firstCommit))
commitTree := gittest.WriteTree(t, cfg, repoPath, []gittest.TreeEntry{
{Path: "LICENSE", Mode: "100644", OID: blobs[2]},
{Path: "README.md", Mode: "100644", OID: blobs[3]},
{Path: "subdir", Mode: "040000", OID: subTree},
})
thirdCommit := gittest.WriteCommit(t, cfg, repoPath, gittest.WithTree(commitTree), gittest.WithParents(secondCommit))
trees := []git.ObjectID{subsubTree, subTree, commitTree}
commits := []git.ObjectID{firstCommit, secondCommit, thirdCommit}
refs := make(map[git.ReferenceName]git.ObjectID)
gittest.WriteRef(t, cfg, repoPath, "refs/heads/first", firstCommit)
refs["refs/heads/first"] = firstCommit
gittest.WriteRef(t, cfg, repoPath, "refs/heads/second", secondCommit)
refs["refs/heads/second"] = secondCommit
gittest.WriteRef(t, cfg, repoPath, "refs/heads/main", thirdCommit)
refs["refs/heads/main"] = thirdCommit
cmdFactory := gittest.NewCommandFactory(t, cfg)
repo := localrepo.New(testhelper.NewLogger(t), gitalycfg.NewLocator(cfg), cmdFactory, nil, repoProto)
gittest.Exec(t, cfg, "-C", repoPath, "gc")
return offloadingRepoSetup{
repoPath: repoPath,
repo: repo,
blobs: blobs,
trees: trees,
commits: commits,
refs: refs,
}
}
func getOffloadingStorageSink(t *testing.T, ctx context.Context, localBucketURL string) (*offloading.Sink, offloading.Bucket) {
bucket, err := blob.OpenBucket(ctx, localBucketURL)
require.NoError(t, err)
sink, err := offloading.NewSink(bucket)
require.NoError(t, err)
return sink, bucket
}
func setupNodeForTransaction(t *testing.T, ctx context.Context, cfg gitalycfg.Cfg, sink *offloading.Sink) *nodeimpl.Manager {
logger := testhelper.SharedLogger(t)
cmdFactory := gittest.NewCommandFactory(t, cfg)
catfileCache := catfile.NewCache(cfg)
t.Cleanup(catfileCache.Stop)
localRepoFactory := localrepo.NewFactory(logger, gitalycfg.NewLocator(cfg), cmdFactory, catfileCache)
dbMgr, err := databasemgr.NewDBManager(
ctx,
cfg.Storages,
keyvalue.NewBadgerStore,
helper.NewNullTickerFactory(),
logger,
)
require.NoError(t, err)
t.Cleanup(dbMgr.Close)
partitionFactoryOptions := []partition.FactoryOption{
partition.WithCmdFactory(cmdFactory),
partition.WithRepoFactory(localRepoFactory),
partition.WithMetrics(partition.NewMetrics(housekeeping.NewMetrics(cfg.Prometheus))),
partition.WithOffloadingSink(sink),
}
node, err := nodeimpl.NewManager(
cfg.Storages,
storagemgr.NewFactory(
logger,
dbMgr,
partition.NewFactory(partitionFactoryOptions...),
gitalycfg.DefaultMaxInactivePartitions,
storagemgr.NewMetrics(cfg.Prometheus),
),
)
require.NoError(t, err)
return node
}
func applyWAL(t *testing.T, ctx context.Context, repo *localrepo.Repo, node storage.Node) {
nodeStorage, err := node.GetStorage(repo.GetStorageName())
require.NoError(t, err)
// Start a transaction to ensure the WAL is fully applied. This test is still
// accessing the repository directly in the storage.
tx, err := nodeStorage.Begin(ctx, storage.TransactionOptions{
ReadOnly: true,
RelativePath: repo.GetRelativePath(),
})
require.NoError(t, err)
defer func() {
require.NoError(t, tx.Rollback(ctx))
}()
}
func downloadFilesToCache(t *testing.T, ctx context.Context, repoPath, cachePath string, sink *offloading.Sink,
repo *localrepo.Repo, cfg gitalycfg.Cfg,
) {
offloadRemoteURL := gittest.Exec(t, cfg, "-C", repoPath, "config", "get", "remote.offload.url")
prefix := filepath.Join(repo.GetRelativePath(), filepath.Base(strings.TrimSpace(string(offloadRemoteURL))))
filesOnSink, err := sink.List(ctx, prefix)
require.NoError(t, err)
require.NoError(t, os.MkdirAll(cachePath, mode.Directory))
for _, name := range filesOnSink {
key := filepath.Join(prefix, name)
require.NoError(t, sink.Download(ctx, key, filepath.Join(cachePath, name)))
}
}
func assertOffloadedRepoObjects(t *testing.T, downloadToCache bool, cachePath string, blobs, trees, commits []git.ObjectID, repoPath string, cfg gitalycfg.Cfg) {
expectedObjects := make([]git.ObjectID, 0, len(trees)+len(commits))
expectedObjects = append(expectedObjects, commits...)
expectedObjects = append(expectedObjects, trees...)
expectedObjectHashes := make([]string, 0, len(blobs)+len(trees)+len(commits))
for _, obj := range expectedObjects {
expectedObjectHashes = append(expectedObjectHashes, string(obj))
}
for _, b := range blobs {
hashPrefix := ""
if !downloadToCache {
// If we choose not to download offloaded files, it means
// blobs are offloaded, hence missing from the repo. Therefore, their hash are prefixed with "?".
hashPrefix = "?"
}
expectedObjectHashes = append(expectedObjectHashes, hashPrefix+string(b))
}
actualObjectHashes := make([]string, 0, len(trees)+len(commits)+len(blobs))
execOpt := gittest.ExecConfig{
Env: []string{fmt.Sprintf("GIT_ALTERNATE_OBJECT_DIRECTORIES=%s", cachePath)},
}
output := gittest.ExecOpts(t, cfg, execOpt, "-C", repoPath, "rev-list", "--objects", "--all", "--missing=print", "--no-object-names")
actualObjectHashes = append(actualObjectHashes, strings.Split(text.ChompBytes(output), "\n")...)
require.ElementsMatch(t, expectedObjectHashes, actualObjectHashes)
}