Skip to content

Git designed for AI agents.

16.6 KB
Raw Blame
package objectpool

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"testing"

	"github.com/feder-cr/git-ai/internal/git"
	"github.com/feder-cr/git-ai/internal/git/catfile"
	"github.com/feder-cr/git-ai/internal/git/gittest"
	"github.com/feder-cr/git-ai/internal/git/housekeeping"
	"github.com/feder-cr/git-ai/internal/git/localrepo"
	"github.com/feder-cr/git-ai/internal/git/stats"
	"github.com/feder-cr/git-ai/internal/gitaly/config"
	"github.com/feder-cr/git-ai/internal/gitaly/storage/keyvalue"
	"github.com/feder-cr/git-ai/internal/gitaly/storage/keyvalue/databasemgr"
	nodeimpl "github.com/feder-cr/git-ai/internal/gitaly/storage/node"
	"github.com/feder-cr/git-ai/internal/gitaly/storage/storagemgr"
	"github.com/feder-cr/git-ai/internal/gitaly/storage/storagemgr/partition"
	"github.com/feder-cr/git-ai/internal/helper"
	"github.com/feder-cr/git-ai/internal/helper/text"
	"github.com/feder-cr/git-ai/internal/log"
	"github.com/feder-cr/git-ai/internal/structerr"
	"github.com/feder-cr/git-ai/internal/testhelper"
	"github.com/feder-cr/git-ai/internal/testhelper/testcfg"
	"github.com/stretchr/testify/require"
)

func TestFetchFromOrigin_dangling(t *testing.T) {
	ctx := testhelper.Context(t)
	testWithAndWithoutTransaction(t, ctx, func(t *testing.T, cfg config.Cfg, newLocalRepo LocalRepoFactory) {
		pool, repo := setupObjectPoolWithCfg(t, ctx, cfg)
		poolPath := gittest.RepositoryPath(t, ctx, pool)
		repoPath := gittest.RepositoryPath(t, ctx, repo)

		// Write some reachable objects into the object pool member and fetch them into the pool.
		blobID := gittest.WriteBlob(t, cfg, repoPath, []byte("contents"))
		treeID := gittest.WriteTree(t, cfg, repoPath, []gittest.TreeEntry{
			{Mode: "100644", OID: blobID, Path: "reachable"},
		})
		commitID := gittest.WriteCommit(t, cfg, repoPath,
			gittest.WithTree(treeID),
			gittest.WithBranch("master"),
		)
		require.NoError(t, pool.FetchFromOrigin(ctx, repo, newLocalRepo))

		// We now write a bunch of objects into the object pool that are not referenced by anything.
		// These are thus "dangling".
		unreachableBlob := gittest.WriteBlob(t, cfg, poolPath, []byte("unreachable"))
		unreachableTree := gittest.WriteTree(t, cfg, poolPath, []gittest.TreeEntry{
			{Mode: "100644", OID: blobID, Path: "unreachable"},
		})
		unreachableCommit := gittest.WriteCommit(t, cfg, poolPath,
			gittest.WithMessage("unreachable"),
			gittest.WithTree(treeID),
		)
		unreachableTag := gittest.WriteTag(t, cfg, poolPath, "unreachable", commitID.Revision(), gittest.WriteTagConfig{
			Message: "unreachable",
		})
		// `WriteTag()` automatically creates a reference and thus makes the annotated tag
		// reachable. We thus delete the reference here again.
		gittest.Exec(t, cfg, "-C", poolPath, "update-ref", "-d", "refs/tags/unreachable")

		// git-fsck(1) should report the newly created unreachable objects as dangling.
		fsckBefore := gittest.Exec(t, cfg, "-C", poolPath, "fsck", "--connectivity-only", "--dangling")
		require.ElementsMatch(t, []string{
			fmt.Sprintf("dangling blob %s", unreachableBlob),
			fmt.Sprintf("dangling tag %s", unreachableTag),
			fmt.Sprintf("dangling commit %s", unreachableCommit),
			fmt.Sprintf("dangling tree %s", unreachableTree),
		}, strings.Split(text.ChompBytes(fsckBefore), "\n"))

		// We expect this second run to convert the dangling objects into non-dangling objects.
		require.NoError(t, pool.FetchFromOrigin(ctx, repo, newLocalRepo))

		// Each of the dangling objects should have gotten a new dangling reference.
		danglingRefs := gittest.Exec(t, cfg, "-C", poolPath, "for-each-ref", "--format=%(refname) %(objectname)", "refs/dangling/")
		require.ElementsMatch(t, []string{
			fmt.Sprintf("refs/dangling/%[1]s %[1]s", unreachableBlob),
			fmt.Sprintf("refs/dangling/%[1]s %[1]s", unreachableTree),
			fmt.Sprintf("refs/dangling/%[1]s %[1]s", unreachableTag),
			fmt.Sprintf("refs/dangling/%[1]s %[1]s", unreachableCommit),
		}, strings.Split(text.ChompBytes(danglingRefs), "\n"))
		// And git-fsck(1) shouldn't report the objects as dangling anymore.
		require.Empty(t, gittest.Exec(t, cfg, "-C", poolPath, "fsck", "--connectivity-only", "--dangling"))
	})
}

func TestFetchFromOrigin_fsck(t *testing.T) {
	ctx := testhelper.Context(t)
	testWithAndWithoutTransaction(t, ctx, func(t *testing.T, cfg config.Cfg, newLocalRepo LocalRepoFactory) {
		pool, repo := setupObjectPoolWithCfg(t, ctx, cfg)
		repoPath, err := repo.Path(ctx)
		require.NoError(t, err)

		require.NoError(t, pool.FetchFromOrigin(ctx, repo, newLocalRepo), "seed pool")

		// We're creating a new commit which has a root tree with duplicate entries. git-mktree(1)
		// allows us to create these trees just fine, but git-fsck(1) complains.
		gittest.WriteCommit(t, cfg, repoPath,
			gittest.WithTreeEntries(
				gittest.TreeEntry{OID: gittest.DefaultObjectHash.EmptyTreeOID, Path: "dup", Mode: "040000"},
				gittest.TreeEntry{OID: gittest.DefaultObjectHash.EmptyTreeOID, Path: "dup", Mode: "040000"},
			),
			gittest.WithBranch("branch"),
		)

		err = pool.FetchFromOrigin(ctx, repo, newLocalRepo)
		require.Error(t, err)
		require.Contains(t, err.Error(), "duplicateEntries: contains duplicate file entries")
	})
}

func TestFetchFromOrigin_deltaIslands(t *testing.T) {
	ctx := testhelper.Context(t)
	testWithAndWithoutTransaction(t, ctx, func(t *testing.T, cfg config.Cfg, newLocalRepo LocalRepoFactory) {
		pool, repo := setupObjectPoolWithCfg(t, ctx, cfg)
		poolPath := gittest.RepositoryPath(t, ctx, pool)
		repoPath := gittest.RepositoryPath(t, ctx, repo)

		require.NoError(t, pool.FetchFromOrigin(ctx, repo, newLocalRepo), "seed pool")
		require.NoError(t, pool.Link(ctx, repo))

		// With multi-pack-indices we don't do full repacks of repositories that
		// aggressively anymore, but in order to test delta islands we need to trigger one.
		// We thus write a second packfile so that `OptimizeRepository()` decides to
		// rewrite packfiles.
		gittest.WriteCommit(t, cfg, poolPath, gittest.WithBranch("irrelevant"))
		gittest.Exec(t, cfg, "-C", poolPath, "repack")

		// The setup of delta islands is done in the normal repository, and thus we pass `false`
		// for `isPoolRepo`. Verification whether we correctly handle repacking though happens in
		// the pool repository.
		gittest.TestDeltaIslands(t, cfg, repoPath, poolPath, false, func() error {
			return pool.FetchFromOrigin(ctx, repo, newLocalRepo)
		})
	})
}

func TestFetchFromOrigin_bitmapHashCache(t *testing.T) {
	ctx := testhelper.Context(t)
	testWithAndWithoutTransaction(t, ctx, func(t *testing.T, cfg config.Cfg, newLocalRepo LocalRepoFactory) {
		pool, repo := setupObjectPoolWithCfg(t, ctx, cfg)
		repoPath, err := repo.Path(ctx)
		require.NoError(t, err)

		gittest.WriteCommit(t, cfg, repoPath, gittest.WithBranch("master"))

		require.NoError(t, pool.FetchFromOrigin(ctx, repo, newLocalRepo))

		bitmaps, err := filepath.Glob(gittest.RepositoryPath(t, ctx, pool, "objects", "pack", "*.bitmap"))
		require.NoError(t, err)
		require.Len(t, bitmaps, 1)

		bitmapInfo, err := stats.BitmapInfoForPath(bitmaps[0])
		require.NoError(t, err)
		require.Equal(t, stats.BitmapInfo{
			Exists:         true,
			Version:        1,
			HasHashCache:   true,
			HasLookupTable: true,
		}, bitmapInfo)
	})
}

func TestFetchFromOrigin_refUpdates(t *testing.T) {
	ctx := testhelper.Context(t)
	testWithAndWithoutTransaction(t, ctx, func(t *testing.T, cfg config.Cfg, newLocalRepo LocalRepoFactory) {
		pool, repo := setupObjectPoolWithCfg(t, ctx, cfg)
		repoPath, err := repo.Path(ctx)
		require.NoError(t, err)

		poolPath := gittest.RepositoryPath(t, ctx, pool)

		// Seed the pool member with some preliminary data.
		oldRefs := map[string]git.ObjectID{}
		oldRefs["heads/csv"] = gittest.WriteCommit(t, cfg, repoPath, gittest.WithBranch("csv"), gittest.WithMessage("old"))
		oldRefs["tags/v1.1.0"] = gittest.WriteTag(t, cfg, repoPath, "v1.1.0", oldRefs["heads/csv"].Revision())

		// We now fetch that data into the object pool and verify that it exists as expected.
		require.NoError(t, pool.FetchFromOrigin(ctx, repo, newLocalRepo))
		for ref, oid := range oldRefs {
			require.Equal(t, oid, gittest.ResolveRevision(t, cfg, poolPath, "refs/remotes/origin/"+ref))
		}

		// Next, we force-overwrite both old references with new objects.
		newRefs := map[string]git.ObjectID{}
		newRefs["heads/csv"] = gittest.WriteCommit(t, cfg, repoPath, gittest.WithBranch("csv"), gittest.WithMessage("new"))
		newRefs["tags/v1.1.0"] = gittest.WriteTag(t, cfg, repoPath, "v1.1.0", newRefs["heads/csv"].Revision(), gittest.WriteTagConfig{
			Force: true,
		})

		// Create a bunch of additional references. This is to trigger OptimizeRepository to indeed
		// repack the loose references as we expect it to in this test. It's debatable whether we
		// should test this at all here given that this is business of the housekeeping package. But
		// it's easy enough to do, so it doesn't hurt.
		for i := 0; i < 32; i++ {
			branchName := fmt.Sprintf("branch-%d", i)
			newRefs["heads/"+branchName] = gittest.WriteCommit(t, cfg, repoPath,
				gittest.WithMessage(strconv.Itoa(i)),
				gittest.WithBranch(branchName),
			)
		}

		// Now we fetch again and verify that all references should have been updated accordingly.
		require.NoError(t, pool.FetchFromOrigin(ctx, repo, newLocalRepo))
		for ref, oid := range newRefs {
			require.Equal(t, oid, gittest.ResolveRevision(t, cfg, poolPath, "refs/remotes/origin/"+ref))
		}
	})
}

func TestFetchFromOrigin_refs(t *testing.T) {
	ctx := testhelper.Context(t)
	testWithAndWithoutTransaction(t, ctx, func(t *testing.T, cfg config.Cfg, newLocalRepo LocalRepoFactory) {
		pool, repo := setupObjectPoolWithCfg(t, ctx, cfg)
		repoPath, err := repo.Path(ctx)
		require.NoError(t, err)

		// Verify that the object pool ain't yet got any references.
		poolPath := gittest.RepositoryPath(t, ctx, pool)
		require.Empty(t, gittest.Exec(t, cfg, "-C", poolPath, "for-each-ref", "--format=%(refname)"))

		// Initialize the repository with a bunch of references.
		commitID := gittest.WriteCommit(t, cfg, repoPath)
		for _, ref := range []git.ReferenceName{"refs/heads/master", "refs/environments/1", "refs/tags/lightweight-tag"} {
			gittest.WriteRef(t, cfg, repoPath, ref, commitID)
		}
		gittest.WriteTag(t, cfg, repoPath, "annotated-tag", commitID.Revision(), gittest.WriteTagConfig{
			Message: "tag message",
		})

		// Fetch from the pool member. This should pull in all references we have just created in
		// that repository into the pool.
		require.NoError(t, pool.FetchFromOrigin(ctx, repo, newLocalRepo))
		require.Equal(t,
			[]string{
				"refs/remotes/origin/environments/1",
				"refs/remotes/origin/heads/master",
				"refs/remotes/origin/tags/annotated-tag",
				"refs/remotes/origin/tags/lightweight-tag",
			},
			strings.Split(text.ChompBytes(gittest.Exec(t, cfg, "-C", poolPath, "for-each-ref", "--format=%(refname)")), "\n"),
		)

		// We don't want to see "FETCH_HEAD" though: it's useless and may take quite some time to
		// write out in Git.
		require.NoFileExists(t, filepath.Join(poolPath, "FETCH_HEAD"))
	})
}

func TestFetchFromOrigin_missingPool(t *testing.T) {
	ctx := testhelper.Context(t)
	testWithAndWithoutTransaction(t, ctx, func(t *testing.T, cfg config.Cfg, newLocalRepo LocalRepoFactory) {
		pool, repo := setupObjectPoolWithCfg(t, ctx, cfg)

		poolPath, err := pool.Repo.Path(ctx)
		require.NoError(t, err)

		// Remove the object pool to assert that we raise an error when fetching into a non-existent
		// object pool.
		require.NoError(t, os.RemoveAll(poolPath))

		require.Equal(t, structerr.NewInvalidArgument("object pool does not exist"), pool.FetchFromOrigin(ctx, repo, newLocalRepo))
		require.False(t, pool.Exists(ctx))
	})
}

func TestObjectPool_logStats(t *testing.T) {
	t.Parallel()

	ctx := testhelper.Context(t)

	type setupData struct {
		pool   *ObjectPool
		fields log.Fields
	}

	for _, tc := range []struct {
		desc           string
		setup          func(t *testing.T, logger log.Logger) setupData
		expectedFields log.Fields
	}{
		{
			desc: "empty object pool",
			setup: func(t *testing.T, logger log.Logger) setupData {
				_, pool, _ := setupObjectPool(t, ctx, withLogger(logger))
				return setupData{
					pool: pool,
					fields: log.Fields{
						"when":                "now",
						"references.dangling": referencedObjectTypes{},
						"references.normal":   referencedObjectTypes{},
						"repository_info": stats.RepositoryInfo{
							IsObjectPool: true,
							References: stats.ReferencesInfo{
								ReferenceBackendName: gittest.DefaultReferenceBackend.Name,
								ReftableTables: gittest.FilesOrReftables(
									nil,
									[]stats.ReftableTable{
										{
											Size:           hashDependentSize(t, 124, 132),
											UpdateIndexMin: 1,
											UpdateIndexMax: 1,
										},
									}),
							},
						},
					},
				}
			},
		},
		{
			desc: "normal reference",
			setup: func(t *testing.T, logger log.Logger) setupData {
				cfg, pool, _ := setupObjectPool(t, ctx, withLogger(logger))
				gittest.WriteCommit(t, cfg, gittest.RepositoryPath(t, ctx, pool), gittest.WithBranch("main"))

				return setupData{
					pool: pool,
					fields: log.Fields{
						"when":                "now",
						"references.dangling": referencedObjectTypes{},
						"references.normal": referencedObjectTypes{
							Commits: 1,
						},
						"repository_info": stats.RepositoryInfo{
							IsObjectPool: true,
							LooseObjects: stats.LooseObjectsInfo{
								Count: 2,
								Size:  hashDependentSize(t, 142, 158),
							},
							References: stats.ReferencesInfo{
								ReferenceBackendName: gittest.DefaultReferenceBackend.Name,
								LooseReferencesCount: gittest.FilesOrReftables[uint64](1, 0),
								ReftableTables: gittest.FilesOrReftables(
									nil,
									[]stats.ReftableTable{
										{
											Size:           hashDependentSize(t, 165, 185),
											UpdateIndexMin: 1,
											UpdateIndexMax: 2,
										},
									}),
							},
						},
					},
				}
			},
		},
		{
			desc: "dangling reference",
			setup: func(t *testing.T, logger log.Logger) setupData {
				cfg, pool, _ := setupObjectPool(t, ctx, withLogger(logger))
				gittest.WriteCommit(t, cfg, gittest.RepositoryPath(t, ctx, pool), gittest.WithReference("refs/dangling/commit"))

				return setupData{
					pool: pool,
					fields: log.Fields{
						"when": "now",
						"references.dangling": referencedObjectTypes{
							Commits: 1,
						},
						"references.normal": referencedObjectTypes{},
						"repository_info": stats.RepositoryInfo{
							IsObjectPool: true,
							LooseObjects: stats.LooseObjectsInfo{
								Count: 2,
								Size:  hashDependentSize(t, 142, 158),
							},
							References: stats.ReferencesInfo{
								ReferenceBackendName: gittest.DefaultReferenceBackend.Name,
								LooseReferencesCount: gittest.FilesOrReftables[uint64](1, 0),
								ReftableTables: gittest.FilesOrReftables(
									nil,
									[]stats.ReftableTable{
										{
											Size:           hashDependentSize(t, 171, 191),
											UpdateIndexMin: 1,
											UpdateIndexMax: 2,
										},
									}),
							},
						},
					},
				}
			},
		},
	} {
		t.Run(tc.desc, func(t *testing.T) {
			logger := testhelper.NewLogger(t)
			hook := testhelper.AddLoggerHook(logger)
			setup := tc.setup(t, logger)

			require.NoError(t, setup.pool.logStats(ctx, "now"))

			logEntries := hook.AllEntries()
			require.Len(t, logEntries, 1)
			require.Equal(t, "pool dangling ref stats", logEntries[0].Message)
			require.Equal(t, setup.fields, logEntries[0].Data)
		})
	}
}

func testWithAndWithoutTransaction(t *testing.T, ctx context.Context, testFunc func(*testing.T, config.Cfg, LocalRepoFactory)) {
	t.Helper()

	t.Run("with transaction", func(t *testing.T) {
		t.Parallel()

		cfg := testcfg.Build(t)

		logger := testhelper.SharedLogger(t)
		cmdFactory := gittest.NewCommandFactory(t, cfg)
		catfileCache := catfile.NewCache(cfg)
		t.Cleanup(catfileCache.Stop)

		dbMgr, err := databasemgr.NewDBManager(ctx, cfg.Storages, keyvalue.NewBadgerStore, helper.NewNullTickerFactory(), logger)
		require.NoError(t, err)
		defer dbMgr.Close()

		localRepoFactory := localrepo.NewFactory(logger, config.NewLocator(cfg), cmdFactory, catfileCache)

		partitionFactoryOptions := []partition.FactoryOption{
			partition.WithCmdFactory(cmdFactory),
			partition.WithRepoFactory(localRepoFactory),
			partition.WithMetrics(partition.NewMetrics(housekeeping.NewMetrics(cfg.Prometheus))),
		}

		node, err := nodeimpl.NewManager(
			cfg.Storages,
			storagemgr.NewFactory(
				logger,
				dbMgr,
				partition.NewFactory(partitionFactoryOptions...),
				config.DefaultMaxInactivePartitions,
				storagemgr.NewMetrics(cfg.Prometheus),
			),
		)
		require.NoError(t, err)
		defer node.Close()

		testFunc(t, cfg, localRepoFactory.Build)
	})

	t.Run("without transaction", func(t *testing.T) {
		t.Parallel()

		cfg := testcfg.Build(t)
		testFunc(t, cfg, nil)
	})
}