Skip to content

Git designed for AI agents.

16.9 KB
Raw Blame
package gitalybackup

import (
	"bytes"
	"encoding/json"
	"fmt"
	"path/filepath"
	"runtime"
	"strconv"
	"testing"

	"github.com/feder-cr/git-ai/internal/backup"
	"github.com/feder-cr/git-ai/internal/git"
	"github.com/feder-cr/git-ai/internal/git/gittest"
	"github.com/feder-cr/git-ai/internal/gitaly/service/setup"
	"github.com/feder-cr/git-ai/internal/gitaly/storage"
	"github.com/feder-cr/git-ai/internal/grpc/client"
	"github.com/feder-cr/git-ai/internal/grpc/metadata"
	"github.com/feder-cr/git-ai/internal/testhelper"
	"github.com/feder-cr/git-ai/internal/testhelper/testcfg"
	"github.com/feder-cr/git-ai/internal/testhelper/testserver"
	"github.com/feder-cr/git-ai/proto/go/gitalypb"
	"github.com/stretchr/testify/require"
)

func TestRestoreSubcommand(t *testing.T) {
	ctx := testhelper.Context(t)

	cfg := testcfg.Build(t)
	testcfg.BuildGitalyHooks(t, cfg)

	cfg.SocketPath = testserver.RunGitalyServer(t, cfg, setup.RegisterAll)
	conn := gittest.DialService(t, ctx, cfg)

	// This is an example of a "dangling" repository (one that was created after a backup was taken) that should be
	// removed after the backup is restored.
	existingRepo, existRepoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{
		RelativePath: "existing_repo",
	})
	gittest.WriteCommit(t, cfg, existRepoPath, gittest.WithBranch(git.DefaultBranch))

	// This pool is also dangling but should not be removed after the backup is restored.
	poolRepo, _ := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{
		RelativePath: gittest.NewObjectPoolName(t),
	})

	// The backupDir contains the artifacts that would've been created as part of a backup.
	backupDir := testhelper.TempDir(t)
	testhelper.WriteFiles(t, backupDir, map[string]any{
		filepath.Join(existingRepo.GetRelativePath() + ".bundle"): gittest.Exec(t, cfg, "-C", existRepoPath, "bundle", "create", "-", "--all"),
		filepath.Join(existingRepo.GetRelativePath() + ".refs"):   gittest.Exec(t, cfg, "-C", existRepoPath, "show-ref", "--head"),
	})
	// backup_ids/ markers written separately so they get different modification times.
	testhelper.WriteFiles(t, backupDir, map[string]any{
		"backup_ids/the-backup-1": "",
	})
	testhelper.WriteFiles(t, backupDir, map[string]any{
		"backup_ids/the-backup-2": "",
	})

	// These repos are the ones being restored, and should exist after the restore.
	var repos []*gitalypb.Repository
	for i := 0; i < 2; i++ {
		repo, _ := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{
			RelativePath: fmt.Sprintf("repo-%d", i),
			Storage:      cfg.Storages[0],
		})

		testhelper.WriteFiles(t, backupDir, map[string]any{
			filepath.Join("manifests", repo.GetStorageName(), repo.GetRelativePath(), "the-backup-2.toml"): fmt.Sprintf(`
object_format = '%[1]s'
head_reference = '%[3]s'

[[steps]]
bundle_path = '%[2]s.bundle'
ref_path = '%[2]s.refs'
custom_hooks_path = '%[2]s/custom_hooks.tar'
`, gittest.DefaultObjectHash.Format, existingRepo.GetRelativePath(), git.DefaultRef.String()),
		})

		repos = append(repos, repo)
	}

	var stdin bytes.Buffer

	encoder := json.NewEncoder(&stdin)
	for _, repo := range repos {
		require.NoError(t, encoder.Encode(map[string]string{
			"address":         cfg.SocketPath,
			"token":           cfg.Auth.Token,
			"storage_name":    repo.GetStorageName(),
			"relative_path":   repo.GetRelativePath(),
			"gl_project_path": repo.GetGlProjectPath(),
		}))
	}

	require.True(t, gittest.RepositoryExists(t, ctx, conn, existingRepo))
	require.True(t, gittest.RepositoryExists(t, ctx, conn, poolRepo))

	ctx, err := storage.InjectGitalyServers(ctx, "default", cfg.SocketPath, cfg.Auth.Token)
	require.NoError(t, err)
	t.Setenv("GITALY_SERVERS", metadata.GetValue(metadata.OutgoingToIncoming(ctx), "gitaly-servers"))

	_, stderr, exitCode := runGitalyBackup(t, ctx, cfg, &stdin,
		"restore",
		"--path",
		backupDir,
		"--parallel",
		strconv.Itoa(runtime.NumCPU()),
		"--parallel-storage",
		"2",
		"--remove-all-repositories",
		existingRepo.GetStorageName(),
	)
	require.Empty(t, stderr)
	require.Zero(t, exitCode)

	require.False(t, gittest.RepositoryExists(t, ctx, conn, existingRepo))
	require.True(t, gittest.RepositoryExists(t, ctx, conn, poolRepo))

	// Ensure the repos were restored correctly.
	for _, repo := range repos {
		repoPath := filepath.Join(cfg.Storages[0].Path, gittest.GetReplicaPath(t, ctx, cfg, repo))
		bundlePath := filepath.Join(backupDir, existingRepo.GetRelativePath()+".bundle")

		output := gittest.Exec(t, cfg, "-C", repoPath, "bundle", "verify", bundlePath)
		require.Contains(t, string(output), "The bundle records a complete history")
	}
}

func TestRestoreSubcommand_latestFallback(t *testing.T) {
	ctx := testhelper.Context(t)

	cfg := testcfg.Build(t)
	testcfg.BuildGitalyHooks(t, cfg)

	cfg.SocketPath = testserver.RunGitalyServer(t, cfg, setup.RegisterAll)
	conn := gittest.DialService(t, ctx, cfg)

	// This is an example of a "dangling" repository (one that was created after a backup was taken) that should be
	// removed after the backup is restored.
	existingRepo, existRepoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{
		RelativePath: "existing_repo",
	})
	gittest.WriteCommit(t, cfg, existRepoPath, gittest.WithBranch(git.DefaultBranch))

	// This pool is also dangling but should not be removed after the backup is restored.
	poolRepo, _ := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{
		RelativePath: gittest.NewObjectPoolName(t),
	})

	// The backupDir contains the artifacts that would've been created as part of a backup.
	backupDir := testhelper.TempDir(t)
	testhelper.WriteFiles(t, backupDir, map[string]any{
		filepath.Join(existingRepo.GetRelativePath() + ".bundle"): gittest.Exec(t, cfg, "-C", existRepoPath, "bundle", "create", "-", "--all"),
		filepath.Join(existingRepo.GetRelativePath() + ".refs"):   gittest.Exec(t, cfg, "-C", existRepoPath, "show-ref", "--head"),
	})

	// These repos are the ones being restored, and should exist after the restore.
	var repos []*gitalypb.Repository
	for i := 0; i < 2; i++ {
		repo, _ := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{
			RelativePath: fmt.Sprintf("repo-%d", i),
			Storage:      cfg.Storages[0],
		})

		testhelper.WriteFiles(t, backupDir, map[string]any{
			filepath.Join("manifests", repo.GetStorageName(), repo.GetRelativePath(), "abc123.toml"): fmt.Sprintf(`
object_format = '%[1]s'
head_reference = '%[3]s'

[[steps]]
bundle_path = '%[2]s.bundle'
ref_path = '%[2]s.refs'
custom_hooks_path = '%[2]s/custom_hooks.tar'
`, gittest.DefaultObjectHash.Format, existingRepo.GetRelativePath(), git.DefaultRef.String()),
		})

		repos = append(repos, repo)
	}

	var stdin bytes.Buffer

	encoder := json.NewEncoder(&stdin)
	for _, repo := range repos {
		require.NoError(t, encoder.Encode(map[string]string{
			"address":         cfg.SocketPath,
			"token":           cfg.Auth.Token,
			"storage_name":    repo.GetStorageName(),
			"relative_path":   repo.GetRelativePath(),
			"gl_project_path": repo.GetGlProjectPath(),
		}))
	}

	require.True(t, gittest.RepositoryExists(t, ctx, conn, existingRepo))
	require.True(t, gittest.RepositoryExists(t, ctx, conn, poolRepo))

	ctx, err := storage.InjectGitalyServers(ctx, "default", cfg.SocketPath, cfg.Auth.Token)
	require.NoError(t, err)
	t.Setenv("GITALY_SERVERS", metadata.GetValue(metadata.OutgoingToIncoming(ctx), "gitaly-servers"))

	_, stderr, exitCode := runGitalyBackup(t, ctx, cfg, &stdin,
		"restore",
		"--path",
		backupDir,
		"--parallel",
		strconv.Itoa(runtime.NumCPU()),
		"--parallel-storage",
		"2",
		"--remove-all-repositories",
		existingRepo.GetStorageName(),
	)
	require.Empty(t, stderr)
	require.Zero(t, exitCode)

	require.False(t, gittest.RepositoryExists(t, ctx, conn, existingRepo))
	require.True(t, gittest.RepositoryExists(t, ctx, conn, poolRepo))

	// Ensure the repos were restored correctly.
	for _, repo := range repos {
		repoPath := filepath.Join(cfg.Storages[0].Path, gittest.GetReplicaPath(t, ctx, cfg, repo))
		bundlePath := filepath.Join(backupDir, existingRepo.GetRelativePath()+".bundle")

		output := gittest.Exec(t, cfg, "-C", repoPath, "bundle", "verify", bundlePath)
		require.Contains(t, string(output), "The bundle records a complete history")
	}
}

func TestRestoreSubcommand_serverSide(t *testing.T) {
	ctx := testhelper.Context(t)

	backupDir := testhelper.TempDir(t)
	backupSink, err := backup.ResolveSink(ctx, backupDir)
	require.NoError(t, err)

	backupLocator := backup.NewLocator(backupSink)

	cfg := testcfg.Build(t)
	testcfg.BuildGitalyHooks(t, cfg)

	cfg.SocketPath = testserver.RunGitalyServer(t, cfg, setup.RegisterAll,
		testserver.WithBackupSink(backupSink),
		testserver.WithBackupLocator(backupLocator),
	)
	conn := gittest.DialService(t, ctx, cfg)

	existingRepo, existRepoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{
		RelativePath: "existing_repo",
	})
	gittest.WriteCommit(t, cfg, existRepoPath, gittest.WithBranch(git.DefaultBranch))

	// This pool is dangling but should not be removed after the backup is restored.
	poolRepo, _ := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{
		RelativePath: gittest.NewObjectPoolName(t),
	})

	testhelper.WriteFiles(t, backupDir, map[string]any{
		filepath.Join(existingRepo.GetRelativePath() + ".bundle"): gittest.Exec(t, cfg, "-C", existRepoPath, "bundle", "create", "-", "--all"),
		filepath.Join(existingRepo.GetRelativePath() + ".refs"):   gittest.Exec(t, cfg, "-C", existRepoPath, "show-ref", "--head"),
	})
	// backup_ids/ markers written separately so they get different modification times.
	testhelper.WriteFiles(t, backupDir, map[string]any{
		"backup_ids/the-backup-1": "",
	})
	testhelper.WriteFiles(t, backupDir, map[string]any{
		"backup_ids/the-backup-2": "",
	})

	var repos []*gitalypb.Repository
	for i := 0; i < 2; i++ {
		repo, _ := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{
			RelativePath: fmt.Sprintf("repo-%d", i),
			Storage:      cfg.Storages[0],
		})

		testhelper.WriteFiles(t, backupDir, map[string]any{
			filepath.Join("manifests", repo.GetStorageName(), repo.GetRelativePath(), "the-backup-2.toml"): fmt.Sprintf(`
object_format = '%[1]s'
head_reference = '%[3]s'

[[steps]]
bundle_path = '%[2]s.bundle'
ref_path = '%[2]s.refs'
custom_hooks_path = '%[2]s/custom_hooks.tar'
`, gittest.DefaultObjectHash.Format, existingRepo.GetRelativePath(), git.DefaultRef.String()),
		})

		repos = append(repos, repo)
	}

	var stdin bytes.Buffer

	encoder := json.NewEncoder(&stdin)
	for _, repo := range repos {
		require.NoError(t, encoder.Encode(map[string]string{
			"address":         cfg.SocketPath,
			"token":           cfg.Auth.Token,
			"storage_name":    repo.GetStorageName(),
			"relative_path":   repo.GetRelativePath(),
			"gl_project_path": repo.GetGlProjectPath(),
		}))
	}

	ctx, err = storage.InjectGitalyServers(ctx, "default", cfg.SocketPath, cfg.Auth.Token)
	require.NoError(t, err)
	t.Setenv("GITALY_SERVERS", metadata.GetValue(metadata.OutgoingToIncoming(ctx), "gitaly-servers"))

	require.True(t, gittest.RepositoryExists(t, ctx, conn, existingRepo))
	require.True(t, gittest.RepositoryExists(t, ctx, conn, poolRepo))

	_, stderr, exitCode := runGitalyBackup(t, ctx, cfg, &stdin,
		"restore",
		"--parallel",
		strconv.Itoa(runtime.NumCPU()),
		"--parallel-storage",
		"2",
		"--remove-all-repositories",
		existingRepo.GetStorageName(),
		"--server-side",
		"true",
	)
	require.Empty(t, stderr)
	require.Zero(t, exitCode)

	require.False(t, gittest.RepositoryExists(t, ctx, conn, existingRepo))
	require.True(t, gittest.RepositoryExists(t, ctx, conn, poolRepo))

	for _, repo := range repos {
		repoPath := filepath.Join(cfg.Storages[0].Path, gittest.GetReplicaPath(t, ctx, cfg, repo))
		bundlePath := filepath.Join(backupDir, existingRepo.GetRelativePath()+".bundle")

		output := gittest.Exec(t, cfg, "-C", repoPath, "bundle", "verify", bundlePath)
		require.Contains(t, string(output), "The bundle records a complete history")
	}
}

func TestRestoreSubcommand_serverSide_latestFallback(t *testing.T) {
	ctx := testhelper.Context(t)

	backupDir := testhelper.TempDir(t)
	backupSink, err := backup.ResolveSink(ctx, backupDir)
	require.NoError(t, err)

	backupLocator := backup.NewLocator(backupSink)

	cfg := testcfg.Build(t)
	testcfg.BuildGitalyHooks(t, cfg)

	cfg.SocketPath = testserver.RunGitalyServer(t, cfg, setup.RegisterAll,
		testserver.WithBackupSink(backupSink),
		testserver.WithBackupLocator(backupLocator),
	)
	conn := gittest.DialService(t, ctx, cfg)

	existingRepo, existRepoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{
		RelativePath: "existing_repo",
	})
	gittest.WriteCommit(t, cfg, existRepoPath, gittest.WithBranch(git.DefaultBranch))

	// This pool is dangling but should not be removed after the backup is restored.
	poolRepo, _ := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{
		RelativePath: gittest.NewObjectPoolName(t),
	})

	testhelper.WriteFiles(t, backupDir, map[string]any{
		filepath.Join(existingRepo.GetRelativePath() + ".bundle"): gittest.Exec(t, cfg, "-C", existRepoPath, "bundle", "create", "-", "--all"),
		filepath.Join(existingRepo.GetRelativePath() + ".refs"):   gittest.Exec(t, cfg, "-C", existRepoPath, "show-ref", "--head"),
	})

	var repos []*gitalypb.Repository
	for i := 0; i < 2; i++ {
		repo, _ := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{
			RelativePath: fmt.Sprintf("repo-%d", i),
			Storage:      cfg.Storages[0],
		})

		testhelper.WriteFiles(t, backupDir, map[string]any{
			filepath.Join("manifests", repo.GetStorageName(), repo.GetRelativePath(), "abc123.toml"): fmt.Sprintf(`
object_format = '%[1]s'
head_reference = '%[3]s'

[[steps]]
bundle_path = '%[2]s.bundle'
ref_path = '%[2]s.refs'
custom_hooks_path = '%[2]s/custom_hooks.tar'
`, gittest.DefaultObjectHash.Format, existingRepo.GetRelativePath(), git.DefaultRef.String()),
		})

		repos = append(repos, repo)
	}

	var stdin bytes.Buffer

	encoder := json.NewEncoder(&stdin)
	for _, repo := range repos {
		require.NoError(t, encoder.Encode(map[string]string{
			"address":         cfg.SocketPath,
			"token":           cfg.Auth.Token,
			"storage_name":    repo.GetStorageName(),
			"relative_path":   repo.GetRelativePath(),
			"gl_project_path": repo.GetGlProjectPath(),
		}))
	}

	ctx, err = storage.InjectGitalyServers(ctx, "default", cfg.SocketPath, cfg.Auth.Token)
	require.NoError(t, err)
	t.Setenv("GITALY_SERVERS", metadata.GetValue(metadata.OutgoingToIncoming(ctx), "gitaly-servers"))

	require.True(t, gittest.RepositoryExists(t, ctx, conn, existingRepo))
	require.True(t, gittest.RepositoryExists(t, ctx, conn, poolRepo))

	_, stderr, exitCode := runGitalyBackup(t, ctx, cfg, &stdin,
		"restore",
		"--parallel",
		strconv.Itoa(runtime.NumCPU()),
		"--parallel-storage",
		"2",
		"--remove-all-repositories",
		existingRepo.GetStorageName(),
		"--server-side",
		"true",
	)
	require.Empty(t, stderr)
	require.Zero(t, exitCode)

	require.False(t, gittest.RepositoryExists(t, ctx, conn, existingRepo))
	require.True(t, gittest.RepositoryExists(t, ctx, conn, poolRepo))

	for _, repo := range repos {
		repoPath := filepath.Join(cfg.Storages[0].Path, gittest.GetReplicaPath(t, ctx, cfg, repo))
		bundlePath := filepath.Join(backupDir, existingRepo.GetRelativePath()+".bundle")

		output := gittest.Exec(t, cfg, "-C", repoPath, "bundle", "verify", bundlePath)
		require.Contains(t, string(output), "The bundle records a complete history")
	}
}

func TestRemoveRepository(t *testing.T) {
	t.Parallel()

	cfg := testcfg.Build(t)
	cfg.SocketPath = testserver.RunGitalyServer(t, cfg, setup.RegisterAll)
	ctx := testhelper.MergeIncomingMetadata(testhelper.Context(t), testcfg.GitalyServersMetadataFromCfg(t, cfg))

	for _, tc := range []struct {
		desc            string
		repositorySetup func() *gitalypb.Repository
	}{
		{
			desc: "with valid repository",
			repositorySetup: func() *gitalypb.Repository {
				repo, _ := gittest.CreateRepository(t, ctx, cfg)
				return repo
			},
		},
		{
			desc: "with non-existent repository",
			repositorySetup: func() *gitalypb.Repository {
				return &gitalypb.Repository{StorageName: "default", RelativePath: "nonexistent"}
			},
		},
	} {
		t.Run(tc.desc, func(t *testing.T) {
			t.Parallel()

			pool := client.NewPool()
			defer testhelper.MustClose(t, pool)

			require.NoError(t, removeRepository(ctx, pool, tc.repositorySetup()))
		})
	}
}

func TestRestoreSubcommand_referenceBackend(t *testing.T) {
	// t.Setenv below rules out t.Parallel, as it does for the other tests here.
	cfg := testcfg.Build(t)
	cfg.SocketPath = testserver.RunGitalyServer(t, cfg, setup.RegisterAll)
	testcfg.BuildGitalyHooks(t, cfg)

	ctx, err := storage.InjectGitalyServers(testhelper.Context(t), "default", cfg.SocketPath, cfg.Auth.Token)
	require.NoError(t, err)
	t.Setenv("GITALY_SERVERS", metadata.GetValue(metadata.OutgoingToIncoming(ctx), "gitaly-servers"))

	stdin := bytes.NewBuffer(nil)

	stdout, _, exitCode := runGitalyBackup(t, ctx, cfg, stdin,
		"restore",
		"--path", testhelper.TempDir(t),
		"--reference-backend", "nonsense",
	)
	require.NotZero(t, exitCode)
	require.Contains(t, stdout, `unknown reference backend: \"nonsense\"`)
}