Skip to content

Git designed for AI agents.

12.6 KB
Raw Blame
package quarantine

import (
	"bytes"
	"context"
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"sync"
	"sync/atomic"
	"testing"

	"github.com/feder-cr/git-ai/internal/git/gittest"
	"github.com/feder-cr/git-ai/internal/gitaly/config"
	"github.com/feder-cr/git-ai/internal/gitaly/storage"
	"github.com/feder-cr/git-ai/internal/testhelper"
	"github.com/feder-cr/git-ai/internal/testhelper/testcfg"
	"github.com/stretchr/testify/require"
)

// The tests of this file replace beforeLink, syncDirFile and syncFile, so they are not parallel.

// writeQuarantine writes a quarantine at dir with one object in each of the fan-out directories
// fanouts and a pack with its index, and returns the relative paths of its files.
func writeQuarantine(t *testing.T, dir string, fanouts []string) []string {
	t.Helper()

	var files []string
	for _, fanout := range fanouts {
		files = append(files, filepath.Join(fanout, "object"+fanout))
	}
	files = append(files, filepath.Join("pack", "pack-1.pack"), filepath.Join("pack", "pack-1.idx"))

	for _, file := range files {
		require.NoError(t, os.MkdirAll(filepath.Join(dir, filepath.Dir(file)), 0o755))
		require.NoError(t, os.WriteFile(filepath.Join(dir, file), []byte(file), 0o444))
	}
	return files
}

// replaceHooks replaces the test hooks of migrate for the test and counts the fsyncs.
func replaceHooks(t *testing.T, before func(targetPath string)) (dirSyncs, fileSyncs *atomic.Int32) {
	t.Helper()

	dirSyncs, fileSyncs = &atomic.Int32{}, &atomic.Int32{}
	originalBefore, originalSyncDir, originalSyncFile := beforeLink, syncDirFile, syncFile
	t.Cleanup(func() { beforeLink, syncDirFile, syncFile = originalBefore, originalSyncDir, originalSyncFile })

	if before != nil {
		beforeLink = before
	}
	syncDirFile = func(f *os.File) error {
		dirSyncs.Add(1)
		return originalSyncDir(f)
	}
	syncFile = func(path string) error {
		fileSyncs.Add(1)
		return originalSyncFile(path)
	}
	return dirSyncs, fileSyncs
}

// TestMigrateRecreatesTargetDirectoriesThatVanish removes each empty target fan-out directory
// right before the first link into it, as housekeeping (git prune-packed) may. Before, the link
// and its rename fallback then failed with ENOENT. The migration creates the directory again and
// links every object.
func TestMigrateRecreatesTargetDirectoriesThatVanish(t *testing.T) {
	ctx := testhelper.Context(t)
	root := t.TempDir()
	source, target := filepath.Join(root, "quarantine"), filepath.Join(root, "objects")
	fanouts := []string{"0a", "1b", "2c"}
	files := writeQuarantine(t, source, fanouts)

	// The target fan-out directories exist and are empty, as housekeeping leaves them.
	require.NoError(t, os.MkdirAll(filepath.Join(target, "pack"), 0o755))
	for _, fanout := range fanouts {
		require.NoError(t, os.Mkdir(filepath.Join(target, fanout), 0o755))
	}

	var removed sync.Map
	replaceHooks(t, func(targetPath string) {
		dir := filepath.Dir(targetPath)
		if filepath.Base(dir) == "pack" {
			return
		}
		if _, done := removed.LoadOrStore(dir, true); !done {
			require.NoError(t, os.Remove(dir))
		}
	})

	require.NoError(t, migrate(ctx, source, target, false))

	for _, file := range files {
		require.FileExists(t, filepath.Join(target, file))
	}
	for _, fanout := range fanouts {
		_, ok := removed.Load(filepath.Join(target, fanout))
		require.True(t, ok, "the directory %s was removed during the migration", fanout)
	}
	require.NoDirExists(t, source)
}

// TestMigrateRecreatesTargetDirectoriesThatVanishAgain removes each target fan-out directory before
// the first three links into it, so it vanishes again after the migration created it again, as
// git prune-packed may each time between the mkdir and the link. git's own writer retries only
// once after its mkdir (create_tmpfile), which a second removal defeats; the migration keeps
// creating the directory until the link holds.
func TestMigrateRecreatesTargetDirectoriesThatVanishAgain(t *testing.T) {
	ctx := testhelper.Context(t)
	root := t.TempDir()
	source, target := filepath.Join(root, "quarantine"), filepath.Join(root, "objects")
	fanouts := []string{"0a", "1b"}
	files := writeQuarantine(t, source, fanouts)
	require.NoError(t, os.MkdirAll(filepath.Join(target, "pack"), 0o755))

	const removals = 3
	var mu sync.Mutex
	removedTimes := map[string]int{}
	replaceHooks(t, func(targetPath string) {
		dir := filepath.Dir(targetPath)
		if filepath.Base(dir) == "pack" {
			return
		}
		mu.Lock()
		defer mu.Unlock()
		if removedTimes[dir] < removals {
			removedTimes[dir]++
			require.NoError(t, os.Remove(dir))
		}
	})

	require.NoError(t, migrate(ctx, source, target, false))

	for _, file := range files {
		require.FileExists(t, filepath.Join(target, file))
	}
	for _, fanout := range fanouts {
		require.Equal(t, removals, removedTimes[filepath.Join(target, fanout)], fanout)
	}
	require.NoDirExists(t, source)
}

// TestMigrateSyncsTheDirectoryThatHoldsTheLink removes each target fan-out directory right before
// the first link into it and creates it again, as housekeeping and then another writer may. The
// link then succeeds into the new directory. The second pass must fsync that directory, not the
// removed one: every descriptor it fsyncs is of the directory now at its path.
func TestMigrateSyncsTheDirectoryThatHoldsTheLink(t *testing.T) {
	ctx := testhelper.Context(t)
	root := t.TempDir()
	source, target := filepath.Join(root, "quarantine"), filepath.Join(root, "objects")
	fanouts := []string{"0a", "1b", "2c"}
	files := writeQuarantine(t, source, fanouts)
	require.NoError(t, os.MkdirAll(target, 0o755))

	var recreated sync.Map
	replaceHooks(t, func(targetPath string) {
		dir := filepath.Dir(targetPath)
		if filepath.Base(dir) == "pack" {
			return
		}
		if _, done := recreated.LoadOrStore(dir, true); !done {
			require.NoError(t, os.Remove(dir))
			require.NoError(t, os.Mkdir(dir, 0o755))
		}
	})

	var synced []string
	originalSyncDir := syncDirFile
	syncDirFile = func(f *os.File) error {
		descriptor, err := f.Stat()
		require.NoError(t, err)
		current, err := os.Stat(f.Name())
		require.NoError(t, err)
		require.True(t, os.SameFile(descriptor, current), "the second pass fsyncs a removed directory: %s", f.Name())
		synced = append(synced, f.Name())
		return originalSyncDir(f)
	}

	require.NoError(t, migrate(ctx, source, target, false))

	for _, file := range files {
		require.FileExists(t, filepath.Join(target, file))
	}
	for _, fanout := range fanouts {
		require.Contains(t, synced, filepath.Join(target, fanout))
	}
}

// TestMigrateSyncsEachDirectoryOnce counts the fsyncs: one per target directory that received a
// link, one for the object directory, and none per file, whose writers made them durable.
func TestMigrateSyncsEachDirectoryOnce(t *testing.T) {
	ctx := testhelper.Context(t)
	root := t.TempDir()
	source, target := filepath.Join(root, "quarantine"), filepath.Join(root, "objects")
	fanouts := []string{"0a", "1b", "2c", "3d"}
	writeQuarantine(t, source, fanouts)
	require.NoError(t, os.MkdirAll(target, 0o755))

	dirSyncs, fileSyncs := replaceHooks(t, nil)
	require.NoError(t, migrate(ctx, source, target, false))

	require.EqualValues(t, len(fanouts)+1+1, dirSyncs.Load(), "the fan-out directories, pack and the object directory")
	require.Zero(t, fileSyncs.Load())
}

// TestMigrateSyncsTheFilesOfAnUntrustedWriter fsyncs every file of a quarantine that a writer
// Gitaly does not control wrote into, and still each directory once.
func TestMigrateSyncsTheFilesOfAnUntrustedWriter(t *testing.T) {
	ctx := testhelper.Context(t)
	root := t.TempDir()
	source, target := filepath.Join(root, "quarantine"), filepath.Join(root, "objects")
	fanouts := []string{"0a", "1b"}
	files := writeQuarantine(t, source, fanouts)
	require.NoError(t, os.MkdirAll(target, 0o755))

	dirSyncs, fileSyncs := replaceHooks(t, nil)
	require.NoError(t, migrate(ctx, source, target, true))

	require.EqualValues(t, len(files), fileSyncs.Load())
	require.EqualValues(t, len(fanouts)+1+1, dirSyncs.Load())
}

// fakeTransaction makes storage.NeedsSync false, as a transaction of the transaction manager does.
type fakeTransaction struct {
	storage.Transaction
}

// TestMigrateDoesNotSyncUnderATransaction leaves durability to the transaction manager.
func TestMigrateDoesNotSyncUnderATransaction(t *testing.T) {
	ctx := storage.ContextWithTransaction(testhelper.Context(t), fakeTransaction{})
	require.False(t, storage.NeedsSync(ctx))

	root := t.TempDir()
	source, target := filepath.Join(root, "quarantine"), filepath.Join(root, "objects")
	files := writeQuarantine(t, source, []string{"0a"})
	require.NoError(t, os.MkdirAll(target, 0o755))

	dirSyncs, fileSyncs := replaceHooks(t, nil)
	require.NoError(t, migrate(ctx, source, target, true))

	require.Zero(t, dirSyncs.Load())
	require.Zero(t, fileSyncs.Load())
	for _, file := range files {
		require.FileExists(t, filepath.Join(target, file))
	}
}

// TestMigrateKeepsThePackOrder links the data of a pack before its index.
func TestMigrateKeepsThePackOrder(t *testing.T) {
	ctx := testhelper.Context(t)
	root := t.TempDir()
	source, target := filepath.Join(root, "quarantine"), filepath.Join(root, "objects")
	writeQuarantine(t, source, nil)
	require.NoError(t, os.MkdirAll(target, 0o755))

	var linked []string
	var mu sync.Mutex
	replaceHooks(t, func(targetPath string) {
		mu.Lock()
		defer mu.Unlock()
		linked = append(linked, filepath.Base(targetPath))
	})

	require.NoError(t, migrate(ctx, source, target, false))
	require.Equal(t, []string{"pack-1.pack", "pack-1.idx"}, linked)
}

// TestMarkUntrustedWriter marks the quarantine whose object directory a hook was given, and only
// while the quarantine exists.
func TestMarkUntrustedWriter(t *testing.T) {
	ctx := testhelper.Context(t)
	cfg := testcfg.Build(t)
	repo, _ := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{SkipCreationViaService: true})
	locator := config.NewLocator(cfg)

	marked, cleanupMarked, err := New(ctx, repo, testhelper.NewLogger(t), locator)
	require.NoError(t, err)
	defer cleanupMarked()
	other, cleanupOther, err := New(ctx, repo, testhelper.NewLogger(t), locator)
	require.NoError(t, err)
	defer cleanupOther()

	MarkUntrustedWriter(marked.dir.Path() + "/")
	require.True(t, marked.untrustedWriter.Load())
	require.False(t, other.untrustedWriter.Load())

	// A path that is no quarantine marks nothing.
	MarkUntrustedWriter(t.TempDir())
	require.False(t, other.untrustedWriter.Load())

	path := other.dir.Path()
	cleanupOther()
	_, registered := quarantines.Load(filepath.Clean(path))
	require.False(t, registered, "a quarantine leaves the registry when it is cleaned up")
}

// TestMigrateWhileGitPrunesPacked runs migrations into a repository while git prune-packed, which
// housekeeping runs after a repack, removes its empty fan-out directories in a loop.
func TestMigrateWhileGitPrunesPacked(t *testing.T) {
	ctx := testhelper.Context(t)
	cfg := testcfg.Build(t)
	_, repoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{SkipCreationViaService: true})
	objects := filepath.Join(repoPath, "objects")

	// Every fan-out directory exists and is empty, the state that prune-packed cleans up.
	for i := 0; i < 256; i++ {
		require.NoError(t, os.MkdirAll(filepath.Join(objects, fmt.Sprintf("%02x", i)), 0o755))
	}

	stop := make(chan struct{})
	var pruned sync.WaitGroup
	pruned.Add(1)
	runs := 0
	go func() {
		defer pruned.Done()
		for {
			select {
			case <-stop:
				return
			default:
			}
			var stderr bytes.Buffer
			cmd := gittest.NewCommand(t, cfg, "-C", repoPath, "prune-packed", "--quiet")
			cmd.Stderr = &stderr
			if err := cmd.Run(); err != nil {
				t.Errorf("prune-packed: %v: %s", err, stderr.String())
				return
			}
			runs++
			// Recreate empty directories, so that the race stays possible.
			for i := 0; i < 256; i += 7 {
				_ = os.Mkdir(filepath.Join(objects, fmt.Sprintf("%02x", i)), 0o755)
			}
		}
	}()

	for i := 0; i < 200; i++ {
		source := filepath.Join(t.TempDir(), "quarantine")
		fanouts := []string{fmt.Sprintf("%02x", (i*7)%256), fmt.Sprintf("%02x", (i*13+1)%256)}
		var files []string
		for _, fanout := range fanouts {
			file := filepath.Join(fanout, fmt.Sprintf("object-%03d", i))
			require.NoError(t, os.MkdirAll(filepath.Join(source, fanout), 0o755))
			require.NoError(t, os.WriteFile(filepath.Join(source, file), []byte(file), 0o444))
			files = append(files, file)
		}

		require.NoError(t, migrate(context.Background(), source, objects, false), "migration %d", i)
		for _, file := range files {
			require.FileExists(t, filepath.Join(objects, file))
		}
	}

	close(stop)
	pruned.Wait()
	require.Positive(t, runs)

	var names []string
	entries, err := os.ReadDir(objects)
	require.NoError(t, err)
	for _, entry := range entries {
		names = append(names, entry.Name())
	}
	sort.Strings(names)
	t.Logf("%d prune-packed runs during 200 migrations", runs)
}