Skip to content

Git designed for AI agents.

12.5 KB
Raw Blame
package mvcc

import (
	"context"
	"fmt"
	"testing"
	"time"

	"github.com/feder-cr/git-ai/internal/testhelper"
	"github.com/stretchr/testify/require"
)

// testConfig prunes eagerly, so that tests only have to age the manifests they care about.
func testConfig() ManifestPruningConfig {
	return ManifestPruningConfig{
		ManifestsToKeep: 1,
		ManifestMinAge:  minEntryAge,
		MaxLockDuration: time.Minute,
	}
}

func TestPruneManifests(t *testing.T) {
	t.Parallel()

	t.Run("prunes manifests past retention and leaves their artifacts", func(t *testing.T) {
		t.Parallel()

		ctx := testhelper.Context(t)
		repo := newMVCCRepo(t)

		artifact := repo.writeArtifact(reftable("a"), minEntryAge+time.Hour)
		repo.writeManifest("obsolete", minEntryAge+time.Hour, artifact)
		repo.setPointer(repo.writeManifest("current", time.Minute))

		result, err := PruneManifests(ctx, repo.path, testConfig())
		require.NoError(t, err)
		require.Equal(t, ManifestPruningResult{Removed: 1}, counts(result))

		repo.requireExists(repo.manifestPath("obsolete"), false)
		repo.requireExists(repo.manifestPath("current"), true)
		// Nothing references it anymore, but reclaiming it is not this run's business.
		repo.requireExists(artifact, true)
	})

	t.Run("keeps the most recent manifests regardless of their age", func(t *testing.T) {
		t.Parallel()

		ctx := testhelper.Context(t)
		repo := newMVCCRepo(t)

		repo.writeManifest("older", minEntryAge+2*time.Hour)
		repo.writeManifest("newer", minEntryAge+time.Hour)
		repo.setPointer(repo.writeManifest("current", time.Minute))

		cfg := testConfig()
		cfg.ManifestsToKeep = 2

		result, err := PruneManifests(ctx, repo.path, cfg)
		require.NoError(t, err)
		require.Equal(t, ManifestPruningResult{Removed: 1}, counts(result))

		repo.requireExists(repo.manifestPath("older"), false)
		repo.requireExists(repo.manifestPath("newer"), true)
	})

	t.Run("keeps manifests that are not old enough", func(t *testing.T) {
		t.Parallel()

		ctx := testhelper.Context(t)
		repo := newMVCCRepo(t)

		// Past the count the policy keeps alive, but younger than the minimum age, which
		// is what a client that is still using a manifest is protected by.
		repo.writeManifest("recent", time.Minute)
		repo.setPointer(repo.writeManifest("current", time.Minute))

		result, err := PruneManifests(ctx, repo.path, testConfig())
		require.NoError(t, err)
		require.Equal(t, ManifestPruningResult{}, counts(result))

		repo.requireExists(repo.manifestPath("recent"), true)
	})

	t.Run("never removes the manifest the pointer names", func(t *testing.T) {
		t.Parallel()

		ctx := testhelper.Context(t)
		repo := newMVCCRepo(t)

		// The policy would prune this one, but the repository would be unopenable without
		// it.
		current := repo.writeManifest("current", minEntryAge+time.Hour)
		repo.writeManifest("newer", time.Minute)
		repo.setPointer(current)

		result, err := PruneManifests(ctx, repo.path, testConfig())
		require.NoError(t, err)
		require.Equal(t, ManifestPruningResult{}, counts(result))

		repo.requireExists(repo.manifestPath("current"), true)
	})

	t.Run("removes a manifest past retention without reading it", func(t *testing.T) {
		t.Parallel()

		ctx := testhelper.Context(t)
		repo := newMVCCRepo(t)

		// Pruning is a matter of policy, so an unreadable manifest goes like any other
		// once it is past retention.
		repo.writeFile(repo.manifestPath("obsolete"), []byte("not a manifest"), minEntryAge+time.Hour)
		repo.setPointer(repo.writeManifest("current", time.Minute))

		result, err := PruneManifests(ctx, repo.path, testConfig())
		require.NoError(t, err)
		require.Equal(t, ManifestPruningResult{Removed: 1}, counts(result))

		repo.requireExists(repo.manifestPath("obsolete"), false)
	})

	t.Run("prunes whatever else is past retention, whatever it is called", func(t *testing.T) {
		t.Parallel()

		ctx := testhelper.Context(t)
		repo := newMVCCRepo(t)

		// What a crashed manifest download leaves behind. Nothing in the directory is
		// recognized by name, so the policy reclaims it like it does a manifest.
		leftover := repo.manifestPath("current") + "1234567890"
		repo.writeFile(leftover, []byte("half a manifest"), minEntryAge+time.Hour)
		repo.setPointer(repo.writeManifest("current", time.Minute))

		result, err := PruneManifests(ctx, repo.path, testConfig())
		require.NoError(t, err)
		require.Equal(t, ManifestPruningResult{Removed: 1}, counts(result))

		repo.requireExists(leftover, false)
	})

	t.Run("keeps a manifest download in progress", func(t *testing.T) {
		t.Parallel()

		ctx := testhelper.Context(t)
		repo := newMVCCRepo(t)

		// A download writes to a temporary file in the manifest directory, and a read
		// downloading one does not hold the repository lock, so its age is all that keeps
		// it from being pruned.
		downloading := repo.manifestPath("current") + "1234567890"
		repo.writeFile(downloading, []byte("half a manifest"), minEntryAge-time.Hour)
		repo.setPointer(repo.writeManifest("current", time.Minute))

		cfg := testConfig()
		cfg.ManifestMinAge = time.Minute

		result, err := PruneManifests(ctx, repo.path, cfg)
		require.NoError(t, err)
		require.Equal(t, ManifestPruningResult{}, counts(result))

		repo.requireExists(downloading, true)
	})

	t.Run("does not lock the repository when there is nothing to prune", func(t *testing.T) {
		t.Parallel()

		ctx := testhelper.Context(t)
		repo := newMVCCRepo(t)

		repo.setPointer(repo.writeManifest("current", time.Minute))

		// A write holds the repository, so a run that needed the lock would report having
		// been skipped.
		unlockWrite, err := LockRepositoryForWrite(ctx, repo.path)
		require.NoError(t, err)
		defer unlockWrite()

		result, err := PruneManifests(ctx, repo.path, testConfig())
		require.NoError(t, err)
		require.Equal(t, ManifestPruningResult{}, counts(result))
	})

	t.Run("gives up when the repository keeps being written to", func(t *testing.T) {
		t.Parallel()

		ctx := testhelper.Context(t)
		repo := newMVCCRepo(t)

		repo.writeManifest("obsolete", minEntryAge+time.Hour)
		repo.setPointer(repo.writeManifest("current", time.Minute))

		unlockWrite, err := LockRepositoryForWrite(ctx, repo.path)
		require.NoError(t, err)
		defer unlockWrite()

		result, err := PruneManifests(ctx, repo.path, testConfig())
		require.NoError(t, err)
		require.Equal(t, ManifestPruningResult{Skipped: true}, counts(result))

		repo.requireExists(repo.manifestPath("obsolete"), true)
	})

	t.Run("waits for a write to finish", func(t *testing.T) {
		t.Parallel()

		ctx := testhelper.Context(t)
		repo := newMVCCRepo(t)

		repo.writeManifest("obsolete", minEntryAge+time.Hour)
		repo.setPointer(repo.writeManifest("current", time.Minute))

		unlockWrite, err := LockRepositoryForWrite(ctx, repo.path)
		require.NoError(t, err)
		go func() {
			time.Sleep(10 * acquireBackoff)
			unlockWrite()
		}()

		cfg := testConfig()
		cfg.MaxAcquireWait = time.Minute

		result, err := PruneManifests(ctx, repo.path, cfg)
		require.NoError(t, err)
		require.Equal(t, ManifestPruningResult{Removed: 1}, counts(result))

		repo.requireExists(repo.manifestPath("obsolete"), false)
	})

	t.Run("stops deleting once it has spent its time budget", func(t *testing.T) {
		t.Parallel()

		ctx := testhelper.Context(t)
		repo := newMVCCRepo(t)

		repo.writeManifest("obsolete", minEntryAge+time.Hour)
		repo.setPointer(repo.writeManifest("current", time.Minute))

		cfg := testConfig()
		cfg.MaxLockDuration = time.Nanosecond

		result, err := PruneManifests(ctx, repo.path, cfg)
		require.NoError(t, err)
		require.Equal(t, ManifestPruningResult{}, counts(result))

		repo.requireExists(repo.manifestPath("obsolete"), true)
	})

	t.Run("does nothing for a repository without MVCC data", func(t *testing.T) {
		t.Parallel()

		ctx := testhelper.Context(t)

		result, err := PruneManifests(ctx, testhelper.TempDir(t), testConfig())
		require.NoError(t, err)
		require.Equal(t, ManifestPruningResult{}, counts(result))
	})

	t.Run("refuses to keep no manifests at all", func(t *testing.T) {
		t.Parallel()

		ctx := testhelper.Context(t)

		cfg := testConfig()
		cfg.ManifestsToKeep = 0

		_, err := PruneManifests(ctx, testhelper.TempDir(t), cfg)
		require.ErrorContains(t, err, "at least one manifest must be kept")
	})

	t.Run("refuses to run without a time budget", func(t *testing.T) {
		t.Parallel()

		ctx := testhelper.Context(t)

		cfg := testConfig()
		cfg.MaxLockDuration = 0

		_, err := PruneManifests(ctx, testhelper.TempDir(t), cfg)
		require.ErrorContains(t, err, "no time budget to delete under")
	})
}

// TestRemoveManifests covers stopping mid-deletion, which is what bounds how long a write waits
// for a run. Both cases need the lock already held, which is why they drive removeManifests rather
// than a whole run.
func TestRemoveManifests(t *testing.T) {
	t.Parallel()

	t.Run("stops when the context is cancelled", func(t *testing.T) {
		t.Parallel()

		ctx, cancel := context.WithCancel(testhelper.Context(t))
		repo := newMVCCRepo(t)

		obsolete := []manifestEntry{{name: repo.writeManifest("obsolete", minEntryAge+time.Hour)}}

		lock := TryLockRepositoryForCollection(repo.path)
		require.NotNil(t, lock)
		defer lock.Unlock()

		cancel()

		require.Zero(t, removeManifests(ctx, repo.mvccPath, obsolete, minEntryAge, lock))
		repo.requireExists(repo.manifestPath("obsolete"), true)
	})

	t.Run("keeps a manifest that was republished while waiting for the lock", func(t *testing.T) {
		t.Parallel()

		ctx := testhelper.Context(t)
		repo := newMVCCRepo(t)

		// The scan found it past retention, but a write has recreated it since: manifests
		// are content-addressed, so publishing one that already exists rewrites this very
		// file.
		obsolete := []manifestEntry{{name: repo.writeManifest("obsolete", minEntryAge+time.Hour)}}
		repo.writeManifest("obsolete", time.Minute)

		lock := TryLockRepositoryForCollection(repo.path)
		require.NotNil(t, lock)
		defer lock.Unlock()

		require.Zero(t, removeManifests(ctx, repo.mvccPath, obsolete, minEntryAge, lock))
		repo.requireExists(repo.manifestPath("obsolete"), true)
	})

	t.Run("stops when a write arrives", func(t *testing.T) {
		t.Parallel()

		ctx := testhelper.Context(t)
		repo := newMVCCRepo(t)

		var obsolete []manifestEntry
		for i := range 10 {
			obsolete = append(obsolete, manifestEntry{
				name: repo.writeManifest(fmt.Sprintf("obsolete-%d", i), minEntryAge+time.Hour),
			})
		}

		lock := TryLockRepositoryForCollection(repo.path)
		require.NotNil(t, lock)

		unlockWrite := lockRepositoryForWriteAsync(t, ctx, repo.path)
		require.Eventually(t, lock.Evicted, time.Minute, time.Millisecond)

		require.Zero(t, removeManifests(ctx, repo.mvccPath, obsolete, minEntryAge, lock))
		repo.requireExists(repo.manifestPath("obsolete-0"), true)

		lock.Unlock()
		unlockWrite()
	})
}

// benchmarkShape is a repository to run the pruning benchmark against.
type benchmarkShape struct {
	// retained is how many manifests the retention policy keeps alive. Every run lists them to
	// work out which manifests have fallen out of the policy.
	retained int
	// obsolete is how many manifests have fallen out of the policy and are to be removed.
	obsolete int
}

// desc names the shape after the two numbers a run's cost depends on.
func (shape benchmarkShape) desc() string {
	return fmt.Sprintf("%d retained, %d obsolete", shape.retained, shape.obsolete)
}

// benchmarkShapes move one variable at a time around a common base, so that what a run costs can
// be attributed rather than only observed.
var benchmarkShapes = []benchmarkShape{
	{retained: 50, obsolete: 50},
	{retained: 500, obsolete: 50},
	{retained: 500, obsolete: 500},
	{retained: 2000, obsolete: 500},
	{retained: 8000, obsolete: 500},
	{retained: 500, obsolete: 2000},
	{retained: 500, obsolete: 8000},
}

// BenchmarkPruneManifests measures a run end to end, including the part that is not under the lock.
func BenchmarkPruneManifests(b *testing.B) {
	ctx := testhelper.Context(b)

	for _, shape := range benchmarkShapes {
		b.Run(shape.desc(), func(b *testing.B) {
			repo := newMVCCRepo(b)

			for i := range shape.retained {
				repo.writeManifest(fmt.Sprintf("retained-%05d", i), time.Duration(i)*time.Minute)
			}
			repo.setPointer(manifestNameFor("retained-00000"))

			cfg := testConfig()
			cfg.ManifestsToKeep = uint(shape.retained)

			for b.Loop() {
				b.StopTimer()
				// Every iteration consumes the manifests it prunes, so they have to be
				// written back. They are older than any retained manifest, so these are
				// the ones the policy drops.
				for j := range shape.obsolete {
					repo.writeManifest(fmt.Sprintf("obsolete-%05d", j), 30*24*time.Hour)
				}
				b.StartTimer()

				result, err := PruneManifests(ctx, repo.path, cfg)
				require.NoError(b, err)
				require.Equal(b, shape.obsolete, result.Removed)
			}
		})
	}
}