Git designed for AI agents.
package mvcc
import (
"context"
"fmt"
"os"
"path/filepath"
"testing"
"time"
"github.com/feder-cr/git-ai/internal/testhelper"
"github.com/stretchr/testify/require"
)
// artifactConfig collects eagerly, so that tests only have to age the entries they care about.
func artifactConfig() ArtifactCollectionConfig {
return ArtifactCollectionConfig{
ArtifactMinAge: minEntryAge,
MaxLockDuration: time.Minute,
}
}
// artifactCounts strips the timings from a result, so it can be compared for what it did.
func artifactCounts(result ArtifactCollectionResult) ArtifactCollectionResult {
result.AcquireDuration = 0
result.LockDuration = 0
return result
}
func TestCollectArtifacts(t *testing.T) {
t.Parallel()
t.Run("removes the artifacts no manifest references", func(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
repo := newMVCCRepo(t)
live := repo.writeArtifact(reftable("a"), minEntryAge+time.Hour)
orphan := repo.writeArtifact(reftable("b"), minEntryAge+time.Hour)
repo.setPointer(repo.writeManifest("current", time.Minute, live))
result, err := CollectArtifacts(ctx, repo.path, artifactConfig())
require.NoError(t, err)
require.Equal(t, ArtifactCollectionResult{Removed: 1}, artifactCounts(result))
repo.requireExists(live, true)
repo.requireExists(orphan, false)
})
t.Run("keeps an artifact any manifest still references", func(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
repo := newMVCCRepo(t)
// Reachability is the union of every manifest on disk, so an artifact only the
// older one references is still reachable.
shared := repo.writeArtifact(reftable("a"), minEntryAge+time.Hour)
repo.writeManifest("older", minEntryAge+time.Hour, shared)
repo.setPointer(repo.writeManifest("current", time.Minute))
result, err := CollectArtifacts(ctx, repo.path, artifactConfig())
require.NoError(t, err)
require.Equal(t, ArtifactCollectionResult{}, artifactCounts(result))
repo.requireExists(shared, true)
})
t.Run("removes anything unreferenced, whatever it is called and wherever it is", func(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
repo := newMVCCRepo(t)
// Nothing below the MVCC directory is recognized by name, and the walk descends
// into whatever directories it finds.
loose := repo.writeArtifact("pack/not-named-like-a-pack", minEntryAge+time.Hour)
nested := repo.writeArtifactUnder("pack/nested", "leftover", minEntryAge+time.Hour)
repo.setPointer(repo.writeManifest("current", time.Minute))
result, err := CollectArtifacts(ctx, repo.path, artifactConfig())
require.NoError(t, err)
require.Equal(t, ArtifactCollectionResult{Removed: 2}, artifactCounts(result))
repo.requireExists(loose, false)
repo.requireExists(nested, false)
})
t.Run("keeps a fetch in progress", func(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
repo := newMVCCRepo(t)
// A fetch writes to a temporary file next to the artifact it is fetching, and it
// outlives the RPC that started it, so its age is all that keeps it from being
// removed.
fetching := repo.writeArtifact(reftable("a")+"1234567890", minEntryAge-time.Hour)
repo.setPointer(repo.writeManifest("current", time.Minute))
cfg := artifactConfig()
cfg.ArtifactMinAge = time.Minute
result, err := CollectArtifacts(ctx, repo.path, cfg)
require.NoError(t, err)
require.Equal(t, ArtifactCollectionResult{}, artifactCounts(result))
repo.requireExists(fetching, true)
})
t.Run("never touches the manifest directory or the pointer", func(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
repo := newMVCCRepo(t)
// Both are past the age floor and neither is referenced by a manifest, so only
// leaving them out of the walk keeps them.
repo.writeManifest("older", minEntryAge+time.Hour)
current := repo.writeManifest("current", minEntryAge+time.Hour)
repo.setPointer(current)
repo.age(manifestPointer, minEntryAge+time.Hour)
result, err := CollectArtifacts(ctx, repo.path, artifactConfig())
require.NoError(t, err)
require.Equal(t, ArtifactCollectionResult{}, artifactCounts(result))
repo.requireExists(repo.manifestPath("older"), true)
repo.requireExists(manifestPointer, true)
})
t.Run("stops when a manifest cannot be read", func(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
repo := newMVCCRepo(t)
orphan := repo.writeArtifact(reftable("a"), minEntryAge+time.Hour)
repo.writeFile(repo.manifestPath("current"), []byte("not a manifest"), time.Minute)
repo.setPointer(manifestNameFor("current"))
_, err := CollectArtifacts(ctx, repo.path, artifactConfig())
require.ErrorContains(t, err, "reading manifest")
repo.requireExists(orphan, true)
})
t.Run("does nothing before the first manifest is published", func(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
repo := newMVCCRepo(t)
orphan := repo.writeArtifact(reftable("a"), minEntryAge+time.Hour)
result, err := CollectArtifacts(ctx, repo.path, artifactConfig())
require.NoError(t, err)
require.Equal(t, ArtifactCollectionResult{}, artifactCounts(result))
repo.requireExists(orphan, true)
})
t.Run("does nothing for a repository without MVCC data", func(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
result, err := CollectArtifacts(ctx, testhelper.TempDir(t), artifactConfig())
require.NoError(t, err)
require.Equal(t, ArtifactCollectionResult{}, artifactCounts(result))
})
t.Run("does not lock the repository when there is nothing to collect", func(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
repo := newMVCCRepo(t)
live := repo.writeArtifact(reftable("a"), minEntryAge+time.Hour)
repo.setPointer(repo.writeManifest("current", time.Minute, live))
unlockWrite, err := LockRepositoryForWrite(ctx, repo.path)
require.NoError(t, err)
defer unlockWrite()
result, err := CollectArtifacts(ctx, repo.path, artifactConfig())
require.NoError(t, err)
require.Equal(t, ArtifactCollectionResult{}, artifactCounts(result))
})
t.Run("gives up when the repository keeps being written to", func(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
repo := newMVCCRepo(t)
orphan := repo.writeArtifact(reftable("a"), minEntryAge+time.Hour)
repo.setPointer(repo.writeManifest("current", time.Minute))
unlockWrite, err := LockRepositoryForWrite(ctx, repo.path)
require.NoError(t, err)
defer unlockWrite()
// Long enough to retry rather than to give up on the first attempt.
cfg := artifactConfig()
cfg.MaxAcquireWait = 10 * acquireBackoff
result, err := CollectArtifacts(ctx, repo.path, cfg)
require.NoError(t, err)
require.Equal(t, ArtifactCollectionResult{Skipped: true}, artifactCounts(result))
repo.requireExists(orphan, true)
})
t.Run("stops deleting once it has spent its time budget", func(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
repo := newMVCCRepo(t)
orphan := repo.writeArtifact(reftable("a"), minEntryAge+time.Hour)
repo.setPointer(repo.writeManifest("current", time.Minute))
cfg := artifactConfig()
cfg.MaxLockDuration = time.Nanosecond
result, err := CollectArtifacts(ctx, repo.path, cfg)
require.NoError(t, err)
require.Equal(t, ArtifactCollectionResult{}, artifactCounts(result))
repo.requireExists(orphan, true)
})
t.Run("refuses to run without a time budget", func(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
cfg := artifactConfig()
cfg.MaxLockDuration = 0
_, err := CollectArtifacts(ctx, testhelper.TempDir(t), cfg)
require.ErrorContains(t, err, "no time budget to delete under")
})
}
// TestReachability covers refreshing the set, which is what a run does under the lock instead of
// reading every manifest again.
func TestReachability(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
repo := newMVCCRepo(t)
first := repo.writeArtifact(reftable("a"), 0)
repo.writeManifest("first", time.Minute, first)
reachable := newReachability()
require.NoError(t, reachable.refresh(ctx, repo.mvccPath))
require.True(t, reachable.references(first))
require.Equal(t, map[string]struct{}{manifestNameFor("first"): {}}, reachable.manifests)
second := repo.writeArtifact(reftable("b"), 0)
repo.writeManifest("second", time.Minute, second)
require.NoError(t, reachable.refresh(ctx, repo.mvccPath))
require.True(t, reachable.references(second))
require.Len(t, reachable.manifests, 2)
// Manifest pruning runs without excluding the scan, so a manifest can go away between
// being listed and being read. A dangling symlink is listed and then fails to open the
// same way.
require.NoError(t, os.Symlink(repo.manifestPath("gone"), filepath.Join(repo.mvccPath, repo.manifestPath("dangling"))))
require.NoError(t, reachable.refresh(ctx, repo.mvccPath))
require.True(t, reachable.references(first))
// A run that is out of time stops without a complete picture, which is what keeps it from
// deleting anything.
cancelled, cancel := context.WithCancel(ctx)
cancel()
require.ErrorIs(t, newReachability().refresh(cancelled, repo.mvccPath), context.Canceled)
}
func TestRemoveStale(t *testing.T) {
t.Parallel()
t.Run("stops when the context is cancelled", func(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithCancel(testhelper.Context(t))
repo := newMVCCRepo(t)
candidates := []artifactEntry{{relativePath: repo.writeArtifact(reftable("a"), minEntryAge+time.Hour)}}
lock := TryLockRepositoryForCollection(repo.path)
require.NotNil(t, lock)
defer lock.Unlock()
cancel()
require.Zero(t, removeStale(ctx, repo.mvccPath, candidates, time.Hour, lock, newReachability()))
repo.requireExists(reftable("a"), true)
})
t.Run("stops when a write arrives", func(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
repo := newMVCCRepo(t)
var candidates []artifactEntry
for i := range 10 {
candidates = append(candidates, artifactEntry{
relativePath: repo.writeArtifact(numberedReftable(i), minEntryAge+time.Hour),
})
}
lock := TryLockRepositoryForCollection(repo.path)
require.NotNil(t, lock)
unlockWrite := lockRepositoryForWriteAsync(t, ctx, repo.path)
require.Eventually(t, lock.Evicted, time.Minute, time.Millisecond)
require.Zero(t, removeStale(ctx, repo.mvccPath, candidates, time.Hour, lock, newReachability()))
repo.requireExists(numberedReftable(0), true)
lock.Unlock()
unlockWrite()
})
}
// artifactShape is a repository to run the collection benchmark against.
type artifactShape struct {
// manifests is how many manifests are on disk. Every one of them has to be read to know
// what is reachable.
manifests int
// artifactsPerManifest is how many artifacts a single manifest references.
artifactsPerManifest int
// garbage is how many unreferenced entries are to be removed.
garbage int
}
func (shape artifactShape) desc() string {
return fmt.Sprintf("%d manifests, %d artifacts each, %d garbage",
shape.manifests, shape.artifactsPerManifest, shape.garbage)
}
// artifactShapes move one variable at a time around a common base, so that what a run costs can be
// attributed rather than only observed.
var artifactShapes = []artifactShape{
{manifests: 50, artifactsPerManifest: 50, garbage: 50},
{manifests: 500, artifactsPerManifest: 200, garbage: 500},
{manifests: 2000, artifactsPerManifest: 200, garbage: 500},
{manifests: 500, artifactsPerManifest: 1000, garbage: 500},
{manifests: 500, artifactsPerManifest: 200, garbage: 2000},
}
// BenchmarkCollectArtifacts measures a run end to end, reading the manifests included.
func BenchmarkCollectArtifacts(b *testing.B) {
ctx := testhelper.Context(b)
for _, shape := range artifactShapes {
b.Run(shape.desc(), func(b *testing.B) {
repo := newMVCCRepo(b)
paths := make([]string, 0, shape.artifactsPerManifest)
for i := range shape.artifactsPerManifest {
paths = append(paths, repo.writeArtifact(numberedReftable(i), minEntryAge+time.Hour))
}
for i := range shape.manifests {
repo.writeManifest(fmt.Sprintf("manifest-%05d", i), time.Duration(i)*time.Minute, paths...)
}
repo.setPointer(manifestNameFor("manifest-00000"))
var runs int
var locked time.Duration
for b.Loop() {
b.StopTimer()
// Every iteration consumes the entries it collects, so they have to be
// written back.
for i := range shape.garbage {
repo.writeArtifact(numberedReftable(shape.artifactsPerManifest+i), minEntryAge+time.Hour)
}
b.StartTimer()
result, err := CollectArtifacts(ctx, repo.path, artifactConfig())
require.NoError(b, err)
require.Equal(b, shape.garbage, result.Removed)
runs++
locked += result.LockDuration
}
// Only the deletions run under the lock, which is what a write can end up
// waiting for, so it is reported separately from the run as a whole.
b.ReportMetric(float64(locked.Nanoseconds())/float64(runs), "ns/locked")
})
}
}