Git designed for AI agents.
package backup
import (
"context"
"fmt"
"io/fs"
"os"
"path/filepath"
"slices"
"strings"
"sync"
"testing"
"testing/fstest"
"time"
"github.com/feder-cr/git-ai/internal/archive"
"github.com/feder-cr/git-ai/internal/gitaly/storage"
"github.com/feder-cr/git-ai/internal/gitaly/storage/mode"
"github.com/feder-cr/git-ai/internal/gitaly/storage/storagemgr/partition/log"
"github.com/feder-cr/git-ai/internal/helper"
"github.com/feder-cr/git-ai/internal/testhelper"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type mockPartition struct {
storage.Partition
manager storage.LogReader
}
func (p *mockPartition) Close() {}
func (p *mockPartition) GetLogReader() storage.LogReader {
return p.manager
}
type mockNode struct {
partitions map[PartitionInfo]*mockPartition
t *testing.T
sync.Mutex
}
func (n *mockNode) GetStorage(storageName string) (storage.Storage, error) {
return mockStorage{storageName: storageName, node: n}, nil
}
type mockStorage struct {
storageName string
node *mockNode
storage.Storage
}
func (m mockStorage) GetPartition(ctx context.Context, partitionID storage.PartitionID) (storage.Partition, error) {
m.node.Lock()
defer m.node.Unlock()
info := PartitionInfo{m.storageName, partitionID}
mgr, ok := m.node.partitions[info]
assert.True(m.node.t, ok)
return mgr, nil
}
type mockLogManager struct {
partitionInfo PartitionInfo
archiver *LogEntryArchiver
notifications []notification
acknowledged []storage.LSN
finalLSN storage.LSN
entryRootPath string
finishFunc func()
finishCount int
sync.Mutex
storage.LogReader
}
func (lm *mockLogManager) AcknowledgePosition(_ storage.PositionType, lsn storage.LSN) error {
lm.Lock()
defer lm.Unlock()
lm.acknowledged = append(lm.acknowledged, lsn)
// If the archiver has completed enough entries to reach our new low water mark,
// send the next notification.
if len(lm.notifications) > 0 && lsn >= lm.notifications[0].sendAt {
lm.SendNotification()
}
// It's possible that the archiver acknowledges a LSN more than once. When the LogEntryArchiver
// processes a notification, it sets the current state's high water mark = notification's high
// water mark and notifies another goroutine for processing. The processing goroutine processes
// log entries sequentially from the current nextLSN -> high watermark. After each iteration, it
// acknowledges the corresponding log entry. If the next notification contains the same high
// watermark as the prior, there's a chance the processing goroutine already handles all log
// entries. It then re-acknowledges the high watermark and skips the notification.
//
// For example:
// - Ingest notification [3, 5]
// - Process 3, ack 3 -> nextLSN = 4
// - Process 4, ack 4 -> nextLSN = 5
// - Process 5, ack 5 -> nextLSN = 6
// - Ingest notification [4, 5] -> re-ack 5
//
// As a result, we allow at most finishCount calls because of redundant acknowledgement. The
// exceeding calls could be ignored.
if lsn == lm.finalLSN && len(lm.notifications) == 0 && lm.finishCount > 0 {
lm.finishCount--
lm.finishFunc()
}
return nil
}
func (lm *mockLogManager) SendNotification() {
n := lm.notifications[0]
lm.archiver.NotifyNewEntries(lm.partitionInfo.StorageName, lm.partitionInfo.PartitionID, n.lowWaterMark, n.highWaterMark)
lm.notifications = lm.notifications[1:]
}
func (lm *mockLogManager) GetEntryPath(lsn storage.LSN) string {
return filepath.Join(partitionPath(lm.entryRootPath, lm.partitionInfo), lsn.String())
}
type notification struct {
lowWaterMark, highWaterMark, sendAt storage.LSN
}
func TestLogEntryArchiver(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
for _, tc := range []struct {
desc string
workerCount uint
partitions []storage.PartitionID
notifications []notification
waitCount int
finalLSN storage.LSN
expectedBackupCount int
expectedLogMessage string
}{
{
desc: "notify one entry",
workerCount: 1,
partitions: []storage.PartitionID{1},
notifications: []notification{
{
lowWaterMark: 1,
highWaterMark: 1,
},
},
finalLSN: 1,
expectedBackupCount: 1,
},
{
desc: "start from later LSN",
workerCount: 1,
partitions: []storage.PartitionID{1},
notifications: []notification{
{
lowWaterMark: 11,
highWaterMark: 11,
},
},
finalLSN: 11,
expectedBackupCount: 1,
},
{
desc: "notify range of entries",
workerCount: 1,
partitions: []storage.PartitionID{1},
notifications: []notification{
{
lowWaterMark: 1,
highWaterMark: 5,
},
},
finalLSN: 5,
expectedBackupCount: 5,
},
{
desc: "increasing high water mark",
workerCount: 1,
partitions: []storage.PartitionID{1},
notifications: []notification{
{
lowWaterMark: 1,
highWaterMark: 5,
},
{
lowWaterMark: 1,
highWaterMark: 6,
},
{
lowWaterMark: 1,
highWaterMark: 7,
},
},
finalLSN: 7,
expectedBackupCount: 7,
},
{
desc: "increasing low water mark",
workerCount: 1,
partitions: []storage.PartitionID{1},
notifications: []notification{
{
lowWaterMark: 1,
highWaterMark: 5,
},
{
lowWaterMark: 2,
highWaterMark: 5,
sendAt: 2,
},
{
lowWaterMark: 3,
highWaterMark: 5,
sendAt: 3,
},
},
finalLSN: 5,
expectedBackupCount: 5,
},
{
desc: "multiple partitions",
workerCount: 1,
partitions: []storage.PartitionID{1, 2, 3},
notifications: []notification{
{
lowWaterMark: 1,
highWaterMark: 3,
},
},
finalLSN: 3,
expectedBackupCount: 9,
},
{
desc: "multiple partitions, multi-threaded",
workerCount: 4,
partitions: []storage.PartitionID{1, 2, 3},
notifications: []notification{
{
lowWaterMark: 1,
highWaterMark: 3,
},
},
finalLSN: 3,
expectedBackupCount: 9,
},
{
desc: "resent items processed once",
workerCount: 1,
partitions: []storage.PartitionID{1},
notifications: []notification{
{
lowWaterMark: 1,
highWaterMark: 3,
},
{
lowWaterMark: 1,
highWaterMark: 3,
sendAt: 3,
},
},
waitCount: 2,
finalLSN: 3,
expectedBackupCount: 3,
},
{
desc: "gap in sequence",
workerCount: 1,
partitions: []storage.PartitionID{1},
notifications: []notification{
{
lowWaterMark: 1,
highWaterMark: 3,
},
{
lowWaterMark: 11,
highWaterMark: 13,
sendAt: 3,
},
},
finalLSN: 13,
expectedBackupCount: 6,
expectedLogMessage: "log entry archiver: gap in log sequence",
},
} {
t.Run(tc.desc, func(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithCancel(ctx)
defer cancel()
entryRootPath := testhelper.TempDir(t)
archivePath := testhelper.TempDir(t)
archiveSink, err := ResolveSink(ctx, archivePath)
require.NoError(t, err)
logger := testhelper.NewLogger(t)
hook := testhelper.AddLoggerHook(logger)
var wg sync.WaitGroup
accessor := &mockNode{
partitions: make(map[PartitionInfo]*mockPartition, len(tc.partitions)),
t: t,
}
node := storage.Node(accessor)
archiver := NewLogEntryArchiver(logger, archiveSink, tc.workerCount, &node)
archiver.Run()
const storageName = "default"
managers := make(map[PartitionInfo]*mockLogManager, len(tc.partitions))
for _, partitionID := range tc.partitions {
info := PartitionInfo{
StorageName: storageName,
PartitionID: partitionID,
}
waitCount := tc.waitCount
if waitCount == 0 {
waitCount = 1
}
wg.Add(waitCount)
manager := &mockLogManager{
partitionInfo: info,
entryRootPath: entryRootPath,
archiver: archiver,
notifications: tc.notifications,
finalLSN: tc.finalLSN,
finishFunc: wg.Done,
finishCount: waitCount,
}
managers[info] = manager
accessor.Lock()
accessor.partitions[info] = &mockPartition{manager: manager}
accessor.Unlock()
// Send partitions in parallel to mimic real usage.
go func() {
sentLSNs := make([]storage.LSN, 0, tc.finalLSN)
for _, notification := range tc.notifications {
for lsn := notification.lowWaterMark; lsn <= notification.highWaterMark; lsn++ {
// Don't recreate entries.
if !slices.Contains(sentLSNs, lsn) {
createEntryDir(t, entryRootPath, info, lsn)
}
sentLSNs = append(sentLSNs, lsn)
}
}
manager.Lock()
defer manager.Unlock()
manager.SendNotification()
}()
}
wg.Wait()
archiver.Close()
cmpDir := testhelper.TempDir(t)
require.NoError(t, os.Mkdir(filepath.Join(cmpDir, storageName), mode.Directory))
for info, partition := range accessor.partitions {
manager := partition.GetLogReader().(*mockLogManager)
lastAck := manager.acknowledged[len(manager.acknowledged)-1]
require.Equal(t, tc.finalLSN, lastAck)
partitionDir := partitionPath(cmpDir, info)
require.NoError(t, os.Mkdir(partitionDir, mode.Directory))
for _, lsn := range manager.acknowledged {
tarPath := filepath.Join(partitionPath(archivePath, info), lsn.String()+".tar")
tar, err := os.Open(tarPath)
require.NoError(t, err)
testhelper.RequireTarState(t, tar, testhelper.DirectoryState{
lsn.String(): {Mode: archive.TarFileMode | archive.ExecuteMode | fs.ModeDir},
filepath.Join(lsn.String(), "LSN"): {Mode: archive.TarFileMode, Content: []byte(lsn.String())},
filepath.Join(lsn.String(), "PARTITION_ID"): {Mode: archive.TarFileMode, Content: []byte(fmt.Sprintf("%d", info.PartitionID))},
})
}
}
if tc.expectedLogMessage != "" {
var logs []string
for _, entry := range hook.AllEntries() {
logs = append(logs, entry.Message)
}
require.Contains(t, logs, tc.expectedLogMessage)
}
testhelper.RequirePromMetrics(t, archiver.backupCounter, buildMetrics(t, tc.expectedBackupCount, 0))
})
}
}
func TestLogEntryArchiver_retry(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithCancel(testhelper.Context(t))
defer cancel()
const lsn = storage.LSN(1)
const workerCount = 1
entryRootPath := testhelper.TempDir(t)
var wg sync.WaitGroup
logger := testhelper.NewLogger(t)
hook := testhelper.AddLoggerHook(logger)
ticker := helper.NewManualTicker()
// Finish waiting when logEntry.backoff is hit after a failure.
tickerFunc := func(time.Duration) helper.Ticker {
wg.Done()
return ticker
}
archivePath := testhelper.TempDir(t)
archiveSink, err := ResolveSink(ctx, archivePath)
require.NoError(t, err)
accessor := &mockNode{
partitions: make(map[PartitionInfo]*mockPartition, 1),
t: t,
}
node := storage.Node(accessor)
archiver := newLogEntryArchiver(logger, archiveSink, workerCount, &node, tickerFunc)
archiver.Run()
defer archiver.Close()
info := PartitionInfo{
StorageName: "default",
PartitionID: 1,
}
manager := &mockLogManager{
partitionInfo: info,
entryRootPath: entryRootPath,
archiver: archiver,
notifications: []notification{
{
lowWaterMark: 1,
highWaterMark: 1,
},
},
finalLSN: 1,
finishFunc: wg.Done,
finishCount: 1,
}
accessor.Lock()
accessor.partitions[info] = &mockPartition{manager: manager}
accessor.Unlock()
wg.Add(1)
manager.SendNotification()
// Wait for initial request to fail and backoff.
wg.Wait()
require.Equal(t, "log entry archiver: failed to backup log entry", hook.LastEntry().Message)
// Create entry so retry will succeed.
createEntryDir(t, entryRootPath, info, lsn)
// Add to wg, this will be decremented when the retry completes.
wg.Add(1)
// Finish backoff.
ticker.Tick()
// Wait for retry to complete.
wg.Wait()
cmpDir := testhelper.TempDir(t)
partitionDir := partitionPath(cmpDir, info)
require.NoError(t, os.MkdirAll(partitionDir, mode.Directory))
tarPath := filepath.Join(partitionPath(archivePath, info), lsn.String()) + ".tar"
tar, err := os.Open(tarPath)
require.NoError(t, err)
testhelper.RequireTarState(t, tar, testhelper.DirectoryState{
lsn.String(): {Mode: archive.TarFileMode | archive.ExecuteMode | fs.ModeDir},
filepath.Join(lsn.String(), "LSN"): {Mode: archive.TarFileMode, Content: []byte(lsn.String())},
filepath.Join(lsn.String(), "PARTITION_ID"): {Mode: archive.TarFileMode, Content: []byte(fmt.Sprintf("%d", info.PartitionID))},
})
testhelper.RequirePromMetrics(t, archiver.backupCounter, buildMetrics(t, 1, 1))
}
func TestLogEntryStore_Query(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
rootPath := t.TempDir()
info := PartitionInfo{
StorageName: "default",
PartitionID: 1,
}
testhelper.WriteFiles(t, rootPath, map[string]any{
archivePath(info, 1): "",
archivePath(info, 2): "",
archivePath(info, 3): "",
archivePath(info, 4): "",
archivePath(info, 8): "",
archivePath(info, 9): "",
})
sink, err := ResolveSink(ctx, rootPath)
require.NoError(t, err)
defer testhelper.MustClose(t, sink)
store := NewLogEntryStore(sink)
var lsns []storage.LSN
it := store.Query(info, storage.LSN(3))
for it.Next(ctx) {
lsns = append(lsns, it.LSN())
}
require.NoError(t, it.Err())
require.Equal(t, []storage.LSN{3, 4, 8, 9}, lsns)
}
func partitionPath(root string, info PartitionInfo) string {
return filepath.Join(root, partitionDir(info))
}
func createEntryDir(t *testing.T, entryRootPath string, partitionInfo PartitionInfo, lsn storage.LSN) {
t.Helper()
partitionPath := partitionPath(entryRootPath, partitionInfo)
require.NoError(t, os.MkdirAll(partitionPath, mode.Directory))
testhelper.CreateFS(t, filepath.Join(partitionPath, lsn.String()), fstest.MapFS{
".": {Mode: mode.Directory},
"LSN": {Mode: mode.File, Data: []byte(lsn.String())},
"PARTITION_ID": {Mode: mode.File, Data: []byte(fmt.Sprintf("%d", partitionInfo.PartitionID))},
})
}
func buildMetrics(t *testing.T, successCt, failCt int) string {
t.Helper()
var builder strings.Builder
_, err := builder.WriteString("# HELP gitaly_wal_backup_count Counter of the number of WAL entries backed up by status\n")
require.NoError(t, err)
_, err = builder.WriteString("# TYPE gitaly_wal_backup_count counter\n")
require.NoError(t, err)
_, err = builder.WriteString(
fmt.Sprintf("gitaly_wal_backup_count{status=\"success\"} %d\n", successCt))
require.NoError(t, err)
if failCt > 0 {
_, err = builder.WriteString(
fmt.Sprintf("gitaly_wal_backup_count{status=\"fail\"} %d\n", failCt))
require.NoError(t, err)
}
return builder.String()
}
// TestLogEntryArchiver_WithRealLogManager runs the log archiver with a real WAL log manager. This test aims to assert
// the flow between the two components to avoid uncaught errors due to mocking. Test setup and verification are
// extremely verbose. There's no reliable way to simulate more sophisticated scenarios because the test could not
// intercept internal states of log archiver or log manager. The log manager doesn't expose the current consumer
// position. Thus, this test verifies the most basic archiving. Unit tests using mocking will cover the rest.
func TestLogEntryArchiver_WithRealLogManager(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
// Setup node
accessor := &mockNode{
partitions: make(map[PartitionInfo]*mockPartition),
t: t,
}
node := storage.Node(accessor)
// Setup archiver
archivePath := testhelper.TempDir(t)
archiveSink, err := ResolveSink(ctx, archivePath)
require.NoError(t, err)
archiver := NewLogEntryArchiver(testhelper.NewLogger(t), archiveSink, 1, &node)
archiver.Run()
// Setup WAL log manager and plug it to storage
const storageName = "default"
var logManagers []*log.Manager
for i := 1; i <= 2; i++ {
info := PartitionInfo{
StorageName: storageName,
PartitionID: storage.PartitionID(i),
}
tracker := log.NewPositionTracker()
require.NoError(t, tracker.Register(log.ConsumerPosition))
logManager := log.NewManager(storageName, storage.PartitionID(i), testhelper.TempDir(t), testhelper.TempDir(t), archiver, tracker)
require.NoError(t, logManager.Initialize(ctx, 0))
accessor.Lock()
accessor.partitions[info] = &mockPartition{manager: logManager}
accessor.Unlock()
logManagers = append(logManagers, logManager)
}
appendLogEntry(t, logManagers[0], map[string][]byte{
"1": []byte("content-1"),
"2": []byte("content-2"),
"3": []byte("content-3"),
})
require.Nil(t, <-logManagers[0].GetNotificationQueue())
// KV operation without any file.
appendLogEntry(t, logManagers[1], map[string][]byte{})
require.Nil(t, <-logManagers[1].GetNotificationQueue())
appendLogEntry(t, logManagers[1], map[string][]byte{
"4": []byte("content-4"),
"5": []byte("content-5"),
"6": []byte("content-6"),
})
require.Nil(t, <-logManagers[1].GetNotificationQueue())
archiver.Close()
cmpDir := testhelper.TempDir(t)
require.NoError(t, os.Mkdir(filepath.Join(cmpDir, storageName), mode.Directory))
lsnPrefix := storage.LSN(1).String()
// Assert manager 1
tarPath := filepath.Join(partitionPath(archivePath, PartitionInfo{storageName, storage.PartitionID(1)}), storage.LSN(1).String()+".tar")
tar, err := os.Open(tarPath)
require.NoError(t, err)
testhelper.RequireTarState(t, tar, testhelper.DirectoryState{
lsnPrefix: {Mode: archive.TarFileMode | archive.ExecuteMode | fs.ModeDir},
filepath.Join(lsnPrefix, "1"): {Mode: archive.TarFileMode, Content: []byte("content-1")},
filepath.Join(lsnPrefix, "2"): {Mode: archive.TarFileMode, Content: []byte("content-2")},
filepath.Join(lsnPrefix, "3"): {Mode: archive.TarFileMode, Content: []byte("content-3")},
})
testhelper.MustClose(t, tar)
// Assert manager 2
tarPath = filepath.Join(partitionPath(archivePath, PartitionInfo{storageName, storage.PartitionID(2)}), lsnPrefix+".tar")
tar, err = os.Open(tarPath)
require.NoError(t, err)
testhelper.RequireTarState(t, tar, testhelper.DirectoryState{
lsnPrefix: {Mode: archive.TarFileMode | archive.ExecuteMode | fs.ModeDir},
})
testhelper.MustClose(t, tar)
lsnPrefix = storage.LSN(2).String()
tarPath = filepath.Join(partitionPath(archivePath, PartitionInfo{storageName, storage.PartitionID(2)}), lsnPrefix+".tar")
tar, err = os.Open(tarPath)
require.NoError(t, err)
testhelper.RequireTarState(t, tar, testhelper.DirectoryState{
lsnPrefix: {Mode: archive.TarFileMode | archive.ExecuteMode | fs.ModeDir},
filepath.Join(lsnPrefix, "4"): {Mode: archive.TarFileMode, Content: []byte("content-4")},
filepath.Join(lsnPrefix, "5"): {Mode: archive.TarFileMode, Content: []byte("content-5")},
filepath.Join(lsnPrefix, "6"): {Mode: archive.TarFileMode, Content: []byte("content-6")},
})
testhelper.MustClose(t, tar)
// Finally, assert the metrics.
testhelper.RequirePromMetrics(t, archiver.backupCounter, buildMetrics(t, 3, 0))
}
func appendLogEntry(t *testing.T, manager *log.Manager, files map[string][]byte) storage.LSN {
t.Helper()
logEntryPath := testhelper.TempDir(t)
for name, value := range files {
path := filepath.Join(logEntryPath, name)
require.NoError(t, os.WriteFile(path, value, mode.File))
}
nextLSN, err := manager.AppendLogEntry(logEntryPath)
require.NoError(t, err)
return nextLSN
}