Skip to content

Git designed for AI agents.

14.0 KB
Raw Blame
package streamcache

import (
	"bytes"
	"context"
	"crypto/rand"
	"errors"
	"fmt"
	"io"
	mathrand "math/rand"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/feder-cr/git-ai/internal/gitaly/config"
	"github.com/feder-cr/git-ai/internal/gitaly/storage/mode"
	"github.com/feder-cr/git-ai/internal/helper/duration"
	"github.com/feder-cr/git-ai/internal/testhelper"
	"github.com/stretchr/testify/require"
)

func newCache(tb testing.TB, dir string, backpressure bool) Cache {
	cache := New(config.StreamCacheConfig{
		Enabled:      true,
		Dir:          dir,
		MaxAge:       duration.Duration(time.Hour),
		Backpressure: backpressure,
	}, testhelper.SharedLogger(tb))
	tb.Cleanup(cache.Stop)

	return cache
}

func innerCache(c Cache) *cache {
	for {
		switch v := c.(type) {
		case *cache:
			return v
		case *minOccurrences:
			c = v.Cache
		case *TestLoggingCache:
			c = v.Cache
		default:
			panic(fmt.Errorf("unexpected cache type: %v", v))
		}
	}
}

func TestCache_writeOneReadMultiple(t *testing.T) {
	t.Parallel()

	for _, backpressure := range []bool{true, false} {
		t.Run(fmt.Sprintf("backpressure-%v", backpressure), func(t *testing.T) {
			t.Parallel()
			ctx := testhelper.Context(t)

			tmp := testhelper.TempDir(t)

			c := newCache(t, tmp, backpressure)

			const (
				key = "test key"
				N   = 10
			)
			content := func(i int) string { return fmt.Sprintf("content %d", i) }

			for i := 0; i < N; i++ {
				t.Run(fmt.Sprintf("read %d", i), func(t *testing.T) {
					buf := &bytes.Buffer{}
					_, created, err := c.Fetch(ctx, key, buf, writeString(content(i)))
					require.NoError(t, err)
					require.Equal(t, i == 0, created, "all calls except the first one should be cache hits")
					require.Equal(t, content(0), buf.String(), "expect cache hits for all i > 0")
				})
			}

			requireCacheFiles(t, tmp, 1)
		})
	}
}

func TestCache_manyConcurrentWrites(t *testing.T) {
	t.Parallel()

	for _, backpressure := range []bool{true, false} {
		t.Run(fmt.Sprintf("backpressure-%v", backpressure), func(t *testing.T) {
			t.Parallel()
			ctx := testhelper.Context(t)

			tmp := testhelper.TempDir(t)

			c := newCache(t, tmp, backpressure)

			const (
				key = "test key"
				N   = 1000
			)
			content := make([]string, N)
			errors := make(chan error, N)
			output := make([]string, N)
			start := make(chan struct{})
			buf := make([]byte, 4096)

			for i := 0; i < N; i++ {
				_, _ = rand.Read(buf) // math/rand.Read always returns len(buf), nil
				content[i] = string(buf)

				go func(i int) {
					errors <- func() error {
						<-start

						buf := &bytes.Buffer{}
						_, _, err := c.Fetch(ctx, key, buf, writeString(content[i]))
						if err != nil {
							return err
						}

						output[i] = buf.String()
						return nil
					}()
				}(i)
			}

			close(start) // Start all goroutines at once

			// Wait for all goroutines to finish
			for i := 0; i < N; i++ {
				require.NoError(t, <-errors)
			}

			for i := 0; i < N; i++ {
				require.Equal(t, output[0], output[i], "all calls to Fetch returned the same bytes")
			}

			require.Contains(t, content, output[0], "data returned by Fetch is not mangled")

			requireCacheFiles(t, tmp, 1)
		})
	}
}

func writeString(s string) func(io.Writer) error {
	return func(w io.Writer) error {
		_, err := io.WriteString(w, s)
		return err
	}
}

func requireCacheFiles(t *testing.T, dir string, n int) {
	t.Helper()

	find := string(testhelper.MustRunCommand(t, nil, "find", dir, "-type", "f"))
	require.Equal(t, n, strings.Count(find, "\n"), "unexpected find output %q", find)
}

func requireCacheEntries(t *testing.T, _c Cache, n int) {
	t.Helper()
	c := innerCache(_c)
	c.m.Lock()
	defer c.m.Unlock()
	require.Len(t, c.index, n)
}

func TestCache_deletedFile(t *testing.T) {
	t.Parallel()

	for _, backpressure := range []bool{true, false} {
		t.Run(fmt.Sprintf("backpressure-%v", backpressure), func(t *testing.T) {
			t.Parallel()
			tmp := testhelper.TempDir(t)
			ctx := testhelper.Context(t)

			c := newCache(t, tmp, backpressure)

			const (
				key = "test key"
			)
			content := func(i int) string { return fmt.Sprintf("content %d", i) }

			buf1 := &bytes.Buffer{}
			_, created, err := c.Fetch(ctx, key, buf1, writeString(content(1)))
			require.NoError(t, err)
			require.True(t, created)

			require.NoError(t, os.RemoveAll(tmp), "wipe out underlying files of cache")
			require.NoError(t, os.MkdirAll(tmp, mode.Directory))

			// File is gone from filesystem but not from cache
			requireCacheFiles(t, tmp, 0)
			requireCacheEntries(t, c, 1)

			buf2 := &bytes.Buffer{}
			_, created, err = c.Fetch(ctx, key, buf2, writeString(content(2)))
			require.NoError(t, err)
			require.True(t, created, "because the first file is gone, cache is forced to create a new entry")

			require.Equal(t, content(1), buf1.String(), "r1 should still see its original pre-wipe contents")
			require.Equal(t, content(2), buf2.String(), "r2 should see the new post-wipe contents")
		})
	}
}

func TestCache_scope(t *testing.T) {
	t.Parallel()

	for _, backpressure := range []bool{true, false} {
		t.Run(fmt.Sprintf("backpressure-%v", backpressure), func(t *testing.T) {
			t.Parallel()
			ctx := testhelper.Context(t)

			tmp := testhelper.TempDir(t)

			const (
				N   = 100
				key = "test key"
			)

			// Intentionally create multiple cache instances sharing one directory,
			// to test that they do not trample on each others files.
			cache := make([]Cache, N)
			input := make([]string, N)
			output := make([]string, N)
			wg := &sync.WaitGroup{}
			wg.Add(N)

			for i := 0; i < N; i++ {
				go func(i int) {
					defer wg.Done()

					input[i] = fmt.Sprintf("test content %d", i)
					cache[i] = newCache(t, tmp, backpressure)

					buf := &bytes.Buffer{}
					_, created, err := cache[i].Fetch(ctx, key, buf, writeString(input[i]))
					require.NoError(t, err)
					require.True(t, created)
					output[i] = buf.String()
				}(i)
			}
			wg.Wait()

			// If different cache instances overwrite their entries, the effect may
			// be order dependent, e.g. "last write wins". We could reverse the order
			// now to catch that possible bug, but then we only test for one kind of
			// bug. Let's shuffle instead, which can catch more hypothetical bugs.
			mathrand.Shuffle(N, func(i, j int) {
				output[i], output[j] = output[j], output[i]
				input[i], input[j] = input[j], input[i]
			})

			for i := 0; i < N; i++ {
				require.Equal(t, input[i], output[i])
			}
		})
	}
}

func TestCache_diskCleanup(t *testing.T) {
	t.Parallel()
	ctx := testhelper.Context(t)

	tmp := testhelper.TempDir(t)

	const (
		key = "test key"
	)

	filestoreCleanTimerCh := make(chan time.Time)
	filestoreClean := func(time.Duration) <-chan time.Time {
		return filestoreCleanTimerCh
	}

	cleanSleepTimerCh := make(chan time.Time)
	cleanSleep := func(time.Duration) <-chan time.Time {
		return cleanSleepTimerCh
	}

	streamCacheCfg := config.StreamCacheConfig{
		Enabled:      true,
		Backpressure: true,
		Dir:          tmp,
		MaxAge:       duration.Duration(0),
	}
	c := newCacheWithSleep(streamCacheCfg, filestoreClean, cleanSleep, testhelper.SharedLogger(t))
	defer c.Stop()

	var removalLock sync.Mutex
	c.removalCond = sync.NewCond(&removalLock)

	content := func(i int) string { return fmt.Sprintf("content %d", i) }

	out1 := &bytes.Buffer{}
	_, created, err := c.Fetch(ctx, key, out1, writeString(content(1)))
	require.NoError(t, err)
	require.True(t, created)
	require.Equal(t, content(1), out1.String())

	// File and index entry should still exist because cleanup goroutines are blocked.
	requireCacheFiles(t, tmp, 1)
	requireCacheEntries(t, c, 1)

	// In order to avoid having to sleep, we instead use the removalCond of the cache. Like
	// this, we can lock the condition before scheduling removal of the cache entry and then
	// wait for the condition to be triggered. Like this, we can wait for removal in an entirely
	// race-free manner.
	removedCh := make(chan struct{})
	removalLock.Lock()
	go func() {
		defer func() {
			removalLock.Unlock()
			close(removedCh)
		}()

		c.removalCond.Wait()
	}()

	// Unblock cleanup goroutines so they run exactly once
	cleanSleepTimerCh <- time.Time{}
	filestoreCleanTimerCh <- time.Time{}

	<-removedCh

	// File and index entry should have been removed by cleanup goroutines.
	requireCacheFiles(t, tmp, 0)
	requireCacheEntries(t, c, 0)

	out2 := &bytes.Buffer{}
	_, created, err = c.Fetch(ctx, key, out2, writeString(content(2)))
	require.NoError(t, err)
	require.True(t, created)
	require.Equal(t, content(2), out2.String(), "Sanity check: no stale value returned by the cache")
}

func TestCache_failedWrite(t *testing.T) {
	t.Parallel()

	for _, backpressure := range []bool{true, false} {
		t.Run(fmt.Sprintf("backpressure-%v", backpressure), func(t *testing.T) {
			t.Parallel()
			ctx := testhelper.Context(t)

			tmp := testhelper.TempDir(t)

			c := newCache(t, tmp, backpressure)

			testCases := []struct {
				desc   string
				create func(io.Writer) error
			}{
				{
					desc:   "create returns error",
					create: func(io.Writer) error { return errors.New("something went wrong") },
				},
				{
					desc:   "create panics",
					create: func(io.Writer) error { panic("oh no") },
				},
			}

			for _, tc := range testCases {
				t.Run(tc.desc, func(t *testing.T) {
					_, created, err := c.Fetch(ctx, tc.desc, io.Discard, tc.create)
					require.Error(t, err)
					require.True(t, created)

					const happy = "all is good"
					buf := &bytes.Buffer{}
					_, created, err = c.Fetch(ctx, tc.desc, buf, writeString(happy))
					require.NoError(t, err)
					require.True(t, created, "because the previous entry failed, a new one should have been created")

					require.Equal(t, happy, buf.String())
				})
			}
		})
	}
}

func TestCache_failCreateFile(t *testing.T) {
	t.Parallel()

	for _, backpressure := range []bool{true, false} {
		t.Run(fmt.Sprintf("backpressure-%v", backpressure), func(t *testing.T) {
			t.Parallel()
			ctx := testhelper.Context(t)

			tmp := testhelper.TempDir(t)

			c := newCache(t, tmp, backpressure)

			createError := errors.New("cannot create file")
			innerCache(c).createFile = func() (namedWriteCloser, error) { return nil, createError }

			_, _, err := c.Fetch(ctx, "key", io.Discard, func(io.Writer) error { return nil })
			require.Equal(t, createError, err)
		})
	}
}

func TestCache_unWriteableFile(t *testing.T) {
	t.Parallel()

	for _, backpressure := range []bool{true, false} {
		t.Run(fmt.Sprintf("backpressure-%v", backpressure), func(t *testing.T) {
			t.Parallel()
			ctx := testhelper.Context(t)

			tmp := testhelper.TempDir(t)

			c := newCache(t, tmp, backpressure)

			innerCache(c).createFile = func() (namedWriteCloser, error) {
				return os.OpenFile(filepath.Join(tmp, "unwriteable"), os.O_RDONLY|os.O_CREATE|os.O_EXCL, mode.File)
			}

			_, _, err := c.Fetch(ctx, "key", io.Discard, func(w io.Writer) error {
				_, err := io.WriteString(w, "hello")
				return err
			})

			var pathErr *os.PathError
			require.True(t, errors.As(err, &pathErr))
			require.Equal(t, "write", pathErr.Op)
		})
	}
}

func TestCache_unCloseableFile(t *testing.T) {
	t.Parallel()

	for _, backpressure := range []bool{true, false} {
		t.Run(fmt.Sprintf("backpressure-%v", backpressure), func(t *testing.T) {
			t.Parallel()
			ctx := testhelper.Context(t)

			tmp := testhelper.TempDir(t)

			c := newCache(t, tmp, backpressure)

			innerCache(c).createFile = func() (namedWriteCloser, error) {
				f, err := os.OpenFile(filepath.Join(tmp, "uncloseable"), os.O_WRONLY|os.O_CREATE|os.O_EXCL, mode.File)
				if err != nil {
					return nil, err
				}
				return f, f.Close() // Already closed so cannot be closed again
			}

			_, _, err := c.Fetch(ctx, "key", io.Discard, func(w io.Writer) error { return nil })

			var pathErr *os.PathError
			require.True(t, errors.As(err, &pathErr))
			require.Equal(t, "close", pathErr.Op)
		})
	}
}

func TestCache_cannotOpenFileForReading(t *testing.T) {
	t.Parallel()

	for _, backpressure := range []bool{true, false} {
		t.Run(fmt.Sprintf("backpressure-%v", backpressure), func(t *testing.T) {
			t.Parallel()
			ctx := testhelper.Context(t)

			tmp := testhelper.TempDir(t)

			c := newCache(t, tmp, backpressure)

			innerCache(c).createFile = func() (namedWriteCloser, error) {
				f, err := os.OpenFile(filepath.Join(tmp, "unopenable"), os.O_WRONLY|os.O_CREATE|os.O_EXCL, mode.File)
				if err != nil {
					return nil, err
				}
				return f, os.Remove(f.Name()) // Removed so cannot be opened
			}

			_, _, err := c.Fetch(ctx, "key", io.Discard, func(w io.Writer) error { return nil })
			err = errors.Unwrap(err)
			var pathErr *os.PathError
			require.True(t, errors.As(err, &pathErr))
			require.Equal(t, "open", pathErr.Op)
		})
	}
}

func TestWaiter(t *testing.T) {
	t.Parallel()
	ctx := testhelper.Context(t)

	w := newWaiter()
	err := errors.New("test error")
	w.SetError(err)
	require.Equal(t, err, w.Wait(ctx))
}

func TestWaiter_cancel(t *testing.T) {
	t.Parallel()
	ctx, cancel := context.WithCancel(testhelper.Context(t))

	w := newWaiter()
	errc := make(chan error, 1)
	go func() { errc <- w.Wait(ctx) }()

	cancel()
	require.Equal(t, context.Canceled, <-errc)
}

func TestNullCache(t *testing.T) {
	t.Parallel()
	ctx := testhelper.Context(t)

	const (
		N         = 1000
		inputSize = 4096
		key       = "key"
	)

	c := NullCache{}
	start := make(chan struct{})
	results := make(chan error, N)

	for i := 0; i < N; i++ {
		go func() {
			results <- func() error {
				input := make([]byte, inputSize)
				n, err := rand.Read(input)
				if err != nil {
					return err
				}
				if n != inputSize {
					return io.ErrShortWrite
				}

				<-start

				output := &bytes.Buffer{}
				_, created, err := c.Fetch(ctx, key, output, func(w io.Writer) error {
					for j := 0; j < len(input); j++ {
						n, err := w.Write(input[j : j+1])
						if err != nil {
							return err
						}
						if n != 1 {
							return io.ErrShortWrite
						}
					}
					return nil
				})
				if err != nil {
					return err
				}

				if !created {
					return errors.New("created should be true")
				}

				if !bytes.Equal(output.Bytes(), input) {
					return errors.New("output does not match input")
				}

				return nil
			}()
		}()
	}

	close(start)
	for i := 0; i < N; i++ {
		require.NoError(t, <-results)
	}
}