Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

16.6 KB
Raw Blame
package filer

import (
	"context"
	"fmt"
	"strconv"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
)

func TestChunkGroupReaderCacheMemory(t *testing.T) {
	budget := NewReaderCacheBudget(8 << 10)
	groups := make([]*ChunkGroup, 32)
	for i := range groups {
		group, err := NewChunkGroup(func(context.Context, string) ([]string, error) { return []string{"unused"}, nil }, newMockChunkCacheForReaderCache(), nil, 128, nil, nil, budget)
		if err != nil {
			t.Fatal(err)
		}
		groups[i] = group
		defer group.Close()
		group.readerCache.fetchChunkDataFn = func(_ context.Context, buffer []byte, _ []string, _ []byte, _ bool, _ bool, _ int64, _ string, _ util_http.RefreshUrlsFunc) (int, error) {
			buffer[0] = 42
			return len(buffer), nil
		}
		buffer := make([]byte, 1)
		n, err := group.readerCache.ReadChunkAt(context.Background(), buffer, fmt.Sprint(i), nil, false, 0, 3<<10, false)
		if err != nil || n != 1 || buffer[0] != 42 {
			t.Fatalf("read %d: n=%d data=%v err=%v", i, n, buffer, err)
		}
		budget.Lock()
		used := budget.used
		budget.Unlock()
		if used > 8<<10 {
			t.Fatalf("shared budget used %d bytes", used)
		}
	}
	groups[0].readerCache.Lock()
	retained := len(groups[0].readerCache.downloaders)
	groups[0].readerCache.Unlock()
	if retained != 0 {
		t.Fatalf("first file still retains %d downloaders", retained)
	}
	for _, group := range groups {
		_ = group.Close()
	}
	budget.Lock()
	defer budget.Unlock()
	if budget.used != 0 {
		t.Fatalf("closed files still reserve %d bytes", budget.used)
	}
}

func TestReaderCacheBudgetInFlight(t *testing.T) {
	for _, prefetch := range []bool{false, true} {
		t.Run(fmt.Sprintf("prefetch=%t", prefetch), func(t *testing.T) {
			budget := NewReaderCacheBudget(8 << 10)
			started := make(chan struct{}, 4)
			gate := make(chan struct{})
			var readers sync.WaitGroup
			var caches []*ReaderCache
			for i := 0; i < 4; i++ {
				rc := NewReaderCache(256, newMockChunkCacheForReaderCache(), func(context.Context, string) ([]string, error) { return []string{"unused"}, nil }, nil, budget)
				caches = append(caches, rc)
				defer rc.destroy()
				rc.fetchChunkDataFn = func(_ context.Context, buffer []byte, _ []string, _ []byte, _ bool, _ bool, _ int64, _ string, _ util_http.RefreshUrlsFunc) (int, error) {
					started <- struct{}{}
					<-gate
					buffer[0] = 42
					return len(buffer), nil
				}
				if prefetch {
					rc.MaybeCache(&Interval[*ChunkView]{Value: &ChunkView{FileId: "chunk", ViewSize: 3 << 10, ChunkSize: 3 << 10}}, 1)
				} else {
					readers.Add(1)
					go func() {
						defer readers.Done()
						buffer := make([]byte, 1)
						n, err := rc.ReadChunkAt(context.Background(), buffer, "chunk", nil, false, 0, 3<<10, false)
						if err != nil || n != 1 || buffer[0] != 42 {
							t.Errorf("read: n=%d data=%v err=%v", n, buffer, err)
						}
					}()
				}
			}
			for i := 0; i < 2; i++ {
				<-started
			}
			select {
			case <-started:
				t.Error("third download allocated before budget was released")
			case <-time.After(50 * time.Millisecond):
			}
			budget.Lock()
			used := budget.used
			budget.Unlock()
			if used != 8<<10 {
				t.Errorf("in-flight reservations = %d, want 8192", used)
			}
			close(gate)
			readers.Wait()
			for i := 0; i < 2; i++ {
				select {
				case <-started:
				case <-time.After(5 * time.Second):
					t.Fatal("download did not resume after eviction")
				}
			}
			for _, rc := range caches {
				rc.destroy()
			}
			budget.Lock()
			defer budget.Unlock()
			if budget.used != 0 {
				t.Errorf("reservations leaked: %d", budget.used)
			}
		})
	}
}

func TestReaderCacheBudgetOversizedChunk(t *testing.T) {
	rc := NewReaderCache(256, newMockChunkCacheForReaderCache(), func(context.Context, string) ([]string, error) {
		t.Error("oversized chunk performed lookup")
		return nil, nil
	}, nil, NewReaderCacheBudget(3<<10))
	defer rc.destroy()
	_, err := rc.ReadChunkAt(context.Background(), make([]byte, 1), "chunk", nil, false, 0, 3<<10, false)
	if err == nil {
		t.Fatal("expected pooled buffer larger than budget to be rejected")
	}
}

func TestReaderCacheEvictionDoesNotHoldCacheLock(t *testing.T) {
	rc := NewReaderCache(2, newMockChunkCacheForReaderCache(), func(context.Context, string) ([]string, error) { return []string{"unused"}, nil }, nil)
	defer rc.destroy()
	rc.fetchChunkDataFn = func(_ context.Context, buffer []byte, _ []string, _ []byte, _ bool, _ bool, _ int64, _ string, _ util_http.RefreshUrlsFunc) (int, error) {
		return len(buffer), nil
	}
	if _, err := rc.ReadChunkAt(context.Background(), make([]byte, 1), "chunk", nil, false, 0, 1024, false); err != nil {
		t.Fatal(err)
	}
	rc.Lock()
	downloader := rc.downloaders["chunk"]
	downloader.wg.Add(1)
	rc.Unlock()
	evicted := make(chan struct{})
	go func() { rc.UnCache("chunk"); close(evicted) }()
	deadline := time.Now().Add(5 * time.Second)
	available := false
	for time.Now().Before(deadline) {
		if rc.TryLock() {
			available = rc.downloaders["chunk"] == nil
			rc.Unlock()
			if available {
				break
			}
		}
		time.Sleep(time.Millisecond)
	}
	downloader.wg.Done()
	<-evicted
	if !available {
		t.Fatal("cache lock held while eviction waited for a reader")
	}
}

func TestReaderCacheFailedPrefetchReleasesBudget(t *testing.T) {
	for _, lookupFailure := range []bool{false, true} {
		t.Run(fmt.Sprintf("lookupFailure=%t", lookupFailure), func(t *testing.T) {
			budget := NewReaderCacheBudget(1024)
			rc := NewReaderCache(1, newMockChunkCacheForReaderCache(), func(context.Context, string) ([]string, error) {
				if lookupFailure {
					return nil, fmt.Errorf("lookup failed")
				}
				return []string{"unused"}, nil
			}, nil, budget)
			defer rc.destroy()
			rc.fetchChunkDataFn = func(_ context.Context, _ []byte, _ []string, _ []byte, _ bool, _ bool, _ int64, _ string, _ util_http.RefreshUrlsFunc) (int, error) {
				return 0, fmt.Errorf("fetch failed")
			}
			rc.MaybeCache(&Interval[*ChunkView]{Value: &ChunkView{FileId: "failed", ViewSize: 1024, ChunkSize: 1024}}, 1)
			deadline := time.Now().Add(5 * time.Second)
			for {
				rc.Lock()
				count := len(rc.downloaders)
				rc.Unlock()
				budget.Lock()
				used := budget.used
				budget.Unlock()
				if count == 0 && used == 0 {
					break
				}
				if time.Now().After(deadline) {
					t.Fatalf("failed prefetch retains %d slots and %d bytes", count, used)
				}
				time.Sleep(time.Millisecond)
			}
		})
	}
}

func TestReaderCacheUnboundedWithoutBudget(t *testing.T) {
	if strconv.IntSize == 32 {
		// Proving the cache has no implicit memory budget needs more than 64
		// in-flight 4 MiB buffers (>256 MiB) at once; the weed/filer test binary
		// already runs near the 4 GiB address-space limit on 32-bit.
		t.Skip("requires >256MiB of in-flight buffers; skipped on 32-bit")
	}
	const readers = 96 // more than 256MiB / 4MiB = 64 buffers
	started := make(chan struct{}, readers)
	gate := make(chan struct{})
	rc := NewReaderCache(256, newMockChunkCacheForReaderCache(), func(context.Context, string) ([]string, error) {
		return []string{"unused"}, nil
	}, nil)
	defer rc.destroy()
	rc.fetchChunkDataFn = func(_ context.Context, buffer []byte, _ []string, _ []byte, _ bool, _ bool, _ int64, _ string, _ util_http.RefreshUrlsFunc) (int, error) {
		started <- struct{}{}
		<-gate
		buffer[0] = 42
		return len(buffer), nil
	}
	var readersWg sync.WaitGroup
	for i := 0; i < readers; i++ {
		readersWg.Add(1)
		go func(i int) {
			defer readersWg.Done()
			rc.ReadChunkAt(context.Background(), make([]byte, 1), fmt.Sprint(i), nil, false, 0, 4<<20, false)
		}(i)
	}
	for i := 0; i < readers; i++ {
		select {
		case <-started:
		case <-time.After(5 * time.Second):
			close(gate)
			readersWg.Wait()
			t.Fatalf("only %d of %d downloads started; an implicit memory budget throttled the reader cache", i, readers)
		}
	}
	close(gate)
	readersWg.Wait()
}

func TestReaderCacheDropsConsumedChunks(t *testing.T) {
	var fetchCount int32
	rc := NewReaderCache(10, newMockChunkCacheForReaderCache(), func(context.Context, string) ([]string, error) {
		return []string{"unused"}, nil
	}, nil)
	defer rc.destroy()
	rc.fetchChunkDataFn = func(_ context.Context, buffer []byte, _ []string, _ []byte, _ bool, _ bool, _ int64, _ string, _ util_http.RefreshUrlsFunc) (int, error) {
		atomic.AddInt32(&fetchCount, 1)
		return len(buffer), nil
	}
	buf := make([]byte, 4<<10)
	if _, err := rc.ReadChunkAt(context.Background(), buf, "chunk", nil, false, 0, 4<<10, false); err != nil {
		t.Fatal(err)
	}
	rc.Lock()
	_, retained := rc.downloaders["chunk"]
	rc.Unlock()
	if retained {
		t.Fatal("fully consumed chunk buffer still retained")
	}
	if _, err := rc.ReadChunkAt(context.Background(), buf, "chunk", nil, false, 0, 4<<10, false); err != nil {
		t.Fatal(err)
	}
	if got := atomic.LoadInt32(&fetchCount); got != 2 {
		t.Fatalf("consumed chunk was not refetched, fetchCount=%d", got)
	}
}

// A prefetched chunk must survive until its reader arrives, then be dropped
// once consumed: the read hits the prefetched buffer (no second fetch) and a
// later read fetches again.
func TestReaderCachePrefetchBufferDroppedAfterRead(t *testing.T) {
	var fetchCount int32
	rc := NewReaderCache(10, newMockChunkCacheForReaderCache(), func(context.Context, string) ([]string, error) {
		return []string{"unused"}, nil
	}, nil)
	defer rc.destroy()
	rc.fetchChunkDataFn = func(_ context.Context, buffer []byte, _ []string, _ []byte, _ bool, _ bool, _ int64, _ string, _ util_http.RefreshUrlsFunc) (int, error) {
		atomic.AddInt32(&fetchCount, 1)
		buffer[0] = 42
		return len(buffer), nil
	}
	rc.MaybeCache(&Interval[*ChunkView]{Value: &ChunkView{FileId: "chunk", ViewSize: 4 << 10, ChunkSize: 4 << 10}}, 1)

	buf := make([]byte, 4<<10)
	n, err := rc.ReadChunkAt(context.Background(), buf, "chunk", nil, false, 0, 4<<10, false)
	if err != nil || n != len(buf) || buf[0] != 42 {
		t.Fatalf("read of prefetched chunk: n=%d data=%d err=%v", n, buf[0], err)
	}
	if got := atomic.LoadInt32(&fetchCount); got != 1 {
		t.Fatalf("read did not hit the prefetched buffer, fetchCount=%d", got)
	}
	rc.Lock()
	_, retained := rc.downloaders["chunk"]
	rc.Unlock()
	if retained {
		t.Fatal("consumed prefetch buffer still retained")
	}
	if _, err := rc.ReadChunkAt(context.Background(), buf, "chunk", nil, false, 0, 4<<10, false); err != nil {
		t.Fatal(err)
	}
	if got := atomic.LoadInt32(&fetchCount); got != 2 {
		t.Fatalf("consumed chunk was not refetched, fetchCount=%d", got)
	}
}

// A consumed chunk survives while another reader is still attached; it is
// dropped once no readers remain, regardless of which reader reached the end.
func TestReaderCacheConsumedBufferSurvivesAttachedReader(t *testing.T) {
	var fetchCount int32
	rc := NewReaderCache(10, newMockChunkCacheForReaderCache(), func(context.Context, string) ([]string, error) {
		return []string{"unused"}, nil
	}, nil)
	defer rc.destroy()
	rc.fetchChunkDataFn = func(_ context.Context, buffer []byte, _ []string, _ []byte, _ bool, _ bool, _ int64, _ string, _ util_http.RefreshUrlsFunc) (int, error) {
		atomic.AddInt32(&fetchCount, 1)
		buffer[0] = 42
		return len(buffer), nil
	}

	// A partial read primes the cacher, then a second reader attaches.
	rc.ReadChunkAt(context.Background(), make([]byte, 1), "chunk", nil, false, 0, 4<<10, false)
	rc.Lock()
	downloader := rc.downloaders["chunk"]
	if downloader == nil {
		rc.Unlock()
		t.Fatal("cacher missing before attach")
	}
	downloader.wg.Add(1)
	atomic.AddInt32(&downloader.readers, 1)
	rc.Unlock()

	buf := make([]byte, 4<<10)
	if n, err := rc.ReadChunkAt(context.Background(), buf, "chunk", nil, false, 0, 4<<10, false); err != nil || n != len(buf) {
		t.Fatalf("full read: n=%d err=%v", n, err)
	}
	rc.Lock()
	_, retained := rc.downloaders["chunk"]
	rc.Unlock()
	if !retained {
		t.Fatal("consumed chunk dropped while a reader was still attached")
	}

	// The attached reader detaches without reading: since the buffer was
	// already consumed, the last detach drops it.
	downloader.wg.Done()
	atomic.AddInt32(&downloader.readers, -1)
	rc.removeConsumed(downloader)
	rc.Lock()
	_, retained = rc.downloaders["chunk"]
	rc.Unlock()
	if retained {
		t.Fatal("consumed chunk retained after last reader detached")
	}
	if got := atomic.LoadInt32(&fetchCount); got != 1 {
		t.Fatalf("attached readers fetched more than once, fetchCount=%d", got)
	}
}

func TestReaderCacheConsumedChunkReleasesBudget(t *testing.T) {
	budget := NewReaderCacheBudget(4 << 10)
	rc := NewReaderCache(10, newMockChunkCacheForReaderCache(), func(context.Context, string) ([]string, error) {
		return []string{"unused"}, nil
	}, nil, budget)
	defer rc.destroy()
	rc.fetchChunkDataFn = func(_ context.Context, buffer []byte, _ []string, _ []byte, _ bool, _ bool, _ int64, _ string, _ util_http.RefreshUrlsFunc) (int, error) {
		return len(buffer), nil
	}
	if _, err := rc.ReadChunkAt(context.Background(), make([]byte, 4<<10), "chunk", nil, false, 0, 4<<10, false); err != nil {
		t.Fatal(err)
	}
	budget.Lock()
	used := budget.used
	budget.Unlock()
	if used != 0 {
		t.Fatalf("consumed chunk still reserves %d bytes", used)
	}
}

// TestReaderCacheReReadAfterEviction verifies that a chunk evicted by budget
// pressure is transparently re-downloaded on the next read and returns the
// correct data. This is the core correctness property of eviction: a reader
// must never observe missing or stale data after a chunk has been evicted.
func TestReaderCacheReReadAfterEviction(t *testing.T) {
	budget := NewReaderCacheBudget(4 << 10) // fits exactly one 4 KiB pooled chunk
	rc := NewReaderCache(256, newMockChunkCacheForReaderCache(), func(context.Context, string) ([]string, error) {
		return []string{"unused"}, nil
	}, nil, budget)
	defer rc.destroy()

	var fetchCount int32
	rc.fetchChunkDataFn = func(_ context.Context, buffer []byte, _ []string, _ []byte, _ bool, _ bool, _ int64, _ string, _ util_http.RefreshUrlsFunc) (int, error) {
		n := atomic.AddInt32(&fetchCount, 1)
		buffer[0] = byte(n) // each download writes a distinct value
		return len(buffer), nil
	}

	// Read chunk "a": triggers download #1, fills the budget.
	buf := make([]byte, 1)
	if n, err := rc.ReadChunkAt(context.Background(), buf, "a", nil, false, 0, 4<<10, false); err != nil || n != 1 || buf[0] != 1 {
		t.Fatalf("first read of 'a': n=%d data=%d err=%v", n, buf[0], err)
	}

	// Read chunk "b": budget only fits one chunk, so "a" is evicted to make room.
	if n, err := rc.ReadChunkAt(context.Background(), buf, "b", nil, false, 0, 4<<10, false); err != nil || n != 1 || buf[0] != 2 {
		t.Fatalf("read of 'b': n=%d data=%d err=%v", n, buf[0], err)
	}

	// "a" should no longer be in the cache.
	rc.Lock()
	_, stillCached := rc.downloaders["a"]
	rc.Unlock()
	if stillCached {
		t.Fatal("chunk 'a' was not evicted by budget pressure")
	}

	// Re-read "a": must trigger download #3 and return the fresh value.
	if n, err := rc.ReadChunkAt(context.Background(), buf, "a", nil, false, 0, 4<<10, false); err != nil || n != 1 || buf[0] != 3 {
		t.Fatalf("re-read of 'a': n=%d data=%d err=%v (expected re-download with value 3)", n, buf[0], err)
	}

	if got := atomic.LoadInt32(&fetchCount); got != 3 {
		t.Fatalf("fetchCount=%d, want 3 (a, b, a-re-read)", got)
	}
}

// A download still waiting for budget is cancelled when its last stream leaves.
func TestReaderCacheBudgetWaitStopsWithItsLastStream(t *testing.T) {
	budget := NewReaderCacheBudget(4 << 10)
	var lookups sync.Map
	rc := NewReaderCache(256, newMockChunkCacheForReaderCache(), func(_ context.Context, fileId string) ([]string, error) {
		lookups.Store(fileId, true)
		return []string{"unused"}, nil
	}, nil, budget)
	defer rc.destroy()

	holding := make(chan struct{})
	var holdingOnce sync.Once
	finish := make(chan struct{})
	rc.fetchChunkDataFn = func(ctx context.Context, buffer []byte, _ []string, _ []byte, _ bool, _ bool, _ int64, _ string, _ util_http.RefreshUrlsFunc) (int, error) {
		holdingOnce.Do(func() { close(holding) })
		select {
		case <-ctx.Done():
			return 0, ctx.Err()
		case <-finish:
			return len(buffer), nil
		}
	}
	defer close(finish)

	// "a" takes the whole budget and its download does not finish
	go rc.ReadChunkAt(context.Background(), make([]byte, 1), "a", nil, false, 0, 3<<10, false)
	<-holding

	var stream chunkStream
	ctx, cancel := context.WithCancel(context.Background())
	readDone := make(chan error, 1)
	go func() {
		_, err := rc.readChunkAt(ctx, &stream, make([]byte, 1), "b", nil, true, 0, 3<<10, false)
		readDone <- err
	}()
	time.Sleep(50 * time.Millisecond)
	if _, looked := lookups.Load("b"); looked {
		t.Fatal(`"b" did not wait for budget`)
	}

	cancel()
	if err := <-readDone; err != context.Canceled {
		t.Fatalf("cancelled read: got %v, want context.Canceled", err)
	}

	released := make(chan struct{})
	go func() {
		rc.releaseStream(&stream)
		close(released)
	}()
	select {
	case <-released:
	case <-time.After(2 * time.Second):
		t.Fatal("releasing the stream waited for budget nobody needs")
	}
}