Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

2.4 KB
Raw Blame
package filer

import (
	"context"
	"fmt"
	"strconv"
	"strings"
	"sync/atomic"
	"testing"

	"github.com/seaweedfs/seaweedfs/weed/util/chunk_cache"
	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
)

// A sequential reader that consumes a chunk in several small ReadAt calls, as
// the S3 GET path does with its 256 KiB copy buffer over 4 MiB chunks, must
// keep its read-ahead at prefetchCount chunks. Each ReadAt triggers a
// prefetch; if chunks already being fetched are not counted toward the
// window, every call starts prefetchCount more downloads and the read-ahead
// runs to the ReaderCache downloader limit, buffering most of the object.
func TestChunkReadAtSequentialPrefetchStaysWithinWindow(t *testing.T) {
	const chunkSize = 64 << 10
	const chunkCount = 64
	const sliceSize = 16 << 10

	var readerChunk atomic.Int64
	var maxLead atomic.Int64
	rc := NewReaderCache(256, (*chunk_cache.TieredChunkCache)(nil), func(context.Context, string) ([]string, error) {
		return []string{"unused"}, nil
	}, nil)
	defer rc.destroy()
	rc.fetchChunkDataFn = func(_ context.Context, buffer []byte, _ []string, _ []byte, _ bool, _ bool, _ int64, fileId string, _ util_http.RefreshUrlsFunc) (int, error) {
		idx, err := strconv.Atoi(strings.TrimPrefix(fileId, "chunk"))
		if err != nil {
			return 0, err
		}
		lead := int64(idx) - readerChunk.Load()
		for {
			cur := maxLead.Load()
			if lead <= cur || maxLead.CompareAndSwap(cur, lead) {
				break
			}
		}
		return len(buffer), nil
	}

	views := NewIntervalList[*ChunkView]()
	for i := 0; i < chunkCount; i++ {
		views.AppendInterval(&Interval[*ChunkView]{
			StartOffset: int64(i * chunkSize),
			StopOffset:  int64((i + 1) * chunkSize),
			Value: &ChunkView{
				FileId:     fmt.Sprintf("chunk%d", i),
				ViewSize:   chunkSize,
				ViewOffset: int64(i * chunkSize),
				ChunkSize:  chunkSize,
			},
		})
	}
	reader := NewChunkReaderAtFromClient(context.Background(), rc, views, chunkSize*chunkCount, DefaultPrefetchCount)
	defer reader.ReleaseStream()

	buf := make([]byte, sliceSize)
	for off := int64(0); off < chunkSize*chunkCount; off += sliceSize {
		readerChunk.Store(off / chunkSize)
		if _, err := reader.ReadAt(buf, off); err != nil && off+sliceSize < chunkSize*chunkCount {
			t.Fatalf("ReadAt(%d): %v", off, err)
		}
	}

	if got := maxLead.Load(); got > DefaultPrefetchCount {
		t.Fatalf("read-ahead reached %d chunks past the reader, want at most %d", got, DefaultPrefetchCount)
	}
}