Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

13.6 KB
Raw Blame
package filer

import (
	"bytes"
	"fmt"
	"io"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"google.golang.org/protobuf/proto"

	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/util"
	"github.com/seaweedfs/seaweedfs/weed/wdclient"
)

func logEntriesAt(tsNs ...int64) []*filer_pb.LogEntry {
	out := make([]*filer_pb.LogEntry, 0, len(tsNs))
	for _, ts := range tsNs {
		out = append(out, &filer_pb.LogEntry{TsNs: ts, Data: []byte("x")})
	}
	return out
}

func encodeLogRecords(t *testing.T, entries []*filer_pb.LogEntry) []byte {
	t.Helper()
	var buf bytes.Buffer
	sizeBuf := make([]byte, 4)
	for _, e := range entries {
		data, err := proto.Marshal(e)
		if err != nil {
			t.Fatal(err)
		}
		util.Uint32toBytes(sizeBuf, uint32(len(data)))
		buf.Write(sizeBuf)
		buf.Write(data)
	}
	return buf.Bytes()
}

func TestPersistedLogCacheHitMiss(t *testing.T) {
	c := newPersistedLogCache(persistedLogCacheMaxBytes)
	var loads int32
	load := func() ([]*filer_pb.LogEntry, bool, error) {
		atomic.AddInt32(&loads, 1)
		return logEntriesAt(1, 2, 3), true, nil
	}

	e1, err := c.getOrLoad("3,01", 1, load)
	if err != nil || len(e1) != 3 {
		t.Fatalf("first getOrLoad: err=%v len=%d", err, len(e1))
	}
	e2, err := c.getOrLoad("3,01", 1, load)
	if err != nil {
		t.Fatal(err)
	}
	if n := atomic.LoadInt32(&loads); n != 1 {
		t.Fatalf("expected loader called once, got %d", n)
	}
	if &e1[0] != &e2[0] {
		t.Fatal("cache should return the same shared slice")
	}
}

func TestPersistedLogCacheNotCachedWhenUncacheable(t *testing.T) {
	c := newPersistedLogCache(persistedLogCacheMaxBytes)
	var loads int32
	// cacheable=false models a chunk-not-found stop: deliver the prefix this
	// round but never pin it, so a transient outage is re-probed next replay.
	load := func() ([]*filer_pb.LogEntry, bool, error) {
		atomic.AddInt32(&loads, 1)
		return logEntriesAt(1, 2), false, nil
	}

	if e, err := c.getOrLoad("3,01", 1, load); err != nil || len(e) != 2 {
		t.Fatalf("first: err=%v len=%d", err, len(e))
	}
	if e, err := c.getOrLoad("3,01", 1, load); err != nil || len(e) != 2 {
		t.Fatalf("second: err=%v len=%d", err, len(e))
	}
	if n := atomic.LoadInt32(&loads); n != 2 {
		t.Fatalf("uncacheable result must re-load every time, got %d loads", n)
	}
}

func TestPersistedLogCacheSingleFlight(t *testing.T) {
	c := newPersistedLogCache(persistedLogCacheMaxBytes)
	var loads int32
	release := make(chan struct{})
	started := make(chan struct{})
	load := func() ([]*filer_pb.LogEntry, bool, error) {
		if atomic.AddInt32(&loads, 1) == 1 {
			close(started)
		}
		<-release // hold the flight open so concurrent callers coalesce
		return logEntriesAt(1), true, nil
	}

	const n = 20
	var wg sync.WaitGroup
	for i := 0; i < n; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			if _, err := c.getOrLoad("3,01", 1, load); err != nil {
				t.Error(err)
			}
		}()
	}
	<-started // the flight is provably open before anyone is released
	time.Sleep(10 * time.Millisecond)
	close(release)
	wg.Wait()

	if got := atomic.LoadInt32(&loads); got != 1 {
		t.Fatalf("expected 1 coalesced load across %d callers, got %d", n, got)
	}
}

func TestPersistedLogCacheEviction(t *testing.T) {
	// One entry estimates to 128 + 100 + 16 = 244 bytes; budget holds one, not two.
	c := newPersistedLogCache(300)
	mk := func(ts int64) func() ([]*filer_pb.LogEntry, bool, error) {
		return func() ([]*filer_pb.LogEntry, bool, error) {
			return []*filer_pb.LogEntry{{TsNs: ts, Data: make([]byte, 100)}}, true, nil
		}
	}

	if _, err := c.getOrLoad("a", 1, mk(1)); err != nil {
		t.Fatal(err)
	}
	if _, err := c.getOrLoad("b", 1, mk(2)); err != nil { // pushes over budget, evicts LRU "a"
		t.Fatal(err)
	}

	c.mu.Lock()
	_, hasA := c.index["a"]
	_, hasB := c.index["b"]
	c.mu.Unlock()
	if hasA {
		t.Error("least-recently-used entry a should have been evicted")
	}
	if !hasB {
		t.Error("most-recently-used entry b should remain")
	}
}

func TestDecodeLogRecords(t *testing.T) {
	want := logEntriesAt(10, 20, 30)
	entries, cacheable, err := decodeLogRecords(encodeLogRecords(t, want))
	if err != nil || !cacheable {
		t.Fatalf("clean chunk: err=%v cacheable=%v", err, cacheable)
	}
	if len(entries) != 3 || entries[0].TsNs != 10 || entries[2].TsNs != 30 {
		t.Fatalf("decoded %v", entries)
	}

	if entries, cacheable, err := decodeLogRecords(nil); err != nil || !cacheable || len(entries) != 0 {
		t.Fatalf("empty chunk: entries=%v cacheable=%v err=%v", entries, cacheable, err)
	}
}

func TestDecodeLogRecordsIncomplete(t *testing.T) {
	full := encodeLogRecords(t, logEntriesAt(10, 20))
	rec1 := len(encodeLogRecords(t, logEntriesAt(10)))

	// stops mid-record: the second record's bytes are cut short
	entries, cacheable, err := decodeLogRecords(full[:len(full)-2])
	if err != errLogChunkIncomplete || cacheable {
		t.Fatalf("mid-record: err=%v cacheable=%v", err, cacheable)
	}
	if len(entries) != 1 || entries[0].TsNs != 10 {
		t.Fatalf("mid-record prefix: %v", entries)
	}

	// stops mid-size-prefix
	if _, _, err := decodeLogRecords(full[:rec1+2]); err != errLogChunkIncomplete {
		t.Fatalf("mid-prefix: err=%v", err)
	}

	// garbage size prefix, e.g. a chunk that starts mid-record
	garbage := make([]byte, 8)
	util.Uint32toBytes(garbage, uint32(maxLogEntrySize)+1)
	if _, _, err := decodeLogRecords(garbage); err != errLogChunkIncomplete {
		t.Fatalf("garbage size: err=%v", err)
	}
}

func logFileEntry(chunks ...*filer_pb.FileChunk) *Entry {
	return &Entry{
		FullPath: "/topics/.system/log/2026-06-10/00-01.f1",
		Attr:     Attr{Mode: 0644},
		Chunks:   chunks,
	}
}

func stubChunkLoader(t *testing.T, chunkData map[string][]byte) *int32 {
	t.Helper()
	var loads int32
	prev := loadLogFileEntriesFn
	loadLogFileEntriesFn = func(_ *wdclient.MasterClient, chunk *filer_pb.FileChunk) ([]*filer_pb.LogEntry, bool, error) {
		atomic.AddInt32(&loads, 1)
		data, ok := chunkData[chunk.GetFileIdString()]
		if !ok {
			t.Errorf("unexpected load of chunk %s", chunk.GetFileIdString())
			return nil, false, fmt.Errorf("unexpected chunk %s", chunk.GetFileIdString())
		}
		return decodeLogRecords(data)
	}
	t.Cleanup(func() { loadLogFileEntriesFn = prev })
	return &loads
}

func collectTs(t *testing.T, iter *LogFileIterator) (got []int64) {
	t.Helper()
	for {
		e, err := iter.getNext()
		if err == io.EOF {
			return
		}
		if err != nil {
			t.Fatal(err)
		}
		got = append(got, e.TsNs)
	}
}

func TestLogFileIteratorChunkedFiltering(t *testing.T) {
	stubChunkLoader(t, map[string][]byte{
		"3,01": encodeLogRecords(t, logEntriesAt(10, 20)),
		"3,02": encodeLogRecords(t, logEntriesAt(30, 40)),
	})
	entry := logFileEntry(
		&filer_pb.FileChunk{FileId: "3,01", Offset: 0, Size: 100, ModifiedTsNs: 25},
		&filer_pb.FileChunk{FileId: "3,02", Offset: 100, Size: 100, ModifiedTsNs: 45},
	)

	iter := newLogFileIterator(nil, newPersistedLogCache(persistedLogCacheMaxBytes), entry, 15, 35)
	got := collectTs(t, iter)
	want := []int64{20, 30} // 10 filtered by startTsNs, 40 cut off by stopTsNs
	if fmt.Sprint(got) != fmt.Sprint(want) {
		t.Fatalf("chunked filtering: got %v, want %v", got, want)
	}
}

func TestLogFileIteratorSkipsColdChunks(t *testing.T) {
	// chunk "3,01" is not in the stub map: loading it would fail the test. Its
	// flush time (ModifiedTsNs) is a full flush interval before startTsNs, so it
	// must be skipped without being read.
	start := int64(20) + int64(LogFlushInterval)
	stubChunkLoader(t, map[string][]byte{
		"3,02": encodeLogRecords(t, logEntriesAt(start+10, start+20)),
	})
	entry := logFileEntry(
		&filer_pb.FileChunk{FileId: "3,01", Offset: 0, Size: 100, ModifiedTsNs: 20},
		&filer_pb.FileChunk{FileId: "3,02", Offset: 100, Size: 100, ModifiedTsNs: start + 25},
	)

	iter := newLogFileIterator(nil, newPersistedLogCache(persistedLogCacheMaxBytes), entry, start, 0)
	got := collectTs(t, iter)
	if fmt.Sprint(got) != fmt.Sprint([]int64{start + 10, start + 20}) {
		t.Fatalf("cold-chunk skip: got %v", got)
	}
}

func TestLogFileIteratorSharesDecodedChunks(t *testing.T) {
	loads := stubChunkLoader(t, map[string][]byte{
		"3,01": encodeLogRecords(t, logEntriesAt(10, 20)),
	})
	entry := logFileEntry(&filer_pb.FileChunk{FileId: "3,01", Offset: 0, Size: 100, ModifiedTsNs: 25})
	cache := newPersistedLogCache(persistedLogCacheMaxBytes)

	for i := 0; i < 3; i++ { // three replays of the same chunk
		iter := newLogFileIterator(nil, cache, entry, 0, 0)
		if got := collectTs(t, iter); fmt.Sprint(got) != fmt.Sprint([]int64{10, 20}) {
			t.Fatalf("replay %d: got %v", i, got)
		}
	}
	if n := atomic.LoadInt32(loads); n != 1 {
		t.Fatalf("expected one shared decode across replays, got %d", n)
	}
}

func TestLogFileIteratorStreamsWhenRecordsSpanChunks(t *testing.T) {
	// One file of four records, flushed in a way that splits the third record
	// across two chunks. The first chunk decodes records 10 and 20 cleanly; the
	// second starts mid-record. The iterator must yield every record exactly
	// once by falling back to a whole-file byte stream.
	all := logEntriesAt(10, 20, 30, 40)
	full := encodeLogRecords(t, all)
	cut := len(encodeLogRecords(t, all[:2])) + 3 // 3 bytes into record 30

	stubChunkLoader(t, map[string][]byte{
		"3,01": full[:cut],
		"3,02": full[cut:],
	})
	prev := newLogFileStreamReader
	newLogFileStreamReader = func(_ *wdclient.MasterClient, _ []*filer_pb.FileChunk) io.Reader {
		return bytes.NewReader(full)
	}
	t.Cleanup(func() { newLogFileStreamReader = prev })

	entry := logFileEntry(
		&filer_pb.FileChunk{FileId: "3,01", Offset: 0, Size: uint64(cut), ModifiedTsNs: 45},
		&filer_pb.FileChunk{FileId: "3,02", Offset: int64(cut), Size: uint64(len(full) - cut), ModifiedTsNs: 45},
	)

	iter := newLogFileIterator(nil, newPersistedLogCache(persistedLogCacheMaxBytes), entry, 0, 0)
	got := collectTs(t, iter)
	if fmt.Sprint(got) != fmt.Sprint([]int64{10, 20, 30, 40}) {
		t.Fatalf("spanning fallback: got %v", got)
	}
}

func TestLogFileIteratorStreamFallbackResumesAfterYielded(t *testing.T) {
	// The first chunk decodes standalone and its records are yielded before the
	// second chunk turns out to be incomplete. The stream fallback re-reads the
	// file from the start, so the already-yielded records must be skipped.
	all := logEntriesAt(10, 20, 30, 40)
	full := encodeLogRecords(t, all)
	chunk1 := len(encodeLogRecords(t, all[:2]))

	stubChunkLoader(t, map[string][]byte{
		"3,01": full[:chunk1],
		"3,02": full[chunk1 : len(full)-2], // truncated mid-record
	})
	prev := newLogFileStreamReader
	newLogFileStreamReader = func(_ *wdclient.MasterClient, _ []*filer_pb.FileChunk) io.Reader {
		return bytes.NewReader(full)
	}
	t.Cleanup(func() { newLogFileStreamReader = prev })

	entry := logFileEntry(
		&filer_pb.FileChunk{FileId: "3,01", Offset: 0, Size: uint64(chunk1), ModifiedTsNs: 45},
		&filer_pb.FileChunk{FileId: "3,02", Offset: int64(chunk1), Size: uint64(len(full) - chunk1), ModifiedTsNs: 45},
	)

	iter := newLogFileIterator(nil, newPersistedLogCache(persistedLogCacheMaxBytes), entry, 0, 0)
	got := collectTs(t, iter)
	if fmt.Sprint(got) != fmt.Sprint([]int64{10, 20, 30, 40}) {
		t.Fatalf("fallback resume: got %v", got)
	}
}

func TestLogFileIteratorLoadErrorPropagates(t *testing.T) {
	boom := fmt.Errorf("boom")
	prev := loadLogFileEntriesFn
	loadLogFileEntriesFn = func(_ *wdclient.MasterClient, _ *filer_pb.FileChunk) ([]*filer_pb.LogEntry, bool, error) {
		return nil, false, boom
	}
	t.Cleanup(func() { loadLogFileEntriesFn = prev })

	entry := logFileEntry(&filer_pb.FileChunk{FileId: "3,01", Offset: 0, Size: 100, ModifiedTsNs: 45})
	iter := newLogFileIterator(nil, newPersistedLogCache(persistedLogCacheMaxBytes), entry, 0, 0)
	if _, err := iter.getNext(); err == nil || err.Error() != "boom" {
		t.Fatalf("expected load error surfaced, got %v", err)
	}
}

func TestPersistedLogCacheIdleEviction(t *testing.T) {
	c := newPersistedLogCache(persistedLogCacheMaxBytes)
	load := func() ([]*filer_pb.LogEntry, bool, error) {
		return logEntriesAt(1), true, nil
	}
	if _, err := c.getOrLoad("idle", 1, load); err != nil {
		t.Fatal(err)
	}
	if _, err := c.getOrLoad("hot", 1, load); err != nil {
		t.Fatal(err)
	}

	c.mu.Lock()
	c.index["idle"].Value.(*logCacheItem).lastUsed = time.Now().Add(-2 * persistedLogCacheIdleTTL)
	c.mu.Unlock()

	c.evictIdle(time.Now().Add(-persistedLogCacheIdleTTL))

	c.mu.Lock()
	_, hasIdle := c.index["idle"]
	_, hasHot := c.index["hot"]
	bytesLeft := c.curBytes
	c.mu.Unlock()
	if hasIdle {
		t.Error("idle entry should have been evicted")
	}
	if !hasHot {
		t.Error("recently used entry should remain")
	}
	if want := estimateEntriesBytes(logEntriesAt(1)); bytesLeft != want {
		t.Errorf("curBytes=%d, want %d", bytesLeft, want)
	}
}

func TestDecodeLogRecordsRejectsImplausibleRecords(t *testing.T) {
	// zeroed region: parses as endless empty records without the size guard
	if _, _, err := decodeLogRecords(make([]byte, 16)); err != errLogChunkIncomplete {
		t.Fatalf("zeroed data: err=%v", err)
	}

	// the writer never produces a zero timestamp
	if _, _, err := decodeLogRecords(encodeLogRecords(t, logEntriesAt(0))); err != errLogChunkIncomplete {
		t.Fatalf("zero ts: err=%v", err)
	}

	// timestamps are strictly increasing within one flushed buffer
	entries, cacheable, err := decodeLogRecords(encodeLogRecords(t, logEntriesAt(10, 10)))
	if err != errLogChunkIncomplete || cacheable {
		t.Fatalf("non-increasing ts: err=%v cacheable=%v", err, cacheable)
	}
	if len(entries) != 1 || entries[0].TsNs != 10 {
		t.Fatalf("non-increasing prefix: %v", entries)
	}
}

func TestPersistedLogCacheLoadLargerThanBudget(t *testing.T) {
	// a load weight above the semaphore size could never be acquired; it must
	// clamp and still run
	c := newPersistedLogCache(persistedLogCacheMaxBytes)
	e, err := c.getOrLoad("3,01", persistedLogCacheLoadBudget*4, func() ([]*filer_pb.LogEntry, bool, error) {
		return logEntriesAt(1), true, nil
	})
	if err != nil || len(e) != 1 {
		t.Fatalf("oversized load: err=%v len=%d", err, len(e))
	}
}