SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package log_buffer
import (
"sync"
"sync/atomic"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)
// TestConcurrentProducerConsumer simulates the integration test scenario:
// - One producer writing messages continuously
// - Multiple consumers reading from different offsets
// - Consumers reading sequentially (like Kafka consumers)
func TestConcurrentProducerConsumer(t *testing.T) {
lb := NewLogBuffer("integration-test", time.Hour, nil, nil, func() {})
lb.hasOffsets = true
const numMessages = 1000
const numConsumers = 2
const messagesPerConsumer = numMessages / numConsumers
// Start producer
producerDone := make(chan bool)
go func() {
for i := 0; i < numMessages; i++ {
entry := &filer_pb.LogEntry{
TsNs: time.Now().UnixNano(),
Key: []byte("key"),
Data: []byte("value"),
Offset: int64(i),
}
if err := lb.AddLogEntryToBuffer(entry); err != nil {
t.Errorf("Failed to add log entry: %v", err)
return
}
time.Sleep(1 * time.Millisecond) // Simulate production rate
}
producerDone <- true
}()
// Start consumers
consumerWg := sync.WaitGroup{}
consumerErrors := make(chan error, numConsumers)
consumedCounts := make([]int64, numConsumers)
for consumerID := 0; consumerID < numConsumers; consumerID++ {
consumerWg.Add(1)
go func(id int, startOffset int64, endOffset int64) {
defer consumerWg.Done()
currentOffset := startOffset
for currentOffset < endOffset {
// Read 10 messages at a time (like integration test)
messages, nextOffset, _, _, err := lb.ReadMessagesAtOffset(currentOffset, 10, 10240)
if err != nil {
consumerErrors <- err
return
}
if len(messages) == 0 {
// No data yet, wait a bit
time.Sleep(5 * time.Millisecond)
continue
}
// Count only messages in this consumer's assigned range
messagesInRange := 0
for i, msg := range messages {
if msg.Offset >= startOffset && msg.Offset < endOffset {
messagesInRange++
expectedOffset := currentOffset + int64(i)
if msg.Offset != expectedOffset {
t.Errorf("Consumer %d: Expected offset %d, got %d", id, expectedOffset, msg.Offset)
}
}
}
atomic.AddInt64(&consumedCounts[id], int64(messagesInRange))
currentOffset = nextOffset
}
}(consumerID, int64(consumerID*messagesPerConsumer), int64((consumerID+1)*messagesPerConsumer))
}
// Wait for producer to finish
<-producerDone
// Wait for consumers (with timeout)
done := make(chan bool)
go func() {
consumerWg.Wait()
done <- true
}()
select {
case <-done:
// Success
case err := <-consumerErrors:
t.Fatalf("Consumer error: %v", err)
case <-time.After(10 * time.Second):
t.Fatal("Timeout waiting for consumers to finish")
}
// Verify all messages were consumed
totalConsumed := int64(0)
for i, count := range consumedCounts {
t.Logf("Consumer %d consumed %d messages", i, count)
totalConsumed += count
}
if totalConsumed != numMessages {
t.Errorf("Expected to consume %d messages, but consumed %d", numMessages, totalConsumed)
}
}
// TestBackwardSeeksWhileProducing simulates consumer rebalancing where
// consumers seek backward to earlier offsets while producer is still writing
func TestBackwardSeeksWhileProducing(t *testing.T) {
lb := NewLogBuffer("backward-seek-test", time.Hour, nil, nil, func() {})
lb.hasOffsets = true
const numMessages = 500
const numSeeks = 10
// Start producer
producerDone := make(chan bool)
go func() {
for i := 0; i < numMessages; i++ {
entry := &filer_pb.LogEntry{
TsNs: time.Now().UnixNano(),
Key: []byte("key"),
Data: []byte("value"),
Offset: int64(i),
}
if err := lb.AddLogEntryToBuffer(entry); err != nil {
t.Errorf("Failed to add log entry: %v", err)
return
}
time.Sleep(1 * time.Millisecond)
}
producerDone <- true
}()
// Consumer that seeks backward periodically
consumerDone := make(chan bool)
readOffsets := make(map[int64]int) // Track how many times each offset was read
go func() {
currentOffset := int64(0)
seeksRemaining := numSeeks
for currentOffset < numMessages {
// Read some messages
messages, nextOffset, _, endOfPartition, err := lb.ReadMessagesAtOffset(currentOffset, 10, 10240)
if err != nil {
// For stateless reads, "offset out of range" means data not in memory yet
// This is expected when reading historical data or before production starts
time.Sleep(5 * time.Millisecond)
continue
}
if len(messages) == 0 {
// No data available yet or caught up to producer
if !endOfPartition {
// Data might be coming, wait
time.Sleep(5 * time.Millisecond)
} else {
// At end of partition, wait for more production
time.Sleep(5 * time.Millisecond)
}
continue
}
// Track read offsets
for _, msg := range messages {
readOffsets[msg.Offset]++
}
// Periodically seek backward (simulating rebalancing)
if seeksRemaining > 0 && nextOffset > 50 && nextOffset%100 == 0 {
seekOffset := nextOffset - 20
t.Logf("Seeking backward from %d to %d", nextOffset, seekOffset)
currentOffset = seekOffset
seeksRemaining--
} else {
currentOffset = nextOffset
}
}
consumerDone <- true
}()
// Wait for both
<-producerDone
<-consumerDone
// Verify each offset was read at least once
for i := int64(0); i < numMessages; i++ {
if readOffsets[i] == 0 {
t.Errorf("Offset %d was never read", i)
}
}
t.Logf("Total unique offsets read: %d out of %d", len(readOffsets), numMessages)
}
// TestHighConcurrencyReads simulates multiple consumers reading from
// different offsets simultaneously (stress test)
func TestHighConcurrencyReads(t *testing.T) {
lb := NewLogBuffer("high-concurrency-test", time.Hour, nil, nil, func() {})
lb.hasOffsets = true
const numMessages = 1000
const numReaders = 10
// Pre-populate buffer
for i := 0; i < numMessages; i++ {
entry := &filer_pb.LogEntry{
TsNs: time.Now().UnixNano(),
Key: []byte("key"),
Data: []byte("value"),
Offset: int64(i),
}
if err := lb.AddLogEntryToBuffer(entry); err != nil {
t.Fatalf("Failed to add log entry: %v", err)
}
}
// Start many concurrent readers at different offsets
wg := sync.WaitGroup{}
errors := make(chan error, numReaders)
for reader := 0; reader < numReaders; reader++ {
wg.Add(1)
go func(startOffset int64) {
defer wg.Done()
// Read 100 messages from this offset
currentOffset := startOffset
readCount := 0
for readCount < 100 && currentOffset < numMessages {
messages, nextOffset, _, _, err := lb.ReadMessagesAtOffset(currentOffset, 10, 10240)
if err != nil {
errors <- err
return
}
// Verify offsets are sequential
for i, msg := range messages {
expected := currentOffset + int64(i)
if msg.Offset != expected {
t.Errorf("Reader at %d: expected offset %d, got %d", startOffset, expected, msg.Offset)
}
}
readCount += len(messages)
currentOffset = nextOffset
}
}(int64(reader * 10))
}
// Wait with timeout
done := make(chan bool)
go func() {
wg.Wait()
done <- true
}()
select {
case <-done:
// Success
case err := <-errors:
t.Fatalf("Reader error: %v", err)
case <-time.After(10 * time.Second):
t.Fatal("Timeout waiting for readers")
}
}
// TestRepeatedReadsAtSameOffset simulates what happens when Kafka
// consumer re-fetches the same offset multiple times (due to timeouts or retries)
func TestRepeatedReadsAtSameOffset(t *testing.T) {
lb := NewLogBuffer("repeated-reads-test", time.Hour, nil, nil, func() {})
lb.hasOffsets = true
const numMessages = 100
// Pre-populate buffer
for i := 0; i < numMessages; i++ {
entry := &filer_pb.LogEntry{
TsNs: time.Now().UnixNano(),
Key: []byte("key"),
Data: []byte("value"),
Offset: int64(i),
}
if err := lb.AddLogEntryToBuffer(entry); err != nil {
t.Fatalf("Failed to add log entry: %v", err)
}
}
// Read the same offset multiple times concurrently
const numReads = 10
const testOffset = int64(50)
wg := sync.WaitGroup{}
results := make([][]*filer_pb.LogEntry, numReads)
for i := 0; i < numReads; i++ {
wg.Add(1)
go func(idx int) {
defer wg.Done()
messages, _, _, _, err := lb.ReadMessagesAtOffset(testOffset, 10, 10240)
if err != nil {
t.Errorf("Read %d error: %v", idx, err)
return
}
results[idx] = messages
}(i)
}
wg.Wait()
// Verify all reads returned the same data
firstRead := results[0]
for i := 1; i < numReads; i++ {
if len(results[i]) != len(firstRead) {
t.Errorf("Read %d returned %d messages, expected %d", i, len(results[i]), len(firstRead))
}
for j := range results[i] {
if results[i][j].Offset != firstRead[j].Offset {
t.Errorf("Read %d message %d has offset %d, expected %d",
i, j, results[i][j].Offset, firstRead[j].Offset)
}
}
}
}
// TestEmptyPartitionPolling simulates consumers polling empty partitions
// waiting for data (common in Kafka)
func TestEmptyPartitionPolling(t *testing.T) {
lb := NewLogBuffer("empty-partition-test", time.Hour, nil, nil, func() {})
lb.hasOffsets = true
lb.bufferStartOffset = 0
lb.offset = 0
// Try to read from empty partition
messages, nextOffset, _, endOfPartition, err := lb.ReadMessagesAtOffset(0, 10, 10240)
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
if len(messages) != 0 {
t.Errorf("Expected 0 messages, got %d", len(messages))
}
if nextOffset != 0 {
t.Errorf("Expected nextOffset=0, got %d", nextOffset)
}
if !endOfPartition {
t.Error("Expected endOfPartition=true for future offset")
}
}