SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package fuse_test
import (
"bytes"
"crypto/rand"
"fmt"
"os"
"path/filepath"
"sync"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestConcurrentFileOperations tests concurrent file operations
func TestConcurrentFileOperations(t *testing.T) {
framework := NewFuseTestFramework(t, DefaultTestConfig())
defer framework.Cleanup()
require.NoError(t, framework.Setup(DefaultTestConfig()))
t.Run("ConcurrentFileWrites", func(t *testing.T) {
testConcurrentFileWrites(t, framework)
})
t.Run("ConcurrentFileReads", func(t *testing.T) {
testConcurrentFileReads(t, framework)
})
t.Run("ConcurrentReadWrite", func(t *testing.T) {
testConcurrentReadWrite(t, framework)
})
t.Run("ConcurrentDirectoryOperations", func(t *testing.T) {
testConcurrentDirectoryOperations(t, framework)
})
t.Run("ConcurrentFileCreation", func(t *testing.T) {
testConcurrentFileCreation(t, framework)
})
}
// testConcurrentFileWrites tests multiple goroutines writing to different files
func testConcurrentFileWrites(t *testing.T, framework *FuseTestFramework) {
numWorkers := 10
filesPerWorker := 5
var wg sync.WaitGroup
var mutex sync.Mutex
errors := make([]error, 0)
// Function to collect errors safely
addError := func(err error) {
mutex.Lock()
defer mutex.Unlock()
errors = append(errors, err)
}
// Start concurrent workers
for worker := 0; worker < numWorkers; worker++ {
wg.Add(1)
go func(workerID int) {
defer wg.Done()
for file := 0; file < filesPerWorker; file++ {
filename := fmt.Sprintf("worker_%d_file_%d.txt", workerID, file)
content := []byte(fmt.Sprintf("Worker %d, File %d - %s", workerID, file, time.Now().String()))
mountPath := filepath.Join(framework.GetMountPoint(), filename)
if err := os.WriteFile(mountPath, content, 0644); err != nil {
addError(fmt.Errorf("worker %d file %d: %v", workerID, file, err))
return
}
// Verify file was written correctly
readContent, err := os.ReadFile(mountPath)
if err != nil {
addError(fmt.Errorf("worker %d file %d read: %v", workerID, file, err))
return
}
if !bytes.Equal(content, readContent) {
addError(fmt.Errorf("worker %d file %d: content mismatch", workerID, file))
return
}
}
}(worker)
}
wg.Wait()
// Check for errors
require.Empty(t, errors, "Concurrent writes failed: %v", errors)
// Verify all files exist and have correct content
for worker := 0; worker < numWorkers; worker++ {
for file := 0; file < filesPerWorker; file++ {
filename := fmt.Sprintf("worker_%d_file_%d.txt", worker, file)
framework.AssertFileExists(filename)
}
}
}
// testConcurrentFileReads tests multiple goroutines reading from the same file
func testConcurrentFileReads(t *testing.T, framework *FuseTestFramework) {
// Create a test file
filename := "concurrent_read_test.txt"
testData := make([]byte, 1024*1024) // 1MB
_, err := rand.Read(testData)
require.NoError(t, err)
framework.CreateTestFile(filename, testData)
numReaders := 20
var wg sync.WaitGroup
var mutex sync.Mutex
errors := make([]error, 0)
addError := func(err error) {
mutex.Lock()
defer mutex.Unlock()
errors = append(errors, err)
}
// Start concurrent readers
for reader := 0; reader < numReaders; reader++ {
wg.Add(1)
go func(readerID int) {
defer wg.Done()
mountPath := filepath.Join(framework.GetMountPoint(), filename)
// Read multiple times
for i := 0; i < 3; i++ {
readData, err := os.ReadFile(mountPath)
if err != nil {
addError(fmt.Errorf("reader %d iteration %d: %v", readerID, i, err))
return
}
if !bytes.Equal(testData, readData) {
addError(fmt.Errorf("reader %d iteration %d: data mismatch", readerID, i))
return
}
}
}(reader)
}
wg.Wait()
require.Empty(t, errors, "Concurrent reads failed: %v", errors)
}
// testConcurrentReadWrite tests simultaneous read and write operations
func testConcurrentReadWrite(t *testing.T, framework *FuseTestFramework) {
filename := "concurrent_rw_test.txt"
initialData := bytes.Repeat([]byte("INITIAL"), 1000)
framework.CreateTestFile(filename, initialData)
var wg sync.WaitGroup
var mutex sync.Mutex
errors := make([]error, 0)
addError := func(err error) {
mutex.Lock()
defer mutex.Unlock()
errors = append(errors, err)
}
mountPath := filepath.Join(framework.GetMountPoint(), filename)
// Start readers
numReaders := 5
for i := 0; i < numReaders; i++ {
wg.Add(1)
go func(readerID int) {
defer wg.Done()
for j := 0; j < 10; j++ {
if err := retryTransientFUSE(func() error {
_, e := os.ReadFile(mountPath)
return e
}); err != nil {
addError(fmt.Errorf("reader %d: %v", readerID, err))
return
}
time.Sleep(10 * time.Millisecond)
}
}(i)
}
// Start writers
numWriters := 2
for i := 0; i < numWriters; i++ {
wg.Add(1)
go func(writerID int) {
defer wg.Done()
for j := 0; j < 5; j++ {
newData := bytes.Repeat([]byte(fmt.Sprintf("WRITER%d", writerID)), 1000)
if err := retryTransientFUSE(func() error {
return os.WriteFile(mountPath, newData, 0644)
}); err != nil {
addError(fmt.Errorf("writer %d: %v", writerID, err))
return
}
time.Sleep(50 * time.Millisecond)
}
}(i)
}
wg.Wait()
require.Empty(t, errors, "Concurrent read/write failed: %v", errors)
// Verify file still exists and is readable
framework.AssertFileExists(filename)
}
// retryTransientFUSE retries op a few times before giving up. A concurrent
// truncating overwrite can leave a short-lived dentry/cache window where the
// entry is momentarily invisible (ENOENT) to another opener; the last error is
// returned so a genuine, persistent failure still surfaces.
func retryTransientFUSE(op func() error) error {
var err error
for attempt := 0; attempt < 5; attempt++ {
if err = op(); err == nil {
return nil
}
time.Sleep(100 * time.Millisecond)
}
return err
}
// testConcurrentDirectoryOperations tests concurrent directory operations
func testConcurrentDirectoryOperations(t *testing.T, framework *FuseTestFramework) {
numWorkers := 8
var wg sync.WaitGroup
var mutex sync.Mutex
errors := make([]error, 0)
addError := func(err error) {
mutex.Lock()
defer mutex.Unlock()
errors = append(errors, err)
}
// Each worker creates a directory tree
for worker := 0; worker < numWorkers; worker++ {
wg.Add(1)
go func(workerID int) {
defer wg.Done()
// Create worker directory
workerDir := fmt.Sprintf("worker_%d", workerID)
mountPath := filepath.Join(framework.GetMountPoint(), workerDir)
if err := os.Mkdir(mountPath, 0755); err != nil {
addError(fmt.Errorf("worker %d mkdir: %v", workerID, err))
return
}
// Create subdirectories and files
for i := 0; i < 5; i++ {
subDir := filepath.Join(mountPath, fmt.Sprintf("subdir_%d", i))
if err := os.Mkdir(subDir, 0755); err != nil {
addError(fmt.Errorf("worker %d subdir %d: %v", workerID, i, err))
return
}
// Create file in subdirectory with retry for transient FUSE errors
testFile := filepath.Join(subDir, "test.txt")
content := []byte(fmt.Sprintf("Worker %d, Subdir %d", workerID, i))
var writeErr error
for attempt := 0; attempt < 3; attempt++ {
if writeErr = os.WriteFile(testFile, content, 0644); writeErr == nil {
break
}
time.Sleep(100 * time.Millisecond)
}
if writeErr != nil {
addError(fmt.Errorf("worker %d file %d: %v", workerID, i, writeErr))
return
}
}
}(worker)
}
wg.Wait()
require.Empty(t, errors, "Concurrent directory operations failed: %v", errors)
// Verify all structures were created
for worker := 0; worker < numWorkers; worker++ {
workerDir := fmt.Sprintf("worker_%d", worker)
mountPath := filepath.Join(framework.GetMountPoint(), workerDir)
info, err := os.Stat(mountPath)
require.NoError(t, err)
assert.True(t, info.IsDir())
// Check subdirectories
for i := 0; i < 5; i++ {
subDir := filepath.Join(mountPath, fmt.Sprintf("subdir_%d", i))
info, err := os.Stat(subDir)
require.NoError(t, err)
assert.True(t, info.IsDir())
testFile := filepath.Join(subDir, "test.txt")
expectedContent := []byte(fmt.Sprintf("Worker %d, Subdir %d", worker, i))
actualContent, err := os.ReadFile(testFile)
require.NoError(t, err)
assert.Equal(t, expectedContent, actualContent)
}
}
}
// testConcurrentFileCreation tests concurrent creation of files in same directory
func testConcurrentFileCreation(t *testing.T, framework *FuseTestFramework) {
// Create test directory
testDir := "concurrent_creation"
framework.CreateTestDir(testDir)
numWorkers := 15
filesPerWorker := 10
var wg sync.WaitGroup
var mutex sync.Mutex
errors := make([]error, 0)
createdFiles := make(map[string]bool)
addError := func(err error) {
mutex.Lock()
defer mutex.Unlock()
errors = append(errors, err)
}
addFile := func(filename string) {
mutex.Lock()
defer mutex.Unlock()
createdFiles[filename] = true
}
// Create files concurrently
for worker := 0; worker < numWorkers; worker++ {
wg.Add(1)
go func(workerID int) {
defer wg.Done()
for file := 0; file < filesPerWorker; file++ {
filename := fmt.Sprintf("file_%d_%d.txt", workerID, file)
relativePath := filepath.Join(testDir, filename)
mountPath := filepath.Join(framework.GetMountPoint(), relativePath)
content := []byte(fmt.Sprintf("Worker %d, File %d, Time: %s",
workerID, file, time.Now().Format(time.RFC3339Nano)))
if err := os.WriteFile(mountPath, content, 0644); err != nil {
addError(fmt.Errorf("worker %d file %d: %v", workerID, file, err))
return
}
addFile(filename)
}
}(worker)
}
wg.Wait()
require.Empty(t, errors, "Concurrent file creation failed: %v", errors)
// Verify all files were created
expectedCount := numWorkers * filesPerWorker
assert.Equal(t, expectedCount, len(createdFiles))
// Read directory and verify count
mountPath := filepath.Join(framework.GetMountPoint(), testDir)
entries, err := os.ReadDir(mountPath)
require.NoError(t, err)
assert.Equal(t, expectedCount, len(entries))
// Verify each file exists and has content
for filename := range createdFiles {
relativePath := filepath.Join(testDir, filename)
framework.AssertFileExists(relativePath)
}
}
// TestStressOperations tests high-load scenarios
func TestStressOperations(t *testing.T) {
framework := NewFuseTestFramework(t, DefaultTestConfig())
defer framework.Cleanup()
require.NoError(t, framework.Setup(DefaultTestConfig()))
t.Run("HighFrequencySmallWrites", func(t *testing.T) {
testHighFrequencySmallWrites(t, framework)
})
t.Run("ManySmallFiles", func(t *testing.T) {
testManySmallFiles(t, framework)
})
}
// testHighFrequencySmallWrites tests many small writes to the same file
func testHighFrequencySmallWrites(t *testing.T, framework *FuseTestFramework) {
filename := "high_freq_writes.txt"
mountPath := filepath.Join(framework.GetMountPoint(), filename)
// Open file for writing
file, err := os.OpenFile(mountPath, os.O_CREATE|os.O_WRONLY, 0644)
require.NoError(t, err)
defer file.Close()
// Perform many small writes
numWrites := 1000
writeSize := 100
totalSize := int64(0)
for i := 0; i < numWrites; i++ {
data := []byte(fmt.Sprintf("Write %04d: %s\n", i, bytes.Repeat([]byte("x"), writeSize-20)))
_, err := file.Write(data)
require.NoError(t, err)
totalSize += int64(len(data))
}
file.Close()
// Verify file size
info, err := os.Stat(mountPath)
require.NoError(t, err)
assert.Equal(t, totalSize, info.Size(), "file size should match total bytes written")
}
// testManySmallFiles tests creating many small files
func testManySmallFiles(t *testing.T, framework *FuseTestFramework) {
testDir := "many_small_files"
framework.CreateTestDir(testDir)
numFiles := 500
var wg sync.WaitGroup
var mutex sync.Mutex
errors := make([]error, 0)
addError := func(err error) {
mutex.Lock()
defer mutex.Unlock()
errors = append(errors, err)
}
// Create files in batches
batchSize := 50
for batch := 0; batch < numFiles/batchSize; batch++ {
wg.Add(1)
go func(batchID int) {
defer wg.Done()
for i := 0; i < batchSize; i++ {
fileNum := batchID*batchSize + i
filename := filepath.Join(testDir, fmt.Sprintf("small_file_%04d.txt", fileNum))
content := []byte(fmt.Sprintf("File %d content", fileNum))
mountPath := filepath.Join(framework.GetMountPoint(), filename)
if err := os.WriteFile(mountPath, content, 0644); err != nil {
addError(fmt.Errorf("file %d: %v", fileNum, err))
return
}
}
}(batch)
}
wg.Wait()
require.Empty(t, errors, "Many small files creation failed: %v", errors)
// Verify directory listing
mountPath := filepath.Join(framework.GetMountPoint(), testDir)
entries, err := os.ReadDir(mountPath)
require.NoError(t, err)
assert.Equal(t, numFiles, len(entries))
}