SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package remote_cache
import (
"fmt"
"testing"
"time"
"github.com/stretchr/testify/require"
)
// TestRemoteCacheBasicCommand tests caching files from remote using command
func TestRemoteCacheBasicCommand(t *testing.T) {
checkServersRunning(t)
testKey := fmt.Sprintf("cache-basic-%d.txt", time.Now().UnixNano())
testData := createTestFile(t, testKey, 1024)
// Uncache first to push to remote
t.Log("Uncaching file to remote...")
uncacheLocal(t, testKey)
// Now cache it back using remote.cache command
t.Log("Caching file from remote using command...")
cmd := fmt.Sprintf("remote.cache -dir=/buckets/%s", testBucket)
output, err := runWeedShellWithOutput(t, cmd)
require.NoError(t, err, "remote.cache command failed")
t.Logf("Cache output: %s", output)
// Verify file is still readable
verifyFileContent(t, testKey, testData)
}
// TestRemoteCacheWithInclude tests caching only matching files
func TestRemoteCacheWithInclude(t *testing.T) {
checkServersRunning(t)
// Create multiple files with different extensions
pdfFile := fmt.Sprintf("doc-%d.pdf", time.Now().UnixNano())
txtFile := fmt.Sprintf("doc-%d.txt", time.Now().UnixNano())
pdfData := createTestFile(t, pdfFile, 512)
txtData := createTestFile(t, txtFile, 512)
// Uncache both
uncacheLocal(t, "*.pdf")
uncacheLocal(t, "*.txt")
time.Sleep(500 * time.Millisecond)
// Cache only PDF files
t.Log("Caching only PDF files...")
cmd := fmt.Sprintf("remote.cache -dir=/buckets/%s -include=*.pdf", testBucket)
output, err := runWeedShellWithOutput(t, cmd)
require.NoError(t, err, "remote.cache with include failed")
t.Logf("Cache output: %s", output)
// Both files should still be readable
verifyFileContent(t, pdfFile, pdfData)
verifyFileContent(t, txtFile, txtData)
}
// TestRemoteCacheWithExclude tests caching excluding pattern
func TestRemoteCacheWithExclude(t *testing.T) {
checkServersRunning(t)
// Create test files
keepFile := fmt.Sprintf("keep-%d.txt", time.Now().UnixNano())
tmpFile := fmt.Sprintf("temp-%d.tmp", time.Now().UnixNano())
keepData := createTestFile(t, keepFile, 512)
tmpData := createTestFile(t, tmpFile, 512)
// Uncache both
uncacheLocal(t, "keep-*")
uncacheLocal(t, "temp-*")
time.Sleep(500 * time.Millisecond)
// Cache excluding .tmp files
t.Log("Caching excluding .tmp files...")
cmd := fmt.Sprintf("remote.cache -dir=/buckets/%s -exclude=*.tmp", testBucket)
output, err := runWeedShellWithOutput(t, cmd)
require.NoError(t, err, "remote.cache with exclude failed")
t.Logf("Cache output: %s", output)
// Both should still be readable
verifyFileContent(t, keepFile, keepData)
verifyFileContent(t, tmpFile, tmpData)
}
// TestRemoteCacheMinSize tests caching files larger than threshold
func TestRemoteCacheMinSize(t *testing.T) {
checkServersRunning(t)
// Create files of different sizes
smallFile := fmt.Sprintf("small-%d.bin", time.Now().UnixNano())
largeFile := fmt.Sprintf("large-%d.bin", time.Now().UnixNano())
smallData := createTestFile(t, smallFile, 100) // 100 bytes
largeData := createTestFile(t, largeFile, 10000) // 10KB
// Uncache both
uncacheLocal(t, "small-*")
uncacheLocal(t, "large-*")
time.Sleep(500 * time.Millisecond)
// Cache only files larger than 1KB
t.Log("Caching files larger than 1KB...")
cmd := fmt.Sprintf("remote.cache -dir=/buckets/%s -minSize=1024", testBucket)
output, err := runWeedShellWithOutput(t, cmd)
require.NoError(t, err, "remote.cache with minSize failed")
t.Logf("Cache output: %s", output)
// Both should still be readable
verifyFileContent(t, smallFile, smallData)
verifyFileContent(t, largeFile, largeData)
}
// TestRemoteCacheMaxSize tests caching files smaller than threshold
func TestRemoteCacheMaxSize(t *testing.T) {
checkServersRunning(t)
// Create files of different sizes
smallFile := fmt.Sprintf("tiny-%d.bin", time.Now().UnixNano())
mediumFile := fmt.Sprintf("medium-%d.bin", time.Now().UnixNano())
smallData := createTestFile(t, smallFile, 500) // 500 bytes
mediumData := createTestFile(t, mediumFile, 5000) // 5KB
// Uncache both
uncacheLocal(t, "tiny-*")
uncacheLocal(t, "medium-*")
time.Sleep(500 * time.Millisecond)
// Cache only files smaller than 2KB
t.Log("Caching files smaller than 2KB...")
cmd := fmt.Sprintf("remote.cache -dir=/buckets/%s -maxSize=2048", testBucket)
output, err := runWeedShellWithOutput(t, cmd)
require.NoError(t, err, "remote.cache with maxSize failed")
t.Logf("Cache output: %s", output)
// Both should still be readable
verifyFileContent(t, smallFile, smallData)
verifyFileContent(t, mediumFile, mediumData)
}
// TestRemoteCacheCombinedFilters tests multiple filters together
func TestRemoteCacheCombinedFilters(t *testing.T) {
checkServersRunning(t)
// Create test files
matchFile := fmt.Sprintf("data-%d.dat", time.Now().UnixNano())
noMatchFile := fmt.Sprintf("skip-%d.txt", time.Now().UnixNano())
matchData := createTestFile(t, matchFile, 2000) // 2KB .dat file
noMatchData := createTestFile(t, noMatchFile, 100) // 100 byte .txt file
// Uncache both
uncacheLocal(t, "data-*")
uncacheLocal(t, "skip-*")
time.Sleep(500 * time.Millisecond)
// Cache .dat files larger than 1KB
t.Log("Caching .dat files larger than 1KB...")
cmd := fmt.Sprintf("remote.cache -dir=/buckets/%s -include=*.dat -minSize=1024", testBucket)
output, err := runWeedShellWithOutput(t, cmd)
require.NoError(t, err, "remote.cache with combined filters failed")
t.Logf("Cache output: %s", output)
// Both should still be readable
verifyFileContent(t, matchFile, matchData)
verifyFileContent(t, noMatchFile, noMatchData)
}
// TestRemoteCacheDryRun tests preview without actual caching
func TestRemoteCacheDryRun(t *testing.T) {
checkServersRunning(t)
testKey := fmt.Sprintf("dryrun-%d.txt", time.Now().UnixNano())
createTestFile(t, testKey, 1024)
// Uncache
uncacheLocal(t, testKey)
time.Sleep(500 * time.Millisecond)
// Run cache in dry-run mode
t.Log("Running cache in dry-run mode...")
cmd := fmt.Sprintf("remote.cache -dir=/buckets/%s -dryRun=true", testBucket)
output, err := runWeedShellWithOutput(t, cmd)
require.NoError(t, err, "remote.cache dry-run failed")
t.Logf("Dry-run output: %s", output)
// File should still be readable (caching happens on-demand anyway)
getFromPrimary(t, testKey)
}
// TestRemoteUncacheBasic tests uncaching files (removing local chunks)
func TestRemoteUncacheBasic(t *testing.T) {
checkServersRunning(t)
testKey := fmt.Sprintf("uncache-basic-%d.txt", time.Now().UnixNano())
testData := createTestFile(t, testKey, 2048)
// Verify file exists
verifyFileContent(t, testKey, testData)
// Uncache it
t.Log("Uncaching file...")
cmd := fmt.Sprintf("remote.uncache -dir=/buckets/%s -include=%s", testBucket, testKey)
output, err := runWeedShellWithOutput(t, cmd)
require.NoError(t, err, "remote.uncache failed")
t.Logf("Uncache output: %s", output)
// File should still be readable (will be fetched from remote)
verifyFileContent(t, testKey, testData)
}
// TestRemoteUncacheWithFilters tests uncaching with include/exclude patterns
func TestRemoteUncacheWithFilters(t *testing.T) {
checkServersRunning(t)
// Create multiple files
file1 := fmt.Sprintf("uncache-filter1-%d.log", time.Now().UnixNano())
file2 := fmt.Sprintf("uncache-filter2-%d.txt", time.Now().UnixNano())
data1 := createTestFile(t, file1, 1024)
data2 := createTestFile(t, file2, 1024)
// Uncache only .log files
t.Log("Uncaching only .log files...")
cmd := fmt.Sprintf("remote.uncache -dir=/buckets/%s -include=*.log", testBucket)
output, err := runWeedShellWithOutput(t, cmd)
require.NoError(t, err, "remote.uncache with filter failed")
t.Logf("Uncache output: %s", output)
// Both should still be readable
verifyFileContent(t, file1, data1)
verifyFileContent(t, file2, data2)
}
// TestRemoteUncacheMinSize tests uncaching files based on size
func TestRemoteUncacheMinSize(t *testing.T) {
checkServersRunning(t)
// Create files of different sizes
smallFile := fmt.Sprintf("uncache-small-%d.bin", time.Now().UnixNano())
largeFile := fmt.Sprintf("uncache-large-%d.bin", time.Now().UnixNano())
smallData := createTestFile(t, smallFile, 500)
largeData := createTestFile(t, largeFile, 5000)
// Uncache only files larger than 2KB
t.Log("Uncaching files larger than 2KB...")
cmd := fmt.Sprintf("remote.uncache -dir=/buckets/%s -minSize=2048", testBucket)
output, err := runWeedShellWithOutput(t, cmd)
require.NoError(t, err, "remote.uncache with minSize failed")
t.Logf("Uncache output: %s", output)
// Both should still be readable
verifyFileContent(t, smallFile, smallData)
verifyFileContent(t, largeFile, largeData)
}
// TestRemoteCacheConcurrency tests cache with different concurrency levels
func TestRemoteCacheConcurrency(t *testing.T) {
checkServersRunning(t)
// Create multiple files
var files []string
var dataMap = make(map[string][]byte)
for i := 0; i < 5; i++ {
key := fmt.Sprintf("concurrent-%d-%d.bin", time.Now().UnixNano(), i)
data := createTestFile(t, key, 1024)
files = append(files, key)
dataMap[key] = data
}
// Uncache all
for _, file := range files {
uncacheLocal(t, file)
}
time.Sleep(1 * time.Second)
// Cache with high concurrency
t.Log("Caching with concurrency=8...")
cmd := fmt.Sprintf("remote.cache -dir=/buckets/%s -concurrent=8", testBucket)
output, err := runWeedShellWithOutput(t, cmd)
require.NoError(t, err, "remote.cache with concurrency failed")
t.Logf("Cache output: %s", output)
// Verify all files are readable
for _, file := range files {
verifyFileContent(t, file, dataMap[file])
}
}