SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package storage
import (
"os"
"path/filepath"
"reflect"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/storage/backend"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
"github.com/seaweedfs/seaweedfs/weed/util"
)
type (
mockBackendStorageFile struct {
backend.DiskFile
datSize int64
}
)
func (df *mockBackendStorageFile) GetStat() (datSize int64, modTime time.Time, err error) {
return df.datSize, time.Now(), nil
}
type (
mockNeedleMapper struct {
NeedleMap
idxSize uint64
}
)
func (nm *mockNeedleMapper) IndexFileSize() (idxSize uint64) {
return nm.idxSize
}
func TestUnUsedSpace(t *testing.T) {
minFreeSpace := util.MinFreeSpace{Type: util.AsPercent, Percent: 1, Raw: "1"}
diskLocation := DiskLocation{
Directory: "/test/",
DirectoryUuid: "1234",
IdxDirectory: "/test/",
DiskType: types.HddType,
MaxVolumeCount: 0,
OriginalMaxVolumeCount: 0,
MinFreeSpace: minFreeSpace,
}
diskLocation.volumes = make(map[needle.VolumeId]*Volume)
volumes := [3]*Volume{
{dir: diskLocation.Directory, dirIdx: diskLocation.IdxDirectory, Collection: "", Id: 0, DataBackend: &mockBackendStorageFile{datSize: 990}, nm: &mockNeedleMapper{idxSize: 10}},
{dir: diskLocation.Directory, dirIdx: diskLocation.IdxDirectory, Collection: "", Id: 1, DataBackend: &mockBackendStorageFile{datSize: 990}, nm: &mockNeedleMapper{idxSize: 10}},
{dir: diskLocation.Directory, dirIdx: diskLocation.IdxDirectory, Collection: "", Id: 2, DataBackend: &mockBackendStorageFile{datSize: 990}, nm: &mockNeedleMapper{idxSize: 10}},
}
for i, vol := range volumes {
diskLocation.SetVolume(needle.VolumeId(i), vol)
}
// Testing when there's still space
unUsedSpace := diskLocation.UnUsedSpace(1200)
if unUsedSpace != 600 {
t.Errorf("unUsedSpace incorrect: %d != %d", unUsedSpace, 1500)
}
// Testing when there's exactly 0 space
unUsedSpace = diskLocation.UnUsedSpace(1000)
if unUsedSpace != 0 {
t.Errorf("unUsedSpace incorrect: %d != %d", unUsedSpace, 0)
}
// Testing when there's negative free space
unUsedSpace = diskLocation.UnUsedSpace(900)
if unUsedSpace != 0 {
t.Errorf("unUsedSpace incorrect: %d != %d", unUsedSpace, 0)
}
}
func TestResolveVolumeIDs(t *testing.T) {
l := DiskLocation{
volumes: map[needle.VolumeId]*Volume{
0: &Volume{},
1: &Volume{},
2: &Volume{},
},
ecVolumes: map[needle.VolumeId]*erasure_coding.EcVolume{
3: &erasure_coding.EcVolume{},
4: &erasure_coding.EcVolume{},
5: &erasure_coding.EcVolume{},
},
}
if got, want := l.VolumeIds(), []needle.VolumeId{0, 1, 2}; !reflect.DeepEqual(got, want) {
t.Errorf("wanted volume IDs %v, got %v", want, got)
}
if got, want := l.EcVolumeIds(), []needle.VolumeId{3, 4, 5}; !reflect.DeepEqual(got, want) {
t.Errorf("wanted EC volume IDs %v, got %v", want, got)
}
}
// The threshold a disk is probed against is picked from that disk's own type,
// so a server mixing media holds each of them to its own latency.
func TestCheckDiskSpaceProbesWithTheDiskTypeThreshold(t *testing.T) {
original := newDiskStatus
defer func() { newDiskStatus = original }()
config := stats.DiskIOProbeConfig{
SlowLatency: 500 * time.Millisecond,
SlowLatencyByDiskType: map[string]time.Duration{
types.HddType: 500 * time.Millisecond,
types.SsdType: 100 * time.Millisecond,
types.NvmeType: 50 * time.Millisecond,
},
}
for diskType, want := range map[string]time.Duration{
"hdd": 500 * time.Millisecond,
"": 500 * time.Millisecond,
"ssd": 100 * time.Millisecond,
"nvme": 50 * time.Millisecond,
"nvme-gen5": 500 * time.Millisecond,
} {
var probed time.Duration
newDiskStatus = func(path string, probeConfig stats.DiskIOProbeConfig) *volume_server_pb.DiskStatus {
probed = probeConfig.SlowLatency
return &volume_server_pb.DiskStatus{Dir: path}
}
location := &DiskLocation{Directory: t.TempDir(), DiskType: types.ToDiskType(diskType)}
location.CheckDiskSpace(config)
if probed != want {
t.Errorf("-disk %q: probed with %v, want %v", diskType, probed, want)
}
}
}
// -dir.idx naming a directory that does not exist yet used to leave every
// volume unable to open its index.
func TestNewDiskLocation_CreatesIdxDirectory(t *testing.T) {
root := t.TempDir()
idxDir := filepath.Join(root, "idx", "nested")
loc := NewDiskLocation(root, 1, util.MinFreeSpace{}, idxDir, types.HardDriveType, nil, stats.DiskIOProbeConfig{})
defer loc.Close()
if _, err := os.Stat(idxDir); err != nil {
t.Fatalf("idx dir not created: %v", err)
}
}