SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
volume: resolve the disk IO slow-latency threshold per disk (#10976)
Chris Lu2d25c39da439
diff --git a/weed/command/scaffold/volume.toml b/weed/command/scaffold/volume.tomlindex 64c350d3a9..4e5f35f91c 100644--- a/weed/command/scaffold/volume.toml+++ b/weed/command/scaffold/volume.toml@@ -21,6 +21,8 @@ percent = 20[volume.disk.io.error]percent = 10+# slow threshold per disk type, matched against each -dir's -disk tag.+# a disk whose type is not listed here uses the hdd threshold.[volume.disk.io.slow.latency]hdd = "500ms"ssd = "100ms"diff --git a/weed/command/volume.go b/weed/command/volume.goindex a2bd91c715..0122227986 100644--- a/weed/command/volume.go+++ b/weed/command/volume.go@@ -293,11 +293,29 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v// Set multiple folders and each folder's max volume count limit'v.folders = strings.Split(volumeFolders, ",")+ // Two locations on one directory would load every volume twice, appending to+ // the same .dat under two independent locks. Compare identity rather than the+ // path, so a symlink or bind mount aliasing an earlier -dir is caught too.+ type seenFolder struct {+ path string+ info os.FileInfo+ }+ var seenFolders []seenFolderfor i, folder := range v.folders {v.folders[i] = util.ResolvePath(folder)if err := util.TestFolderWritable(v.folders[i]); err != nil {glog.Fatalf("Check Data Folder(-dir) Writable %s : %s", v.folders[i], err)}+ folderInfo, err := os.Stat(v.folders[i])+ if err != nil {+ glog.Fatalf("Check Data Folder(-dir) %s : %s", v.folders[i], err)+ }+ for _, seen := range seenFolders {+ if os.SameFile(seen.info, folderInfo) {+ glog.Fatalf("Data Folder(-dir) %s and %s are the same directory", seen.path, v.folders[i])+ }+ }+ seenFolders = append(seenFolders, seenFolder{v.folders[i], folderInfo})}// set max@@ -396,24 +414,18 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v// Determine volume server ID: if not specified, use ip:portvolumeServerId := util.GetVolumeServerId(*v.id, *v.ip, *v.port)- var slowLatency time.Duration-- switch *v.diskType {- case "hdd":- slowLatency = *v.diskHDDIOSlowLatency- case "ssd":- slowLatency = *v.diskSSDIOSlowLatency- case "nvme":- slowLatency = *v.diskNVMEIOSlowLatency- default:- slowLatency = *v.diskHDDIOSlowLatency- }+diskProbeConfig := stats_collect.DiskIOProbeConfig{Enabled: *v.diskIOProbe,Timeout: *v.diskIOTimeout,Interval: *v.diskIOInterval,- SlowLatency: slowLatency,+ SlowLatency: *v.diskHDDIOSlowLatency,+ SlowLatencyByDiskType: map[string]time.Duration{+ types.HddType: *v.diskHDDIOSlowLatency,+ types.SsdType: *v.diskSSDIOSlowLatency,+ types.NvmeType: *v.diskNVMEIOSlowLatency,+ },Window: *v.diskIOWindow,MinSamples: *v.diskIOMinSamples,@@ -425,10 +437,6 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, vRecoveryCoef: *v.diskRecoveryCoef,}- if diskProbeConfig.Enabled && len(v.folders) > 1 {- glog.Warningf("disk IO probe is disabled for multiple volume directories: %v", v.folders)- diskProbeConfig.Enabled = false- }volumeServer := weed_server.NewVolumeServer(volumeMux, publicVolumeMux,*v.ip, *v.port, *v.portGrpc, *v.publicUrl, volumeServerId,v.folders, v.folderMaxLimits, minFreeSpaces, diskTypes, folderTags,diff --git a/weed/stats/disk.go b/weed/stats/disk.goindex 260901ef81..a75cbbab41 100644--- a/weed/stats/disk.go+++ b/weed/stats/disk.go@@ -27,6 +27,9 @@ type DiskIOProbeConfig struct {// latency above this threshold is considered slowSlowLatency time.Duration+ // per disk type overrides for SlowLatency, keyed by the -disk tag+ SlowLatencyByDiskType map[string]time.Duration+// rolling observation windowWindow time.Duration@@ -64,6 +67,15 @@ func DefaultDiskIOProbeConfig() DiskIOProbeConfig {}}+// SlowLatencyFor returns the slow threshold configured for a disk type,+// falling back to SlowLatency when the type has no entry of its own.+func (config DiskIOProbeConfig) SlowLatencyFor(diskType string) time.Duration {+ if slowLatency, found := config.SlowLatencyByDiskType[diskType]; found {+ return slowLatency+ }+ return config.SlowLatency+}+type ioSample struct {ts time.Timelatency time.Durationdiff --git a/weed/stats/disk_test.go b/weed/stats/disk_test.goindex cc34573563..abbe26dea9 100644--- a/weed/stats/disk_test.go+++ b/weed/stats/disk_test.go@@ -6,6 +6,7 @@ import ("os""path/filepath""testing"+ "time""github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb")@@ -67,3 +68,29 @@ func TestSetDiskStatusRequiresRepeatedSuccessesToRecover(t *testing.T) {t.Fatalf("expected disk error to recover after %d successful checks: %s", recoveryChecks, disk.Error)}}++func TestSlowLatencyForFallsBackToDefault(t *testing.T) {+ config := DiskIOProbeConfig{+ SlowLatency: 500 * time.Millisecond,+ SlowLatencyByDiskType: map[string]time.Duration{+ "hdd": 500 * time.Millisecond,+ "nvme": 50 * time.Millisecond,+ },+ }++ for diskType, want := range map[string]time.Duration{+ "hdd": 500 * time.Millisecond,+ "nvme": 50 * time.Millisecond,+ "ssd": 500 * time.Millisecond,+ "nvme-gen5": 500 * time.Millisecond,+ } {+ if got := config.SlowLatencyFor(diskType); got != want {+ t.Errorf("disk type %q: got %v, want %v", diskType, got, want)+ }+ }++ empty := DiskIOProbeConfig{SlowLatency: 100 * time.Millisecond}+ if got := empty.SlowLatencyFor("nvme"); got != 100*time.Millisecond {+ t.Errorf("unconfigured table: got %v, want %v", got, 100*time.Millisecond)+ }+}diff --git a/weed/storage/disk_location.go b/weed/storage/disk_location.goindex 4eadb03f73..d8b83a0ca0 100644--- a/weed/storage/disk_location.go+++ b/weed/storage/disk_location.go@@ -671,9 +671,13 @@ func (l *DiskLocation) UnUsedSpace(volumeSizeLimit uint64) (unUsedSpace uint64)return}+// newDiskStatus is a seam letting a test observe the config CheckDiskSpace probes with.+var newDiskStatus = stats.NewDiskStatusOnStart+func (l *DiskLocation) CheckDiskSpace(config stats.DiskIOProbeConfig) {+ config.SlowLatency = config.SlowLatencyFor(l.DiskType.ReadableString())if dir, e := filepath.Abs(l.Directory); e == nil {- s := stats.NewDiskStatusOnStart(dir, config)+ s := newDiskStatus(dir, config)if len(s.Error) != 0 {l.isDiskUnavailable.Store(true)stats.VolumeServerDiskErrorGauge.WithLabelValues(l.Directory, "error").Set(1)diff --git a/weed/storage/disk_location_test.go b/weed/storage/disk_location_test.goindex 554aec6a82..2825af356b 100644--- a/weed/storage/disk_location_test.go+++ b/weed/storage/disk_location_test.go@@ -5,6 +5,8 @@ import ("testing""time"+ "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"+ "github.com/seaweedfs/seaweedfs/weed/stats""github.com/seaweedfs/seaweedfs/weed/storage/backend""github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding""github.com/seaweedfs/seaweedfs/weed/storage/needle"@@ -100,3 +102,40 @@ func TestResolveVolumeIDs(t *testing.T) {t.Errorf("wanted EC volume IDs %v, got %v", want, got)}}++// The threshold a disk is probed against is picked from that disk's own type,+// so a server mixing media holds each of them to its own latency.+func TestCheckDiskSpaceProbesWithTheDiskTypeThreshold(t *testing.T) {+ original := newDiskStatus+ defer func() { newDiskStatus = original }()++ config := stats.DiskIOProbeConfig{+ SlowLatency: 500 * time.Millisecond,+ SlowLatencyByDiskType: map[string]time.Duration{+ types.HddType: 500 * time.Millisecond,+ types.SsdType: 100 * time.Millisecond,+ types.NvmeType: 50 * time.Millisecond,+ },+ }++ for diskType, want := range map[string]time.Duration{+ "hdd": 500 * time.Millisecond,+ "": 500 * time.Millisecond,+ "ssd": 100 * time.Millisecond,+ "nvme": 50 * time.Millisecond,+ "nvme-gen5": 500 * time.Millisecond,+ } {+ var probed time.Duration+ newDiskStatus = func(path string, probeConfig stats.DiskIOProbeConfig) *volume_server_pb.DiskStatus {+ probed = probeConfig.SlowLatency+ return &volume_server_pb.DiskStatus{Dir: path}+ }++ location := &DiskLocation{Directory: t.TempDir(), DiskType: types.ToDiskType(diskType)}+ location.CheckDiskSpace(config)++ if probed != want {+ t.Errorf("-disk %q: probed with %v, want %v", diskType, probed, want)+ }+ }+}diff --git a/weed/storage/types/volume_disk_type.go b/weed/storage/types/volume_disk_type.goindex 3a5e140554..a28d9137b2 100644--- a/weed/storage/types/volume_disk_type.go+++ b/weed/storage/types/volume_disk_type.go@@ -16,6 +16,7 @@ const (HardDriveType DiskType = ""HddType = "hdd"SsdType = "ssd"+ NvmeType = "nvme")func ToDiskType(vt string) (diskType DiskType) {diff --git a/weed/storage/types/volume_disk_type_test.go b/weed/storage/types/volume_disk_type_test.gonew file mode 100644index 0000000000..f53ec8230f--- /dev/null+++ b/weed/storage/types/volume_disk_type_test.go@@ -0,0 +1,21 @@+package types++import "testing"++// The disk IO probe keys its per-disk-type settings by ReadableString, so a+// disk configured as "hdd" has to come back out as "hdd" and not as the empty+// string HardDriveType is stored as.+func TestReadableStringRoundTrip(t *testing.T) {+ for configured, want := range map[string]string{+ "": HddType,+ "hdd": HddType,+ "HDD": HddType,+ "ssd": SsdType,+ "nvme": NvmeType,+ "nvme-gen5": "nvme-gen5",+ } {+ if got := ToDiskType(configured).ReadableString(); got != want {+ t.Errorf("-disk %q: got %q, want %q", configured, got, want)+ }+ }+}