Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

volume: resolve the disk IO slow-latency threshold per disk (#10976)

Chris Lu2d25c39da439
diff --git a/weed/command/scaffold/volume.toml b/weed/command/scaffold/volume.toml
index 64c350d3a9..4e5f35f91c 100644
--- a/weed/command/scaffold/volume.toml
+++ b/weed/command/scaffold/volume.toml
@@ -21,6 +21,8 @@ percent = 20
[volume.disk.io.error]
percent = 10
+# slow threshold per disk type, matched against each -dir's -disk tag.
+# a disk whose type is not listed here uses the hdd threshold.
[volume.disk.io.slow.latency]
hdd = "500ms"
ssd = "100ms"
diff --git a/weed/command/volume.go b/weed/command/volume.go
index a2bd91c715..0122227986 100644
--- a/weed/command/volume.go
+++ b/weed/command/volume.go
@@ -293,11 +293,29 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
// Set multiple folders and each folder's max volume count limit'
v.folders = strings.Split(volumeFolders, ",")
+ // Two locations on one directory would load every volume twice, appending to
+ // the same .dat under two independent locks. Compare identity rather than the
+ // path, so a symlink or bind mount aliasing an earlier -dir is caught too.
+ type seenFolder struct {
+ path string
+ info os.FileInfo
+ }
+ var seenFolders []seenFolder
for i, folder := range v.folders {
v.folders[i] = util.ResolvePath(folder)
if err := util.TestFolderWritable(v.folders[i]); err != nil {
glog.Fatalf("Check Data Folder(-dir) Writable %s : %s", v.folders[i], err)
}
+ folderInfo, err := os.Stat(v.folders[i])
+ if err != nil {
+ glog.Fatalf("Check Data Folder(-dir) %s : %s", v.folders[i], err)
+ }
+ for _, seen := range seenFolders {
+ if os.SameFile(seen.info, folderInfo) {
+ glog.Fatalf("Data Folder(-dir) %s and %s are the same directory", seen.path, v.folders[i])
+ }
+ }
+ seenFolders = append(seenFolders, seenFolder{v.folders[i], folderInfo})
}
// set max
@@ -396,24 +414,18 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
// Determine volume server ID: if not specified, use ip:port
volumeServerId := util.GetVolumeServerId(*v.id, *v.ip, *v.port)
- var slowLatency time.Duration
-
- switch *v.diskType {
- case "hdd":
- slowLatency = *v.diskHDDIOSlowLatency
- case "ssd":
- slowLatency = *v.diskSSDIOSlowLatency
- case "nvme":
- slowLatency = *v.diskNVMEIOSlowLatency
- default:
- slowLatency = *v.diskHDDIOSlowLatency
- }
+
diskProbeConfig := stats_collect.DiskIOProbeConfig{
Enabled: *v.diskIOProbe,
Timeout: *v.diskIOTimeout,
Interval: *v.diskIOInterval,
- SlowLatency: slowLatency,
+ SlowLatency: *v.diskHDDIOSlowLatency,
+ SlowLatencyByDiskType: map[string]time.Duration{
+ types.HddType: *v.diskHDDIOSlowLatency,
+ types.SsdType: *v.diskSSDIOSlowLatency,
+ types.NvmeType: *v.diskNVMEIOSlowLatency,
+ },
Window: *v.diskIOWindow,
MinSamples: *v.diskIOMinSamples,
@@ -425,10 +437,6 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
RecoveryCoef: *v.diskRecoveryCoef,
}
- if diskProbeConfig.Enabled && len(v.folders) > 1 {
- glog.Warningf("disk IO probe is disabled for multiple volume directories: %v", v.folders)
- diskProbeConfig.Enabled = false
- }
volumeServer := weed_server.NewVolumeServer(volumeMux, publicVolumeMux,
*v.ip, *v.port, *v.portGrpc, *v.publicUrl, volumeServerId,
v.folders, v.folderMaxLimits, minFreeSpaces, diskTypes, folderTags,
diff --git a/weed/stats/disk.go b/weed/stats/disk.go
index 260901ef81..a75cbbab41 100644
--- a/weed/stats/disk.go
+++ b/weed/stats/disk.go
@@ -27,6 +27,9 @@ type DiskIOProbeConfig struct {
// latency above this threshold is considered slow
SlowLatency time.Duration
+ // per disk type overrides for SlowLatency, keyed by the -disk tag
+ SlowLatencyByDiskType map[string]time.Duration
+
// rolling observation window
Window time.Duration
@@ -64,6 +67,15 @@ func DefaultDiskIOProbeConfig() DiskIOProbeConfig {
}
}
+// SlowLatencyFor returns the slow threshold configured for a disk type,
+// falling back to SlowLatency when the type has no entry of its own.
+func (config DiskIOProbeConfig) SlowLatencyFor(diskType string) time.Duration {
+ if slowLatency, found := config.SlowLatencyByDiskType[diskType]; found {
+ return slowLatency
+ }
+ return config.SlowLatency
+}
+
type ioSample struct {
ts time.Time
latency time.Duration
diff --git a/weed/stats/disk_test.go b/weed/stats/disk_test.go
index cc34573563..abbe26dea9 100644
--- a/weed/stats/disk_test.go
+++ b/weed/stats/disk_test.go
@@ -6,6 +6,7 @@ import (
"os"
"path/filepath"
"testing"
+ "time"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
)
@@ -67,3 +68,29 @@ func TestSetDiskStatusRequiresRepeatedSuccessesToRecover(t *testing.T) {
t.Fatalf("expected disk error to recover after %d successful checks: %s", recoveryChecks, disk.Error)
}
}
+
+func TestSlowLatencyForFallsBackToDefault(t *testing.T) {
+ config := DiskIOProbeConfig{
+ SlowLatency: 500 * time.Millisecond,
+ SlowLatencyByDiskType: map[string]time.Duration{
+ "hdd": 500 * time.Millisecond,
+ "nvme": 50 * time.Millisecond,
+ },
+ }
+
+ for diskType, want := range map[string]time.Duration{
+ "hdd": 500 * time.Millisecond,
+ "nvme": 50 * time.Millisecond,
+ "ssd": 500 * time.Millisecond,
+ "nvme-gen5": 500 * time.Millisecond,
+ } {
+ if got := config.SlowLatencyFor(diskType); got != want {
+ t.Errorf("disk type %q: got %v, want %v", diskType, got, want)
+ }
+ }
+
+ empty := DiskIOProbeConfig{SlowLatency: 100 * time.Millisecond}
+ if got := empty.SlowLatencyFor("nvme"); got != 100*time.Millisecond {
+ t.Errorf("unconfigured table: got %v, want %v", got, 100*time.Millisecond)
+ }
+}
diff --git a/weed/storage/disk_location.go b/weed/storage/disk_location.go
index 4eadb03f73..d8b83a0ca0 100644
--- a/weed/storage/disk_location.go
+++ b/weed/storage/disk_location.go
@@ -671,9 +671,13 @@ func (l *DiskLocation) UnUsedSpace(volumeSizeLimit uint64) (unUsedSpace uint64)
return
}
+// newDiskStatus is a seam letting a test observe the config CheckDiskSpace probes with.
+var newDiskStatus = stats.NewDiskStatusOnStart
+
func (l *DiskLocation) CheckDiskSpace(config stats.DiskIOProbeConfig) {
+ config.SlowLatency = config.SlowLatencyFor(l.DiskType.ReadableString())
if dir, e := filepath.Abs(l.Directory); e == nil {
- s := stats.NewDiskStatusOnStart(dir, config)
+ s := newDiskStatus(dir, config)
if len(s.Error) != 0 {
l.isDiskUnavailable.Store(true)
stats.VolumeServerDiskErrorGauge.WithLabelValues(l.Directory, "error").Set(1)
diff --git a/weed/storage/disk_location_test.go b/weed/storage/disk_location_test.go
index 554aec6a82..2825af356b 100644
--- a/weed/storage/disk_location_test.go
+++ b/weed/storage/disk_location_test.go
@@ -5,6 +5,8 @@ import (
"testing"
"time"
+ "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+ "github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/storage/backend"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
@@ -100,3 +102,40 @@ func TestResolveVolumeIDs(t *testing.T) {
t.Errorf("wanted EC volume IDs %v, got %v", want, got)
}
}
+
+// The threshold a disk is probed against is picked from that disk's own type,
+// so a server mixing media holds each of them to its own latency.
+func TestCheckDiskSpaceProbesWithTheDiskTypeThreshold(t *testing.T) {
+ original := newDiskStatus
+ defer func() { newDiskStatus = original }()
+
+ config := stats.DiskIOProbeConfig{
+ SlowLatency: 500 * time.Millisecond,
+ SlowLatencyByDiskType: map[string]time.Duration{
+ types.HddType: 500 * time.Millisecond,
+ types.SsdType: 100 * time.Millisecond,
+ types.NvmeType: 50 * time.Millisecond,
+ },
+ }
+
+ for diskType, want := range map[string]time.Duration{
+ "hdd": 500 * time.Millisecond,
+ "": 500 * time.Millisecond,
+ "ssd": 100 * time.Millisecond,
+ "nvme": 50 * time.Millisecond,
+ "nvme-gen5": 500 * time.Millisecond,
+ } {
+ var probed time.Duration
+ newDiskStatus = func(path string, probeConfig stats.DiskIOProbeConfig) *volume_server_pb.DiskStatus {
+ probed = probeConfig.SlowLatency
+ return &volume_server_pb.DiskStatus{Dir: path}
+ }
+
+ location := &DiskLocation{Directory: t.TempDir(), DiskType: types.ToDiskType(diskType)}
+ location.CheckDiskSpace(config)
+
+ if probed != want {
+ t.Errorf("-disk %q: probed with %v, want %v", diskType, probed, want)
+ }
+ }
+}
diff --git a/weed/storage/types/volume_disk_type.go b/weed/storage/types/volume_disk_type.go
index 3a5e140554..a28d9137b2 100644
--- a/weed/storage/types/volume_disk_type.go
+++ b/weed/storage/types/volume_disk_type.go
@@ -16,6 +16,7 @@ const (
HardDriveType DiskType = ""
HddType = "hdd"
SsdType = "ssd"
+ NvmeType = "nvme"
)
func ToDiskType(vt string) (diskType DiskType) {
diff --git a/weed/storage/types/volume_disk_type_test.go b/weed/storage/types/volume_disk_type_test.go
new file mode 100644
index 0000000000..f53ec8230f
--- /dev/null
+++ b/weed/storage/types/volume_disk_type_test.go
@@ -0,0 +1,21 @@
+package types
+
+import "testing"
+
+// The disk IO probe keys its per-disk-type settings by ReadableString, so a
+// disk configured as "hdd" has to come back out as "hdd" and not as the empty
+// string HardDriveType is stored as.
+func TestReadableStringRoundTrip(t *testing.T) {
+ for configured, want := range map[string]string{
+ "": HddType,
+ "hdd": HddType,
+ "HDD": HddType,
+ "ssd": SsdType,
+ "nvme": NvmeType,
+ "nvme-gen5": "nvme-gen5",
+ } {
+ if got := ToDiskType(configured).ReadableString(); got != want {
+ t.Errorf("-disk %q: got %q, want %q", configured, got, want)
+ }
+ }
+}