SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package stats
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"google.golang.org/protobuf/proto"
)
type DiskIOProbeConfig struct {
// enable/disable disk io probing
Enabled bool
// timeout for a single io probe
Timeout time.Duration
// probe interval
Interval time.Duration
// latency above this threshold is considered slow
SlowLatency time.Duration
// per disk type overrides for SlowLatency, keyed by the -disk tag
SlowLatencyByDiskType map[string]time.Duration
// rolling observation window
Window time.Duration
// minimum number of samples required before evaluating health
MinSamples int
// percentage of slow operations required to mark disk degraded
SlowPercent float64
// percentage of failed operations required to mark disk degraded
ErrorPercent float64
// recovery coef defines the hysteresis threshold for recovering from degraded state.
RecoveryCoef float64
// statfs failures before alerting
MaxStatFailures int
}
func DefaultDiskIOProbeConfig() DiskIOProbeConfig {
return DiskIOProbeConfig{
Enabled: false,
Timeout: 2 * time.Second,
Interval: 30 * time.Second,
SlowLatency: 100 * time.Millisecond,
Window: time.Minute,
MinSamples: 10,
SlowPercent: 20,
ErrorPercent: 10,
RecoveryCoef: 0.5,
MaxStatFailures: 3,
}
}
// SlowLatencyFor returns the slow threshold configured for a disk type,
// falling back to SlowLatency when the type has no entry of its own.
func (config DiskIOProbeConfig) SlowLatencyFor(diskType string) time.Duration {
if slowLatency, found := config.SlowLatencyByDiskType[diskType]; found {
return slowLatency
}
return config.SlowLatency
}
type ioSample struct {
ts time.Time
latency time.Duration
failed bool
slow bool
}
type diskState struct {
mu sync.Mutex
// statfs probe
isChecking bool
statFailureCount int
statSuccessCount int
lastStatErr error
lastGoodStatus *volume_server_pb.DiskStatus
hasLastGood bool
samples []ioSample
// IO probe
isIOChecking bool
ioCheckID uint64
lastIOCheck time.Time
ioFailureCount int
lastIOErr error
ioDegraded bool
}
var diskRegistry sync.Map
const (
// statfs timeout
diskTimeout = 500 * time.Millisecond
)
func NewDiskStatus(path string) (disk *volume_server_pb.DiskStatus) {
disk = &volume_server_pb.DiskStatus{Dir: path}
fillInDiskStatus(disk)
if disk.PercentUsed > 95 {
glog.V(0).Infof("disk status: %v", disk)
}
return
}
func NewDiskStatusOnStart(path string, config DiskIOProbeConfig) (disk *volume_server_pb.DiskStatus) {
disk = &volume_server_pb.DiskStatus{Dir: path}
diskProbe(disk, config)
if disk.PercentUsed > 95 {
glog.V(0).Infof("disk status: %v", disk)
}
return
}
func diskProbe(disk *volume_server_pb.DiskStatus, config DiskIOProbeConfig) {
actual, _ := diskRegistry.LoadOrStore(disk.Dir, &diskState{})
state := actual.(*diskState)
state.mu.Lock()
if state.isChecking {
state.statFailureCount++
state.statSuccessCount = 0
state.lastStatErr = errors.New("statfs still in progress")
state.updateErrorLocked(disk, config)
state.applyLastGoodStatusLocked(disk)
state.mu.Unlock()
return
}
state.isChecking = true
state.mu.Unlock()
ctx, cancel := context.WithTimeout(context.Background(), diskTimeout)
defer cancel()
probe := &volume_server_pb.DiskStatus{Dir: disk.Dir}
ch := make(chan error, 1)
go func() {
err := fillInDiskStatus(probe)
// Clear the flag only once statfs has actually returned, so a stuck disk
// keeps a single outstanding probe instead of spawning a new one each tick.
state.mu.Lock()
state.isChecking = false
state.mu.Unlock()
ch <- err
}()
var probeErr error
select {
case probeErr = <-ch:
case <-ctx.Done():
// Leave isChecking set; the probe goroutine clears it when statfs returns.
state.mu.Lock()
state.statFailureCount++
state.statSuccessCount = 0
state.lastStatErr = errors.New("statfs timeout")
state.updateErrorLocked(disk, config)
state.applyLastGoodStatusLocked(disk)
state.mu.Unlock()
return
}
state.mu.Lock()
if probeErr != nil {
state.statFailureCount++
state.statSuccessCount = 0
state.lastStatErr = probeErr
state.updateErrorLocked(disk, config)
state.applyLastGoodStatusLocked(disk)
state.mu.Unlock()
return
}
state.observeStatSuccessLocked(config)
disk.All = probe.All
disk.Free = probe.Free
disk.Used = probe.Used
disk.PercentFree = probe.PercentFree
disk.PercentUsed = probe.PercentUsed
state.rememberGoodStatusLocked(disk)
if !config.Enabled {
state.ioFailureCount = 0
state.lastIOErr = nil
state.ioDegraded = false
state.updateErrorLocked(disk, config)
state.mu.Unlock()
return
}
shouldCheckIO := false
ioCheckID := uint64(0)
if time.Since(state.lastIOCheck) >= config.Interval {
if state.isIOChecking {
degraded := state.observeLatencyLocked(
config.Timeout,
true,
config,
)
state.ioDegraded = degraded
state.lastIOCheck = time.Now()
if degraded {
state.ioFailureCount++
state.lastIOErr = errors.New("disk io still in progress")
} else {
state.ioFailureCount = 0
state.lastIOErr = nil
}
} else {
state.isIOChecking = true
state.ioCheckID++
ioCheckID = state.ioCheckID
state.lastIOCheck = time.Now()
shouldCheckIO = true
}
}
state.updateErrorLocked(disk, config)
state.mu.Unlock()
if shouldCheckIO {
checkDiskIOLatency(state, disk, ioCheckID, config)
}
}
type ioProbeResult struct {
err error
latency time.Duration
}
func checkDiskIOLatency(state *diskState, disk *volume_server_pb.DiskStatus, checkID uint64, config DiskIOProbeConfig) {
ctx, cancel := context.WithTimeout(context.Background(), config.Timeout)
defer cancel()
ch := make(chan ioProbeResult, 1)
go func() {
start := time.Now()
err := runIOProbe(disk.Dir)
ch <- ioProbeResult{
err: err,
latency: time.Since(start),
}
state.mu.Lock()
if state.ioCheckID == checkID {
state.isIOChecking = false
state.lastIOCheck = time.Now()
}
state.mu.Unlock()
}()
select {
case result := <-ch:
state.mu.Lock()
defer state.mu.Unlock()
if state.ioCheckID != checkID {
return
}
state.isIOChecking = false
state.lastIOCheck = time.Now()
if result.err != nil {
degraded := state.observeLatencyLocked(
config.Timeout,
true,
config,
)
state.ioDegraded = degraded
if degraded {
state.ioFailureCount++
state.lastIOErr = result.err
} else {
state.ioFailureCount = 0
state.lastIOErr = nil
}
state.updateErrorLocked(disk, config)
return
}
degraded := state.observeLatencyLocked(
result.latency,
false,
config,
)
state.ioDegraded = degraded
if degraded {
state.ioFailureCount++
state.lastIOErr = fmt.Errorf(
"disk io degradation detected latency=%v",
result.latency,
)
} else {
state.ioFailureCount = 0
state.lastIOErr = nil
}
state.updateErrorLocked(disk, config)
glog.V(1).Infof(
"disk io latency dir=%s latency=%v ioFailureCount=%d degraded=%v samples=%d error=%v",
disk.Dir,
result.latency,
state.ioFailureCount,
degraded,
len(state.samples),
state.lastIOErr,
)
case <-ctx.Done():
state.mu.Lock()
if state.ioCheckID == checkID {
degraded := state.observeLatencyLocked(
config.Timeout,
true,
config,
)
state.ioDegraded = degraded
if degraded {
state.ioFailureCount++
state.lastIOErr = errors.New("disk io timeout")
} else {
state.ioFailureCount = 0
state.lastIOErr = nil
}
state.lastIOCheck = time.Now()
state.updateErrorLocked(disk, config)
}
state.mu.Unlock()
}
}
func runIOProbe(dir string) error {
path := filepath.Join(dir, ".disk-health-check")
f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0644)
if err != nil {
return err
}
defer f.Close()
if _, err := f.WriteAt(make([]byte, 4096), 0); err != nil {
return err
}
return f.Sync()
}
func (s *diskState) observeLatencyLocked(latency time.Duration, failed bool, config DiskIOProbeConfig) bool {
now := time.Now()
sample := ioSample{
ts: now,
latency: latency,
failed: failed,
slow: latency >= config.SlowLatency,
}
s.samples = append(s.samples, sample)
cutoff := now.Add(-config.Window)
idx := 0
for _, sample := range s.samples {
if sample.ts.After(cutoff) {
s.samples[idx] = sample
idx++
}
}
s.samples = s.samples[:idx]
total := len(s.samples)
if total < config.MinSamples {
return s.ioDegraded
}
var slowCount int
var failedCount int
for _, sample := range s.samples {
if sample.slow {
slowCount++
}
if sample.failed {
failedCount++
}
}
slowPercent := float64(slowCount) / float64(total) * 100
errorPercent := float64(failedCount) / float64(total) * 100
if !s.ioDegraded {
return slowPercent >= config.SlowPercent ||
errorPercent >= config.ErrorPercent
}
recoverSlowPercent := config.SlowPercent * config.RecoveryCoef
recoverErrorPercent := config.ErrorPercent * config.RecoveryCoef
return slowPercent > recoverSlowPercent ||
errorPercent > recoverErrorPercent
}
func (s *diskState) observeStatSuccessLocked(config DiskIOProbeConfig) {
if s.statFailureCount < config.MaxStatFailures {
s.statFailureCount = 0
s.statSuccessCount = 0
s.lastStatErr = nil
return
}
s.statSuccessCount++
if s.statSuccessCount >= statRecoverySuccesses(config) {
s.statFailureCount = 0
s.statSuccessCount = 0
s.lastStatErr = nil
}
}
func statRecoverySuccesses(config DiskIOProbeConfig) int {
if config.MaxStatFailures <= 1 {
return 1
}
if config.RecoveryCoef <= 0 {
return config.MaxStatFailures * 2
}
successes := int(float64(config.MaxStatFailures) * (1/config.RecoveryCoef - 1))
if successes < 1 {
return 1
}
return successes
}
func (s *diskState) updateErrorLocked(disk *volume_server_pb.DiskStatus, config DiskIOProbeConfig) {
var problems []string
if s.statFailureCount >= config.MaxStatFailures &&
s.lastStatErr != nil {
problems = append(
problems,
fmt.Sprintf("statfs: %v", s.lastStatErr),
)
}
if config.Enabled && s.ioDegraded && s.lastIOErr != nil {
problems = append(
problems,
fmt.Sprintf("io: %v", s.lastIOErr),
)
}
if len(problems) == 0 {
disk.Error = ""
return
}
disk.Error =
"disk health check failed: " +
strings.Join(problems, "; ")
}
func (s *diskState) rememberGoodStatusLocked(disk *volume_server_pb.DiskStatus) {
s.lastGoodStatus = proto.Clone(disk).(*volume_server_pb.DiskStatus)
s.hasLastGood = true
}
func (s *diskState) applyLastGoodStatusLocked(disk *volume_server_pb.DiskStatus) {
if !s.hasLastGood || disk.Error != "" {
return
}
dir := disk.Dir
errorMessage := disk.Error
proto.Reset(disk)
proto.Merge(disk, s.lastGoodStatus)
disk.Dir = dir
disk.Error = errorMessage
}