Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

10.7 KB
Raw Blame
package stats

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/glog"
	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
	"google.golang.org/protobuf/proto"
)

type DiskIOProbeConfig struct {
	// enable/disable disk io probing
	Enabled bool

	// timeout for a single io probe
	Timeout time.Duration

	// probe interval
	Interval time.Duration

	// latency above this threshold is considered slow
	SlowLatency time.Duration

	// per disk type overrides for SlowLatency, keyed by the -disk tag
	SlowLatencyByDiskType map[string]time.Duration

	// rolling observation window
	Window time.Duration

	// minimum number of samples required before evaluating health
	MinSamples int

	// percentage of slow operations required to mark disk degraded
	SlowPercent float64

	// percentage of failed operations required to mark disk degraded
	ErrorPercent float64

	// recovery coef defines the hysteresis threshold for recovering from degraded state.
	RecoveryCoef float64

	// statfs failures before alerting
	MaxStatFailures int
}

func DefaultDiskIOProbeConfig() DiskIOProbeConfig {
	return DiskIOProbeConfig{
		Enabled:     false,
		Timeout:     2 * time.Second,
		Interval:    30 * time.Second,
		SlowLatency: 100 * time.Millisecond,

		Window:     time.Minute,
		MinSamples: 10,

		SlowPercent:  20,
		ErrorPercent: 10,
		RecoveryCoef: 0.5,

		MaxStatFailures: 3,
	}
}

// SlowLatencyFor returns the slow threshold configured for a disk type,
// falling back to SlowLatency when the type has no entry of its own.
func (config DiskIOProbeConfig) SlowLatencyFor(diskType string) time.Duration {
	if slowLatency, found := config.SlowLatencyByDiskType[diskType]; found {
		return slowLatency
	}
	return config.SlowLatency
}

type ioSample struct {
	ts      time.Time
	latency time.Duration
	failed  bool
	slow    bool
}

type diskState struct {
	mu sync.Mutex

	// statfs probe
	isChecking       bool
	statFailureCount int
	statSuccessCount int
	lastStatErr      error
	lastGoodStatus   *volume_server_pb.DiskStatus
	hasLastGood      bool

	samples []ioSample

	// IO probe
	isIOChecking   bool
	ioCheckID      uint64
	lastIOCheck    time.Time
	ioFailureCount int
	lastIOErr      error
	ioDegraded     bool
}

var diskRegistry sync.Map

const (
	// statfs timeout
	diskTimeout = 500 * time.Millisecond
)

func NewDiskStatus(path string) (disk *volume_server_pb.DiskStatus) {
	disk = &volume_server_pb.DiskStatus{Dir: path}
	fillInDiskStatus(disk)
	if disk.PercentUsed > 95 {
		glog.V(0).Infof("disk status: %v", disk)
	}
	return
}

func NewDiskStatusOnStart(path string, config DiskIOProbeConfig) (disk *volume_server_pb.DiskStatus) {
	disk = &volume_server_pb.DiskStatus{Dir: path}
	diskProbe(disk, config)
	if disk.PercentUsed > 95 {
		glog.V(0).Infof("disk status: %v", disk)
	}
	return
}

func diskProbe(disk *volume_server_pb.DiskStatus, config DiskIOProbeConfig) {
	actual, _ := diskRegistry.LoadOrStore(disk.Dir, &diskState{})
	state := actual.(*diskState)

	state.mu.Lock()
	if state.isChecking {
		state.statFailureCount++
		state.statSuccessCount = 0
		state.lastStatErr = errors.New("statfs still in progress")
		state.updateErrorLocked(disk, config)
		state.applyLastGoodStatusLocked(disk)
		state.mu.Unlock()
		return
	}
	state.isChecking = true
	state.mu.Unlock()

	ctx, cancel := context.WithTimeout(context.Background(), diskTimeout)
	defer cancel()

	probe := &volume_server_pb.DiskStatus{Dir: disk.Dir}
	ch := make(chan error, 1)
	go func() {
		err := fillInDiskStatus(probe)
		// Clear the flag only once statfs has actually returned, so a stuck disk
		// keeps a single outstanding probe instead of spawning a new one each tick.
		state.mu.Lock()
		state.isChecking = false
		state.mu.Unlock()
		ch <- err
	}()

	var probeErr error
	select {
	case probeErr = <-ch:
	case <-ctx.Done():
		// Leave isChecking set; the probe goroutine clears it when statfs returns.
		state.mu.Lock()
		state.statFailureCount++
		state.statSuccessCount = 0
		state.lastStatErr = errors.New("statfs timeout")
		state.updateErrorLocked(disk, config)
		state.applyLastGoodStatusLocked(disk)
		state.mu.Unlock()
		return
	}

	state.mu.Lock()

	if probeErr != nil {
		state.statFailureCount++
		state.statSuccessCount = 0
		state.lastStatErr = probeErr
		state.updateErrorLocked(disk, config)
		state.applyLastGoodStatusLocked(disk)
		state.mu.Unlock()
		return
	}

	state.observeStatSuccessLocked(config)
	disk.All = probe.All
	disk.Free = probe.Free
	disk.Used = probe.Used
	disk.PercentFree = probe.PercentFree
	disk.PercentUsed = probe.PercentUsed
	state.rememberGoodStatusLocked(disk)

	if !config.Enabled {
		state.ioFailureCount = 0
		state.lastIOErr = nil
		state.ioDegraded = false
		state.updateErrorLocked(disk, config)
		state.mu.Unlock()
		return
	}

	shouldCheckIO := false
	ioCheckID := uint64(0)

	if time.Since(state.lastIOCheck) >= config.Interval {
		if state.isIOChecking {
			degraded := state.observeLatencyLocked(
				config.Timeout,
				true,
				config,
			)

			state.ioDegraded = degraded
			state.lastIOCheck = time.Now()

			if degraded {
				state.ioFailureCount++
				state.lastIOErr = errors.New("disk io still in progress")
			} else {
				state.ioFailureCount = 0
				state.lastIOErr = nil
			}
		} else {
			state.isIOChecking = true
			state.ioCheckID++
			ioCheckID = state.ioCheckID
			state.lastIOCheck = time.Now()
			shouldCheckIO = true
		}
	}

	state.updateErrorLocked(disk, config)
	state.mu.Unlock()

	if shouldCheckIO {
		checkDiskIOLatency(state, disk, ioCheckID, config)
	}
}

type ioProbeResult struct {
	err     error
	latency time.Duration
}

func checkDiskIOLatency(state *diskState, disk *volume_server_pb.DiskStatus, checkID uint64, config DiskIOProbeConfig) {
	ctx, cancel := context.WithTimeout(context.Background(), config.Timeout)
	defer cancel()

	ch := make(chan ioProbeResult, 1)

	go func() {
		start := time.Now()
		err := runIOProbe(disk.Dir)
		ch <- ioProbeResult{
			err:     err,
			latency: time.Since(start),
		}

		state.mu.Lock()
		if state.ioCheckID == checkID {
			state.isIOChecking = false
			state.lastIOCheck = time.Now()
		}
		state.mu.Unlock()
	}()

	select {
	case result := <-ch:
		state.mu.Lock()
		defer state.mu.Unlock()

		if state.ioCheckID != checkID {
			return
		}

		state.isIOChecking = false
		state.lastIOCheck = time.Now()

		if result.err != nil {
			degraded := state.observeLatencyLocked(
				config.Timeout,
				true,
				config,
			)

			state.ioDegraded = degraded

			if degraded {
				state.ioFailureCount++
				state.lastIOErr = result.err
			} else {
				state.ioFailureCount = 0
				state.lastIOErr = nil
			}

			state.updateErrorLocked(disk, config)
			return
		}

		degraded := state.observeLatencyLocked(
			result.latency,
			false,
			config,
		)

		state.ioDegraded = degraded

		if degraded {
			state.ioFailureCount++
			state.lastIOErr = fmt.Errorf(
				"disk io degradation detected latency=%v",
				result.latency,
			)
		} else {
			state.ioFailureCount = 0
			state.lastIOErr = nil
		}

		state.updateErrorLocked(disk, config)

		glog.V(1).Infof(
			"disk io latency dir=%s latency=%v ioFailureCount=%d degraded=%v samples=%d error=%v",
			disk.Dir,
			result.latency,
			state.ioFailureCount,
			degraded,
			len(state.samples),
			state.lastIOErr,
		)

	case <-ctx.Done():
		state.mu.Lock()

		if state.ioCheckID == checkID {
			degraded := state.observeLatencyLocked(
				config.Timeout,
				true,
				config,
			)

			state.ioDegraded = degraded

			if degraded {
				state.ioFailureCount++
				state.lastIOErr = errors.New("disk io timeout")
			} else {
				state.ioFailureCount = 0
				state.lastIOErr = nil
			}

			state.lastIOCheck = time.Now()
			state.updateErrorLocked(disk, config)
		}

		state.mu.Unlock()
	}
}

func runIOProbe(dir string) error {
	path := filepath.Join(dir, ".disk-health-check")

	f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0644)
	if err != nil {
		return err
	}
	defer f.Close()

	if _, err := f.WriteAt(make([]byte, 4096), 0); err != nil {
		return err
	}

	return f.Sync()
}

func (s *diskState) observeLatencyLocked(latency time.Duration, failed bool, config DiskIOProbeConfig) bool {
	now := time.Now()

	sample := ioSample{
		ts:      now,
		latency: latency,
		failed:  failed,
		slow:    latency >= config.SlowLatency,
	}

	s.samples = append(s.samples, sample)

	cutoff := now.Add(-config.Window)

	idx := 0
	for _, sample := range s.samples {
		if sample.ts.After(cutoff) {
			s.samples[idx] = sample
			idx++
		}
	}

	s.samples = s.samples[:idx]

	total := len(s.samples)

	if total < config.MinSamples {
		return s.ioDegraded
	}

	var slowCount int
	var failedCount int

	for _, sample := range s.samples {
		if sample.slow {
			slowCount++
		}

		if sample.failed {
			failedCount++
		}
	}

	slowPercent := float64(slowCount) / float64(total) * 100
	errorPercent := float64(failedCount) / float64(total) * 100

	if !s.ioDegraded {
		return slowPercent >= config.SlowPercent ||
			errorPercent >= config.ErrorPercent
	}

	recoverSlowPercent := config.SlowPercent * config.RecoveryCoef
	recoverErrorPercent := config.ErrorPercent * config.RecoveryCoef

	return slowPercent > recoverSlowPercent ||
		errorPercent > recoverErrorPercent
}

func (s *diskState) observeStatSuccessLocked(config DiskIOProbeConfig) {
	if s.statFailureCount < config.MaxStatFailures {
		s.statFailureCount = 0
		s.statSuccessCount = 0
		s.lastStatErr = nil
		return
	}

	s.statSuccessCount++
	if s.statSuccessCount >= statRecoverySuccesses(config) {
		s.statFailureCount = 0
		s.statSuccessCount = 0
		s.lastStatErr = nil
	}
}

func statRecoverySuccesses(config DiskIOProbeConfig) int {
	if config.MaxStatFailures <= 1 {
		return 1
	}
	if config.RecoveryCoef <= 0 {
		return config.MaxStatFailures * 2
	}

	successes := int(float64(config.MaxStatFailures) * (1/config.RecoveryCoef - 1))
	if successes < 1 {
		return 1
	}
	return successes
}

func (s *diskState) updateErrorLocked(disk *volume_server_pb.DiskStatus, config DiskIOProbeConfig) {
	var problems []string

	if s.statFailureCount >= config.MaxStatFailures &&
		s.lastStatErr != nil {
		problems = append(
			problems,
			fmt.Sprintf("statfs: %v", s.lastStatErr),
		)
	}

	if config.Enabled && s.ioDegraded && s.lastIOErr != nil {
		problems = append(
			problems,
			fmt.Sprintf("io: %v", s.lastIOErr),
		)
	}

	if len(problems) == 0 {
		disk.Error = ""
		return
	}

	disk.Error =
		"disk health check failed: " +
			strings.Join(problems, "; ")
}

func (s *diskState) rememberGoodStatusLocked(disk *volume_server_pb.DiskStatus) {
	s.lastGoodStatus = proto.Clone(disk).(*volume_server_pb.DiskStatus)
	s.hasLastGood = true
}

func (s *diskState) applyLastGoodStatusLocked(disk *volume_server_pb.DiskStatus) {
	if !s.hasLastGood || disk.Error != "" {
		return
	}

	dir := disk.Dir
	errorMessage := disk.Error
	proto.Reset(disk)
	proto.Merge(disk, s.lastGoodStatus)
	disk.Dir = dir
	disk.Error = errorMessage
}