SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package shell
import (
"context"
"flag"
"fmt"
"math"
"strings"
"sync"
"github.com/dustin/go-humanize"
"github.com/dustin/go-humanize/english"
"github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"io"
)
func init() {
Commands = append(Commands, &commandClusterStatus{})
}
// Map of volume_id -> [volume replicas] with stat details.
type VolumeReplicaStats struct {
Id string
VolumeId uint32
Files uint64
FilesDeleted uint64
TotalSize uint64
}
type RegularVolumesStats map[uint32][]*VolumeReplicaStats
// Map of ec_volume_id -> stat details.
type EcVolumeStats struct {
VolumeId uint32
Files uint64
FilesDeleted uint64
TotalSize uint64
}
type EcVolumesStats map[uint32]*EcVolumeStats
type commandClusterStatus struct{}
type ClusterStatusPrinter struct {
writer io.Writer
writerMu sync.Mutex
humanize bool
maxParallelization int
locked bool
lockHeld bool
lockHolder string
lockMessage string
collections []string
topology *master_pb.TopologyInfo
volumeSizeLimitMb uint64
regularVolumesStats RegularVolumesStats
ecVolumesStats EcVolumesStats
}
func (c *commandClusterStatus) Name() string {
return "cluster.status"
}
func (c *commandClusterStatus) Help() string {
return `outputs a quick overview of the cluster status`
}
func (c *commandClusterStatus) HasTag(CommandTag) bool {
return false
}
func (c *commandClusterStatus) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
flags := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
humanize := flags.Bool("humanize", true, "human-readable output")
includeFiles := flags.Bool("files", false, "include detailed file metrics, from all volume servers")
maxParallelization := flags.Int("maxParallelization", DefaultMaxParallelization, "run up to X tasks in parallel, whenever possible")
if err = flags.Parse(args); err != nil {
return err
}
collections, err := ListCollectionNames(commandEnv, true, true)
if err != nil {
return err
}
topology, volumeSizeLimitMb, err := collectTopologyInfo(commandEnv, 0)
if err != nil {
return err
}
lockHolder, lockMessage, lockHeld := commandEnv.shellLockHolder()
sp := &ClusterStatusPrinter{
writer: writer,
humanize: *humanize,
maxParallelization: *maxParallelization,
locked: commandEnv.isLocked(),
lockHeld: lockHeld,
lockHolder: lockHolder,
lockMessage: lockMessage,
collections: collections,
topology: topology,
volumeSizeLimitMb: volumeSizeLimitMb,
}
if *includeFiles {
if err := sp.loadFileStats(commandEnv); err != nil {
return err
}
}
sp.Print()
return nil
}
func (sp *ClusterStatusPrinter) uint64(n uint64) string {
if !sp.humanize {
return fmt.Sprintf("%d", n)
}
return humanize.Comma(int64(n))
}
func (sp *ClusterStatusPrinter) int(n int) string {
return sp.uint64(uint64(n))
}
func (sp *ClusterStatusPrinter) uint64Plural(n uint64, str string) string {
if !sp.humanize {
return fmt.Sprintf("%s(s)", str)
}
uin := math.MaxInt
if n < math.MaxInt {
uin = int(n)
}
return english.PluralWord(int(uin), str, "")
}
func (sp *ClusterStatusPrinter) plural(n int, str string) string {
return sp.uint64Plural(uint64(n), str)
}
func (sp *ClusterStatusPrinter) bytes(b uint64) string {
if !sp.humanize {
return fmt.Sprintf("%d %s", b, sp.plural(int(b), "byte"))
}
return fmt.Sprintf("%s", humanize.Bytes(b))
}
func (sp *ClusterStatusPrinter) uint64Ratio(a, b uint64) string {
var p float64
if b != 0 {
if a%b == 0 {
// Avoid float precision issues on integer ratios.
p = float64(a / b)
} else {
p = float64(a) / float64(b)
}
}
if !sp.humanize {
return fmt.Sprintf("%.02f", p)
}
return fmt.Sprintf("%s", humanize.FtoaWithDigits(p, 2))
}
func (sp *ClusterStatusPrinter) intRatio(a, b int) string {
return sp.uint64Ratio(uint64(a), uint64(b))
}
func (sp *ClusterStatusPrinter) uint64Pct(a, b uint64) string {
var p float64
if b != 0 {
if a%b == 0 {
// avoid float rounding errors on exact ratios
p = float64(a / b * 100)
} else {
p = 100 * float64(a) / float64(b)
}
}
if !sp.humanize {
return fmt.Sprintf("%.02f%%", p)
}
return fmt.Sprintf("%s%%", humanize.FtoaWithDigits(p, 2))
}
func (sp *ClusterStatusPrinter) intPct(a, b int) string {
return sp.uint64Pct(uint64(a), uint64(b))
}
func (sp *ClusterStatusPrinter) write(format string, a ...any) {
sp.writerMu.Lock()
defer sp.writerMu.Unlock()
format = strings.TrimRight(format, " ")
if len(format) == 0 {
format = "\n"
}
fmt.Fprintf(sp.writer, format, a...)
last := format[len(format)-1:]
if last != "\n" && last != "\r" {
fmt.Fprint(sp.writer, "\n")
}
}
func (sp *ClusterStatusPrinter) Print() {
sp.write("")
sp.printClusterInfo()
sp.printVolumeInfo()
sp.printStorageInfo()
sp.printFilesInfo()
}
func (sp *ClusterStatusPrinter) loadFileStats(commandEnv *CommandEnv) error {
sp.regularVolumesStats = RegularVolumesStats{}
sp.ecVolumesStats = EcVolumesStats{}
var statsMu, writerMu sync.Mutex
var progressTotal, progressDone uint64
ewg := NewErrorWaitGroup(sp.maxParallelization)
updateProgress := func() {
writerMu.Lock()
defer writerMu.Unlock()
progressDone++
sp.write("collecting file stats: %s \r", sp.uint64Pct(progressDone, progressTotal))
}
for _, dci := range sp.topology.DataCenterInfos {
for _, ri := range dci.RackInfos {
for _, dni := range ri.DataNodeInfos {
for _, d := range dni.DiskInfos {
statsMu.Lock()
progressTotal += uint64(len(d.VolumeInfos))
progressTotal += uint64(len(d.EcShardInfos))
statsMu.Unlock()
for _, v := range d.VolumeInfos {
ewg.Add(func() error {
defer updateProgress()
// Collect regular volume stats
err := operation.WithVolumeServerClient(false, pb.NewServerAddressWithGrpcPort(dni.Id, int(dni.GrpcPort)), commandEnv.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
resp, reqErr := volumeServerClient.VolumeStatus(context.Background(), &volume_server_pb.VolumeStatusRequest{
VolumeId: uint32(v.Id),
})
if reqErr != nil {
return reqErr
}
statsMu.Lock()
defer statsMu.Unlock()
if resp != nil {
if _, ok := sp.regularVolumesStats[v.Id]; !ok {
sp.regularVolumesStats[v.Id] = []*VolumeReplicaStats{}
}
sp.regularVolumesStats[v.Id] = append(sp.regularVolumesStats[v.Id], &VolumeReplicaStats{
Id: dni.Id,
VolumeId: v.Id,
Files: resp.FileCount,
FilesDeleted: resp.FileDeletedCount,
TotalSize: resp.VolumeSize,
})
}
return nil
})
return err
})
}
for _, eci := range d.EcShardInfos {
ewg.Add(func() error {
defer updateProgress()
// Collect EC shard stats
statsMu.Lock()
_, ok := sp.ecVolumesStats[eci.Id]
statsMu.Unlock()
if ok {
// this EC volume has been already processed, likely on a different node
return nil
}
err := operation.WithVolumeServerClient(false, pb.NewServerAddressWithGrpcPort(dni.Id, int(dni.GrpcPort)), commandEnv.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
resp, reqErr := volumeServerClient.VolumeEcShardsInfo(context.Background(), &volume_server_pb.VolumeEcShardsInfoRequest{
VolumeId: uint32(eci.Id),
})
if reqErr != nil {
return reqErr
}
statsMu.Lock()
defer statsMu.Unlock()
if resp != nil {
sp.ecVolumesStats[eci.Id] = &EcVolumeStats{
VolumeId: eci.Id,
Files: resp.FileCount,
FilesDeleted: resp.FileDeletedCount,
TotalSize: resp.VolumeSize,
}
}
return nil
})
return err
})
}
}
}
}
}
err := ewg.Wait()
sp.write("")
return err
}
func (sp *ClusterStatusPrinter) printClusterInfo() {
dcs := len(sp.topology.DataCenterInfos)
racks := 0
nodes := 0
disks := 0
for _, dci := range sp.topology.DataCenterInfos {
racks += len(dci.RackInfos)
for _, ri := range dci.RackInfos {
for _, dni := range ri.DataNodeInfos {
nodes++
// The master keys DiskInfos by disk type, so multiple
// same-type physical disks on one node collapse into a single
// entry. Count physical disks via SplitByPhysicalDisk so a node
// with N disks of one type reports N, not 1.
for _, di := range dni.DiskInfos {
disks += len(di.SplitByPhysicalDisk())
}
}
}
}
status := "unlocked"
if sp.lockHeld || sp.locked {
status = "LOCKED"
if sp.lockHolder != "" {
status += " by " + sp.lockHolder
}
if sp.lockMessage != "" {
status += " (" + sp.lockMessage + ")"
}
}
sp.write("cluster:")
sp.write("\tid: %s", sp.topology.Id)
sp.write("\tstatus: %s", status)
sp.write("\tnodes: %s", sp.int(nodes))
sp.write("\ttopology: %s %s, %s %s on %s %s",
sp.int(dcs), sp.plural(dcs, "DC"),
sp.int(disks), sp.plural(disks, "disk"),
sp.int(racks), sp.plural(racks, "rack"))
sp.write("")
}
func (sp *ClusterStatusPrinter) printVolumeInfo() {
collections := len(sp.collections)
var maxVolumes uint64
volumeIds := map[needle.VolumeId]bool{}
ecVolumeIds := map[needle.VolumeId]bool{}
var replicas, roReplicas, rwReplicas, ecShards int
for _, dci := range sp.topology.DataCenterInfos {
for _, ri := range dci.RackInfos {
for _, dni := range ri.DataNodeInfos {
for _, di := range dni.DiskInfos {
maxVolumes += uint64(di.MaxVolumeCount)
for _, vi := range di.VolumeInfos {
vid := needle.VolumeId(vi.Id)
volumeIds[vid] = true
replicas++
if vi.ReadOnly {
roReplicas++
} else {
rwReplicas++
}
}
for _, eci := range di.EcShardInfos {
vid := needle.VolumeId(eci.Id)
ecVolumeIds[vid] = true
ecShards += erasure_coding.GetShardCount(eci)
}
}
}
}
}
volumes := len(volumeIds)
ecVolumes := len(ecVolumeIds)
totalVolumes := volumes + ecVolumes
sp.write("volumes:")
sp.write("\ttotal: %s %s, %s %s",
sp.int(totalVolumes), sp.plural(totalVolumes, "volume"),
sp.int(collections), sp.plural(collections, "collection"))
sp.write("\tmax size: %s", sp.bytes(sp.volumeSizeLimitMb*1024*1024))
sp.write("\tregular: %s/%s %s on %s %s, %s writable (%s), %s read-only (%s)",
sp.int(volumes), sp.uint64(maxVolumes), sp.plural(volumes, "volume"),
sp.int(replicas), sp.plural(replicas, "replica"),
sp.int(rwReplicas), sp.intPct(rwReplicas, replicas),
sp.int(roReplicas), sp.intPct(roReplicas, replicas))
sp.write("\tEC: %s EC %s on %s %s (%s shards/volume)",
sp.int(ecVolumes), sp.plural(ecVolumes, "volume"),
sp.int(ecShards), sp.plural(ecShards, "shard"),
sp.intRatio(ecShards, ecVolumes))
sp.write("")
}
func (sp *ClusterStatusPrinter) printStorageInfo() {
perVolumeSize := map[needle.VolumeId]uint64{}
perEcVolumeSize := map[needle.VolumeId]uint64{}
var rawVolumeSize, rawEcVolumeSize uint64
for _, dci := range sp.topology.DataCenterInfos {
for _, ri := range dci.RackInfos {
for _, dni := range ri.DataNodeInfos {
for _, di := range dni.DiskInfos {
for _, vi := range di.VolumeInfos {
vid := needle.VolumeId(vi.Id)
perVolumeSize[vid] = vi.Size
rawVolumeSize += vi.Size
}
for _, eci := range di.EcShardInfos {
vid := needle.VolumeId(eci.Id)
var size uint64
for _, ss := range eci.ShardSizes {
size += uint64(ss)
}
perEcVolumeSize[vid] += size
rawEcVolumeSize += size
}
}
}
}
}
// normalize EC logical volume sizes given shard settings
for vid := range perEcVolumeSize {
perEcVolumeSize[vid] = perEcVolumeSize[vid] * erasure_coding.DataShardsCount / erasure_coding.TotalShardsCount
}
var volumeSize, ecVolumeSize uint64
for _, s := range perVolumeSize {
volumeSize += s
}
for _, s := range perEcVolumeSize {
ecVolumeSize += s
}
totalSize := volumeSize + ecVolumeSize
totalRawSize := rawVolumeSize + rawEcVolumeSize
sp.write("storage:")
sp.write("\ttotal: %s (%s raw, %s)", sp.bytes(totalSize), sp.bytes(totalRawSize), sp.uint64Pct(totalRawSize, totalSize))
sp.write("\tregular volumes: %s (%s raw, %s)", sp.bytes(volumeSize), sp.bytes(rawVolumeSize), sp.uint64Pct(rawVolumeSize, volumeSize))
sp.write("\tEC volumes: %s (%s raw, %s)", sp.bytes(ecVolumeSize), sp.bytes(rawEcVolumeSize), sp.uint64Pct(rawEcVolumeSize, ecVolumeSize))
sp.write("")
}
func (sp *ClusterStatusPrinter) printFilesInfo() {
if len(sp.regularVolumesStats) == 0 && len(sp.ecVolumesStats) == 0 {
return
}
var regularFilesTotal, regularFilesDeleted, regularFilesSize uint64
for _, replicaStats := range sp.regularVolumesStats {
rc := uint64(len(replicaStats))
var volumeFilesTotal, volumeFilesSize, volumeFilesDeleted uint64
for _, rs := range replicaStats {
volumeFilesTotal += rs.Files
volumeFilesSize += rs.TotalSize
volumeFilesDeleted += rs.FilesDeleted
}
regularFilesTotal += (volumeFilesTotal / rc)
regularFilesSize += (volumeFilesSize / rc)
regularFilesDeleted += (volumeFilesDeleted / rc)
}
regularFiles := regularFilesTotal - regularFilesDeleted
var avgRegularFileSize uint64
if regularFilesTotal != 0 {
avgRegularFileSize = regularFilesSize / regularFilesTotal
}
var ecFilesTotal, ecFilesDeleted, ecFilesSize uint64
for _, ecStats := range sp.ecVolumesStats {
ecFilesTotal += ecStats.Files
ecFilesSize += ecStats.TotalSize
ecFilesDeleted += ecStats.FilesDeleted
}
ecFiles := ecFilesTotal - ecFilesDeleted
var avgEcFileSize uint64
if ecFilesTotal != 0 {
avgEcFileSize = ecFilesSize / ecFilesTotal
}
files := regularFiles + ecFiles
filesDeleted := regularFilesDeleted + ecFilesDeleted
filesTotal := regularFilesTotal + ecFilesTotal
filesSize := regularFilesSize + ecFilesSize
var avgFileSize uint64
if filesTotal != 0 {
avgFileSize = filesSize / filesTotal
}
sp.write("files:")
sp.write("\ttotal: %s %s, %s readable (%s), %s deleted (%s), avg %s per file",
sp.uint64(filesTotal), sp.uint64Plural(filesTotal, "file"),
sp.uint64(files), sp.uint64Pct(files, filesTotal),
sp.uint64(filesDeleted), sp.uint64Pct(filesDeleted, filesTotal),
sp.bytes(avgFileSize))
sp.write("\tregular: %s %s, %s readable (%s), %s deleted (%s), avg %s per file",
sp.uint64(regularFilesTotal), sp.uint64Plural(regularFilesTotal, "file"),
sp.uint64(regularFiles), sp.uint64Pct(regularFiles, regularFilesTotal),
sp.uint64(regularFilesDeleted), sp.uint64Pct(regularFilesDeleted, regularFilesTotal),
sp.bytes(avgRegularFileSize))
sp.write("\tEC: %s %s, %s readable (%s), %s deleted (%s), avg %s per file",
sp.uint64(ecFilesTotal), sp.uint64Plural(ecFilesTotal, "file"),
sp.uint64(ecFiles), sp.uint64Pct(ecFiles, ecFilesTotal),
sp.uint64(ecFilesDeleted), sp.uint64Pct(ecFilesDeleted, ecFilesTotal),
sp.bytes(avgEcFileSize))
sp.write("")
}