SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package pluginworker
import (
"context"
"errors"
"fmt"
"sort"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/admin/topology"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
workertypes "github.com/seaweedfs/seaweedfs/weed/worker/types"
"google.golang.org/grpc"
)
// CollectVolumeMetricsFromMasters dials the provided master addresses in order
// until one returns a usable volume list, then converts that into per-volume
// health metrics, an active-topology view, and a replica-location map.
func CollectVolumeMetricsFromMasters(
ctx context.Context,
masterAddresses []string,
collectionFilter string,
grpcDialOption grpc.DialOption,
) ([]*workertypes.VolumeHealthMetrics, *topology.ActiveTopology, map[uint32][]workertypes.ReplicaLocation, error) {
if grpcDialOption == nil {
return nil, nil, nil, fmt.Errorf("grpc dial option is not configured")
}
if len(masterAddresses) == 0 {
return nil, nil, nil, fmt.Errorf("no master addresses provided in cluster context")
}
for _, masterAddress := range masterAddresses {
response, err := FetchVolumeList(ctx, masterAddress, grpcDialOption)
if err != nil {
glog.Warningf("Plugin worker failed master volume list at %s: %v", masterAddress, err)
continue
}
metrics, activeTopology, replicaMap, buildErr := buildVolumeMetrics(response, collectionFilter)
if buildErr != nil {
// Configuration errors (e.g. invalid regex) will fail on every master,
// so return immediately instead of masking them with retries.
if isConfigError(buildErr) {
return nil, nil, nil, buildErr
}
glog.Warningf("Plugin worker failed to build metrics from master %s: %v", masterAddress, buildErr)
continue
}
return metrics, activeTopology, replicaMap, nil
}
return nil, nil, nil, fmt.Errorf("failed to load topology from all provided masters")
}
// FetchVolumeList dials the given master address (trying both the address as
// given and the gRPC port variant) and returns the master's volume list. Used
// by detection helpers that already know which master address to talk to.
// FetchDefaultReplicaPlacement returns the master's configured default replication
// (GetMasterConfiguration), used by detectors as the replica-placement fallback so
// the plugin path matches the shell. Returns "" if it cannot be fetched, so callers
// fall back to even spread rather than failing detection.
func FetchDefaultReplicaPlacement(ctx context.Context, masterAddresses []string, grpcDialOption grpc.DialOption) string {
if grpcDialOption == nil {
return ""
}
for _, address := range masterAddresses {
for _, candidate := range MasterAddressCandidates(address) {
if ctx.Err() != nil {
return ""
}
dialCtx, cancelDial := context.WithTimeout(ctx, 5*time.Second)
conn, err := pb.GrpcDial(dialCtx, candidate, false, grpcDialOption)
cancelDial()
if err != nil {
continue
}
client := master_pb.NewSeaweedClient(conn)
callCtx, cancelCall := context.WithTimeout(ctx, 10*time.Second)
resp, callErr := client.GetMasterConfiguration(callCtx, &master_pb.GetMasterConfigurationRequest{})
cancelCall()
_ = conn.Close()
if callErr == nil {
return resp.DefaultReplication
}
}
}
return ""
}
func FetchVolumeList(ctx context.Context, address string, grpcDialOption grpc.DialOption) (*master_pb.VolumeListResponse, error) {
var lastErr error
for _, candidate := range MasterAddressCandidates(address) {
if ctx.Err() != nil {
return nil, ctx.Err()
}
dialCtx, cancelDial := context.WithTimeout(ctx, 5*time.Second)
conn, err := pb.GrpcDial(dialCtx, candidate, false, grpcDialOption)
cancelDial()
if err != nil {
lastErr = err
continue
}
client := master_pb.NewSeaweedClient(conn)
callCtx, cancelCall := context.WithTimeout(ctx, 10*time.Second)
response, callErr := pb.CollectVolumeList(callCtx, client, &master_pb.VolumeListRequest{})
cancelCall()
_ = conn.Close()
if callErr == nil {
return response, nil
}
lastErr = callErr
}
if lastErr == nil {
lastErr = fmt.Errorf("no valid master address candidate")
}
return nil, lastErr
}
func buildVolumeMetrics(
response *master_pb.VolumeListResponse,
collectionFilter string,
) ([]*workertypes.VolumeHealthMetrics, *topology.ActiveTopology, map[uint32][]workertypes.ReplicaLocation, error) {
if response == nil || response.TopologyInfo == nil {
return nil, nil, nil, fmt.Errorf("volume list response has no topology info")
}
activeTopology := topology.NewActiveTopology(10)
if err := activeTopology.UpdateTopology(response.TopologyInfo); err != nil {
return nil, nil, nil, err
}
collectionMatcher, err := wildcard.CompileCollectionMatcher(collectionFilter)
if err != nil {
return nil, nil, nil, &configError{err: err}
}
volumeSizeLimitBytes := uint64(response.VolumeSizeLimitMb) * 1024 * 1024
now := time.Now()
metrics := make([]*workertypes.VolumeHealthMetrics, 0, 256)
replicaMap := make(map[uint32][]workertypes.ReplicaLocation)
for _, dc := range response.TopologyInfo.DataCenterInfos {
for _, rack := range dc.RackInfos {
for _, node := range rack.DataNodeInfos {
for diskType, diskInfo := range node.DiskInfos {
for _, volume := range diskInfo.VolumeInfos {
// Build replica map from ALL volumes BEFORE collection filtering,
// since replicas may span filtered/unfiltered nodes.
replicaMap[volume.Id] = append(replicaMap[volume.Id], workertypes.ReplicaLocation{
DataCenter: dc.Id,
Rack: rack.Id,
NodeID: node.Id,
Host: pb.NewServerAddressFromDataNode(node).ToHost(),
})
if !collectionMatcher.Matches(volume.Collection) {
continue
}
metric := &workertypes.VolumeHealthMetrics{
VolumeID: volume.Id,
Server: node.Id,
ServerAddress: string(pb.NewServerAddressFromDataNode(node)),
DiskType: diskType,
DiskId: volume.DiskId,
DataCenter: dc.Id,
Rack: rack.Id,
Collection: volume.Collection,
Size: volume.Size,
DeletedBytes: volume.DeletedByteCount,
LastModified: time.Unix(volume.ModifiedAtSecond, 0),
ReplicaCount: 1,
ExpectedReplicas: int(volume.ReplicaPlacement),
IsReadOnly: volume.ReadOnly,
HasRemoteCopy: volume.RemoteStorageName != "",
}
if metric.Size > 0 {
metric.GarbageRatio = float64(metric.DeletedBytes) / float64(metric.Size)
}
if volumeSizeLimitBytes > 0 {
metric.FullnessRatio = float64(metric.Size) / float64(volumeSizeLimitBytes)
}
metric.Age = now.Sub(metric.LastModified)
metrics = append(metrics, metric)
}
}
}
}
}
replicaCounts := make(map[uint32]int)
for _, metric := range metrics {
replicaCounts[metric.VolumeID]++
}
for _, metric := range metrics {
metric.ReplicaCount = replicaCounts[metric.VolumeID]
}
return metrics, activeTopology, replicaMap, nil
}
// configError wraps configuration errors that should not be retried across masters.
type configError struct {
err error
}
func (e *configError) Error() string { return e.err.Error() }
func (e *configError) Unwrap() error { return e.err }
func isConfigError(err error) bool {
var ce *configError
return errors.As(err, &ce)
}
// MasterAddressCandidates returns address forms to try when dialing a master:
// the address as given plus the gRPC variant (port + 10000). Both are tried
// because callers may pass either an HTTP-port or gRPC-port address.
func MasterAddressCandidates(address string) []string {
trimmed := strings.TrimSpace(address)
if trimmed == "" {
return nil
}
candidateSet := map[string]struct{}{
trimmed: {},
}
converted := pb.ServerToGrpcAddress(trimmed)
candidateSet[converted] = struct{}{}
candidates := make([]string, 0, len(candidateSet))
for candidate := range candidateSet {
candidates = append(candidates, candidate)
}
sort.Strings(candidates)
return candidates
}