Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

8.4 KB
Raw Blame
package erasure_coding

import (
	"context"
	"fmt"
	"sort"

	"github.com/seaweedfs/seaweedfs/weed/operation"
	"github.com/seaweedfs/seaweedfs/weed/pb"
	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
	"google.golang.org/grpc"
)

type ServerShardInventory struct {
	Bits       ShardBits
	QueryError error
	// BlockSize is the shard block layout this holder reports for the volume —
	// the one it will serve reads through. A binary predating the uniform
	// layout drops the field off the wire and reports 0, which is how
	// RequireAgreedBlockLayout tells such a holder from one that agrees.
	BlockSize int64
}

// Query errors are recorded per-server and treated as zero shards rather
// than aborting the scan, so the caller still sees partial coverage from
// healthy peers when one server is down. The caller gates destructive
// actions on RequireRecoverableShardSet against the returned union.
func VerifyShardsAcrossServers(ctx context.Context, volumeID uint32,
	servers []string, dialOption grpc.DialOption) (
	union ShardBits, perServer map[string]ServerShardInventory) {

	perServer = make(map[string]ServerShardInventory, len(servers))

	for _, server := range servers {
		if server == "" {
			continue
		}
		if _, seen := perServer[server]; seen {
			continue
		}

		var inv ServerShardInventory

		callErr := operation.WithVolumeServerClient(false, pb.ServerAddress(server), dialOption,
			func(client volume_server_pb.VolumeServerClient) error {
				resp, e := client.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{
					VolumeId: volumeID,
				})
				if e != nil {
					return e
				}
				for _, s := range resp.EcShardInfos {
					if s.VolumeId != volumeID || s.ShardId >= MaxShardCount {
						continue
					}
					inv.Bits = inv.Bits.Set(ShardId(s.ShardId))
				}
				inv.BlockSize = resp.GetEcShardConfig().GetBlockSize()
				return nil
			})
		if callErr != nil {
			inv.QueryError = callErr
		}

		perServer[server] = inv
		union = ShardBits(uint32(union) | uint32(inv.Bits))
	}

	return union, perServer
}

// RequireRecoverableShardSet gates source-volume deletion after EC encode:
// a non-empty .dat may only be deleted when enough distinct shards exist to
// reconstruct the volume (>= dataShards). A full set returns (false, nil); a
// degraded-but-recoverable set returns (true, nil) so the caller can warn and
// proceed -- the missing shards can be rebuilt from the survivors, while
// keeping the source next to live shards is the more dangerous mixed state.
// Below dataShards it returns an error and the source must be kept.
// dataShards/totalShards are passed as parameters (not derived from the
// package constants) so enterprise builds with custom EC ratios share this
// helper verbatim.
func RequireRecoverableShardSet(volumeID uint32, shardsPresent ShardBits, dataShards, totalShards int) (degraded bool, err error) {
	if totalShards <= 0 || totalShards > MaxShardCount {
		return false, fmt.Errorf("invalid totalShards %d for volume %d (must be in [1, %d])",
			totalShards, volumeID, MaxShardCount)
	}
	if dataShards <= 0 || dataShards > totalShards {
		return false, fmt.Errorf("invalid dataShards %d for volume %d (must be in [1, %d])",
			dataShards, volumeID, totalShards)
	}
	var missing []int
	for id := 0; id < totalShards; id++ {
		if !shardsPresent.Has(ShardId(id)) {
			missing = append(missing, id)
		}
	}
	if len(missing) == 0 {
		return false, nil
	}
	if totalShards-len(missing) >= dataShards {
		return true, nil
	}
	sort.Ints(missing)
	return false, fmt.Errorf("EC shard set unrecoverable for volume %d: %d/%d shards present, need %d to reconstruct, missing shard ids %v",
		volumeID, totalShards-len(missing), totalShards, dataShards, missing)
}

// RequireAgreedBlockLayout gates source-volume deletion on every holder that
// answered agreeing with the layout the shards were encoded under.
//
// The uniform block layout lives in a `.vif` field that older volume servers
// have never heard of: such a server parses the file, silently discards the
// unknown field, and mounts the volume on the legacy 1GiB/1MiB striping. Its
// reads then land at the wrong shard offsets and return wrong bytes, and
// nothing else in the encode path notices — the shard files are the same
// length under either layout. Asking each holder which layout it is serving is
// the one question that separates the two, and asking it here means the answer
// arrives while the source .dat is still on disk.
//
// Holders that could not be reached, or that report no shard of this volume,
// are skipped: RequireRecoverableShardSet already covers a missing holder, and
// one that answered nothing has promised nothing.
func RequireAgreedBlockLayout(volumeID uint32, encodedBlockSize int64, perServer map[string]ServerShardInventory) error {
	disagreeing := make([]string, 0, len(perServer))
	for server, inv := range perServer {
		if inv.QueryError != nil || inv.Bits.Count() == 0 {
			continue
		}
		if inv.BlockSize != encodedBlockSize {
			disagreeing = append(disagreeing, fmt.Sprintf("%s serves block size %d", server, inv.BlockSize))
		}
	}
	if len(disagreeing) == 0 {
		return nil
	}
	sort.Strings(disagreeing)
	return fmt.Errorf("volume %d was encoded with shard block size %d but %v; upgrade every volume server to a build that understands the uniform shard block layout before encoding",
		volumeID, encodedBlockSize, disagreeing)
}

func SummarizeShardInventory(perServer map[string]ServerShardInventory) string {
	servers := make([]string, 0, len(perServer))
	for s := range perServer {
		servers = append(servers, s)
	}
	sort.Strings(servers)

	var b []byte
	for i, s := range servers {
		if i > 0 {
			b = append(b, ' ')
		}
		inv := perServer[s]
		b = append(b, s...)
		b = append(b, '=')
		b = append(b, '[')
		ids := make([]int, 0)
		for id := 0; id < MaxShardCount; id++ {
			if inv.Bits.Has(ShardId(id)) {
				ids = append(ids, id)
			}
		}
		for j, id := range ids {
			if j > 0 {
				b = append(b, ' ')
			}
			b = append(b, []byte(fmt.Sprintf("%d", id))...)
		}
		if inv.QueryError != nil {
			if len(ids) > 0 {
				b = append(b, ' ')
			}
			b = append(b, []byte("ERR:"+inv.QueryError.Error())...)
		}
		b = append(b, ']')
	}
	return string(b)
}

// VerifyShardsOnServer confirms one server really holds the named shards of a
// specific (collection, volume), for callers about to delete another copy.
//
// Collection is checked, unlike in VerifyShardsAcrossServers: the inventory RPC
// is keyed by volume id alone, so a server answering for volume N says nothing
// about which collection's volume N it means. Approving a delete on the
// strength of a different collection's shard would remove the last real copy —
// the exact outcome the caller is trying to prevent.
//
// A server that cannot be queried is unknown, not confirmed, and returns an
// error: treating an unreachable peer as proof of a surviving copy is how a
// network blip becomes data loss.
// CollectShardsOnServer asks one volume server which shards of the volume it
// actually serves right now — its live inventory, as opposed to what the
// master's possibly stale topology claims for it.
func CollectShardsOnServer(ctx context.Context, collection string, volumeID uint32,
	server string, dialOption grpc.DialOption) (present ShardBits, err error) {
	err = operation.WithVolumeServerClient(false, pb.ServerAddress(server), dialOption,
		func(client volume_server_pb.VolumeServerClient) error {
			resp, e := client.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{
				VolumeId: volumeID,
			})
			if e != nil {
				return e
			}
			for _, s := range resp.EcShardInfos {
				if s.VolumeId != volumeID || s.Collection != collection || s.ShardId >= MaxShardCount {
					continue
				}
				present = present.Set(ShardId(s.ShardId))
			}
			return nil
		})
	return present, err
}

func VerifyShardsOnServer(ctx context.Context, collection string, volumeID uint32,
	server string, shardIDs []uint32, dialOption grpc.DialOption) error {

	if server == "" {
		return fmt.Errorf("no server given to verify volume %d shard(s) %v", volumeID, shardIDs)
	}

	present, callErr := CollectShardsOnServer(ctx, collection, volumeID, server, dialOption)
	if callErr != nil {
		return fmt.Errorf("verify volume %d shard(s) %v on %s: %w", volumeID, shardIDs, server, callErr)
	}

	for _, sid := range shardIDs {
		if !present.Has(ShardId(sid)) {
			return fmt.Errorf("%s does not hold ec shard %d.%d of collection %q", server, volumeID, sid, collection)
		}
	}
	return nil
}