SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package ec
import (
"fmt"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
"github.com/stretchr/testify/assert"
)
func TestSelectVolumeIdsFromTopology(t *testing.T) {
// Topology volumeSizeLimit:1000 MB hdd(volume:28/30 active:28 free:2 remote:0)
// DataCenter DefaultDataCenter hdd(volume:28/30 active:28 free:2 remote:0)
// Rack DefaultRack hdd(volume:28/30 active:28 free:2 remote:0)
// DataNode seaweedfs-volume-0.seaweedfs-volume.sea:8080 hdd(volume:14/15 active:14 free:1 remote:0)
// Disk hdd(volume:14/15 active:14 free:1 remote:0) id:0
// volume Id:1, Size:23520, ReplicaPlacement:001, Collection:, Version:3, Ttl:, FileCount:3, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770366011
// volume Id:2, Size:28536, ReplicaPlacement:001, Collection:, Version:3, Ttl:, FileCount:1, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770365291
// volume Id:3, Size:100752, ReplicaPlacement:001, Collection:, Version:3, Ttl:, FileCount:2, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770364931
// volume Id:4, Size:3112, ReplicaPlacement:001, Collection:, Version:3, Ttl:, FileCount:4, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770300850
// volume Id:5, Size:145208, ReplicaPlacement:001, Collection:, Version:3, Ttl:, FileCount:2, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770365652
// volume Id:6, Size:146456, ReplicaPlacement:001, Collection:, Version:3, Ttl:, FileCount:4, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770364812
// volume Id:7, Size:8, ReplicaPlacement:001, Collection:encrypt-data, Version:3, Ttl:, FileCount:0, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770300824
// volume Id:8, Size:808, ReplicaPlacement:001, Collection:encrypt-data, Version:3, Ttl:, FileCount:2, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770300796
// volume Id:9, Size:8, ReplicaPlacement:001, Collection:encrypt-data, Version:3, Ttl:, FileCount:0, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770300814
// volume Id:10, Size:1048582008, ReplicaPlacement:001, Collection:test, Version:3, Ttl:, FileCount:125, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770364737
// volume Id:11, Size:1048582008, ReplicaPlacement:001, Collection:test, Version:3, Ttl:, FileCount:125, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770364796
// volume Id:12, Size:1048582008, ReplicaPlacement:001, Collection:test, Version:3, Ttl:, FileCount:125, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770364829
// volume Id:13, Size:570428616, ReplicaPlacement:001, Collection:test, Version:3, Ttl:, FileCount:68, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770366062
// volume Id:14, Size:478153400, ReplicaPlacement:001, Collection:test, Version:3, Ttl:, FileCount:57, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770366020
// Disk hdd {Size:4194776448 FileCount:518 DeletedFileCount:0 DeletedBytes:0}
// DataNode seaweedfs-volume-0.seaweedfs-volume.sea:8080 {Size:4194776448 FileCount:518 DeletedFileCount:0 DeletedBytes:0}
// DataCenter DefaultDataCenter hdd(volume:28/30 active:28 free:2 remote:0)
// Rack DefaultRack hdd(volume:28/30 active:28 free:2 remote:0)
// DataNode seaweedfs-volume-1.seaweedfs-volume.sea:8080 hdd(volume:14/15 active:14 free:1 remote:0)
// Disk hdd(volume:14/15 active:14 free:1 remote:0) id:0
// volume Id:1, Size:23520, ReplicaPlacement:001, Collection:, Version:3, Ttl:, FileCount:3, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770366012
// volume Id:2, Size:28536, ReplicaPlacement:001, Collection:, Version:3, Ttl:, FileCount:1, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770365292
// volume Id:3, Size:100752, ReplicaPlacement:001, Collection:, Version:3, Ttl:, FileCount:2, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770364931
// volume Id:4, Size:3112, ReplicaPlacement:001, Collection:, Version:3, Ttl:, FileCount:4, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770300850
// volume Id:5, Size:145208, ReplicaPlacement:001, Collection:, Version:3, Ttl:, FileCount:2, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770365653
// volume Id:6, Size:146456, ReplicaPlacement:001, Collection:, Version:3, Ttl:, FileCount:4, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770364812
// volume Id:7, Size:8, ReplicaPlacement:001, Collection:encrypt-data, Version:3, Ttl:, FileCount:0, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770300824
// volume Id:8, Size:808, ReplicaPlacement:001, Collection:encrypt-data, Version:3, Ttl:, FileCount:2, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770300796
// volume Id:9, Size:8, ReplicaPlacement:001, Collection:encrypt-data, Version:3, Ttl:, FileCount:0, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770300814
// volume Id:10, Size:1048582008, ReplicaPlacement:001, Collection:test, Version:3, Ttl:, FileCount:125, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770364737
// volume Id:11, Size:1048582008, ReplicaPlacement:001, Collection:test, Version:3, Ttl:, FileCount:125, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770364796
// volume Id:12, Size:1048582008, ReplicaPlacement:001, Collection:test, Version:3, Ttl:, FileCount:125, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770364828
// volume Id:13, Size:570428616, ReplicaPlacement:001, Collection:test, Version:3, Ttl:, FileCount:68, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770366062
// volume Id:14, Size:478153400, ReplicaPlacement:001, Collection:test, Version:3, Ttl:, FileCount:57, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770366020
// Disk hdd {Size:4194776448 FileCount:518 DeletedFileCount:0 DeletedBytes:0}
// DataNode seaweedfs-volume-1.seaweedfs-volume.sea:8080 {Size:4194776448 FileCount:518 DeletedFileCount:0 DeletedBytes:0}
// Rack DefaultRack {Size:8389552896 FileCount:1036 DeletedFileCount:0 DeletedBytes:0}
// DataCenter DefaultDataCenter {Size:8389552896 FileCount:1036 DeletedFileCount:0 DeletedBytes:0}
// total size:8389552896 file_count:1036
topologyInfo := &master_pb.TopologyInfo{
Id: "DefaultDataCenter",
DataCenterInfos: []*master_pb.DataCenterInfo{
{
Id: "DefaultDataCenter",
RackInfos: []*master_pb.RackInfo{
{
Id: "DefaultRack",
DataNodeInfos: []*master_pb.DataNodeInfo{
{
Id: "seaweedfs-volume-0.seaweedfs-volume.sea:8080",
DiskInfos: map[string]*master_pb.DiskInfo{
"hdd": {
Type: "hdd",
FreeVolumeCount: 1,
VolumeInfos: []*master_pb.VolumeInformationMessage{
{Id: 1, Size: 23520, Collection: "", ModifiedAtSecond: 1770366011},
{Id: 2, Size: 28536, Collection: "", ModifiedAtSecond: 1770365291},
{Id: 3, Size: 100752, Collection: "", ModifiedAtSecond: 1770364931},
{Id: 4, Size: 3112, Collection: "", ModifiedAtSecond: 1770300850},
{Id: 5, Size: 145208, Collection: "", ModifiedAtSecond: 1770365652},
{Id: 6, Size: 146456, Collection: "", ModifiedAtSecond: 1770364812},
{Id: 7, Size: 8, Collection: "encrypt-data", ModifiedAtSecond: 1770300824},
{Id: 8, Size: 808, Collection: "encrypt-data", ModifiedAtSecond: 1770300796},
{Id: 9, Size: 8, Collection: "encrypt-data", ModifiedAtSecond: 1770300814},
{Id: 10, Size: 1048582008, Collection: "test", ModifiedAtSecond: 1770364737},
{Id: 11, Size: 1048582008, Collection: "test", ModifiedAtSecond: 1770364796},
{Id: 12, Size: 1048582008, Collection: "test", ModifiedAtSecond: 1770364829},
{Id: 13, Size: 570428616, Collection: "test", ModifiedAtSecond: 1770366062},
{Id: 14, Size: 478153400, Collection: "test", ModifiedAtSecond: 1770366020},
},
},
},
},
{
Id: "seaweedfs-volume-1.seaweedfs-volume.sea:8080",
DiskInfos: map[string]*master_pb.DiskInfo{
"hdd": {
Type: "hdd",
FreeVolumeCount: 1,
VolumeInfos: []*master_pb.VolumeInformationMessage{
{Id: 1, Size: 23520, Collection: "", ModifiedAtSecond: 1770366012},
{Id: 2, Size: 28536, Collection: "", ModifiedAtSecond: 1770365292},
{Id: 3, Size: 100752, Collection: "", ModifiedAtSecond: 1770364931},
{Id: 4, Size: 3112, Collection: "", ModifiedAtSecond: 1770300850},
{Id: 5, Size: 145208, Collection: "", ModifiedAtSecond: 1770365653},
{Id: 6, Size: 146456, Collection: "", ModifiedAtSecond: 1770364812},
{Id: 7, Size: 8, Collection: "encrypt-data", ModifiedAtSecond: 1770300824},
{Id: 8, Size: 808, Collection: "encrypt-data", ModifiedAtSecond: 1770300796},
{Id: 9, Size: 8, Collection: "encrypt-data", ModifiedAtSecond: 1770300814},
{Id: 10, Size: 1048582008, Collection: "test", ModifiedAtSecond: 1770364737},
{Id: 11, Size: 1048582008, Collection: "test", ModifiedAtSecond: 1770364796},
{Id: 12, Size: 1048582008, Collection: "test", ModifiedAtSecond: 1770364828},
{Id: 13, Size: 570428616, Collection: "test", ModifiedAtSecond: 1770366062},
{Id: 14, Size: 478153400, Collection: "test", ModifiedAtSecond: 1770366020},
},
},
},
},
},
},
},
},
},
}
volumeSizeLimitMb := uint64(1000)
collectionMatcher, _ := wildcard.CompileCollectionMatcher(".*")
// ec.encode -force
// force means we ignore the check for 4+ volume servers.
// But in this unit test we are testing selectVolumeIdsFromTopology, which selects volumes.
// The -force flag is handled in the caller of this function.
// Default values
quietPeriod := time.Hour
fullPercentage := 95.0
verbose := true
// Mock time to be slightly after the latest modification time in the test data
// The latest ModifiedAtSecond in the test data is 1770366062 (Volume 13)
// Let's set now to be 1770366062 + quietPeriod.Seconds() + 100
nowUnixSeconds := int64(1770366062) + int64(quietPeriod.Seconds()) + 100
quietSeconds := int64(quietPeriod.Seconds())
// Test case 1: Select all volumes with sufficient size
// Note: The logic requires (FreeVolumeCount >= 2) unless the volume is already selected?
// Wait, distinct volume IDs.
// Logic:
// if diskInfo.FreeVolumeCount < 2 { skip }
// In the test data, FreeVolumeCount is 1. So it should skip all volumes basically?
// Let's check the code:
/*
// check free disk space
if good, found := vidMap[v.Id]; found {
if good {
if diskInfo.FreeVolumeCount < 2 {
// ...
vidMap[v.Id] = false
noFreeDisk++
}
}
} else {
if diskInfo.FreeVolumeCount < 2 {
// ...
vidMap[v.Id] = false
noFreeDisk++
} else {
// ...
vidMap[v.Id] = true
}
}
*/
// Yes, if FreeVolumeCount < 2, it marks the volume as bad (false in vidMap).
// In the provided topology, FreeVolumeCount is 1 ("free:1"), so it is < 2.
// So ALL volumes should be skipped due to insufficient free disk space.
vids, _ := SelectVolumeIdsFromTopology(topologyInfo, volumeSizeLimitMb, collectionMatcher, nil, quietSeconds, nowUnixSeconds, fullPercentage, verbose)
assert.Equal(t, 0, len(vids), "Should select 0 volumes because FreeVolumeCount is 1 (less than 2)")
// Test case 2: If we had enough free space
topologyInfo.DataCenterInfos[0].RackInfos[0].DataNodeInfos[0].DiskInfos["hdd"].FreeVolumeCount = 2
topologyInfo.DataCenterInfos[0].RackInfos[0].DataNodeInfos[1].DiskInfos["hdd"].FreeVolumeCount = 2
// Update expected selection based on size and other filters
// Size threshold: 95% of 1000MB = 950MB = 996147200 bytes
// Volumes >= 950MB:
// Vol 10: 1048582008 > 950MB
// Vol 11: 1048582008 > 950MB
// Vol 12: 1048582008 > 950MB
// Vol 13: 570428616 < 950MB
// Vol 14: 478153400 < 950MB
// ... others are small
// So expected volumes: 10, 11, 12.
vids, _ = SelectVolumeIdsFromTopology(topologyInfo, volumeSizeLimitMb, collectionMatcher, nil, quietSeconds, nowUnixSeconds, fullPercentage, verbose)
expectedVids := []needle.VolumeId{10, 11, 12}
assert.Equal(t, len(expectedVids), len(vids), "Should select 3 volumes")
// Check content
vidMap := make(map[needle.VolumeId]bool)
for _, vid := range vids {
vidMap[vid] = true
}
for _, vid := range expectedVids {
assert.True(t, vidMap[vid], "Volume %d should be selected", vid)
}
}
func TestEcEncodeNodeCountCheck(t *testing.T) {
topologyInfo := &master_pb.TopologyInfo{
Id: "DefaultDataCenter",
DataCenterInfos: []*master_pb.DataCenterInfo{
{
Id: "DefaultDataCenter",
RackInfos: []*master_pb.RackInfo{
{
Id: "DefaultRack",
DataNodeInfos: []*master_pb.DataNodeInfo{
{Id: "node1"},
{Id: "node2"},
},
},
},
},
},
}
nodeCount := 0
EachDataNode(topologyInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) {
nodeCount++
})
// Default parity shards count is 4
minNodeCount := 4
// Case 1: Without -force
forceChanges := false
willProceed := forceChanges || nodeCount >= minNodeCount
assert.False(t, willProceed, "Should NOT proceed with %d nodes (min %d) without force", nodeCount, minNodeCount)
// Case 2: With -force
forceChanges = true
willProceed = forceChanges || nodeCount >= minNodeCount
assert.True(t, willProceed, "Should proceed with -force even with %d nodes", nodeCount)
}
func TestChunkVolumeIds(t *testing.T) {
vids := []needle.VolumeId{101, 102, 103, 104, 105}
assert.Equal(t, [][]needle.VolumeId{
{101, 102},
{103, 104},
{105},
}, ChunkVolumeIds(vids, 2))
assert.Equal(t, [][]needle.VolumeId{vids}, ChunkVolumeIds(vids, 0))
}
func ecShardVisibilityTestTopology(nodes ...*master_pb.DataNodeInfo) *master_pb.TopologyInfo {
return &master_pb.TopologyInfo{
DataCenterInfos: []*master_pb.DataCenterInfo{{
Id: "dc1",
RackInfos: []*master_pb.RackInfo{{
Id: "rack1",
DataNodeInfos: nodes,
}},
}},
}
}
func TestCollectEcShardBitsByNode(t *testing.T) {
allBits := uint32(1)<<erasure_coding.TotalShardsCount - 1
// One node reports the volume's shards split across two disk types; the
// other node reports nothing for it.
node1 := &master_pb.DataNodeInfo{
Id: "node1:8080",
DiskInfos: map[string]*master_pb.DiskInfo{
"": {
MaxVolumeCount: 10,
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
{Id: 1, EcIndexBits: allBits & 0x00ff, DiskId: 0},
},
},
"ssd": {
Type: "ssd",
MaxVolumeCount: 10,
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
{Id: 1, EcIndexBits: allBits &^ 0x00ff, DiskId: 1},
{Id: 2, EcIndexBits: allBits, DiskId: 1},
},
},
},
}
node2 := &master_pb.DataNodeInfo{
Id: "node2:8080",
DiskInfos: map[string]*master_pb.DiskInfo{"": {MaxVolumeCount: 10}},
}
topo := ecShardVisibilityTestTopology(node1, node2)
byNode := CollectEcShardBitsByNode(topo, needle.VolumeId(1))
assert.Len(t, byNode, 1)
assert.Equal(t, erasure_coding.ShardBits(allBits), byNode[pb.NewServerAddressFromDataNode(node1)])
assert.Empty(t, CollectEcShardBitsByNode(topo, needle.VolumeId(3)))
}
func TestCollectEcShardBitsByNode_MixedGenerations(t *testing.T) {
allBits := uint32(1)<<erasure_coding.TotalShardsCount - 1
nodeWithGeneration := func(id string, ts int64) *master_pb.DataNodeInfo {
return &master_pb.DataNodeInfo{
Id: id,
DiskInfos: map[string]*master_pb.DiskInfo{
"": {
MaxVolumeCount: 10,
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
{Id: 1, EcIndexBits: allBits, DiskId: 0, EncodeTsNs: ts},
},
},
},
}
}
// A full orphaned older generation on node2 must not count: neither toward
// the registration union nor as a second holder.
fresh := nodeWithGeneration("node1:8080", 200)
orphan := nodeWithGeneration("node2:8080", 100)
byNode := CollectEcShardBitsByNode(ecShardVisibilityTestTopology(fresh, orphan), needle.VolumeId(1))
assert.Len(t, byNode, 1)
assert.Equal(t, erasure_coding.ShardBits(allBits), byNode[pb.NewServerAddressFromDataNode(fresh)])
// Un-stamped entries are the legacy generation zero: dropped when a stamped
// generation exists, all counted when nothing is stamped.
legacy := nodeWithGeneration("node2:8080", 0)
byNode = CollectEcShardBitsByNode(ecShardVisibilityTestTopology(fresh, legacy), needle.VolumeId(1))
assert.Len(t, byNode, 1)
assert.Equal(t, erasure_coding.ShardBits(allBits), byNode[pb.NewServerAddressFromDataNode(fresh)])
byNode = CollectEcShardBitsByNode(
ecShardVisibilityTestTopology(nodeWithGeneration("node1:8080", 0), nodeWithGeneration("node2:8080", 0)),
needle.VolumeId(1))
assert.Len(t, byNode, 2)
}
func TestEcShardsClumpedOnOneNode(t *testing.T) {
allBits := uint32(1)<<erasure_coding.TotalShardsCount - 1
holderNode := func() *master_pb.DataNodeInfo {
return &master_pb.DataNodeInfo{
Id: "node1:8080",
DiskInfos: map[string]*master_pb.DiskInfo{
"": {
MaxVolumeCount: 10,
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
{Id: 1, EcIndexBits: allBits, DiskId: 0},
},
},
},
}
}
emptyNode := func(id string, maxVolumeCount, volumeCount int64) *master_pb.DataNodeInfo {
return &master_pb.DataNodeInfo{
Id: id,
DiskInfos: map[string]*master_pb.DiskInfo{
"": {MaxVolumeCount: maxVolumeCount, VolumeCount: volumeCount},
},
}
}
// All shards on one node while another node has free slots: clumped.
node1 := holderNode()
topo := ecShardVisibilityTestTopology(node1, emptyNode("node2:8080", 10, 0))
holder, clumped := ecShardsClumpedOnOneNode(topo, needle.VolumeId(1), types.HardDriveType)
assert.True(t, clumped)
assert.Equal(t, pb.NewServerAddressFromDataNode(node1), holder)
// The only other node has no free slots: the balance could not have spread
// the shards, so deletion may proceed.
topo = ecShardVisibilityTestTopology(holderNode(), emptyNode("node2:8080", 1, 1))
_, clumped = ecShardsClumpedOnOneNode(topo, needle.VolumeId(1), types.HardDriveType)
assert.False(t, clumped)
// Single-node cluster: not a clump.
topo = ecShardVisibilityTestTopology(holderNode())
_, clumped = ecShardsClumpedOnOneNode(topo, needle.VolumeId(1), types.HardDriveType)
assert.False(t, clumped)
// Shards spread across two nodes: not a clump.
spread1 := holderNode()
spread1.DiskInfos[""].EcShardInfos[0].EcIndexBits = allBits & 0x007f
spread2 := emptyNode("node2:8080", 10, 0)
spread2.DiskInfos[""].EcShardInfos = []*master_pb.VolumeEcShardInformationMessage{
{Id: 1, EcIndexBits: allBits &^ 0x007f, DiskId: 0},
}
topo = ecShardVisibilityTestTopology(spread1, spread2)
_, clumped = ecShardsClumpedOnOneNode(topo, needle.VolumeId(1), types.HardDriveType)
assert.False(t, clumped)
// No shards visible at all: the recoverability check owns that case.
topo = ecShardVisibilityTestTopology(emptyNode("node1:8080", 10, 0), emptyNode("node2:8080", 10, 0))
_, clumped = ecShardsClumpedOnOneNode(topo, needle.VolumeId(1), types.HardDriveType)
assert.False(t, clumped)
// A full orphaned older generation on another node must not pose as a
// second holder and defeat the clump detection.
fresh := holderNode()
fresh.DiskInfos[""].EcShardInfos[0].EncodeTsNs = 200
orphan := emptyNode("node2:8080", 10, 0)
orphan.DiskInfos[""].EcShardInfos = []*master_pb.VolumeEcShardInformationMessage{
{Id: 1, EcIndexBits: allBits, DiskId: 0, EncodeTsNs: 100},
}
topo = ecShardVisibilityTestTopology(fresh, orphan, emptyNode("node3:8080", 10, 0))
holder, clumped = ecShardsClumpedOnOneNode(topo, needle.VolumeId(1), types.HardDriveType)
assert.True(t, clumped)
assert.Equal(t, pb.NewServerAddressFromDataNode(fresh), holder)
}
// A volume that lived on ssd gets its shards written beside it, on ssd, while
// -diskType still defaults to hdd. The pre-delete check must count them anyway:
// scoping the count to the hdd bucket saw a complete set as zero shards and
// aborted the encode, leaving the volume as both a .dat and a full shard set.
func TestEcShardCountIgnoresDiskTypeOfTheShards(t *testing.T) {
allBits := uint32(1)<<erasure_coding.TotalShardsCount - 1
node := &master_pb.DataNodeInfo{
Id: "node1:8080",
DiskInfos: map[string]*master_pb.DiskInfo{
"": {MaxVolumeCount: 10},
"ssd": {Type: "ssd", MaxVolumeCount: 10, EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{{Id: 1, EcIndexBits: allBits, DiskId: 1}}},
},
}
topo := ecShardVisibilityTestTopology(node)
var union erasure_coding.ShardBits
for _, bits := range CollectEcShardBitsByNode(topo, needle.VolumeId(1)) {
union |= bits
}
assert.Equal(t, erasure_coding.TotalShardsCount, union.Count(),
"shards on a non-default medium must still be counted")
degraded, err := erasure_coding.RequireRecoverableShardSet(
1, union, erasure_coding.DataShardsCount, erasure_coding.TotalShardsCount)
assert.NoError(t, err, "a complete shard set must not read as unrecoverable")
assert.False(t, degraded)
// The decode's node view unions across buckets for the same reason: a
// decode scoped to the hdd bucket would report this decodable volume as
// having no shards at all.
byNode, _ := collectEcNodeShardsInfo(topo, needle.VolumeId(1))
assert.Len(t, byNode, 1, "decode discovery must see shards on a non-default medium")
for _, si := range byNode {
assert.Equal(t, erasure_coding.TotalShardsCount, si.Count(),
"decode discovery must include every shard on the non-default medium")
}
}
// The message an aborted deletion leaves behind is all the operator has to go
// on, so it has to name which shards were found and not only how many: a set
// holding 0-9 and one holding 4-13 are both "10 shards", and only the ids say
// whether what survived can rebuild the volume.
func TestEcShardSummaryNamesTheShardIds(t *testing.T) {
byNode := map[pb.ServerAddress]erasure_coding.ShardBits{
"node2:8080": erasure_coding.ShardBits(0).Set(1).Set(2),
"node1:8080": erasure_coding.ShardBits(0).Set(0).Set(12),
}
assert.Equal(t, []string{
"node1:8080=2 shards [0 12]",
"node2:8080=2 shards [1 2]",
}, ecShardSummaryByNode(byNode))
}
// A retried ec.encode clears the leftover shards of an interrupted run before
// the source health check, but the topology snapshot predates that sweep: the
// leftovers still depress FreeVolumeCount on the source disk. The counts must
// refund those slots, or the retry refuses the very volume it just cleaned up
// after. Numbers mirror the interruption-matrix failure: 8 slots, 4 volumes,
// 8 shards of other EC volumes, plus 14 leftover shards of volume 3, so the
// master charges ceil(22/10)=3 slots and reports free=1.
func TestCollectSourceFreeVolumeCountsRefundsClearedShards(t *testing.T) {
newTopo := func() *master_pb.TopologyInfo {
return &master_pb.TopologyInfo{
DataCenterInfos: []*master_pb.DataCenterInfo{{
Id: "dc1",
RackInfos: []*master_pb.RackInfo{{
Id: "rack0",
DataNodeInfos: []*master_pb.DataNodeInfo{{
Id: "127.0.0.1:8110",
Address: "127.0.0.1:8110",
DiskInfos: map[string]*master_pb.DiskInfo{
"hdd": {
Type: "hdd",
MaxVolumeCount: 8,
VolumeCount: 4,
FreeVolumeCount: 1,
VolumeInfos: []*master_pb.VolumeInformationMessage{
{Id: 3}, {Id: 6},
},
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
{Id: 1, EcIndexBits: 0b00000010001, DiskId: 0}, // shards 0,4
{Id: 1, EcIndexBits: 0b10000000100, DiskId: 1}, // shards 2,10
{Id: 2, EcIndexBits: 0b00000010001, DiskId: 0},
{Id: 2, EcIndexBits: 0b10000000100, DiskId: 1},
{Id: 3, EcIndexBits: 0b11111111111111, DiskId: 0}, // interrupted run's leftovers
},
},
},
}},
}},
}},
}
}
counts := collectSourceFreeVolumeCounts(newTopo(), []needle.VolumeId{3}, nil)
assert.Equal(t, 3, counts["3-127.0.0.1:8110"],
"the cleared leftovers' ceil(22/10)-ceil(8/10)=2 slots must be refunded")
counts = collectSourceFreeVolumeCounts(newTopo(), []needle.VolumeId{6}, nil)
assert.Equal(t, 1, counts["6-127.0.0.1:8110"],
"no refund when the encoded volume has no leftover shards")
skipped := map[pb.ServerAddress]struct{}{"127.0.0.1:8110": {}}
counts = collectSourceFreeVolumeCounts(newTopo(), []needle.VolumeId{3}, skipped)
assert.Equal(t, 1, counts["3-127.0.0.1:8110"],
"no refund on a node the sweep skipped as unreachable")
}
func TestEcShardPlacementViolations(t *testing.T) {
node := func(id string, shardBits uint32) *master_pb.DataNodeInfo {
return &master_pb.DataNodeInfo{
Id: id,
DiskInfos: map[string]*master_pb.DiskInfo{
"": {
MaxVolumeCount: 10,
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
{Id: 1, EcIndexBits: shardBits, DiskId: 0},
},
},
},
}
}
topo := &master_pb.TopologyInfo{
DataCenterInfos: []*master_pb.DataCenterInfo{{
Id: "dc1",
RackInfos: []*master_pb.RackInfo{
{Id: "rack1", DataNodeInfos: []*master_pb.DataNodeInfo{
node("n1:8080", 0b0011), // shards 0,1
node("n2:8080", 0b0100), // shard 2
}},
{Id: "rack2", DataNodeInfos: []*master_pb.DataNodeInfo{
node("n3:8080", 0b1000), // shard 3
}},
},
}},
}
vid := needle.VolumeId(1)
violationsOf := func(topo *master_pb.TopologyInfo, rp *super_block.ReplicaPlacement) []string {
return ecShardPlacementViolations(vid, CollectEcShardBitsByNode(topo, vid), ecShardRackByNode(topo), rp)
}
// No caps requested: nothing to violate.
assert.Empty(t, violationsOf(topo, nil))
assert.Empty(t, violationsOf(topo, &super_block.ReplicaPlacement{}))
// Node cap: n1 holds 2 shards, over the requested 1 per node.
violations := violationsOf(topo, &super_block.ReplicaPlacement{SameRackCount: 1})
assert.Len(t, violations, 1)
assert.Contains(t, violations[0], "per node")
// Rack cap: rack1 holds 3 shards, over the requested 2 per rack; the
// same-named rack in another dc is a different failure domain.
topo2 := &master_pb.TopologyInfo{
DataCenterInfos: []*master_pb.DataCenterInfo{
{Id: "dc1", RackInfos: []*master_pb.RackInfo{
{Id: "rack1", DataNodeInfos: []*master_pb.DataNodeInfo{
node("n1:8080", 0b0111), // shards 0,1,2
}},
}},
{Id: "dc2", RackInfos: []*master_pb.RackInfo{
{Id: "rack1", DataNodeInfos: []*master_pb.DataNodeInfo{
node("n9:8080", 0b1000), // shard 3
}},
}},
},
}
violations = violationsOf(topo2, &super_block.ReplicaPlacement{DiffRackCount: 2})
assert.Len(t, violations, 1)
assert.Contains(t, violations[0], "dc1:rack1")
// Caps that fit the layout: no violations.
assert.Empty(t, violationsOf(topo, &super_block.ReplicaPlacement{SameRackCount: 2, DiffRackCount: 3}))
}
func TestVerifyEcShardsBeforeDeletePlacement(t *testing.T) {
oldAttempts, oldInterval := verifyEcShardAttempts, verifyEcShardRetryInterval
verifyEcShardAttempts, verifyEcShardRetryInterval = 2, time.Millisecond
t.Cleanup(func() {
verifyEcShardAttempts, verifyEcShardRetryInterval = oldAttempts, oldInterval
})
vid := needle.VolumeId(1)
topologyWith := func(nodeShards map[string]uint32) *master_pb.TopologyInfo {
var nodes []*master_pb.DataNodeInfo
for id, bits := range nodeShards {
si := erasure_coding.NewShardsInfo()
for sid := 0; sid < erasure_coding.TotalShardsCount; sid++ {
if bits&(1<<sid) != 0 {
si.Set(erasure_coding.NewShardInfo(erasure_coding.ShardId(sid), 1))
}
}
nodes = append(nodes, &master_pb.DataNodeInfo{
Id: id,
DiskInfos: map[string]*master_pb.DiskInfo{"": {
MaxVolumeCount: 100,
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{{
Id: uint32(vid),
EcIndexBits: si.Bitmap(),
ShardSizes: si.SizesInt64(),
EncodeTsNs: 1,
}},
}},
})
}
return &master_pb.TopologyInfo{DataCenterInfos: []*master_pb.DataCenterInfo{{
Id: "dc1",
RackInfos: []*master_pb.RackInfo{{Id: "rack1", DataNodeInfos: nodes}},
}}}
}
envWith := func(topo *master_pb.TopologyInfo) *Env {
return &Env{FetchTopology: func(delay time.Duration) (*master_pb.TopologyInfo, uint64, error) {
return topo, 0, nil
}}
}
rp := &super_block.ReplicaPlacement{SameRackCount: 1}
// One shard per node: the requested cap is met, deletion is allowed.
onePerNode := map[string]uint32{}
for i := 0; i < erasure_coding.TotalShardsCount; i++ {
onePerNode[fmt.Sprintf("n%d:8080", i)] = 1 << i
}
assert.NoError(t, verifyEcShardsBeforeDelete(envWith(topologyWith(onePerNode)), []needle.VolumeId{vid}, types.HardDriveType, false, rp))
// All 14 shards on one node: over the cap, so the gate refuses to let the
// source volume be deleted.
err := verifyEcShardsBeforeDelete(envWith(topologyWith(map[string]uint32{"n1:8080": 0x3FFF})), []needle.VolumeId{vid}, types.HardDriveType, false, rp)
assert.Error(t, err)
assert.Contains(t, err.Error(), "does not meet the requested replica placement limits")
}