SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package ec
import (
"bytes"
"fmt"
"reflect"
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
)
// TestEcCheckReplication exercises the ec.check.replication classification and reporting over a
// fake topology. EC volume 1 is fully replicated, EC volume 2 is
// under-replicated, and EC volume 3 is over-replicated.
func TestEcCheckReplication(t *testing.T) {
nodes := []*master_pb.DataNodeInfo{
{
Address: "localhost:1111",
DiskInfos: map[string]*master_pb.DiskInfo{
"d1": {
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
{
Id: 1,
EcIndexBits: 0b0000000011111,
},
{
Id: 2,
EcIndexBits: 0b0000000011111,
},
{
Id: 3,
EcIndexBits: 0b11111111111111,
},
},
},
},
},
{
Address: "localhost:2222",
DiskInfos: map[string]*master_pb.DiskInfo{
"d2": {
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
{
Id: 1,
EcIndexBits: 0b11111111100000,
},
{
Id: 2,
EcIndexBits: 0b11111000000000,
},
{
Id: 3,
EcIndexBits: 0b00000000001111,
},
},
},
},
},
}
testCases := []struct {
name string
showDetails bool
volumeIDMap map[uint32]bool
want string
wantErr error
}{
{
name: "default",
showDetails: false,
want: `Found 1/3 under-replicated EC volumes: [2]
Found 1/3 over-replicated EC volumes: [3]
`,
},
{
name: "single healthy volume",
showDetails: false,
volumeIDMap: map[uint32]bool{1: true},
want: `EC volumes are healthy.
`,
},
{
name: "filtered by volume ID",
showDetails: false,
volumeIDMap: map[uint32]bool{1: true, 2: true, 5: true},
want: `Found 1/2 under-replicated EC volumes: [2]
`,
},
{
name: "no valid volume IDs",
showDetails: false,
volumeIDMap: map[uint32]bool{5: true, 6: true},
want: "",
wantErr: fmt.Errorf("no EC volumes found"),
},
{
name: "with details",
showDetails: true,
want: `Found 1/3 under-replicated EC volumes: [2]
Found 1/3 over-replicated EC volumes: [3]
Shards map for under-replicated EC volume 2 (10/14 shards):
00 => [localhost:1111]
01 => [localhost:1111]
02 => [localhost:1111]
03 => [localhost:1111]
04 => [localhost:1111]
05 is missing
06 is missing
07 is missing
08 is missing
09 => [localhost:2222]
10 (parity) => [localhost:2222]
11 (parity) => [localhost:2222]
12 (parity) => [localhost:2222]
13 (parity) => [localhost:2222]
Shards map for over-replicated EC volume 3 (18/14 shards):
00 => [localhost:1111 localhost:2222]
01 => [localhost:1111 localhost:2222]
02 => [localhost:1111 localhost:2222]
03 => [localhost:1111 localhost:2222]
04 => [localhost:1111]
05 => [localhost:1111]
06 => [localhost:1111]
07 => [localhost:1111]
08 => [localhost:1111]
09 => [localhost:1111]
10 (parity) => [localhost:1111]
11 (parity) => [localhost:1111]
12 (parity) => [localhost:1111]
13 (parity) => [localhost:1111]
`,
},
}
for _, tc := range testCases {
var buf bytes.Buffer
runner := &ecCheckReplicationRunner{
writer: &buf,
dataNodes: nodes,
volumeIDMap: tc.volumeIDMap,
}
gotErr := runner.checkEcVolumes(tc.showDetails)
got := buf.String()
if got != tc.want {
t.Errorf("%s: got %v, want %v", tc.name, got, tc.want)
}
if !reflect.DeepEqual(gotErr, tc.wantErr) {
t.Errorf("%s: got error %v, want %v", tc.name, gotErr, tc.wantErr)
}
}
}
// TestEcCheckReplicationNoVolumes checks the empty-topology behavior: an unfiltered
// run over a cluster with no EC volumes is healthy (informational message, no
// error), while a run filtered to volume IDs that are not EC volumes errors.
func TestEcCheckReplicationNoVolumes(t *testing.T) {
// a data node reporting a normal volume but no EC shards
nodes := []*master_pb.DataNodeInfo{
{
Address: "localhost:1111",
DiskInfos: map[string]*master_pb.DiskInfo{"d1": {}},
},
}
t.Run("unfiltered", func(t *testing.T) {
var buf bytes.Buffer
runner := &ecCheckReplicationRunner{writer: &buf, dataNodes: nodes}
if err := runner.checkEcVolumes(false); err != nil {
t.Errorf("got error %v, want nil", err)
}
if got, want := buf.String(), "No EC volumes found.\n"; got != want {
t.Errorf("got %q, want %q", got, want)
}
})
t.Run("filtered no match", func(t *testing.T) {
var buf bytes.Buffer
runner := &ecCheckReplicationRunner{writer: &buf, dataNodes: nodes, volumeIDMap: map[uint32]bool{5: true}}
if err := runner.checkEcVolumes(false); err == nil {
t.Errorf("got nil error, want %q", "no EC volumes found")
}
if got := buf.String(); got != "" {
t.Errorf("got output %q, want none", got)
}
})
}
// TestEcCheckReplicationUnexpectedShard verifies that a shard id beyond a volume's
// expected data+parity range is flagged as over-replication and surfaced in the
// details map, rather than being silently ignored. Volume 20 is a healthy 10+4
// (shards 0..13 on nodeA) with a stray shard 14 on nodeB.
func TestEcCheckReplicationUnexpectedShard(t *testing.T) {
nodes := []*master_pb.DataNodeInfo{
{
Address: "nodeA",
DiskInfos: map[string]*master_pb.DiskInfo{
"d1": {
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
{Id: 20, EcIndexBits: 0b11111111111111}, // shards 0..13
},
},
},
},
{
Address: "nodeB",
DiskInfos: map[string]*master_pb.DiskInfo{
"d2": {
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
{Id: 20, EcIndexBits: 0b100000000000000}, // stray shard 14
},
},
},
},
}
want := `Found 1/1 over-replicated EC volumes: [20]
Shards map for over-replicated EC volume 20 (15/14 shards):
00 => [nodeA]
01 => [nodeA]
02 => [nodeA]
03 => [nodeA]
04 => [nodeA]
05 => [nodeA]
06 => [nodeA]
07 => [nodeA]
08 => [nodeA]
09 => [nodeA]
10 (parity) => [nodeA]
11 (parity) => [nodeA]
12 (parity) => [nodeA]
13 (parity) => [nodeA]
14 (unexpected) => [nodeB]
`
var buf bytes.Buffer
runner := &ecCheckReplicationRunner{writer: &buf, dataNodes: nodes}
if err := runner.checkEcVolumes(true); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got := buf.String(); got != want {
t.Errorf("unexpected shard:\n got: %q\nwant: %q", got, want)
}
}
// TestEcCheckReplicationMixedAnomalies verifies that a volume with both a missing
// shard and a duplicated shard is reported in both lists. Volume 30 has shard 5
// missing everywhere (under-replicated) and shard 0 on two nodes (over-replicated).
func TestEcCheckReplicationMixedAnomalies(t *testing.T) {
nodes := []*master_pb.DataNodeInfo{
{
Address: "nodeA",
DiskInfos: map[string]*master_pb.DiskInfo{
"d1": {
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
{Id: 30, EcIndexBits: 0b11111111011111}, // shards 0..13 except 5
},
},
},
},
{
Address: "nodeB",
DiskInfos: map[string]*master_pb.DiskInfo{
"d2": {
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
{Id: 30, EcIndexBits: 0b00000000000001}, // duplicate of shard 0
},
},
},
},
}
want := `Found 1/1 under-replicated EC volumes: [30]
Found 1/1 over-replicated EC volumes: [30]
`
var buf bytes.Buffer
runner := &ecCheckReplicationRunner{writer: &buf, dataNodes: nodes}
if err := runner.checkEcVolumes(false); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got := buf.String(); got != want {
t.Errorf("mixed anomalies:\n got: %q\nwant: %q", got, want)
}
}