Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

ec.decode: finish the cleanup an interrupted decode left behind (#10767)

Chris Lu829064af71aa
diff --git a/weed/ec/ec_decode.go b/weed/ec/ec_decode.go
index c236b35d84..a208305295 100644
--- a/weed/ec/ec_decode.go
+++ b/weed/ec/ec_decode.go
@@ -35,6 +35,29 @@ func DoEcDecode(env *Env, topoInfo *master_pb.TopologyInfo, collection string, v
return fmt.Errorf("no EC shards found for volume %d", vid)
}
+ // A decode interrupted while deleting the shards leaves the regenerated
+ // volume in place with the set half gone, so a re-run finds both and fails
+ // re-collecting the first shard the interrupted run removed.
+ //
+ // Finding a volume beside the shards is not enough to act on: an encode
+ // interrupted before it deleted the original leaves the same shape, as does
+ // a decode killed while generating, whose volume may be half written. Both
+ // of those leave the shard set COMPLETE. Only the deletion phase can remove
+ // a data shard, so require one to be gone -- that is also exactly the state
+ // no decode can recover from, which makes finishing the cleanup the only
+ // move left rather than a choice between two.
+ if _, found := missingDataShard(nodeToEcShardsInfo, dataShards); found {
+ holder, hasVolume := regularVolumeHolder(topoInfo, vid)
+ if !hasVolume {
+ return fmt.Errorf("volume %d cannot be decoded: data shards are missing and no decoded volume exists to finish", vid)
+ }
+ if err := verifyDecodedVolumeBeforeDelete(env.GrpcDialOption, holder, vid); err != nil {
+ return fmt.Errorf("volume %d is already decoded on %s but did not verify, keeping its ec shards: %v", vid, holder, err)
+ }
+ fmt.Printf("volume %d is already decoded on %s; deleting the ec shards the interrupted run left behind\n", vid, holder)
+ return unmountAndDeleteEcShardsWithPrefix("deleteDecodedEcShards", env.GrpcDialOption, collection, nodeToEcShardsInfo, vid)
+ }
+
var originalShardCounts map[pb.ServerAddress]int
if diskUsageState != nil {
originalShardCounts = make(map[pb.ServerAddress]int, len(nodeToEcShardsInfo))
@@ -145,6 +168,48 @@ func unmountAndDeleteEcShardsWithPrefix(prefix string, grpcDialOption grpc.DialO
return ewg.Wait()
}
+// missingDataShard reports the first data shard absent from every holder.
+// Parity shards are not enough to answer this: the decode rebuilds the volume
+// from the data shards, so one of those going missing is what makes a re-run
+// impossible.
+func missingDataShard(nodeToShardsInfo map[pb.ServerAddress]*erasure_coding.ShardsInfo, dataShards int) (erasure_coding.ShardId, bool) {
+ var present erasure_coding.ShardBits
+ for _, si := range nodeToShardsInfo {
+ for _, id := range si.Ids() {
+ present = present.Set(id)
+ }
+ }
+ for id := 0; id < dataShards; id++ {
+ if !present.Has(erasure_coding.ShardId(id)) {
+ return erasure_coding.ShardId(id), true
+ }
+ }
+ return 0, false
+}
+
+// regularVolumeHolder returns a server already serving vid as a regular
+// volume. A decode only finds one when an earlier run was interrupted between
+// regenerating the volume and deleting the shards it came from.
+func regularVolumeHolder(topoInfo *master_pb.TopologyInfo, vid needle.VolumeId) (pb.ServerAddress, bool) {
+ var holder pb.ServerAddress
+ found := false
+ EachDataNode(topoInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) {
+ if found {
+ return
+ }
+ for _, diskInfo := range dn.DiskInfos {
+ for _, vi := range diskInfo.VolumeInfos {
+ if needle.VolumeId(vi.Id) == vid {
+ holder = pb.NewServerAddressFromDataNode(dn)
+ found = true
+ return
+ }
+ }
+ }
+ })
+ return holder, found
+}
+
func verifyDecodedVolumeBeforeDelete(grpcDialOption grpc.DialOption, target pb.ServerAddress, vid needle.VolumeId) error {
var resp *volume_server_pb.ReadVolumeFileStatusResponse
if err := operation.WithVolumeServerClient(false, target, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
diff --git a/weed/ec/ec_decode_test.go b/weed/ec/ec_decode_test.go
new file mode 100644
index 0000000000..bf73126e86
--- /dev/null
+++ b/weed/ec/ec_decode_test.go
@@ -0,0 +1,91 @@
+package ec
+
+import (
+ "testing"
+
+ "github.com/seaweedfs/seaweedfs/weed/pb"
+ "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+)
+
+// shardsOn builds one holder's inventory.
+func shardsOn(ids ...int) *erasure_coding.ShardsInfo {
+ si := erasure_coding.NewShardsInfo()
+ for _, id := range ids {
+ si.Set(erasure_coding.NewShardInfo(erasure_coding.ShardId(id), 1024))
+ }
+ return si
+}
+
+// missingDataShard decides whether a decode may delete the shards of a volume
+// that already exists elsewhere. Only the deletion phase of a decode removes a
+// data shard, so the answer separates "an earlier decode was interrupted while
+// cleaning up" from the two states that look the same from the outside: an
+// encode interrupted before it deleted the original, and a decode killed while
+// generating. Both of those leave every shard in place, and the volume beside
+// them may be the untouched original or a half-written one -- neither is safe
+// to trade for the shards.
+func TestMissingDataShard(t *testing.T) {
+ const dataShards = 10
+
+ tests := []struct {
+ name string
+ holders map[pb.ServerAddress]*erasure_coding.ShardsInfo
+ wantID erasure_coding.ShardId
+ wantMissed bool
+ }{
+ {
+ name: "complete set on one holder decodes, so nothing is finished",
+ holders: map[pb.ServerAddress]*erasure_coding.ShardsInfo{
+ "server1:8080": shardsOn(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13),
+ },
+ },
+ {
+ name: "complete set spread across holders is still complete",
+ holders: map[pb.ServerAddress]*erasure_coding.ShardsInfo{
+ "server1:8080": shardsOn(0, 1, 2, 3, 4),
+ "server2:8080": shardsOn(5, 6, 7, 8, 9),
+ "server3:8080": shardsOn(10, 11, 12, 13),
+ },
+ },
+ {
+ name: "parity gone is not the deletion phase: the volume still decodes",
+ holders: map[pb.ServerAddress]*erasure_coding.ShardsInfo{
+ "server1:8080": shardsOn(0, 1, 2, 3, 4, 5, 6, 7, 8, 9),
+ },
+ },
+ {
+ name: "a data shard gone is a decode that was interrupted mid-cleanup",
+ holders: map[pb.ServerAddress]*erasure_coding.ShardsInfo{
+ "server1:8080": shardsOn(0, 1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 12, 13),
+ },
+ wantID: 6,
+ wantMissed: true,
+ },
+ {
+ name: "the first gap is reported, not the last",
+ holders: map[pb.ServerAddress]*erasure_coding.ShardsInfo{
+ "server1:8080": shardsOn(0, 3, 4, 5, 6, 7, 8, 9),
+ },
+ wantID: 1,
+ wantMissed: true,
+ },
+ {
+ name: "no holders at all leaves every data shard missing",
+ holders: map[pb.ServerAddress]*erasure_coding.ShardsInfo{},
+ wantID: 0,
+ wantMissed: true,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ id, missed := missingDataShard(tt.holders, dataShards)
+ if missed != tt.wantMissed {
+ t.Fatalf("missingDataShard = %v, want %v", missed, tt.wantMissed)
+ }
+ if missed && id != tt.wantID {
+ t.Errorf("missing shard id = %d, want %d", id, tt.wantID)
+ }
+ })
+ }
+}