SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
ec.balance: verify shard landed on destination before deleting the source (#9858)
Chris Lua549580e6517
diff --git a/weed/worker/tasks/ec_balance/ec_balance_task.go b/weed/worker/tasks/ec_balance/ec_balance_task.goindex f9692d7f2f..cf7190f820 100644--- a/weed/worker/tasks/ec_balance/ec_balance_task.go+++ b/weed/worker/tasks/ec_balance/ec_balance_task.go@@ -12,6 +12,7 @@ import ("github.com/seaweedfs/seaweedfs/weed/pb""github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb""github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"+ "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding""github.com/seaweedfs/seaweedfs/weed/worker/types""github.com/seaweedfs/seaweedfs/weed/worker/types/base""google.golang.org/grpc"@@ -98,6 +99,16 @@ func (t *ECBalanceTask) Execute(ctx context.Context, params *worker_pb.TaskParamreturn fmt.Errorf("copy and mount shard: %v", err)}+ // Step 1.5: confirm the destination actually registered the shard(s)+ // before removing them from the source. A copy/mount RPC can return OK+ // while the shard isn't loadable on the destination; deleting the source+ // then would lose the shard. On a mismatch we keep the source so the+ // scanner retries (the move is reported failed).+ t.reportProgress(40.0, "Verifying EC shard(s) on destination")+ if err := t.verifyShardsOnDestination(ctx, params.VolumeId, targetAddr, source.ShardIds); err != nil {+ return err+ }+// Step 2: Unmount shard on sourcet.reportProgress(50.0, "Unmounting EC shard from source")if err := t.unmountShard(ctx, params.VolumeId, sourceAddr, source.ShardIds); err != nil {@@ -179,6 +190,25 @@ func (t *ECBalanceTask) unmountShard(ctx context.Context, volumeID uint32, addr})}+// verifyShardsOnDestination confirms targetAddr has registered every shard in+// shardIDs for volumeID, so the caller can safely delete the source copies.+func (t *ECBalanceTask) verifyShardsOnDestination(ctx context.Context, volumeID uint32, targetAddr pb.ServerAddress, shardIDs []uint32) error {+ _, perServer := erasure_coding.VerifyShardsAcrossServers(ctx, volumeID, []string{string(targetAddr)}, t.grpcDialOption)+ inv, ok := perServer[string(targetAddr)]+ if !ok {+ return fmt.Errorf("verify shard(s) on destination %s for volume %d: no inventory returned", targetAddr, volumeID)+ }+ if inv.QueryError != nil {+ return fmt.Errorf("verify shard(s) on destination %s for volume %d: %v", targetAddr, volumeID, inv.QueryError)+ }+ for _, sid := range shardIDs {+ if !inv.Bits.Has(erasure_coding.ShardId(sid)) {+ return fmt.Errorf("destination %s missing EC shard %d.%d after copy/mount; keeping source", targetAddr, volumeID, sid)+ }+ }+ return nil+}+// deleteShard deletes EC shards from a serverfunc (t *ECBalanceTask) deleteShard(ctx context.Context, volumeID uint32, collection string, addr pb.ServerAddress, shardIDs []uint32) error {return operation.WithVolumeServerClient(false, addr, t.grpcDialOption,