SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
shell: confirm trim survivors still serve the volume before deleting (#12590)
Chris Lua3b68aa37cc8
diff --git a/weed/shell/command_volume_fix_replication.go b/weed/shell/command_volume_fix_replication.goindex 7cb38fd3ba..f1e2d69804 100644--- a/weed/shell/command_volume_fix_replication.go+++ b/weed/shell/command_volume_fix_replication.go@@ -382,6 +382,18 @@ func (c *commandVolumeFixReplication) deleteOneVolume(commandEnv *CommandEnv, wrbreak}}+ // A survivor can go away while the indexes are read.+ for _, replicaB := range replicas {+ if checkErr != nil {+ break+ }+ if replicaB.location.dataNode == replica.location.dataNode {+ continue+ }+ if _, checkErr = readVolumeStatus(commandEnv.option.GrpcDialOption, pb.NewServerAddressFromDataNode(replicaB.location.dataNode), needle.VolumeId(replicaB.info.Id)); checkErr != nil {+ fmt.Fprintf(writer, "skip deleting volume %d from %s: %s no longer serves it: %v\n", replica.info.Id, replica.location.dataNode.Id, replicaB.location.dataNode.Id, checkErr)+ }+ }if checkErr != nil {continue}diff --git a/weed/shell/command_volume_fix_replication_test.go b/weed/shell/command_volume_fix_replication_test.goindex 5f9d711629..23e06938af 100644--- a/weed/shell/command_volume_fix_replication_test.go+++ b/weed/shell/command_volume_fix_replication_test.go@@ -1,10 +1,18 @@package shellimport (+ "context"+ "io"+ "sync/atomic""testing""github.com/seaweedfs/seaweedfs/weed/pb/master_pb"+ "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"+ "github.com/seaweedfs/seaweedfs/weed/storage/needle_map""github.com/seaweedfs/seaweedfs/weed/storage/super_block"+ "github.com/seaweedfs/seaweedfs/weed/storage/types"+ "google.golang.org/grpc/codes"+ "google.golang.org/grpc/status")type testcase struct {@@ -731,3 +739,71 @@ func TestClassifyReplicaSet(t *testing.T) {})}}++type trimTestVolumeServer struct {+ volume_server_pb.UnimplementedVolumeServerServer+ afterIndex func()+ gone atomic.Bool+ deletes atomic.Int32+}++func (s *trimTestVolumeServer) CopyFile(_ *volume_server_pb.CopyFileRequest, stream volume_server_pb.VolumeServer_CopyFileServer) error {+ if s.gone.Load() {+ return status.Error(codes.NotFound, "volume not found")+ }+ if err := stream.Send(&volume_server_pb.CopyFileResponse{FileContent: needle_map.ToBytes(1, types.ToOffset(8), 4)}); err != nil {+ return err+ }+ if s.afterIndex != nil {+ s.afterIndex()+ }+ return nil+}++func (s *trimTestVolumeServer) VolumeStatus(context.Context, *volume_server_pb.VolumeStatusRequest) (*volume_server_pb.VolumeStatusResponse, error) {+ if s.gone.Load() {+ return nil, status.Error(codes.NotFound, "volume not found")+ }+ return &volume_server_pb.VolumeStatusResponse{}, nil+}++func (s *trimTestVolumeServer) VolumeDelete(context.Context, *volume_server_pb.VolumeDeleteRequest) (*volume_server_pb.VolumeDeleteResponse, error) {+ s.deletes.Add(1)+ return &volume_server_pb.VolumeDeleteResponse{}, nil+}++func TestDeleteOneVolumeRechecksSurvivorsBeforeDelete(t *testing.T) {+ for _, tc := range []struct {+ name string+ survivorGoes bool+ wantDeleted int+ }{+ {name: "survivor stays", wantDeleted: 1},+ {name: "survivor gone after its index read", survivorGoes: true, wantDeleted: 0},+ } {+ t.Run(tc.name, func(t *testing.T) {+ doomed, survivor := &trimTestVolumeServer{}, &trimTestVolumeServer{}+ if tc.survivorGoes {+ survivor.afterIndex = func() { survivor.gone.Store(true) }+ }+ commandEnv, _ := configureReplicationTestEnv()+ replicas := []*VolumeReplica{+ {location: &location{dataNode: startConfigureReplicationTestVolumeServer(t, doomed, 1)}, info: &master_pb.VolumeInformationMessage{Id: 1}},+ {location: &location{dataNode: startConfigureReplicationTestVolumeServer(t, survivor, 1)}, info: &master_pb.VolumeInformationMessage{Id: 1}},+ }+ pickDoomed := func([]*VolumeReplica, *super_block.ReplicaPlacement) *VolumeReplica { return replicas[0] }++ c := &commandVolumeFixReplication{}+ deleted, err := c.deleteOneVolume(commandEnv, io.Discard, true, true, []uint32{1}, map[uint32][]*VolumeReplica{1: replicas}, pickDoomed)+ if err != nil {+ t.Fatalf("deleteOneVolume: %v", err)+ }+ if tc.survivorGoes && !survivor.gone.Load() {+ t.Fatal("the survivor never served its index")+ }+ if deleted != tc.wantDeleted || int(doomed.deletes.Load()) != tc.wantDeleted {+ t.Fatalf("deleted=%d VolumeDelete calls=%d, want %d", deleted, doomed.deletes.Load(), tc.wantDeleted)+ }+ })+ }+}