SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
master: shed assigns retryably until volume servers register capacity (#11032)
Chris Lud3b8030a69de
diff --git a/weed/server/master_grpc_server_assign.go b/weed/server/master_grpc_server_assign.goindex 0f037dcd18..02f45269a8 100644--- a/weed/server/master_grpc_server_assign.go+++ b/weed/server/master_grpc_server_assign.go@@ -117,7 +117,7 @@ func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequestfid, count, dnList, shouldGrow, err := ms.Topo.PickForWrite(req.Count, option, vl, req.ExpectedDataSize)if shouldGrow && !initiatedGrow && !ms.option.VolumeGrowthDisabled && vl.AddGrowRequestIfAbsent() {initiatedGrow = true- if err != nil && ms.Topo.AvailableSpaceFor(option) <= 0 {+ if err != nil && ms.Topo.AvailableSpaceFor(option) <= 0 && ms.Topo.CapacityFor(option) > 0 {err = fmt.Errorf("%s and no free volumes left for %s", err.Error(), option.String())}ms.volumeGrowthRequestChan <- &topology.VolumeGrowRequest{@@ -136,7 +136,14 @@ func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequest}if shouldGrow {if ms.Topo.AvailableSpaceFor(option) <= 0 {- break // out of space: surface the real error, not a retryable shed+ if ms.Topo.CapacityFor(option) > 0 {+ break // out of space: surface the real error, not a retryable shed+ }+ // No capacity registered for this disk type yet, typically a+ // just-started cluster whose volume servers have not+ // heartbeated. Shed retryably so the first write rides out+ // the startup window instead of failing outright.+ return nil, status.Errorf(codes.ResourceExhausted, "no volume server capacity registered yet for %s", option.String())}// Only the initiator waits, and only while the growth it triggered// is still pending: followers shed fast so a herd doesn't pin adiff --git a/weed/server/master_grpc_server_assign_test.go b/weed/server/master_grpc_server_assign_test.goindex 7b65002f8a..acd8ea60a7 100644--- a/weed/server/master_grpc_server_assign_test.go+++ b/weed/server/master_grpc_server_assign_test.go@@ -175,9 +175,24 @@ func TestAssignAbortsOnCancel(t *testing.T) {// Out of space, Assign fails fast with the real error rather than masking it as// a retryable "growth in progress".func TestAssignFailsFastWhenOutOfSpace(t *testing.T) {- ms := newLeaderMaster() // no data nodes -> no free space+ ms := newLeaderMaster()+ // One slot, already taken by another collection's volume: capacity is+ // registered but genuinely exhausted.+ dn := ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").+ GetOrCreateDataNode("127.0.0.1", 8080, 18080, "127.0.0.1", "dn1", map[string]uint32{"": 1})+ rp, err := super_block.NewReplicaPlacementFromString("000")+ require.NoError(t, err)+ v := storage.VolumeInfo{+ Id: needle.VolumeId(1),+ Collection: "other",+ Version: needle.GetCurrentVersion(),+ ReplicaPlacement: rp,+ Ttl: needle.EMPTY_TTL,+ }+ dn.UpdateVolumes([]storage.VolumeInfo{v})+ ms.Topo.RegisterVolumeLayout(v, dn)- req := &master_pb.AssignRequest{Count: 1, Replication: "000"}+ req := &master_pb.AssignRequest{Count: 1, Replication: "000", Collection: "fresh"}start := time.Now()resp, err := ms.Assign(context.Background(), req)@@ -187,7 +202,29 @@ func TestAssignFailsFastWhenOutOfSpace(t *testing.T) {require.Nil(t, resp)if st, ok := status.FromError(err); ok {assert.NotEqual(t, codes.Unavailable, st.Code())+ assert.NotEqual(t, codes.ResourceExhausted, st.Code())}assert.Contains(t, err.Error(), "no free volumes left")assert.Less(t, elapsed, 2*time.Second)}++// A topology with no registered capacity is a cluster whose volume servers have+// not heartbeated yet, not one that is full: the first write to a fresh bucket+// races the volume server registration at startup, so Assign must shed with a+// retryable code instead of failing the write outright.+func TestAssignShedsRetryablyBeforeCapacityRegisters(t *testing.T) {+ ms := newLeaderMaster() // no data nodes: nothing has heartbeated yet++ req := &master_pb.AssignRequest{Count: 1, Replication: "000"}++ start := time.Now()+ resp, err := ms.Assign(context.Background(), req)+ elapsed := time.Since(start)++ require.Error(t, err)+ require.Nil(t, resp)+ st, ok := status.FromError(err)+ require.True(t, ok)+ assert.Equal(t, codes.ResourceExhausted, st.Code())+ assert.Less(t, elapsed, 2*time.Second)+}diff --git a/weed/topology/node.go b/weed/topology/node.goindex dc53007898..8d82fdfe2c 100644--- a/weed/topology/node.go+++ b/weed/topology/node.go@@ -313,6 +313,13 @@ func (n *NodeImpl) AvailableSpaceFor(option *VolumeGrowOption) int64 {return freeVolumeSlotCount}+// CapacityFor is the total registered volume slots for the option's disk type;+// zero means no volume server has reported capacity for it yet.+func (n *NodeImpl) CapacityFor(option *VolumeGrowOption) int64 {+ t := n.getOrCreateDisk(option.DiskType)+ return atomic.LoadInt64(&t.maxVolumeCount) + atomic.LoadInt64(&t.remoteVolumeCount)+}+// AvailableSpaceForReservation returns available space considering existing reservationsfunc (n *NodeImpl) AvailableSpaceForReservation(option *VolumeGrowOption) int64 {baseAvailable := n.AvailableSpaceFor(option)