SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
placement: move the target picker out of the shell package (#10580)
Chris Luaceab0802e8e
diff --git a/weed/shell/volume_placement.go b/weed/placement/placement.gosimilarity index 78%rename from weed/shell/volume_placement.gorename to weed/placement/placement.goindex e2a78aef45..274fb3e132 100644--- a/weed/shell/volume_placement.go+++ b/weed/placement/placement.go@@ -1,4 +1,4 @@-package shell+package placementimport ("sort"@@ -46,7 +46,7 @@ type PlacementPreference struct {// since each pick would see the capacity its predecessors already took. Callers// therefore pass a snapshot they own and may mutate.func PickTarget(topo *master_pb.TopologyInfo, pref PlacementPreference) *master_pb.DataNodeInfo {- nodes := collectVolumeServersByDcRackNode(topo, pref.AnchorDataCenter, "", "")+ nodes := candidateNodes(topo, pref.AnchorDataCenter)var srcDc, srcRack stringfor _, n := range nodes {@@ -57,7 +57,7 @@ func PickTarget(topo *master_pb.TopologyInfo, pref PlacementPreference) *master_}type candidate struct {- node *Node+ node *nodelocality int // 0 same rack, 1 same dc, 2 elsewherebytes uint64hasBytes bool@@ -69,15 +69,13 @@ func PickTarget(topo *master_pb.TopologyInfo, pref PlacementPreference) *master_if n.info.Id == pref.Source || pref.Exclude[n.info.Id] {continue}- if !n.hasFreeVolumeSlot(pref.DiskType) {+ d := n.disk(pref.DiskType)+ if d == nil || d.VolumeCount >= d.MaxVolumeCount {continue}c := candidate{node: n}- _, free, ok := n.diskBytes(pref.DiskType)- c.bytes, c.hasBytes = free, ok- if d, found := n.info.DiskInfos[string(pref.DiskType)]; found && d != nil {- c.slots = d.MaxVolumeCount - d.VolumeCount- }+ c.bytes, c.hasBytes = d.DiskFreeBytes, d.DiskTotalBytes != 0+ c.slots = d.MaxVolumeCount - d.VolumeCountswitch {case srcRack != "" && n.rack == srcRack && n.dc == srcDc:c.locality = 0@@ -145,3 +143,35 @@ func max64(a, b int64) int64 {}return b}++// node is a volume server as a placement snapshot sees it: where it sits and+// what it has free. Deliberately smaller than the balancer's Node, which also+// carries the volumes it holds -- placement never needs those.+type node struct {+ info *master_pb.DataNodeInfo+ dc string+ rack string+}++func (n *node) disk(diskType types.DiskType) *master_pb.DiskInfo {+ d, found := n.info.DiskInfos[string(diskType)]+ if !found {+ return nil+ }+ return d+}++// candidateNodes flattens the topology, keeping one data center when anchored.+func candidateNodes(topo *master_pb.TopologyInfo, anchorDataCenter string) (nodes []*node) {+ for _, dc := range topo.GetDataCenterInfos() {+ if anchorDataCenter != "" && dc.GetId() != anchorDataCenter {+ continue+ }+ for _, rack := range dc.GetRackInfos() {+ for _, dn := range rack.GetDataNodeInfos() {+ nodes = append(nodes, &node{info: dn, dc: dc.GetId(), rack: rack.GetId()})+ }+ }+ }+ return nodes+}diff --git a/weed/shell/volume_placement_test.go b/weed/placement/placement_test.gosimilarity index 99%rename from weed/shell/volume_placement_test.gorename to weed/placement/placement_test.goindex f831ca3476..f36faa8fcd 100644--- a/weed/shell/volume_placement_test.go+++ b/weed/placement/placement_test.go@@ -1,4 +1,4 @@-package shell+package placementimport ("testing"