SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
perf(weed/topology): diff a heartbeat without copying the volume map (#10608)
Chris Lu2ec899bdee9f
diff --git a/weed/topology/data_node.go b/weed/topology/data_node.goindex d20b6194f9..a1ad201a9b 100644--- a/weed/topology/data_node.go+++ b/weed/topology/data_node.go@@ -78,22 +78,18 @@ func (dn *DataNode) doAddOrUpdateVolume(v storage.VolumeInfo) (isNew, isChanged// used in master to notify master clients of these changes.func (dn *DataNode) UpdateVolumes(actualVolumes []storage.VolumeInfo) (newVolumes, deletedVolumes, changedVolumes []storage.VolumeInfo) {- actualVolumeMap := make(map[needle.VolumeId]storage.VolumeInfo)+ actualVolumeIds := make(map[needle.VolumeId]struct{}, len(actualVolumes))for _, v := range actualVolumes {- actualVolumeMap[v.Id] = v+ actualVolumeIds[v.Id] = struct{}{}}dn.Lock()defer dn.Unlock()- existingVolumes := dn.getVolumes()-- for _, v := range existingVolumes {- vid := v.Id- if _, ok := actualVolumeMap[vid]; !ok {- glog.V(0).Infoln("Deleting volume id:", vid)- disk := dn.getOrCreateDisk(v.DiskType)- disk.DeleteVolumeById(vid)+ for _, c := range dn.children {+ disk := c.(*Disk)+ for _, v := range disk.RemoveVolumesNotIn(actualVolumeIds) {+ glog.V(0).Infoln("Deleting volume id:", v.Id)deletedVolumes = append(deletedVolumes, v)deltaDiskUsage := &DiskUsageCounts{}diff --git a/weed/topology/disk.go b/weed/topology/disk.goindex e77715e88b..a7f819eb92 100644--- a/weed/topology/disk.go+++ b/weed/topology/disk.go@@ -211,6 +211,21 @@ func (d *Disk) GetVolumes() (ret []storage.VolumeInfo) {return ret}+// RemoveVolumesNotIn drops the volumes whose ids are absent from keep and+// returns them, so a heartbeat can be diffed without first copying the whole+// volume map out.+func (d *Disk) RemoveVolumesNotIn(keep map[needle.VolumeId]struct{}) (removed []storage.VolumeInfo) {+ d.Lock()+ defer d.Unlock()+ for vid, v := range d.volumes {+ if _, ok := keep[vid]; !ok {+ removed = append(removed, v)+ delete(d.volumes, vid)+ }+ }+ return removed+}+func (d *Disk) GetVolumesById(id needle.VolumeId) (storage.VolumeInfo, error) {d.RLock()defer d.RUnlock()