SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
fix(master): re-register volumes missing from the lookup index
Chris Lu0f1e50f9ec08
diff --git a/weed/topology/topology.go b/weed/topology/topology.goindex b99bbd816e..ddba44d2db 100644--- a/weed/topology/topology.go+++ b/weed/topology/topology.go@@ -586,6 +586,19 @@ func (t *Topology) SyncDataNodeRegistration(volumes []*master_pb.VolumeInformati}diskType := types.ToDiskType(v.DiskType)vl := t.GetVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl, diskType)+ // Self-heal: a volume reported by the data node but missing from the+ // lookup index is re-registered. This repairs the split left by a+ // disconnect/reconnect race, where UnRegisterDataNode dropped the volume+ // from vid2location but the reconnecting full heartbeat skipped it+ // (still in the disk map, so UpdateVolumes did not report it as new).+ // Without this, the volume stays visible in volume.list/admin UI yet+ // LookupVolume returns "volume id not found".+ if !vl.HasDataNode(v.Id, dn) {+ // vl is already resolved above; call it directly instead of+ // RegisterVolumeLayout, which would repeat the GetVolumeLayout lookup.+ vl.RegisterVolume(&v, dn)+ vl.EnsureCorrectWritables(&v)+ }if vl.UpdateVolumeSize(v.Id, v.Size, v.CompactRevision) {vl.AdjustActiveVolumeCountAfterRecovery(v.Id)}diff --git a/weed/topology/topology_test.go b/weed/topology/topology_test.goindex 415a7198dc..99124a600d 100644--- a/weed/topology/topology_test.go+++ b/weed/topology/topology_test.go@@ -573,3 +573,56 @@ func TestLookupDataNodeByAddress(t *testing.T) {t.Fatalf("address must be unregistered after UnRegisterDataNode, got %v", got)}}++// TestSyncDataNodeRegistrationReRegistersMissingVolume reproduces the divergence+// where a volume present on a data node (shown by volume.list / admin UI) is+// missing from the lookup index, which surfaces as "volume id not found" on+// LookupVolume. SetVolumeUnavailable (used by+// UnRegisterDataNode on a disconnect) drops the volume from the index, and the+// reconnecting full heartbeat used to skip it because it was no longer "new" to+// the disk map. The full heartbeat now self-heals.+func TestSyncDataNodeRegistrationReRegistersMissingVolume(t *testing.T) {+ topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)++ dc := topo.GetOrCreateDataCenter("dc1")+ rack := dc.GetOrCreateRack("rack1")+ dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", map[string]uint32{"": 25})++ vid := needle.VolumeId(18994)+ volumeMessage := &master_pb.VolumeInformationMessage{+ Id: uint32(vid),+ Size: 100,+ Collection: "drr",+ ReplicaPlacement: uint32(0),+ Version: uint32(needle.GetCurrentVersion()),+ Ttl: 0,+ }++ // Initial full heartbeat registers the volume in the lookup index.+ topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{volumeMessage}, dn)+ if got := topo.Lookup("", vid); len(got) != 1 {+ t.Fatalf("after registration: lookup %d got %v, want 1 location", vid, got)+ }++ // Drop the volume from the index the way UnRegisterDataNode does, but leave+ // it in the data node's disk map (the reconnecting heartbeat did not report+ // it as new).+ rp, _ := super_block.NewReplicaPlacementFromString("000")+ vl := topo.GetVolumeLayout("drr", rp, needle.EMPTY_TTL, types.HardDriveType)+ vl.SetVolumeUnavailable(dn, vid)++ // The empty entry must be removed, otherwise Lookup returns a non-nil empty+ // list that still reads as "not found".+ if got := topo.Lookup("", vid); got != nil {+ t.Fatalf("after SetVolumeUnavailable: expected lookup miss, got %v", got)+ }+ if _, err := dn.GetVolumesById(vid); err != nil {+ t.Fatalf("volume %d should still be in the data node disk map: %v", vid, err)+ }++ // The next full heartbeat re-registers the volume even though it is not new.+ topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{volumeMessage}, dn)+ if got := topo.Lookup("", vid); len(got) != 1 {+ t.Fatalf("after self-heal: lookup %d got %v, want 1 location", vid, got)+ }+}diff --git a/weed/topology/volume_layout.go b/weed/topology/volume_layout.goindex b25097e71a..2beedd82e7 100644--- a/weed/topology/volume_layout.go+++ b/weed/topology/volume_layout.go@@ -502,6 +502,25 @@ func (vl *VolumeLayout) Lookup(vid needle.VolumeId) []*DataNode {return nil}+// HasDataNode reports whether the layout already lists dn as a location for vid.+// Used to detect a volume that is present on a data node but missing from the+// lookup index, so it can be re-registered.+func (vl *VolumeLayout) HasDataNode(vid needle.VolumeId, dn *DataNode) bool {+ vl.accessLock.RLock()+ defer vl.accessLock.RUnlock()++ location, ok := vl.vid2location[vid]+ if !ok {+ return false+ }+ for _, n := range location.list {+ if n.Ip == dn.Ip && n.Port == dn.Port {+ return true+ }+ }+ return false+}+func (vl *VolumeLayout) ListVolumeServers() (nodes []*DataNode) {vl.accessLock.RLock()defer vl.accessLock.RUnlock()@@ -750,10 +769,21 @@ func (vl *VolumeLayout) SetVolumeUnavailable(dn *DataNode, vid needle.VolumeId)if location.Remove(dn) {vl.readonlyVolumes.Remove(vid, dn)vl.oversizedVolumes.Remove(vid, dn)+ wasWritable := falseif location.Length() < vl.rp.GetCopyCount() {glog.V(0).Infoln("Volume", vid, "has", location.Length(), "replica, less than required", vl.rp.GetCopyCount())- return vl.removeFromWritable(vid)+ wasWritable = vl.removeFromWritable(vid)+ }+ if location.Length() == 0 {+ // Drop the now-empty entry. Otherwise Lookup returns a non-nil+ // empty location list, which surfaces as "volume id not found"+ // even though the volume still appears in volume.list/admin UI.+ // Mirrors UnRegisterVolume.+ delete(vl.vid2location, vid)+ delete(vl.sizeTracking, vid)+ vl.removeFromCrowded(vid)}+ return wasWritable}}return false