Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

test: wait for the master to register the volume servers before failover tests run (#10871)

Chris Lu301d83cc7a84
diff --git a/test/fuse_failover/framework_test.go b/test/fuse_failover/framework_test.go
index 28c26a2504..5ab1338d8b 100644
--- a/test/fuse_failover/framework_test.go
+++ b/test/fuse_failover/framework_test.go
@@ -109,6 +109,7 @@ func startFailoverCluster(t testing.TB, numVolumes, numMounts int) *failoverClus
require.NoError(t, c.waitForMount(mp, 30*time.Second),
"mount %d not ready\n%s", i, c.tailLog(fmt.Sprintf("mount%d", i)))
}
+ require.NoError(t, c.WaitForVolumeServers(60*time.Second))
return c
}
@@ -344,6 +345,62 @@ func (c *failoverCluster) WaitForHolders(vid uint32, count int, timeout time.Dur
}
}
+// WaitForVolumeServers blocks until the master's topology lists every volume
+// server. A volume server accepts connections well before the master knows it
+// exists: a lone master only elects itself once its bootstrap check expires,
+// and heartbeats sent before that are refused and retried with backoff, so
+// registration lands seconds after the process is up. Until it does the
+// topology has no data node at all, an assign fails with "no free volumes
+// left", and the mount reports that as ENOSPC instead of retrying.
+func (c *failoverCluster) WaitForVolumeServers(timeout time.Duration) error {
+ deadline := time.Now().Add(timeout)
+ for {
+ registered := c.registeredVolumeServers()
+ var missing []string
+ for i := range c.volumePorts {
+ if address := c.VolumeServerAddress(i); !registered[address] {
+ missing = append(missing, address)
+ }
+ }
+ if len(missing) == 0 {
+ return nil
+ }
+ if time.Now().After(deadline) {
+ return fmt.Errorf("volume servers %v not registered with the master within %v\n%s",
+ missing, timeout, c.MasterGet("/dir/status?pretty=y"))
+ }
+ time.Sleep(200 * time.Millisecond)
+ }
+}
+
+// registeredVolumeServers is the set of volume server addresses the master
+// currently holds in its topology.
+func (c *failoverCluster) registeredVolumeServers() map[string]bool {
+ var status struct {
+ Topology struct {
+ DataCenters []struct {
+ Racks []struct {
+ DataNodes []struct {
+ Url string `json:"Url"`
+ } `json:"DataNodes"`
+ } `json:"Racks"`
+ } `json:"DataCenters"`
+ } `json:"Topology"`
+ }
+ if err := json.Unmarshal([]byte(c.MasterGet("/dir/status")), &status); err != nil {
+ return nil
+ }
+ registered := make(map[string]bool)
+ for _, dc := range status.Topology.DataCenters {
+ for _, rack := range dc.Racks {
+ for _, dn := range rack.DataNodes {
+ registered[dn.Url] = true
+ }
+ }
+ }
+ return registered
+}
+
// volumeIndexOf maps a server address back to its index, or -1.
func (c *failoverCluster) volumeIndexOf(address string) int {
for i := range c.volumePorts {
diff --git a/test/fuse_failover/volume_failover_test.go b/test/fuse_failover/volume_failover_test.go
index 60dc5b7cff..f5208e0ed3 100644
--- a/test/fuse_failover/volume_failover_test.go
+++ b/test/fuse_failover/volume_failover_test.go
@@ -190,7 +190,9 @@ func TestReadWithVolumeServerDown(t *testing.T) {
t.Logf("read %s (held by volume%d) with volume%d down in %v", name, victim, victim, elapsed)
require.NoError(t, c.StartVolume(victim))
- time.Sleep(3 * time.Second) // let the master see the heartbeat again
+ // The next iteration picks its victim from the master's placement view,
+ // so wait for the restarted server to be in it again.
+ require.NoError(t, c.WaitForVolumeServers(60*time.Second))
}
}