Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

9.9 KB
Raw Blame
package multi_master

import (
	"testing"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/pb"
)

const (
	// Election timeout is 3s in our cluster config; allow generous margin.
	leaderElectionTimeout = 30 * time.Second

	// Losing a quorum is much slower to show than winning one. goraft only
	// notices on an election-timeout ticker, after its peers have been quiet
	// for a full timeout, and has been measured taking over 30s to step down.
	leaderStepDownTimeout = 60 * time.Second
)

// TestLeaderDownAndRecoverQuickly verifies that when the leader is stopped and
// restarted quickly, the cluster re-elects a leader and the restarted node
// rejoins as a follower. TopologyId must be consistent across all nodes.
func TestLeaderDownAndRecoverQuickly(t *testing.T) {
	mc := StartMasterCluster(t)

	// Record initial state.
	leaderIdx, leaderAddr := mc.FindLeader()
	if leaderIdx < 0 {
		t.Fatal("no leader found after cluster start")
	}
	t.Logf("initial leader: node %d at %s", leaderIdx, leaderAddr)

	topologyId, err := mc.GetTopologyId(leaderIdx)
	if err != nil || topologyId == "" {
		t.Fatalf("failed to get initial TopologyId: %v", err)
	}
	t.Logf("initial TopologyId: %s", topologyId)

	// Stop the leader.
	mc.StopNode(leaderIdx)
	t.Logf("stopped leader node %d", leaderIdx)

	// Wait for a new leader from the remaining 2 nodes.
	newLeaderIdx, newLeaderAddr, err := mc.WaitForNewLeader(leaderAddr, leaderElectionTimeout)
	if err != nil {
		mc.DumpLogs()
		t.Fatalf("new leader not elected after stopping old leader: %v", err)
	}
	t.Logf("new leader: node %d at %s", newLeaderIdx, newLeaderAddr)

	// Restart the old leader quickly.
	mc.StartNode(leaderIdx)
	if err := mc.WaitForNodeReady(leaderIdx, waitTimeout); err != nil {
		mc.DumpLogs()
		t.Fatalf("restarted node %d not ready: %v", leaderIdx, err)
	}
	t.Logf("restarted node %d", leaderIdx)

	// Give raft time to settle.
	time.Sleep(3 * time.Second)

	// Verify leader is stable.
	finalLeaderIdx, _ := mc.FindLeader()
	if finalLeaderIdx < 0 {
		mc.DumpLogs()
		t.Fatal("no leader after restarting old leader node")
	}

	// Verify TopologyId is consistent across all nodes.
	assertTopologyIdConsistent(t, mc, topologyId)
}

// TestLeaderDownSlowRecover verifies that when the leader goes down and takes
// a long time to come back, the remaining 2 nodes elect a new leader and the
// cluster continues to function. When the slow node returns, it rejoins.
func TestLeaderDownSlowRecover(t *testing.T) {
	mc := StartMasterCluster(t)

	leaderIdx, leaderAddr := mc.FindLeader()
	if leaderIdx < 0 {
		t.Fatal("no leader found")
	}
	topologyId, err := mc.GetTopologyId(leaderIdx)
	if err != nil || topologyId == "" {
		t.Fatalf("failed to get initial TopologyId: %v", err)
	}
	t.Logf("initial leader: node %d, TopologyId: %s", leaderIdx, topologyId)

	// Stop the leader.
	mc.StopNode(leaderIdx)

	// Wait for a new leader.
	newLeaderIdx, _, err := mc.WaitForNewLeader(leaderAddr, leaderElectionTimeout)
	if err != nil {
		mc.DumpLogs()
		t.Fatalf("new leader not elected: %v", err)
	}
	t.Logf("new leader: node %d", newLeaderIdx)

	// Verify cluster functions with only 2 nodes (quorum is 2/3).
	cs, err := mc.GetClusterStatus(newLeaderIdx)
	if err != nil {
		mc.DumpLogs()
		t.Fatalf("cannot get cluster status from new leader: %v", err)
	}
	if !cs.IsLeader {
		t.Fatalf("node %d claims not to be leader", newLeaderIdx)
	}

	// Simulate slow recovery: wait significantly longer than election timeout.
	t.Log("simulating slow recovery (10 seconds)...")
	time.Sleep(10 * time.Second)

	// Verify leader is still stable during the outage.
	stableLeaderIdx, _ := mc.FindLeader()
	if stableLeaderIdx < 0 {
		mc.DumpLogs()
		t.Fatal("leader lost during extended outage of one node")
	}

	// Restart the downed node.
	mc.StartNode(leaderIdx)
	if err := mc.WaitForNodeReady(leaderIdx, waitTimeout); err != nil {
		mc.DumpLogs()
		t.Fatalf("slow-recovered node %d not ready: %v", leaderIdx, err)
	}

	time.Sleep(3 * time.Second)
	assertTopologyIdConsistent(t, mc, topologyId)
}

// TestTwoMastersDownAndRestart verifies that when 2 of 3 masters go down
// (losing quorum), the cluster cannot elect a leader. When both restart,
// a leader is elected and TopologyId is preserved.
func TestTwoMastersDownAndRestart(t *testing.T) {
	mc := StartMasterCluster(t)

	leaderIdx, _ := mc.FindLeader()
	if leaderIdx < 0 {
		t.Fatal("no leader found")
	}
	topologyId, err := mc.GetTopologyId(leaderIdx)
	if err != nil || topologyId == "" {
		t.Fatalf("failed to get initial TopologyId: %v", err)
	}
	t.Logf("initial TopologyId: %s", topologyId)

	// Determine which 2 nodes to stop (stop the leader + one follower).
	down1 := leaderIdx
	down2 := (leaderIdx + 1) % 3
	survivor := (leaderIdx + 2) % 3
	t.Logf("stopping nodes %d and %d, keeping node %d", down1, down2, survivor)

	mc.StopNode(down1)
	mc.StopNode(down2)

	// The surviving node alone cannot form a quorum, so it has to give up the
	// leadership it is still claiming.
	if err := mc.WaitForNoLeader(leaderStepDownTimeout); err != nil {
		mc.DumpLogs()
		t.Fatalf("expected no leader with only 1 of 3 nodes: %v", err)
	}

	// Restart both downed nodes.
	mc.StartNode(down1)
	mc.StartNode(down2)
	for _, i := range []int{down1, down2} {
		if err := mc.WaitForNodeReady(i, waitTimeout); err != nil {
			mc.DumpLogs()
			t.Fatalf("restarted node %d not ready: %v", i, err)
		}
	}

	// Wait for leader election.
	if err := mc.WaitForLeader(leaderElectionTimeout); err != nil {
		mc.DumpLogs()
		t.Fatalf("no leader after restarting 2 downed nodes: %v", err)
	}

	time.Sleep(3 * time.Second)
	assertTopologyIdConsistent(t, mc, topologyId)
}

// TestAllMastersDownAndRestart verifies that when all 3 masters are stopped
// and restarted, the cluster elects a leader and all nodes agree on a
// TopologyId. With RaftResumeState=false (default), raft state is cleared on
// restart. The TopologyId is recovered from snapshots when available; on a
// short-lived cluster that hasn't taken snapshots on all nodes, a new
// TopologyId may be generated — but all nodes must still agree.
func TestAllMastersDownAndRestart(t *testing.T) {
	mc := StartMasterCluster(t)

	leaderIdx, _ := mc.FindLeader()
	if leaderIdx < 0 {
		t.Fatal("no leader found")
	}
	topologyId, _ := mc.GetTopologyId(leaderIdx)
	if topologyId == "" {
		t.Fatal("no TopologyId on initial leader")
	}
	t.Logf("initial TopologyId: %s", topologyId)

	// Stop all nodes.
	for i := range 3 {
		mc.StopNode(i)
	}
	t.Log("all nodes stopped")

	time.Sleep(2 * time.Second)

	// Restart all nodes.
	for i := range 3 {
		mc.StartNode(i)
	}
	for i := range 3 {
		if err := mc.WaitForNodeReady(i, waitTimeout); err != nil {
			mc.DumpLogs()
			t.Fatalf("node %d not ready after full restart: %v", i, err)
		}
	}

	// Wait for leader.
	if err := mc.WaitForLeader(leaderElectionTimeout); err != nil {
		mc.DumpLogs()
		t.Fatalf("no leader after full cluster restart: %v", err)
	}

	newLeaderIdx, _ := mc.FindLeader()
	t.Logf("leader after full restart: node %d", newLeaderIdx)

	time.Sleep(3 * time.Second)

	// All nodes must agree on a TopologyId (may differ from original if
	// snapshots were not yet taken on all nodes before shutdown).
	newTopologyId, err := mc.GetTopologyId(newLeaderIdx)
	if err != nil || newTopologyId == "" {
		mc.DumpLogs()
		t.Fatal("no TopologyId after full restart")
	}
	if newTopologyId == topologyId {
		t.Logf("TopologyId preserved across full restart: %s", topologyId)
	} else {
		t.Logf("TopologyId changed (expected for short-lived cluster without snapshots): %s -> %s", topologyId, newTopologyId)
	}
	assertTopologyIdConsistent(t, mc, newTopologyId)
}

// TestLeaderConsistencyAcrossNodes verifies that all nodes agree on who the
// leader is and report the same TopologyId.
func TestLeaderConsistencyAcrossNodes(t *testing.T) {
	mc := StartMasterCluster(t)

	// Allow cluster to stabilize.
	time.Sleep(3 * time.Second)

	leaderIdx, leaderAddr := mc.FindLeader()
	if leaderIdx < 0 {
		t.Fatal("no leader found")
	}
	t.Logf("leader: node %d at %s", leaderIdx, leaderAddr)

	// Every node should agree on the leader.
	for i := range 3 {
		cs, err := mc.GetClusterStatus(i)
		if err != nil {
			t.Fatalf("node %d cluster/status error: %v", i, err)
		}
		if i == leaderIdx {
			if !cs.IsLeader {
				t.Errorf("node %d should be leader but IsLeader=false", i)
			}
		} else {
			if cs.IsLeader {
				t.Errorf("node %d should not be leader but IsLeader=true", i)
			}
			// cs.Leader is a ServerAddress like "127.0.0.1:10000.20000";
			// convert to HTTP address for comparison with leaderAddr.
			leaderHttp := pb.ServerAddress(cs.Leader).ToHttpAddress()
			if leaderHttp != leaderAddr {
				t.Errorf("node %d reports leader %q (http: %s), expected %q", i, cs.Leader, leaderHttp, leaderAddr)
			}
		}
	}

	// All nodes should have the same TopologyId.
	topologyId, _ := mc.GetTopologyId(leaderIdx)
	if topologyId == "" {
		t.Fatal("leader has no TopologyId")
	}
	assertTopologyIdConsistent(t, mc, topologyId)
}

// assertTopologyIdConsistent waits until all running nodes report the expected
// TopologyId. A restarted node only discovers it after the leader has admitted
// it back into raft — /dir/status is proxied to the leader, so a node that has
// not yet learned the leader reports its own empty id. That admission trails
// HTTP readiness by several seconds, so a one-shot check races the propagation.
func assertTopologyIdConsistent(t *testing.T, mc *MasterCluster, expectedId string) {
	t.Helper()
	deadline := time.Now().Add(waitTimeout)
	for time.Now().Before(deadline) {
		consistent := true
		for i := range 3 {
			if !mc.IsNodeRunning(i) {
				continue
			}
			if id, err := mc.GetTopologyId(i); err != nil || id != expectedId {
				consistent = false
				break
			}
		}
		if consistent {
			return
		}
		time.Sleep(waitTick)
	}
	// Timed out: report what each node actually sees.
	for i := range 3 {
		if !mc.IsNodeRunning(i) {
			continue
		}
		id, err := mc.GetTopologyId(i)
		if err != nil {
			t.Errorf("node %d: failed to get TopologyId: %v", i, err)
			continue
		}
		if id != expectedId {
			t.Errorf("node %d: TopologyId=%q, expected %q", i, id, expectedId)
		}
	}
}