Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

5.4 KB
Raw Blame
package multi_master

import (
	"fmt"
	"math/rand/v2"
	"os"
	"strconv"
	"testing"
	"time"
)

// chaosRounds is one stop or start per round, enough to walk in and out of
// quorum several times without turning this into a soak test.
const chaosRounds = 12

// TestRandomStartStopElection bounces masters at random and holds the election
// to the two things it must never get wrong: two masters claiming leadership at
// once, and a quorum that comes back without agreeing on one. The cluster's
// identity has to survive the whole walk — a master that re-mints a TopologyId
// here is the split brain SetTopologyId kills its peers over.
func TestRandomStartStopElection(t *testing.T) {
	for _, impl := range raftImplementations {
		t.Run(impl.name, func(t *testing.T) {
			seed := chaosSeed(t)
			t.Logf("seed %d — replay this walk with MULTI_MASTER_IT_SEED=%d", seed, seed)
			rng := rand.New(rand.NewPCG(seed, seed))

			mc := NewMasterCluster(t, impl.raftHashicorp)
			for i := range 3 {
				mc.StartNode(i)
			}
			if err := mc.WaitForLeader(waitTimeout); err != nil {
				mc.DumpLogs()
				t.Fatalf("cluster did not elect a leader: %v", err)
			}
			topologyId, err := mc.WaitForTopologyId(waitTimeout)
			if err != nil {
				mc.DumpLogs()
				t.Fatalf("no initial TopologyId: %v", err)
			}

			for round := 1; round <= chaosRounds; round++ {
				target := rng.IntN(3)
				if mc.IsNodeRunning(target) {
					mc.StopNode(target)
					t.Logf("round %d: stopped master %d, %d left running", round, target, runningMasters(mc))
				} else {
					mc.StartNode(target)
					t.Logf("round %d: started master %d, %d now running", round, target, runningMasters(mc))
				}

				if err := waitForSettledElection(mc, leaderElectionTimeout); err != nil {
					mc.DumpLogs()
					t.Fatalf("round %d (seed %d): %v", round, seed, err)
				}

				// /dir/status proxies to the leader, so this reads the value the
				// cluster as a whole is carrying, not any one master's copy.
				if runningMasters(mc) >= 2 {
					id, err := mc.WaitForTopologyId(leaderElectionTimeout)
					if err != nil {
						mc.DumpLogs()
						t.Fatalf("round %d (seed %d): %v", round, seed, err)
					}
					if id != topologyId {
						mc.DumpLogs()
						t.Fatalf("round %d (seed %d): TopologyId changed from %s to %s", round, seed, topologyId, id)
					}
				}
			}

			// Everything back up, so the walk ends on a full cluster.
			for i := range 3 {
				mc.StartNode(i)
			}
			if _, err := waitForCommonLeader(mc, leaderElectionTimeout); err != nil {
				mc.DumpLogs()
				t.Fatalf("cluster did not recover after the walk (seed %d): %v", seed, err)
			}
			for i := range 3 {
				if err := waitForPeerCount(mc, i, 2, leaderElectionTimeout); err != nil {
					mc.DumpLogs()
					t.Fatalf("master %d does not see the full cluster (seed %d): %v", i, seed, err)
				}
			}
			id, err := mc.WaitForTopologyId(leaderElectionTimeout)
			if err != nil {
				mc.DumpLogs()
				t.Fatalf("no TopologyId after the walk (seed %d): %v", seed, err)
			}
			if id != topologyId {
				mc.DumpLogs()
				t.Fatalf("TopologyId changed from %s to %s over the walk (seed %d)", topologyId, id, seed)
			}
		})
	}
}

// waitForSettledElection waits for a quorum to agree on one leader, and fails
// if two masters claim leadership across consecutive polls — anything shorter
// than that is a master on its way down.
//
// Below a quorum there is nothing to wait for: the walk moves on. A master that
// has lost its quorum can keep claiming leadership for tens of seconds under
// goraft, and it cannot commit anything in that window, so stepping down is a
// liveness question rather than a safety one. TestTwoMastersDownAndRestart
// holds that direction to account.
func waitForSettledElection(mc *MasterCluster, timeout time.Duration) error {
	if runningMasters(mc) < 2 {
		return nil
	}

	var lastErr error
	splitPolls := 0
	deadline := time.Now().Add(timeout)
	for time.Now().Before(deadline) {
		claims, err := leaderClaims(mc)
		if err != nil {
			lastErr = err
		}
		if len(claims) > 1 {
			splitPolls++
			if splitPolls > 1 {
				return fmt.Errorf("masters %v all claim leadership", claims)
			}
			time.Sleep(waitTick)
			continue
		}
		splitPolls = 0

		if _, err := commonLeader(mc); err == nil {
			return nil
		} else {
			lastErr = err
		}
		time.Sleep(waitTick)
	}
	if lastErr == nil {
		lastErr = fmt.Errorf("cluster did not settle within %v", timeout)
	}
	return lastErr
}

// leaderClaims returns the running masters that call themselves leader. The
// error reports masters that did not answer at all, which is a reason to keep
// waiting rather than a verdict.
func leaderClaims(mc *MasterCluster) (claims []int, err error) {
	for i := range 3 {
		if !mc.IsNodeRunning(i) {
			continue
		}
		cs, statusErr := mc.GetClusterStatus(i)
		if statusErr != nil {
			err = fmt.Errorf("master %d: %w", i, statusErr)
			continue
		}
		if cs.IsLeader {
			claims = append(claims, i)
		}
	}
	return claims, err
}

func runningMasters(mc *MasterCluster) int {
	count := 0
	for i := range 3 {
		if mc.IsNodeRunning(i) {
			count++
		}
	}
	return count
}

// chaosSeed picks the walk. It is random by default and always logged, so a
// failure names the seed that reproduces it.
func chaosSeed(t *testing.T) uint64 {
	t.Helper()
	if v := os.Getenv("MULTI_MASTER_IT_SEED"); v != "" {
		seed, err := strconv.ParseUint(v, 10, 64)
		if err != nil {
			t.Fatalf("MULTI_MASTER_IT_SEED %q: %v", v, err)
		}
		return seed
	}
	return uint64(time.Now().UnixNano())
}