Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

13.7 KB
Raw Blame
package multi_master

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/seaweedfs/seaweedfs/test/testutil"
)

const (
	waitTimeout = 30 * time.Second
	waitTick    = 200 * time.Millisecond
)

// masterNode represents a single master process in the cluster.
type masterNode struct {
	port     int
	grpcPort int
	dataDir  string
	cmd      *exec.Cmd
	logFile  string
	stopped  bool
	// peersStr overrides the cluster-wide peer list for this node, so a test
	// can start a master that only knows about a subset of the cluster.
	peersStr string
	// raftBootstrap starts the master with -raftBootstrap, the way the helm
	// chart renders master.raftBootstrap on every master, every restart.
	raftBootstrap bool
}

// MasterCluster manages a 3-node master raft cluster for integration tests.
type MasterCluster struct {
	t          testing.TB
	weedBinary string
	baseDir    string
	logsDir    string
	keepLogs   bool

	nodes [3]*masterNode
	mu    sync.Mutex

	// peers string shared by all nodes, e.g. "127.0.0.1:9333,127.0.0.1:9334,127.0.0.1:9335"
	peersStr string

	// raftHashicorp starts the masters with -raftHashicorp
	raftHashicorp bool
}

// clusterStatus is the JSON returned by /cluster/status.
type clusterStatus struct {
	IsLeader bool     `json:"IsLeader"`
	Leader   string   `json:"Leader"`
	Peers    []string `json:"Peers"`
}

// StartMasterCluster boots a 3-node master raft cluster and waits for a leader.
func StartMasterCluster(t testing.TB) *MasterCluster {
	t.Helper()

	mc := NewMasterCluster(t, false)
	for i := range 3 {
		mc.StartNode(i)
	}

	if err := mc.WaitForLeader(waitTimeout); err != nil {
		mc.DumpLogs()
		mc.StopAll()
		t.Fatalf("cluster did not elect a leader: %v", err)
	}

	// Wait for TopologyId to be generated and propagated. This is async
	// after leader election, and we need it committed before tests can
	// reliably stop/restart nodes.
	if _, err := mc.WaitForTopologyId(waitTimeout); err != nil {
		mc.DumpLogs()
		mc.StopAll()
		t.Fatalf("TopologyId not generated: %v", err)
	}

	return mc
}

// NewMasterCluster allocates ports and data directories for a 3-node master
// cluster without starting anything, so a test can choose what each node comes
// up with.
func NewMasterCluster(t testing.TB, raftHashicorp bool) *MasterCluster {
	t.Helper()

	weedBinary, err := findOrBuildWeedBinary()
	if err != nil {
		t.Fatalf("resolve weed binary: %v", err)
	}

	keepLogs := os.Getenv("MULTI_MASTER_IT_KEEP_LOGS") == "1"
	baseDir, err := os.MkdirTemp("", "seaweedfs_multi_master_it_")
	if err != nil {
		t.Fatalf("create temp dir: %v", err)
	}
	logsDir := filepath.Join(baseDir, "logs")
	os.MkdirAll(logsDir, 0o755)

	// Allocate 3 mini-safe ports (each guarantees port+10000 is also free).
	httpPorts, err := testutil.AllocateMiniPorts(3)
	if err != nil {
		t.Fatalf("allocate ports: %v", err)
	}
	var nodes [3]*masterNode
	var peerParts []string
	for i, hp := range httpPorts {
		dataDir := filepath.Join(baseDir, fmt.Sprintf("m%d", i))
		os.MkdirAll(dataDir, 0o755)
		nodes[i] = &masterNode{
			port:     hp,
			grpcPort: hp + testutil.GrpcPortOffset,
			dataDir:  dataDir,
			logFile:  filepath.Join(logsDir, fmt.Sprintf("master%d.log", i)),
		}
		peerParts = append(peerParts, fmt.Sprintf("127.0.0.1:%d", hp))
	}

	mc := &MasterCluster{
		t:             t,
		weedBinary:    weedBinary,
		baseDir:       baseDir,
		logsDir:       logsDir,
		keepLogs:      keepLogs,
		nodes:         nodes,
		peersStr:      strings.Join(peerParts, ","),
		raftHashicorp: raftHashicorp,
	}

	t.Cleanup(func() {
		mc.StopAll()
	})
	return mc
}

// SetNodePeers narrows the peer list node i starts with, mirroring a
// StatefulSet whose replica count changed under a running master.
func (mc *MasterCluster) SetNodePeers(i int, peers string) {
	mc.mu.Lock()
	defer mc.mu.Unlock()
	mc.nodes[i].peersStr = peers
}

// SetRaftBootstrap makes node i start with -raftBootstrap.
func (mc *MasterCluster) SetRaftBootstrap(i int) {
	mc.mu.Lock()
	defer mc.mu.Unlock()
	mc.nodes[i].raftBootstrap = true
}

// StartNode starts the master process at the given index (0–2).
func (mc *MasterCluster) StartNode(i int) {
	mc.t.Helper()
	mc.mu.Lock()
	defer mc.mu.Unlock()

	n := mc.nodes[i]
	if n.cmd != nil && !n.stopped {
		return // already running
	}

	logFile, err := os.OpenFile(n.logFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
	if err != nil {
		mc.t.Fatalf("create log for node %d: %v", i, err)
	}

	peersStr := n.peersStr
	if peersStr == "" {
		peersStr = mc.peersStr
	}
	args := []string{
		"master",
		"-ip=127.0.0.1",
		"-port=" + strconv.Itoa(n.port),
		"-port.grpc=" + strconv.Itoa(n.grpcPort),
		"-mdir=" + n.dataDir,
		"-peers=" + peersStr,
		"-electionTimeout=3s",
		"-volumeSizeLimitMB=32",
		"-defaultReplication=000",
	}
	if mc.raftHashicorp {
		args = append(args, "-raftHashicorp")
	}
	if n.raftBootstrap {
		args = append(args, "-raftBootstrap")
	}

	n.cmd = exec.Command(mc.weedBinary, args...)
	n.cmd.Dir = mc.baseDir
	n.cmd.Stdout = logFile
	n.cmd.Stderr = logFile
	n.stopped = false
	if err := n.cmd.Start(); err != nil {
		mc.t.Fatalf("start node %d: %v", i, err)
	}
}

// StopNode gracefully stops the master at the given index.
func (mc *MasterCluster) StopNode(i int) {
	mc.mu.Lock()
	defer mc.mu.Unlock()
	mc.stopNodeLocked(i)
}

func (mc *MasterCluster) stopNodeLocked(i int) {
	n := mc.nodes[i]
	if n.cmd == nil || n.stopped {
		return
	}
	n.stopped = true
	_ = n.cmd.Process.Signal(os.Interrupt)
	done := make(chan error, 1)
	go func() { done <- n.cmd.Wait() }()
	select {
	case <-time.After(10 * time.Second):
		_ = n.cmd.Process.Kill()
		<-done
	case <-done:
	}
}

// StopAll stops all running master nodes.
func (mc *MasterCluster) StopAll() {
	mc.mu.Lock()
	defer mc.mu.Unlock()
	for i := range 3 {
		mc.stopNodeLocked(i)
	}
	if !mc.keepLogs && !mc.t.Failed() {
		os.RemoveAll(mc.baseDir)
	} else if mc.baseDir != "" {
		mc.t.Logf("multi-master logs kept at %s", mc.baseDir)
	}
}

// NodeURL returns the HTTP URL for node i.
func (mc *MasterCluster) NodeURL(i int) string {
	return fmt.Sprintf("http://127.0.0.1:%d", mc.nodes[i].port)
}

// NodeAddress returns "127.0.0.1:port" for node i.
func (mc *MasterCluster) NodeAddress(i int) string {
	return fmt.Sprintf("127.0.0.1:%d", mc.nodes[i].port)
}

// NodeGRPCAddress returns "127.0.0.1:grpcPort" for node i.
func (mc *MasterCluster) NodeGRPCAddress(i int) string {
	return fmt.Sprintf("127.0.0.1:%d", mc.nodes[i].grpcPort)
}

// IsNodeRunning returns true if the node at index i has a live process.
func (mc *MasterCluster) IsNodeRunning(i int) bool {
	mc.mu.Lock()
	defer mc.mu.Unlock()
	n := mc.nodes[i]
	return n.cmd != nil && !n.stopped
}

// GetClusterStatus fetches /cluster/status from node i.
func (mc *MasterCluster) GetClusterStatus(i int) (*clusterStatus, error) {
	client := &http.Client{Timeout: 2 * time.Second}
	resp, err := client.Get(mc.NodeURL(i) + "/cluster/status")
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, _ := io.ReadAll(resp.Body)
	var cs clusterStatus
	if err := json.Unmarshal(body, &cs); err != nil {
		return nil, fmt.Errorf("parse cluster/status: %w (body: %s)", err, string(body))
	}
	return &cs, nil
}

// GetTopologyId fetches the TopologyId from /dir/status on node i.
func (mc *MasterCluster) GetTopologyId(i int) (string, error) {
	client := &http.Client{Timeout: 2 * time.Second}
	resp, err := client.Get(mc.NodeURL(i) + "/dir/status")
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	body, _ := io.ReadAll(resp.Body)
	var raw map[string]any
	if err := json.Unmarshal(body, &raw); err != nil {
		return "", err
	}
	if id, ok := raw["TopologyId"].(string); ok {
		return id, nil
	}
	return "", nil
}

// FindLeader returns the index of the leader node and its address.
// Returns -1 if no leader is found.
func (mc *MasterCluster) FindLeader() (int, string) {
	for i := range 3 {
		if !mc.IsNodeRunning(i) {
			continue
		}
		cs, err := mc.GetClusterStatus(i)
		if err != nil {
			continue
		}
		if cs.IsLeader {
			return i, mc.NodeAddress(i)
		}
	}
	return -1, ""
}

// WaitForLeader polls until a leader is elected or timeout.
func (mc *MasterCluster) WaitForLeader(timeout time.Duration) error {
	deadline := time.Now().Add(timeout)
	for time.Now().Before(deadline) {
		if idx, _ := mc.FindLeader(); idx >= 0 {
			return nil
		}
		time.Sleep(waitTick)
	}
	return fmt.Errorf("no leader elected within %v", timeout)
}

// WaitForNoLeader waits until no running master claims leadership. goraft only
// checks whether it still has a quorum on an election-timeout ticker, and needs
// its peers to go quiet for a full timeout first, so a master that has lost its
// quorum can keep claiming leadership for tens of seconds. It cannot commit
// anything in that window.
func (mc *MasterCluster) WaitForNoLeader(timeout time.Duration) error {
	deadline := time.Now().Add(timeout)
	for time.Now().Before(deadline) {
		idx, _ := mc.FindLeader()
		if idx < 0 {
			return nil
		}
		time.Sleep(waitTick)
	}
	idx, addr := mc.FindLeader()
	return fmt.Errorf("master %d at %s still claims leadership after %v", idx, addr, timeout)
}

// WaitForNewLeader waits for a leader that is different from the given address.
func (mc *MasterCluster) WaitForNewLeader(oldLeaderAddr string, timeout time.Duration) (int, string, error) {
	deadline := time.Now().Add(timeout)
	for time.Now().Before(deadline) {
		idx, addr := mc.FindLeader()
		if idx >= 0 && addr != oldLeaderAddr {
			return idx, addr, nil
		}
		time.Sleep(waitTick)
	}
	return -1, "", fmt.Errorf("no new leader (different from %s) within %v", oldLeaderAddr, timeout)
}

// WaitForTopologyId waits until the leader reports a non-empty TopologyId, and
// returns it. It is only readable once the leader has applied the raft entry
// carrying it, which lands after the election it won.
func (mc *MasterCluster) WaitForTopologyId(timeout time.Duration) (string, error) {
	deadline := time.Now().Add(timeout)
	for time.Now().Before(deadline) {
		if idx, _ := mc.FindLeader(); idx >= 0 {
			if id, err := mc.GetTopologyId(idx); err == nil && id != "" {
				return id, nil
			}
		}
		time.Sleep(waitTick)
	}
	return "", fmt.Errorf("TopologyId not available within %v", timeout)
}

// WaitForNodeReady waits for node i to respond to HTTP.
func (mc *MasterCluster) WaitForNodeReady(i int, timeout time.Duration) error {
	client := &http.Client{Timeout: 1 * time.Second}
	deadline := time.Now().Add(timeout)
	for time.Now().Before(deadline) {
		resp, err := client.Get(mc.NodeURL(i) + "/cluster/status")
		if err == nil {
			resp.Body.Close()
			return nil
		}
		time.Sleep(waitTick)
	}
	return fmt.Errorf("node %d not ready within %v", i, timeout)
}

// LogContains reports whether node i's log holds the given text.
func (mc *MasterCluster) LogContains(i int, text string) bool {
	b, err := os.ReadFile(mc.nodes[i].logFile)
	if err != nil {
		return false
	}
	return strings.Contains(string(b), text)
}

// topologyIdLine matches every line a master logs when it learns a TopologyId,
// whichever raft implementation applied it.
var topologyIdLine = regexp.MustCompile(`TopologyId[^:]*: ([0-9a-f-]{36})`)

// NodeTopologyIds returns the TopologyIds node i has logged. /dir/status is
// proxied to the leader, so a master's own view of the cluster identity is only
// visible in its log, and that is where a fork shows up.
func (mc *MasterCluster) NodeTopologyIds(i int) []string {
	b, err := os.ReadFile(mc.nodes[i].logFile)
	if err != nil {
		return nil
	}
	var ids []string
	for _, m := range topologyIdLine.FindAllStringSubmatch(string(b), -1) {
		ids = append(ids, m[1])
	}
	return ids
}

// WaitForNodeTopologyIds waits until every master has logged a TopologyId and
// returns what each one saw.
func (mc *MasterCluster) WaitForNodeTopologyIds(timeout time.Duration) ([3][]string, error) {
	var ids [3][]string
	deadline := time.Now().Add(timeout)
	for {
		missing := -1
		for i := range 3 {
			ids[i] = mc.NodeTopologyIds(i)
			if len(ids[i]) == 0 {
				missing = i
			}
		}
		if missing < 0 {
			return ids, nil
		}
		if !time.Now().Before(deadline) {
			return ids, fmt.Errorf("master %d logged no TopologyId within %v", missing, timeout)
		}
		time.Sleep(waitTick)
	}
}

// DumpLogs prints the tail of all master logs.
func (mc *MasterCluster) DumpLogs() {
	for i := range 3 {
		mc.t.Logf("=== master%d log tail ===\n%s", i, mc.tailLog(i))
	}
}

func (mc *MasterCluster) tailLog(i int) string {
	f, err := os.Open(mc.nodes[i].logFile)
	if err != nil {
		return "(no log)"
	}
	defer f.Close()
	scanner := bufio.NewScanner(f)
	lines := make([]string, 0, 50)
	for scanner.Scan() {
		lines = append(lines, scanner.Text())
		if len(lines) > 50 {
			lines = lines[1:]
		}
	}
	return strings.Join(lines, "\n")
}

func findOrBuildWeedBinary() (string, error) {
	if fromEnv := os.Getenv("WEED_BINARY"); fromEnv != "" {
		if isExecutableFile(fromEnv) {
			return fromEnv, nil
		}
		return "", fmt.Errorf("WEED_BINARY not executable: %s", fromEnv)
	}

	repoRoot := ""
	if _, file, _, ok := runtime.Caller(0); ok {
		repoRoot = filepath.Clean(filepath.Join(filepath.Dir(file), "..", ".."))
	}
	if repoRoot == "" {
		return "", fmt.Errorf("unable to detect repository root")
	}

	// Check if already built
	binDir := filepath.Join(os.TempDir(), "seaweedfs_multi_master_it_bin")
	os.MkdirAll(binDir, 0o755)
	binPath := filepath.Join(binDir, "weed")
	if isExecutableFile(binPath) {
		return binPath, nil
	}

	cmd := exec.Command("go", "build", "-o", binPath, ".")
	cmd.Dir = filepath.Join(repoRoot, "weed")
	var out bytes.Buffer
	cmd.Stdout = &out
	cmd.Stderr = &out
	if err := cmd.Run(); err != nil {
		return "", fmt.Errorf("build weed binary: %w\n%s", err, out.String())
	}
	return binPath, nil
}

func isExecutableFile(path string) bool {
	info, err := os.Stat(path)
	if err != nil || info.IsDir() {
		return false
	}
	return info.Mode().Perm()&0o111 != 0
}