Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

19.6 KB
Raw Blame
//go:build linux || darwin

package fuse_failover

import (
	"encoding/json"
	"fmt"
	"io"
	"net"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"testing"
	"time"

	"github.com/seaweedfs/seaweedfs/test/testutil"
	"github.com/seaweedfs/seaweedfs/weed/pb"
	"github.com/stretchr/testify/require"
)

// failoverCluster runs 1 master, N volume servers, 1 filer and M FUSE mounts,
// with the master defaulting to 001 replication so every chunk lands on two
// distinct volume servers. Individual volume servers can be stopped and started
// while IO is in flight, which is what the Docker Swarm reports in discussion
// 10206 exercise: a volume server disappears mid-append and the mounts must
// keep reading from the surviving replica and keep writing on another volume.
type failoverCluster struct {
	t          testing.TB
	baseDir    string
	weedBinary string

	masterPort     int
	masterGrpcPort int
	filerPort      int
	filerGrpcPort  int
	volumePorts    []int
	volumeGrpcPort []int
	volumeDirs     []string

	masterCmd   *exec.Cmd
	filerCmd    *exec.Cmd
	volumeCmds  []*exec.Cmd
	mountCmds   []*exec.Cmd
	mountPoints []string
	logFiles    []*os.File

	mu          sync.Mutex
	waits       map[*exec.Cmd]chan error
	cleanupOnce sync.Once
}

func startFailoverCluster(t testing.TB, numVolumes, numMounts int) *failoverCluster {
	require.GreaterOrEqual(t, numVolumes, 2, "001 replication needs at least 2 volume servers")
	require.GreaterOrEqual(t, numMounts, 1)

	binary := findWeedBinary()
	if binary == "" {
		t.Skip("weed binary not found; set WEED_BINARY or ensure it is on PATH")
	}
	baseDir, err := os.MkdirTemp("", "seaweedfs_fuse_failover_")
	require.NoError(t, err)

	c := &failoverCluster{
		t:              t,
		baseDir:        baseDir,
		weedBinary:     binary,
		volumePorts:    make([]int, numVolumes),
		volumeGrpcPort: make([]int, numVolumes),
		volumeDirs:     make([]string, numVolumes),
		volumeCmds:     make([]*exec.Cmd, numVolumes),
		mountCmds:      make([]*exec.Cmd, numMounts),
		mountPoints:    make([]string, numMounts),
	}
	t.Cleanup(c.Stop)

	ports, err := testutil.AllocatePorts(4 + 2*numVolumes)
	require.NoError(t, err)
	c.masterPort, c.masterGrpcPort = ports[0], ports[1]
	c.filerPort, c.filerGrpcPort = ports[2], ports[3]
	for i := 0; i < numVolumes; i++ {
		c.volumePorts[i] = ports[4+2*i]
		c.volumeGrpcPort[i] = ports[5+2*i]
		c.volumeDirs[i] = filepath.Join(baseDir, fmt.Sprintf("volume%d", i))
		require.NoError(t, os.MkdirAll(c.volumeDirs[i], 0755))
	}

	require.NoError(t, c.startMaster())
	require.NoError(t, c.waitForTCP(c.masterCmd, "master",
		fmt.Sprintf("127.0.0.1:%d", c.masterPort), 30*time.Second))

	for i := 0; i < numVolumes; i++ {
		require.NoError(t, c.StartVolume(i))
	}

	require.NoError(t, c.startFiler())
	require.NoError(t, c.waitForTCP(c.filerCmd, "filer",
		fmt.Sprintf("127.0.0.1:%d", c.filerGrpcPort), 30*time.Second))

	for i := 0; i < numMounts; i++ {
		mp := filepath.Join(baseDir, fmt.Sprintf("mount%d", i))
		require.NoError(t, os.MkdirAll(mp, 0755))
		c.mountPoints[i] = mp
		require.NoError(t, c.startMount(i))
		require.NoError(t, c.waitForMount(mp, 30*time.Second),
			"mount %d not ready\n%s", i, c.tailLog(fmt.Sprintf("mount%d", i)))
	}
	require.NoError(t, c.WaitForVolumeServers(60*time.Second))

	// A mounted filesystem and registered volume servers do not yet guarantee
	// that the lock ring has settled or the first replicated volume is ready.
	// Exercise the write/read path during setup so the append latency checks
	// measure failover on a running cluster rather than its cold start.
	probeName := ".failover-ready"
	probeData := []byte("ready\n")
	probePath := filepath.Join(c.MountDir(0), probeName)
	require.NoError(t, os.WriteFile(probePath, probeData, 0644),
		"readiness write failed\n%s", c.tailLog("mount0"))
	for i := 0; i < numMounts; i++ {
		_, ready := waitForContent(filepath.Join(c.MountDir(i), probeName), probeData, convergeTimeout)
		require.True(t, ready, "mount %d did not observe readiness write\n%s", i, c.tailLog(fmt.Sprintf("mount%d", i)))
	}
	require.NoError(t, os.Remove(probePath))
	return c
}

func (c *failoverCluster) MountDir(i int) string { return c.mountPoints[i] }

func (c *failoverCluster) Stop() {
	if c == nil {
		return
	}
	c.cleanupOnce.Do(func() {
		for i := len(c.mountCmds) - 1; i >= 0; i-- {
			c.stopCmd(c.mountCmds[i], syscall.SIGTERM)
			_ = exec.Command("fusermount3", "-u", c.mountPoints[i]).Run()
			_ = exec.Command("fusermount", "-u", c.mountPoints[i]).Run()
		}
		c.stopCmd(c.filerCmd, syscall.SIGTERM)
		for i := len(c.volumeCmds) - 1; i >= 0; i-- {
			c.stopCmd(c.volumeCmds[i], syscall.SIGTERM)
		}
		c.stopCmd(c.masterCmd, syscall.SIGTERM)

		c.mu.Lock()
		for _, f := range c.logFiles {
			_ = f.Close()
		}
		c.mu.Unlock()
		c.copyLogsForCI()
		if !c.t.Failed() {
			os.RemoveAll(c.baseDir)
		}
	})
}

// KillVolume drops a volume server without letting it deregister, the closest
// local equivalent of a Swarm task vanishing from the overlay network.
func (c *failoverCluster) KillVolume(i int) {
	c.stopCmd(c.volumeCmds[i], syscall.SIGKILL)
	c.volumeCmds[i] = nil
}

// StartVolume (re)starts volume server i on its original ports and data dir.
func (c *failoverCluster) StartVolume(i int) error {
	cmd := exec.Command(c.weedBinary,
		"-logdir="+filepath.Join(c.baseDir, "logs"),
		"volume",
		"-ip=127.0.0.1",
		"-ip.bind=127.0.0.1",
		"-port="+strconv.Itoa(c.volumePorts[i]),
		"-port.grpc="+strconv.Itoa(c.volumeGrpcPort[i]),
		"-master="+c.masterAddress(),
		"-dir="+c.volumeDirs[i],
		"-dataCenter=dc1",
		"-rack=rack1",
		"-max=10",
	)
	c.volumeCmds[i] = cmd
	if err := c.startCmd(cmd, fmt.Sprintf("volume%d", i)); err != nil {
		return err
	}
	return c.waitForTCP(cmd, fmt.Sprintf("volume%d", i),
		fmt.Sprintf("127.0.0.1:%d", c.volumePorts[i]), 30*time.Second)
}

func (c *failoverCluster) startMaster() error {
	c.masterCmd = exec.Command(c.weedBinary,
		"-logdir="+filepath.Join(c.baseDir, "logs"),
		"master",
		"-ip=127.0.0.1",
		"-ip.bind=127.0.0.1",
		"-port="+strconv.Itoa(c.masterPort),
		"-port.grpc="+strconv.Itoa(c.masterGrpcPort),
		"-mdir="+filepath.Join(c.baseDir, "master"),
		"-defaultReplication=001",
		"-volumeSizeLimitMB=64",
	)
	return c.startCmd(c.masterCmd, "master")
}

func (c *failoverCluster) startFiler() error {
	filerDir := filepath.Join(c.baseDir, "filer")
	if err := os.MkdirAll(filerDir, 0755); err != nil {
		return fmt.Errorf("create filer dir: %w", err)
	}
	c.filerCmd = exec.Command(c.weedBinary,
		"-logdir="+filepath.Join(c.baseDir, "logs"),
		"filer",
		"-ip=127.0.0.1",
		"-ip.bind=127.0.0.1",
		"-port="+strconv.Itoa(c.filerPort),
		"-port.grpc="+strconv.Itoa(c.filerGrpcPort),
		"-master="+c.masterAddress(),
		"-defaultReplicaPlacement=001",
		"-defaultStoreDir="+filerDir,
	)
	return c.startCmd(c.filerCmd, "filer")
}

func (c *failoverCluster) startMount(idx int) error {
	cacheDir := filepath.Join(c.baseDir, fmt.Sprintf("cache%d", idx))
	if err := os.MkdirAll(cacheDir, 0755); err != nil {
		return fmt.Errorf("create cache dir: %w", err)
	}
	// Chunk-level detail needs -v=4; keep CI at -v=2 so the logs stay small.
	verbosity := os.Getenv("FUSE_FAILOVER_MOUNT_V")
	if verbosity == "" {
		verbosity = "2"
	}
	c.mountCmds[idx] = exec.Command(c.weedBinary,
		"-logdir="+filepath.Join(c.baseDir, "logs"),
		"-v="+verbosity,
		"mount",
		"-filer="+c.filerAddress(),
		"-dir="+c.mountPoints[idx],
		"-filer.path=/",
		"-dirAutoCreate",
		"-allowOthers=false",
		"-replication=001",
		"-cacheDir="+cacheDir,
	)
	return c.startCmd(c.mountCmds[idx], fmt.Sprintf("mount%d", idx))
}

// MasterGet fetches a master HTTP endpoint, e.g. "/dir/status?pretty=y" or
// "/dir/lookup?volumeId=6", so a failing test can show where the replicas are.
func (c *failoverCluster) MasterGet(path string) string {
	resp, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d%s", c.masterPort, path))
	if err != nil {
		return fmt.Sprintf("(master %s failed: %v)", path, err)
	}
	defer resp.Body.Close()
	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return fmt.Sprintf("(master %s read failed: %v)", path, err)
	}
	return string(body)
}

// FilerGet reads a file back through the filer's own HTTP handler: a view of
// the chunk list that neither mount's cache can colour.
func (c *failoverCluster) FilerGet(path string) ([]byte, error) {
	resp, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d%s", c.filerPort, path))
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("filer %s: %s", path, resp.Status)
	}
	return body, nil
}

// VolumeServerAddress is the address volume server i registers with the master.
func (c *failoverCluster) VolumeServerAddress(i int) string {
	return fmt.Sprintf("127.0.0.1:%d", c.volumePorts[i])
}

// FileVolumeIds returns the volume ids backing a file, read from the filer's
// own entry rather than inferred, so a test can tell which servers a given
// file actually depends on. Manifests are resolved first: a manifest chunk's
// own fid names the volume holding the manifest, not the data.
func (c *failoverCluster) FileVolumeIds(path string) ([]uint32, error) {
	body, err := c.FilerGet(path + "?metadata=true&resolveManifest=true")
	if err != nil {
		return nil, err
	}
	var entry struct {
		Chunks []struct {
			FileId string `json:"file_id"`
			Fid    struct {
				VolumeId uint32 `json:"volume_id"`
			} `json:"fid"`
		} `json:"chunks"`
	}
	if err = json.Unmarshal(body, &entry); err != nil {
		return nil, fmt.Errorf("decode entry %s: %w", path, err)
	}
	seen := make(map[uint32]bool)
	var vids []uint32
	for _, chunk := range entry.Chunks {
		vid := chunk.Fid.VolumeId
		if vid == 0 && chunk.FileId != "" {
			parsed, parseErr := strconv.ParseUint(strings.SplitN(chunk.FileId, ",", 2)[0], 10, 32)
			if parseErr != nil {
				return nil, fmt.Errorf("parse file id %s: %w", chunk.FileId, parseErr)
			}
			vid = uint32(parsed)
		}
		if !seen[vid] {
			seen[vid] = true
			vids = append(vids, vid)
		}
	}
	return vids, nil
}

// FileChunkList returns the filer's chunk list for a file: each chunk's file
// id, logical offset, size, and the volume id it lives on. Used to pinpoint
// which chunk covers a corrupted byte range and which volume server holds it.
func (c *failoverCluster) FileChunkList(path string) ([]struct {
	FileId   string
	Offset   int64
	Size     uint64
	VolumeId uint32
}, error) {
	body, err := c.FilerGet(path + "?metadata=true&resolveManifest=true")
	if err != nil {
		return nil, err
	}
	var entry struct {
		Chunks []struct {
			FileId string `json:"file_id"`
			Offset int64  `json:"offset"`
			Size   uint64 `json:"size"`
			Fid    struct {
				VolumeId uint32 `json:"volume_id"`
			} `json:"fid"`
		} `json:"chunks"`
	}
	if err = json.Unmarshal(body, &entry); err != nil {
		return nil, fmt.Errorf("decode entry %s: %w", path, err)
	}
	out := make([]struct {
		FileId   string
		Offset   int64
		Size     uint64
		VolumeId uint32
	}, len(entry.Chunks))
	for i, ch := range entry.Chunks {
		vid := ch.Fid.VolumeId
		if vid == 0 && ch.FileId != "" {
			parsed, parseErr := strconv.ParseUint(strings.SplitN(ch.FileId, ",", 2)[0], 10, 32)
			if parseErr != nil {
				return nil, fmt.Errorf("parse file id %s: %w", ch.FileId, parseErr)
			}
			vid = uint32(parsed)
		}
		out[i] = struct {
			FileId   string
			Offset   int64
			Size     uint64
			VolumeId uint32
		}{ch.FileId, ch.Offset, ch.Size, vid}
	}
	return out, nil
}

// VolumeHolders returns the volume server addresses the master currently lists
// for a volume id.
func (c *failoverCluster) VolumeHolders(vid uint32) ([]string, error) {
	var lookup struct {
		Locations []struct {
			Url string `json:"url"`
		} `json:"locations"`
	}
	body := c.MasterGet(fmt.Sprintf("/dir/lookup?volumeId=%d", vid))
	if err := json.Unmarshal([]byte(body), &lookup); err != nil {
		return nil, fmt.Errorf("decode lookup for volume %d: %w (%s)", vid, err, body)
	}
	holders := make([]string, 0, len(lookup.Locations))
	for _, loc := range lookup.Locations {
		holders = append(holders, loc.Url)
	}
	return holders, nil
}

// WaitForHolders polls the master until it lists exactly count servers for a
// volume. The master only drops a dead node after three missed heartbeats, so a
// test that depends on the cluster's view having caught up has to wait for it.
func (c *failoverCluster) WaitForHolders(vid uint32, count int, timeout time.Duration) ([]string, error) {
	deadline := time.Now().Add(timeout)
	for {
		holders, err := c.VolumeHolders(vid)
		if err == nil && len(holders) == count {
			return holders, nil
		}
		if time.Now().After(deadline) {
			return holders, fmt.Errorf("volume %d still has %d holders (%v), want %d", vid, len(holders), holders, count)
		}
		time.Sleep(500 * time.Millisecond)
	}
}

// WaitForVolumeServers blocks until the master's topology lists every volume
// server. A volume server accepts connections well before the master knows it
// exists: a lone master only elects itself once its bootstrap check expires,
// and heartbeats sent before that are refused and retried with backoff, so
// registration lands seconds after the process is up. Until it does the
// topology has no data node at all, an assign fails with "no free volumes
// left", and the mount reports that as ENOSPC instead of retrying.
func (c *failoverCluster) WaitForVolumeServers(timeout time.Duration) error {
	deadline := time.Now().Add(timeout)
	for {
		registered := c.registeredVolumeServers()
		var missing []string
		for i := range c.volumePorts {
			if address := c.VolumeServerAddress(i); !registered[address] {
				missing = append(missing, address)
			}
		}
		if len(missing) == 0 {
			return nil
		}
		if time.Now().After(deadline) {
			return fmt.Errorf("volume servers %v not registered with the master within %v\n%s",
				missing, timeout, c.MasterGet("/dir/status?pretty=y"))
		}
		time.Sleep(200 * time.Millisecond)
	}
}

// registeredVolumeServers is the set of volume server addresses the master
// currently holds in its topology.
func (c *failoverCluster) registeredVolumeServers() map[string]bool {
	var status struct {
		Topology struct {
			DataCenters []struct {
				Racks []struct {
					DataNodes []struct {
						Url string `json:"Url"`
					} `json:"DataNodes"`
				} `json:"Racks"`
			} `json:"DataCenters"`
		} `json:"Topology"`
	}
	if err := json.Unmarshal([]byte(c.MasterGet("/dir/status")), &status); err != nil {
		return nil
	}
	registered := make(map[string]bool)
	for _, dc := range status.Topology.DataCenters {
		for _, rack := range dc.Racks {
			for _, dn := range rack.DataNodes {
				registered[dn.Url] = true
			}
		}
	}
	return registered
}

// volumeIndexOf maps a server address back to its index, or -1.
func (c *failoverCluster) volumeIndexOf(address string) int {
	for i := range c.volumePorts {
		if c.VolumeServerAddress(i) == address {
			return i
		}
	}
	return -1
}

// FileIsOn reports whether any of path's chunks live on the given volume
// server, i.e. whether taking that server down actually costs this file a
// replica.
func (c *failoverCluster) FileIsOn(path, serverAddress string) (bool, error) {
	vids, err := c.FileVolumeIds(path)
	if err != nil {
		return false, err
	}
	for _, vid := range vids {
		holders, holdersErr := c.VolumeHolders(vid)
		if holdersErr != nil {
			return false, holdersErr
		}
		for _, holder := range holders {
			if holder == serverAddress {
				return true, nil
			}
		}
	}
	return false, nil
}

func (c *failoverCluster) masterAddress() string {
	return string(pb.NewServerAddress("127.0.0.1", c.masterPort, c.masterGrpcPort))
}

func (c *failoverCluster) filerAddress() string {
	return string(pb.NewServerAddress("127.0.0.1", c.filerPort, c.filerGrpcPort))
}

func (c *failoverCluster) startCmd(cmd *exec.Cmd, name string) error {
	logPath := filepath.Join(c.baseDir, "logs")
	if err := os.MkdirAll(logPath, 0755); err != nil {
		return fmt.Errorf("create log dir: %w", err)
	}
	// Append so a restarted volume server keeps the log of its earlier run.
	logFile, err := os.OpenFile(filepath.Join(logPath, name+".log"),
		os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
	if err != nil {
		return err
	}
	c.mu.Lock()
	c.logFiles = append(c.logFiles, logFile)
	c.mu.Unlock()
	cmd.Stdout = logFile
	cmd.Stderr = logFile
	if err := cmd.Start(); err != nil {
		return err
	}
	// Reap in the background and publish the result: Signal(0) succeeds for a
	// zombie, so an unreaped child that died at startup would otherwise look
	// alive until the readiness timeout expired.
	ch := make(chan error, 1)
	c.mu.Lock()
	if c.waits == nil {
		c.waits = make(map[*exec.Cmd]chan error)
	}
	c.waits[cmd] = ch
	c.mu.Unlock()
	go func() {
		ch <- cmd.Wait()
		close(ch)
	}()
	return nil
}

// waitChan returns the channel carrying cmd's exit, or nil if it was never
// started through startCmd. It stays readable after the exit is consumed.
func (c *failoverCluster) waitChan(cmd *exec.Cmd) chan error {
	if cmd == nil {
		return nil
	}
	c.mu.Lock()
	defer c.mu.Unlock()
	return c.waits[cmd]
}

func (c *failoverCluster) tailLog(name string) string {
	data, err := os.ReadFile(filepath.Join(c.baseDir, "logs", name+".log"))
	if err != nil {
		return fmt.Sprintf("(log %s not available: %v)", name, err)
	}
	const maxTail = 8192
	if len(data) > maxTail {
		data = data[len(data)-maxTail:]
	}
	return string(data)
}

func (c *failoverCluster) copyLogsForCI() {
	// One directory per test: subtests share a log dir name otherwise, and the
	// last one to finish would overwrite the logs of the one that failed.
	ciLogDir := filepath.Join("/tmp/seaweedfs-fuse-failover-logs",
		strings.ReplaceAll(c.t.Name(), "/", "_"))
	os.MkdirAll(ciLogDir, 0755)
	entries, err := os.ReadDir(filepath.Join(c.baseDir, "logs"))
	if err != nil {
		return
	}
	for _, e := range entries {
		data, err := os.ReadFile(filepath.Join(c.baseDir, "logs", e.Name()))
		if err != nil {
			continue
		}
		os.WriteFile(filepath.Join(ciLogDir, e.Name()), data, 0644)
	}
}

func (c *failoverCluster) waitForTCP(cmd *exec.Cmd, name, addr string, timeout time.Duration) error {
	deadline := time.Now().Add(timeout)
	for time.Now().Before(deadline) {
		conn, err := net.DialTimeout("tcp", addr, time.Second)
		if err == nil {
			conn.Close()
			return nil
		}
		if ch := c.waitChan(cmd); ch != nil {
			select {
			case waitErr := <-ch:
				return fmt.Errorf("%s exited before listening on %s: %v\n%s",
					name, addr, waitErr, c.tailLog(name))
			default:
			}
		}
		time.Sleep(200 * time.Millisecond)
	}
	return fmt.Errorf("service at %s not ready within timeout\n%s", addr, c.tailLog(name))
}

func (c *failoverCluster) waitForMount(mountPoint string, timeout time.Duration) error {
	parentDir := filepath.Dir(mountPoint)
	deadline := time.Now().Add(timeout)
	for time.Now().Before(deadline) {
		parentStat, err := os.Stat(parentDir)
		if err != nil {
			time.Sleep(200 * time.Millisecond)
			continue
		}
		mountStat, err := os.Stat(mountPoint)
		if err != nil {
			time.Sleep(200 * time.Millisecond)
			continue
		}
		if parentStat.Sys().(*syscall.Stat_t).Dev != mountStat.Sys().(*syscall.Stat_t).Dev {
			return nil
		}
		time.Sleep(200 * time.Millisecond)
	}
	return fmt.Errorf("mount point %s not ready within timeout (FUSE not detected)", mountPoint)
}

func findWeedBinary() string {
	if env := os.Getenv("WEED_BINARY"); env != "" {
		if _, err := os.Stat(env); err == nil {
			return env
		}
	}
	if p, err := exec.LookPath("weed"); err == nil {
		return p
	}
	return ""
}

// stopCmd signals cmd and waits for the reaper goroutine started by startCmd to
// report its exit, escalating to SIGKILL if it does not go quietly.
func (c *failoverCluster) stopCmd(cmd *exec.Cmd, sig syscall.Signal) {
	if cmd == nil || cmd.Process == nil {
		return
	}
	_ = cmd.Process.Signal(sig)
	done := c.waitChan(cmd)
	if done == nil {
		return
	}
	select {
	case <-done:
	case <-time.After(10 * time.Second):
		_ = cmd.Process.Signal(syscall.SIGKILL)
		<-done
	}
}