Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

14.5 KB
Raw Blame
//go:build integration

package shell

import (
	"bytes"
	"context"
	"fmt"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials/insecure"
	"io"
	"net"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"testing"
	"time"
)

// Opt-in: runs report-only and confirmed-purge scans against real services
// after a volume is deleted, and checks that a stopped server blocks the purge.
func TestVolumeFsckFindsReferenceAfterRealVolumeDeletion(t *testing.T) {
	weed := buildWeedForIntegration(t)
	base := t.TempDir()
	for _, name := range []string{"logs", "master", "volume", "filer"} {
		if err := os.MkdirAll(filepath.Join(base, name), 0o755); err != nil {
			t.Fatal(err)
		}
	}

	portPairs := freeIntegrationPortPairs(t, 3)
	masterHTTP, masterGRPC := portPairs[0][0], portPairs[0][1]
	volumeHTTP, volumeGRPC := portPairs[1][0], portPairs[1][1]
	filerHTTP, filerGRPC := portPairs[2][0], portPairs[2][1]
	masterAddress := fmt.Sprintf("127.0.0.1:%d", masterHTTP)

	startIntegrationProcess(t, base, weed, "master",
		"master", "-ip=127.0.0.1", "-ip.bind=127.0.0.1",
		fmt.Sprintf("-port=%d", masterHTTP), fmt.Sprintf("-port.grpc=%d", masterGRPC),
		"-mdir="+filepath.Join(base, "master"), "-defaultReplication=000", "-volumeSizeLimitMB=64")
	waitIntegrationTCP(t, masterHTTP, filepath.Join(base, "logs", "master.log"))
	waitIntegrationTCP(t, masterGRPC, filepath.Join(base, "logs", "master.log"))

	volumeProcess := startIntegrationProcess(t, base, weed, "volume",
		"volume", "-ip=127.0.0.1", "-ip.bind=127.0.0.1",
		fmt.Sprintf("-port=%d", volumeHTTP), fmt.Sprintf("-port.grpc=%d", volumeGRPC),
		"-master="+masterAddress, "-dir="+filepath.Join(base, "volume"), "-max=10")
	waitIntegrationTCP(t, volumeHTTP, filepath.Join(base, "logs", "volume.log"))
	waitIntegrationTCP(t, volumeGRPC, filepath.Join(base, "logs", "volume.log"))
	waitForMasterVolumeRegistration(t, masterHTTP, volumeHTTP, filepath.Join(base, "logs", "volume.log"))

	startIntegrationProcess(t, base, weed, "filer",
		"filer", "-ip=127.0.0.1", "-ip.bind=127.0.0.1",
		fmt.Sprintf("-port=%d", filerHTTP), fmt.Sprintf("-port.grpc=%d", filerGRPC),
		"-master="+masterAddress, "-defaultReplicaPlacement=000",
		"-defaultStoreDir="+filepath.Join(base, "filer"))
	waitIntegrationTCP(t, filerHTTP, filepath.Join(base, "logs", "filer.log"))
	waitIntegrationTCP(t, filerGRPC, filepath.Join(base, "logs", "filer.log"))

	filerURL := fmt.Sprintf("http://127.0.0.1:%d", filerHTTP)
	request, err := http.NewRequest(http.MethodPut, filerURL+"/fsck-probe.txt", strings.NewReader("live integration data"))
	if err != nil {
		t.Fatal(err)
	}
	response, err := http.DefaultClient.Do(request)
	if err != nil {
		t.Fatalf("upload probe through Filer: %v", err)
	}
	response.Body.Close()
	if response.StatusCode != http.StatusCreated && response.StatusCode != http.StatusAccepted {
		t.Fatalf("Filer PUT status = %d, want 201 or 202", response.StatusCode)
	}

	filerAddress := fmt.Sprintf("127.0.0.1:%d", filerHTTP)
	meta := runIntegrationShell(t, weed, masterAddress, filerAddress, "lock\nfs.meta.cat /fsck-probe.txt\nunlock\n")
	fidPattern := regexp.MustCompile(`"fid"\s*:\s*\{\s*"volumeId"\s*:\s*(\d+)`)
	match := fidPattern.FindStringSubmatch(meta)
	if len(match) != 2 {
		t.Fatalf("could not read chunk volume ID from Filer metadata: %s", meta)
	}
	volumeID := match[1]

	// Delete only the physical volume. The Filer entry intentionally remains
	// so fsck must discover a dangling reference rather than a missing file.
	deleteOutput := runIntegrationShell(t, weed, masterAddress, filerAddress,
		fmt.Sprintf("lock\nvolume.delete -node=127.0.0.1:%d -volumeId=%s\nunlock\n", volumeHTTP, volumeID))
	recentOutput := runIntegrationShell(t, weed, masterAddress, filerAddress,
		fmt.Sprintf("lock\nvolume.fsck -findMissingChunksInFiler -volumeId=%s\nunlock\n", volumeID))
	if strings.Contains(recentOutput, "/fsck-probe.txt") {
		t.Fatalf("default cutoff unexpectedly reported a just-written file: %q", recentOutput)
	}

	fsckOutput := runIntegrationShell(t, weed, masterAddress, filerAddress,
		fmt.Sprintf("lock\nvolume.fsck -findMissingChunksInFiler -volumeId=%s -cutoffTimeAgo=0\nunlock\n", volumeID))
	if !strings.Contains(fsckOutput, "scanning filer references for selected volume(s)") {
		t.Fatalf("fsck did not treat the deleted ID as a missing-volume target; delete=%q fsck=%q", deleteOutput, fsckOutput)
	}
	if !strings.Contains(fsckOutput, "/fsck-probe.txt") || !strings.Contains(fsckOutput, "volume not found") {
		t.Fatalf("fsck did not report the stale Filer reference: %q", fsckOutput)
	}

	scopedPurgeOutput := runIntegrationShell(t, weed, masterAddress, filerAddress,
		"lock\nvolume.fsck -findMissingChunksInFiler -reallyDeleteFilerEntries -collection=logs -cutoffTimeAgo=0\nunlock\n")
	if !strings.Contains(scopedPurgeOutput, "collection cannot be verified for volumes absent from master") {
		t.Fatalf("fsck did not warn that a missing volume cannot be collection-scoped: %q", scopedPurgeOutput)
	}
	if strings.Contains(scopedPurgeOutput, "deleting path /fsck-probe.txt after volume not found") {
		t.Fatalf("collection-filtered fsck queued an unverified missing-volume path: %q", scopedPurgeOutput)
	}

	// Report-only fsck must not delete the stale Filer entry.
	metaAfter := runIntegrationShell(t, weed, masterAddress, filerAddress, "lock\nfs.meta.cat /fsck-probe.txt\nunlock\n")
	if !strings.Contains(metaAfter, "fsck-probe.txt") {
		t.Fatalf("report-only fsck changed or removed the Filer entry: %q", metaAfter)
	}
	// An entry whose chunk manifest lived on the retired volume is unreadable too.
	filerConn, err := grpc.NewClient(fmt.Sprintf("127.0.0.1:%d", filerGRPC), grpc.WithTransportCredentials(insecure.NewCredentials()))
	if err != nil {
		t.Fatalf("dial Filer gRPC: %v", err)
	}
	defer filerConn.Close()
	filerClient := filer_pb.NewSeaweedFilerClient(filerConn)
	volumeIDUint, err := strconv.ParseUint(volumeID, 10, 32)
	if err != nil {
		t.Fatalf("parse volume ID %q: %v", volumeID, err)
	}
	manifestChunk := &filer_pb.FileChunk{
		Fid:             &filer_pb.FileId{VolumeId: uint32(volumeIDUint), FileKey: 987654321, Cookie: 1},
		Size:            21,
		IsChunkManifest: true,
	}
	manifestChunk.FileId = manifestChunk.GetFileIdString()
	now := time.Now().Unix()
	if err := filer_pb.CreateEntry(context.Background(), filerClient, &filer_pb.CreateEntryRequest{
		Directory: "/",
		Entry: &filer_pb.Entry{
			Name:       "manifest-probe.txt",
			Attributes: &filer_pb.FuseAttributes{FileSize: 21, FileMode: 0o644, Mtime: now, Crtime: now},
			Chunks:     []*filer_pb.FileChunk{manifestChunk},
		},
	}); err != nil {
		t.Fatalf("create unresolved-manifest probe: %v", err)
	}

	purgeOutput := runIntegrationShell(t, weed, masterAddress, filerAddress,
		fmt.Sprintf("lock\nvolume.fsck -findMissingChunksInFiler -reallyDeleteFilerEntries -confirmMissingVolumeDeletion -volumeId=%s -cutoffTimeAgo=0\nunlock\n", volumeID))
	if !strings.Contains(purgeOutput, "ResolveChunkManifest failed") {
		t.Fatalf("fsck did not surface the unresolved manifest: %q", purgeOutput)
	}
	if !strings.Contains(purgeOutput, "deleting path /manifest-probe.txt") {
		t.Fatalf("confirmed purge did not queue the entry whose manifest was on the retired volume: %q", purgeOutput)
	}
	manifestAfterPurge, err := http.Get(filerURL + "/manifest-probe.txt")
	if err != nil {
		t.Fatalf("read manifest probe after confirmed purge: %v", err)
	}
	manifestAfterPurge.Body.Close()
	if manifestAfterPurge.StatusCode != http.StatusNotFound {
		t.Fatalf("confirmed purge manifest probe GET status = %d, want 404", manifestAfterPurge.StatusCode)
	}

	if !strings.Contains(purgeOutput, "deleting path /fsck-probe.txt after volume not found") {
		t.Fatalf("confirmed purge did not queue the direct missing-volume entry: %q", purgeOutput)
	}
	probeAfterPurge, err := http.Get(filerURL + "/fsck-probe.txt")
	if err != nil {
		t.Fatalf("read Filer probe after confirmed purge: %v", err)
	}
	probeAfterPurge.Body.Close()
	if probeAfterPurge.StatusCode != http.StatusNotFound {
		t.Fatalf("confirmed purge Filer GET status = %d, want 404", probeAfterPurge.StatusCode)
	}
	// A volume still listed by the master but whose server is unreachable is
	// not proof that its references are stale. Purge must fail before touching
	// the Filer entry when the volume index cannot be read.
	offlineRequest, err := http.NewRequest(http.MethodPut, filerURL+"/offline-probe.txt", strings.NewReader("offline safety probe"))
	if err != nil {
		t.Fatal(err)
	}
	offlineResponse, err := http.DefaultClient.Do(offlineRequest)
	if err != nil {
		t.Fatalf("upload offline safety probe: %v", err)
	}
	offlineResponse.Body.Close()
	if offlineResponse.StatusCode != http.StatusCreated && offlineResponse.StatusCode != http.StatusAccepted {
		t.Fatalf("offline safety PUT status = %d", offlineResponse.StatusCode)
	}
	offlineMeta := runIntegrationShell(t, weed, masterAddress, filerAddress, "lock\nfs.meta.cat /offline-probe.txt\nunlock\n")
	offlineMatch := fidPattern.FindStringSubmatch(offlineMeta)
	if len(offlineMatch) != 2 {
		t.Fatalf("could not read offline probe volume ID: %s", offlineMeta)
	}
	if err := volumeProcess.Process.Kill(); err != nil {
		t.Fatalf("stop volume server: %v", err)
	}
	waitIntegrationTCPClosed(t, volumeHTTP)

	shellCtx, cancelShell := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancelShell()
	fsckCmd := exec.CommandContext(shellCtx, weed, "shell", "-master="+masterAddress, "-filer="+filerAddress)
	fsckCmd.Stdin = strings.NewReader(fmt.Sprintf("lock\nvolume.fsck -findMissingChunksInFiler -reallyDeleteFilerEntries -volumeId=%s -cutoffTimeAgo=0\nunlock\n", offlineMatch[1]))
	fsckBytes, fsckErr := fsckCmd.CombinedOutput()
	offlineFsckOutput := string(fsckBytes)
	if fsckErr == nil {
		t.Fatalf("fsck unexpectedly purged metadata for a volume server outage: %q", offlineFsckOutput)
	}
	if !strings.Contains(offlineFsckOutput, "confirmMissingVolumeDeletion") && !strings.Contains(offlineFsckOutput, "failed to collect file ids") {
		t.Fatalf("fsck failed for an unexpected reason: err=%v output=%q", fsckErr, offlineFsckOutput)
	}
	offlineMetaAfter := runIntegrationShell(t, weed, masterAddress, filerAddress, "lock\nfs.meta.cat /offline-probe.txt\nunlock\n")
	if !strings.Contains(offlineMetaAfter, "offline-probe.txt") {
		t.Fatalf("offline-volume fsck removed the live Filer entry: %q", offlineMetaAfter)
	}
}

func buildWeedForIntegration(t *testing.T) string {
	t.Helper()
	// Reuse the CI/integration binary when supplied; otherwise build this checkout.
	if binary := os.Getenv("WEED_BINARY"); binary != "" {
		if _, err := os.Stat(binary); err != nil {
			t.Fatalf("WEED_BINARY=%q is unavailable: %v", binary, err)
		}
		return binary
	}
	_, source, _, ok := runtime.Caller(0)
	if !ok {
		t.Fatal("resolve integration test source path")
	}
	root := filepath.Clean(filepath.Join(filepath.Dir(source), "..", ".."))
	binary := filepath.Join(t.TempDir(), "weed")
	cmd := exec.Command("go", "build", "-o", binary, "./weed")
	cmd.Dir = root
	if out, err := cmd.CombinedOutput(); err != nil {
		t.Fatalf("build weed: %v\n%s", err, out)
	}
	return binary
}

// freeIntegrationPortPairs reserves HTTP ports with the CLI's conventional
// HTTP+10000 gRPC mapping. The low range also keeps the derived ports valid
// for weed shell's master/Filer client configuration.
func freeIntegrationPortPairs(t *testing.T, count int) [][2]int {
	t.Helper()
	var listeners []net.Listener
	defer func() {
		for _, listener := range listeners {
			listener.Close()
		}
	}()

	pairs := make([][2]int, 0, count)
	for httpPort := 20000; httpPort < 30000 && len(pairs) < count; httpPort++ {
		grpcPort := httpPort + 10000
		httpListener, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", httpPort))
		if err != nil {
			continue
		}
		grpcListener, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", grpcPort))
		if err != nil {
			httpListener.Close()
			continue
		}
		listeners = append(listeners, httpListener, grpcListener)
		pairs = append(pairs, [2]int{httpPort, grpcPort})
	}
	if len(pairs) != count {
		t.Fatalf("reserved %d of %d required HTTP/gRPC port pairs", len(pairs), count)
	}
	return pairs
}

func startIntegrationProcess(t *testing.T, base, weed, name string, args ...string) *exec.Cmd {
	t.Helper()
	logFile, err := os.Create(filepath.Join(base, "logs", name+".log"))
	if err != nil {
		t.Fatal(err)
	}
	cmd := exec.Command(weed, args...)
	cmd.Stdout = logFile
	cmd.Stderr = logFile
	if err := cmd.Start(); err != nil {
		logFile.Close()
		t.Fatalf("start %s: %v", name, err)
	}
	t.Cleanup(func() {
		if cmd.Process != nil {
			_ = cmd.Process.Kill()
		}
		_ = cmd.Wait()
		_ = logFile.Close()
	})
	return cmd
}

// waitForMasterVolumeRegistration prevents racing a successful HTTP listener
// start against the volume server's first heartbeat to the master.
func waitForMasterVolumeRegistration(t *testing.T, masterPort, volumePort int, logPath string) {
	t.Helper()
	deadline := time.Now().Add(20 * time.Second)
	wantAddress := fmt.Sprintf("127.0.0.1:%d", volumePort)
	for time.Now().Before(deadline) {
		response, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d/dir/status", masterPort))
		if err == nil {
			body, readErr := io.ReadAll(response.Body)
			response.Body.Close()
			if readErr == nil && bytes.Contains(body, []byte(wantAddress)) {
				return
			}
		}
		time.Sleep(100 * time.Millisecond)
	}
	log, _ := os.ReadFile(logPath)
	t.Fatalf("master did not register volume server %s; log:\n%s", wantAddress, bytes.TrimSpace(log))
}

func waitIntegrationTCPClosed(t *testing.T, port int) {
	t.Helper()
	deadline := time.Now().Add(10 * time.Second)
	for time.Now().Before(deadline) {
		conn, err := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", port), 100*time.Millisecond)
		if err != nil {
			return
		}
		conn.Close()
		time.Sleep(50 * time.Millisecond)
	}
	t.Fatalf("service still listens on port %d after process was stopped", port)
}

func waitIntegrationTCP(t *testing.T, port int, logPath string) {
	t.Helper()
	deadline := time.Now().Add(20 * time.Second)
	for time.Now().Before(deadline) {
		conn, err := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", port), 200*time.Millisecond)
		if err == nil {
			conn.Close()
			return
		}
		time.Sleep(100 * time.Millisecond)
	}
	log, _ := os.ReadFile(logPath)
	t.Fatalf("service did not listen on port %d; log:\n%s", port, bytes.TrimSpace(log))
}

func runIntegrationShell(t *testing.T, weed, master, filer, input string) string {
	t.Helper()
	cmd := exec.Command(weed, "shell", "-master="+master, "-filer="+filer)
	cmd.Stdin = strings.NewReader(input)
	out, err := cmd.CombinedOutput()
	if err != nil {
		t.Fatalf("weed shell failed: %v\n%s", err, out)
	}
	return string(out)
}