Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

6.5 KB
Raw Blame
package command

import (
	"context"
	"net"
	"strings"
	"sync"
	"sync/atomic"
	"testing"

	"github.com/seaweedfs/seaweedfs/weed/pb"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/util"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials/insecure"
)

// kvFilerServer implements just the KV calls of the filer gRPC API over an
// in-memory map, mirroring the real server's behavior of returning an empty
// response, not an error, for a missing key.
type kvFilerServer struct {
	filer_pb.UnimplementedSeaweedFilerServer
	mu sync.Mutex
	kv map[string][]byte
}

func (s *kvFilerServer) KvGet(_ context.Context, req *filer_pb.KvGetRequest) (*filer_pb.KvGetResponse, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	return &filer_pb.KvGetResponse{Value: s.kv[string(req.Key)]}, nil
}

func (s *kvFilerServer) KvPut(_ context.Context, req *filer_pb.KvPutRequest) (*filer_pb.KvPutResponse, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.kv[string(req.Key)] = req.Value
	return &filer_pb.KvPutResponse{}, nil
}

type failingKvFilerServer struct {
	filer_pb.UnimplementedSeaweedFilerServer
	calls atomic.Int32
}

func (s *failingKvFilerServer) KvGet(context.Context, *filer_pb.KvGetRequest) (*filer_pb.KvGetResponse, error) {
	if s.calls.Add(1) == 1 {
		return &filer_pb.KvGetResponse{Error: "kv store unavailable"}, nil
	}
	return &filer_pb.KvGetResponse{}, nil
}

// a miss reads as ErrNotFound so the retried attempt skips the KEK retry loop
func (s *failingKvFilerServer) LookupDirectoryEntry(context.Context, *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) {
	return &filer_pb.LookupDirectoryEntryResponse{}, nil
}

func startKvFiler(t *testing.T) (pb.ServerAddress, grpc.DialOption) {
	return startFiler(t, &kvFilerServer{kv: map[string][]byte{}})
}

func startFiler(t *testing.T, server filer_pb.SeaweedFilerServer) (pb.ServerAddress, grpc.DialOption) {
	t.Helper()
	listener, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatalf("listen: %v", err)
	}
	grpcServer := grpc.NewServer()
	filer_pb.RegisterSeaweedFilerServer(grpcServer, server)
	go func() { _ = grpcServer.Serve(listener) }()
	t.Cleanup(func() { grpcServer.Stop() })
	grpcPort := listener.Addr().(*net.TCPAddr).Port
	return pb.NewServerAddressWithGrpcPort("127.0.0.1:8888", grpcPort),
		grpc.WithTransportCredentials(insecure.NewCredentials())
}

// An upgraded backup finds no destination-scoped checkpoint yet and must
// resume from the historical name+directory key; once its own key holds a
// value, the historical key — possibly still advanced by another backup that
// shared it — must no longer influence it.
func TestBackupOffset_HistoricalFallbackThenIndependence(t *testing.T) {
	filerAddr, dial := startKvFiler(t)

	s := &stubSink{name: "s3", dir: "/", destination: "endpoint-a|bucket-a|/"}
	sinkId, legacySinkId := backupCheckpointIds("/", s, false)

	// pre-upgrade state: only the shared historical checkpoint exists
	if err := setOffset(dial, filerAddr, BackupKeyPrefix, legacySinkId, 111); err != nil {
		t.Fatalf("seed historical offset: %v", err)
	}
	got, err := getOffsetWithFallback(dial, filerAddr, BackupKeyPrefix, sinkId, BackupKeyPrefix, legacySinkId)
	if err != nil || got != 111 {
		t.Fatalf("fallback read = (%d, %v), want (111, nil)", got, err)
	}

	// the backup checkpoints under its own key and prefers it from then on
	if err := setOffset(dial, filerAddr, BackupKeyPrefix, sinkId, 222); err != nil {
		t.Fatalf("write destination-scoped offset: %v", err)
	}
	got, err = getOffsetWithFallback(dial, filerAddr, BackupKeyPrefix, sinkId, BackupKeyPrefix, legacySinkId)
	if err != nil || got != 222 {
		t.Fatalf("destination-scoped read = (%d, %v), want (222, nil)", got, err)
	}

	// another backup advancing the historical key no longer moves this one
	if err := setOffset(dial, filerAddr, BackupKeyPrefix, legacySinkId, 333); err != nil {
		t.Fatalf("advance historical offset: %v", err)
	}
	got, err = getOffsetWithFallback(dial, filerAddr, BackupKeyPrefix, sinkId, BackupKeyPrefix, legacySinkId)
	if err != nil || got != 222 {
		t.Fatalf("read after historical advance = (%d, %v), want (222, nil)", got, err)
	}
}

// The collision scenario from the report, over the real KV wire path: two
// backups whose configurations differ only in endpoint and bucket write
// their checkpoints without disturbing each other's.
func TestBackupOffset_DistinctDestinationsDoNotShare(t *testing.T) {
	filerAddr, dial := startKvFiler(t)

	idA, legacyA := backupCheckpointIds("/", &stubSink{name: "s3", dir: "/", destination: "s3.us-west-004.backblazeb2.com|seaweed-backup-a|/"}, false)
	idB, legacyB := backupCheckpointIds("/", &stubSink{name: "s3", dir: "/", destination: "s3.us-east-005.backblazeb2.com|seaweed-backup-b|/"}, false)

	if err := setOffset(dial, filerAddr, BackupKeyPrefix, idA, 1000); err != nil {
		t.Fatalf("write A: %v", err)
	}
	if err := setOffset(dial, filerAddr, BackupKeyPrefix, idB, 2000); err != nil {
		t.Fatalf("write B: %v", err)
	}

	if got, err := getOffsetWithFallback(dial, filerAddr, BackupKeyPrefix, idA, BackupKeyPrefix, legacyA); err != nil || got != 1000 {
		t.Fatalf("A reads (%d, %v), want (1000, nil)", got, err)
	}
	if got, err := getOffsetWithFallback(dial, filerAddr, BackupKeyPrefix, idB, BackupKeyPrefix, legacyB); err != nil || got != 2000 {
		t.Fatalf("B reads (%d, %v), want (2000, nil)", got, err)
	}
}

func TestFilerBackup_OffsetReadFailureFailsAttempt(t *testing.T) {
	server := &failingKvFilerServer{}
	filerAddr, dial := startFiler(t, server)

	config := util.GetViper()
	config.Set("sink.local.enabled", true)
	config.Set("sink.local.directory", t.TempDir())
	t.Cleanup(func() {
		config.Set("sink.local.enabled", false)
		config.Set("sink.local.directory", "")
	})

	options := filerBackupOptions
	address := string(filerAddr)
	options.filer = &address

	resumeTsNs, err := doFilerBackup(dial, &options, nil, nil, nil, 1, 1, 0)
	if err == nil || !strings.Contains(err.Error(), "kv store unavailable") {
		t.Fatalf("doFilerBackup error = %v, want the offset read failure", err)
	}
	if resumeTsNs != 0 {
		t.Fatalf("resume position = %d, want 0", resumeTsNs)
	}

	// the next attempt re-reads the checkpoint and gets past it once the KV
	// recovers, so it must fail later, not on the offset read
	_, err = doFilerBackup(dial, &options, nil, nil, nil, 1, 2, resumeTsNs)
	if err == nil || strings.Contains(err.Error(), "kv store unavailable") {
		t.Fatalf("retried doFilerBackup error = %v, want a later-stage failure", err)
	}
	if got := server.calls.Load(); got < 2 {
		t.Fatalf("checkpoint reads across attempts = %d, want a re-read", got)
	}
}