Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

14.4 KB
Raw Blame
package erasure_coding

import (
	"context"
	"math"
	"strings"
	"testing"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
	ecstorage "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
	workertypes "github.com/seaweedfs/seaweedfs/weed/worker/types"
	"google.golang.org/protobuf/proto"
)

func TestDecodeErasureCodingTaskParamsFromPayload(t *testing.T) {
	expected := &worker_pb.TaskParams{
		TaskId:     "task-ec-1",
		VolumeId:   88,
		Collection: "images",
		Sources: []*worker_pb.TaskSource{
			{
				Node:     "10.0.0.1:8080",
				VolumeId: 88,
			},
		},
		Targets: []*worker_pb.TaskTarget{
			{
				Node:     "10.0.0.2:8080",
				VolumeId: 88,
				ShardIds: []uint32{0, 10},
			},
		},
		TaskParams: &worker_pb.TaskParams_ErasureCodingParams{
			ErasureCodingParams: &worker_pb.ErasureCodingTaskParams{
				DataShards:    ecstorage.DataShardsCount,
				ParityShards:  ecstorage.ParityShardsCount,
				WorkingDir:    "/tmp/ec-work",
				CleanupSource: true,
			},
		},
	}
	payload, err := proto.Marshal(expected)
	if err != nil {
		t.Fatalf("marshal payload: %v", err)
	}

	job := &plugin_pb.JobSpec{
		JobId: "job-from-admin",
		Parameters: map[string]*plugin_pb.ConfigValue{
			"task_params_pb": {Kind: &plugin_pb.ConfigValue_BytesValue{BytesValue: payload}},
		},
	}

	actual, err := decodeErasureCodingTaskParams(job)
	if err != nil {
		t.Fatalf("decodeErasureCodingTaskParams() err = %v", err)
	}
	if !proto.Equal(expected, actual) {
		t.Fatalf("decoded params mismatch\nexpected: %+v\nactual:   %+v", expected, actual)
	}
}

func TestDecodeErasureCodingTaskParamsFallback(t *testing.T) {
	targetServers := make([]string, 0, ecstorage.TotalShardsCount)
	for i := 0; i < ecstorage.TotalShardsCount; i++ {
		targetServers = append(targetServers, "10.0.0."+string(rune('a'+i))+":8080")
	}

	job := &plugin_pb.JobSpec{
		JobId: "job-ec-2",
		Parameters: map[string]*plugin_pb.ConfigValue{
			"volume_id": {
				Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 7},
			},
			"source_server": {
				Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "127.0.0.1:8080"},
			},
			"target_servers": {
				Kind: &plugin_pb.ConfigValue_StringList{
					StringList: &plugin_pb.StringList{Values: targetServers},
				},
			},
			"collection": {
				Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "videos"},
			},
		},
	}

	params, err := decodeErasureCodingTaskParams(job)
	if err != nil {
		t.Fatalf("decodeErasureCodingTaskParams() err = %v", err)
	}
	if params.TaskId != "job-ec-2" || params.VolumeId != 7 || params.Collection != "videos" {
		t.Fatalf("unexpected basic params: %+v", params)
	}
	if len(params.Sources) != 1 || params.Sources[0].Node != "127.0.0.1:8080" {
		t.Fatalf("unexpected sources: %+v", params.Sources)
	}
	if len(params.Targets) != ecstorage.TotalShardsCount {
		t.Fatalf("unexpected target count: %d", len(params.Targets))
	}
	if params.GetErasureCodingParams() == nil {
		t.Fatalf("expected fallback erasure coding params")
	}
}

func TestDeriveErasureCodingWorkerConfig(t *testing.T) {
	values := map[string]*plugin_pb.ConfigValue{
		"quiet_for_seconds": {
			Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 720},
		},
		"fullness_ratio": {
			Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0.92},
		},
		"min_size_mb": {
			Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 128},
		},
	}

	cfg := deriveErasureCodingWorkerConfig(values)
	if cfg.TaskConfig.QuietForSeconds != 720 {
		t.Fatalf("expected quiet_for_seconds 720, got %d", cfg.TaskConfig.QuietForSeconds)
	}
	if cfg.TaskConfig.FullnessRatio != 0.92 {
		t.Fatalf("expected fullness_ratio 0.92, got %v", cfg.TaskConfig.FullnessRatio)
	}
	if cfg.TaskConfig.MinSizeMB != 128 {
		t.Fatalf("expected min_size_mb 128, got %d", cfg.TaskConfig.MinSizeMB)
	}
	if cfg.TaskConfig.FreeSlotsDeadband != 0 {
		t.Fatalf("expected default free_slots_deadband 0, got %d", cfg.TaskConfig.FreeSlotsDeadband)
	}

	values["free_slots_deadband"] = &plugin_pb.ConfigValue{
		Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 12},
	}
	cfg = deriveErasureCodingWorkerConfig(values)
	if cfg.TaskConfig.FreeSlotsDeadband != 12 {
		t.Fatalf("expected free_slots_deadband 12, got %d", cfg.TaskConfig.FreeSlotsDeadband)
	}
	if cfg.TaskConfig.MaxShardTransfers != nil {
		t.Fatalf("missing max_shard_transfers must stay unset, got %d", *cfg.TaskConfig.MaxShardTransfers)
	}
}

func TestDeriveErasureCodingWorkerConfigMaxShardTransfers(t *testing.T) {
	if got := deriveErasureCodingWorkerConfig(nil).TaskConfig.MaxShardTransfers; got != nil {
		t.Fatalf("nil values: got %d, want unset", *got)
	}
	if got := deriveErasureCodingWorkerConfig(map[string]*plugin_pb.ConfigValue{}).TaskConfig.MaxShardTransfers; got != nil {
		t.Fatalf("missing key: got %d, want unset", *got)
	}

	cfg := deriveErasureCodingWorkerConfig(map[string]*plugin_pb.ConfigValue{
		"max_shard_transfers": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
	})
	if cfg.TaskConfig.MaxShardTransfers == nil || *cfg.TaskConfig.MaxShardTransfers != 0 {
		t.Fatalf("0: got %v, want 0 (sequential)", cfg.TaskConfig.MaxShardTransfers)
	}

	cfg = deriveErasureCodingWorkerConfig(map[string]*plugin_pb.ConfigValue{
		"max_shard_transfers": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 4}},
	})
	if cfg.TaskConfig.MaxShardTransfers == nil || *cfg.TaskConfig.MaxShardTransfers != 4 {
		t.Fatalf("4: got %v, want 4", cfg.TaskConfig.MaxShardTransfers)
	}

	cfg = deriveErasureCodingWorkerConfig(map[string]*plugin_pb.ConfigValue{
		"max_shard_transfers": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: -3}},
	})
	if cfg.TaskConfig.MaxShardTransfers == nil || *cfg.TaskConfig.MaxShardTransfers != 0 {
		t.Fatalf("negative: got %v, want 0", cfg.TaskConfig.MaxShardTransfers)
	}

	cfg = deriveErasureCodingWorkerConfig(map[string]*plugin_pb.ConfigValue{
		"max_shard_transfers": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: math.MaxInt32 + 1}},
	})
	if cfg.TaskConfig.MaxShardTransfers == nil || *cfg.TaskConfig.MaxShardTransfers != math.MaxInt32 {
		t.Fatalf("above int32: got %v, want clamp to %d", cfg.TaskConfig.MaxShardTransfers, int64(math.MaxInt32))
	}
}

func TestBuildErasureCodingProposal(t *testing.T) {
	params := &worker_pb.TaskParams{
		TaskId:     "ec-task-1",
		VolumeId:   99,
		Collection: "c1",
		Sources: []*worker_pb.TaskSource{
			{
				Node:     "source-a:8080",
				VolumeId: 99,
			},
		},
		Targets: []*worker_pb.TaskTarget{
			{
				Node:     "target-a:8080",
				VolumeId: 99,
				ShardIds: []uint32{0, 10},
			},
			{
				Node:     "target-b:8080",
				VolumeId: 99,
				ShardIds: []uint32{1, 11},
			},
		},
		TaskParams: &worker_pb.TaskParams_ErasureCodingParams{
			ErasureCodingParams: &worker_pb.ErasureCodingTaskParams{
				DataShards:   ecstorage.DataShardsCount,
				ParityShards: ecstorage.ParityShardsCount,
			},
		},
	}
	result := &workertypes.TaskDetectionResult{
		TaskID:      "ec-task-1",
		TaskType:    workertypes.TaskTypeErasureCoding,
		VolumeID:    99,
		Server:      "source-a",
		Collection:  "c1",
		Priority:    workertypes.TaskPriorityLow,
		Reason:      "quiet and full",
		TypedParams: params,
	}

	proposal, err := buildErasureCodingProposal(result, "")
	if err != nil {
		t.Fatalf("buildErasureCodingProposal() err = %v", err)
	}
	if proposal.JobType != "erasure_coding" {
		t.Fatalf("unexpected job type %q", proposal.JobType)
	}
	if proposal.Parameters["task_params_pb"] == nil {
		t.Fatalf("expected serialized task params")
	}
	if proposal.Labels["source_node"] != "source-a:8080" {
		t.Fatalf("unexpected source label %q", proposal.Labels["source_node"])
	}
}

func TestErasureCodingHandlerRejectsUnsupportedJobType(t *testing.T) {
	handler := NewErasureCodingHandler(nil, "")
	err := handler.Detect(context.Background(), &plugin_pb.RunDetectionRequest{
		JobType: "vacuum",
	}, noopDetectionSender{})
	if err == nil {
		t.Fatalf("expected detect job type mismatch error")
	}

	err = handler.Execute(context.Background(), &plugin_pb.ExecuteJobRequest{
		Job: &plugin_pb.JobSpec{JobId: "job-1", JobType: "vacuum"},
	}, noopExecutionSender{})
	if err == nil {
		t.Fatalf("expected execute job type mismatch error")
	}
}

func TestEmitErasureCodingDetectionDecisionTraceNoTasks(t *testing.T) {
	sender := &recordingDetectionSender{}
	config := NewDefaultConfig()
	config.QuietForSeconds = 5 * 60
	config.MinSizeMB = 30
	config.FullnessRatio = 0.91

	metrics := []*workertypes.VolumeHealthMetrics{
		{
			VolumeID:      20,
			Size:          0,
			Age:           218*time.Hour + 41*time.Minute,
			FullnessRatio: 0,
		},
		{
			VolumeID:      27,
			Size:          uint64(16 * 1024 * 1024 / 10),
			Age:           91*time.Hour + time.Minute,
			FullnessRatio: 0.002,
		},
		{
			VolumeID:      12,
			Size:          0,
			Age:           219*time.Hour + 49*time.Minute,
			FullnessRatio: 0,
		},
	}

	if err := emitErasureCodingDetectionDecisionTrace(sender, metrics, config, nil, 0, false); err != nil {
		t.Fatalf("emitErasureCodingDetectionDecisionTrace error: %v", err)
	}
	if len(sender.events) < 4 {
		t.Fatalf("expected at least 4 detection events, got %d", len(sender.events))
	}

	if sender.events[0].Source != plugin_pb.ActivitySource_ACTIVITY_SOURCE_DETECTOR {
		t.Fatalf("expected detector source, got %v", sender.events[0].Source)
	}
	if !strings.Contains(sender.events[0].Message, "EC detection: No tasks created for 3 volumes") {
		t.Fatalf("unexpected summary message: %q", sender.events[0].Message)
	}
	if !strings.Contains(sender.events[1].Message, "ERASURE CODING: Volume 20: size=0.0MB") {
		t.Fatalf("unexpected first detail message: %q", sender.events[1].Message)
	}
}

func TestErasureCodingDescriptorExposesFreeSlotsDeadband(t *testing.T) {
	descriptor := NewErasureCodingHandler(nil, "").Descriptor()
	if descriptor == nil || descriptor.WorkerConfigForm == nil {
		t.Fatalf("expected worker config form in descriptor")
	}
	if descriptor.DescriptorVersion != 1 {
		t.Fatalf("descriptor version = %d, want 1", descriptor.DescriptorVersion)
	}
	if !workerConfigFormHasField(descriptor.WorkerConfigForm, "free_slots_deadband") {
		t.Fatal("worker config form omitted free_slots_deadband")
	}
	var sectionID, prev string
	for _, section := range descriptor.WorkerConfigForm.Sections {
		if section == nil {
			continue
		}
		for _, field := range section.Fields {
			if field != nil && field.Name == "free_slots_deadband" {
				sectionID = section.SectionId
				if field.Label != "Free Volume Deadband (M)" {
					t.Fatalf("free_slots_deadband label = %q, want Free Volume Deadband (M)", field.Label)
				}
			}
			if field != nil && field.Name == "replica_placement" {
				prev = section.SectionId
			}
		}
	}
	if sectionID != "placement" {
		t.Fatalf("free_slots_deadband section = %q, want placement", sectionID)
	}
	if !workerConfigFormHasField(descriptor.WorkerConfigForm, "max_shard_transfers") {
		t.Fatal("worker config form omitted max_shard_transfers")
	}
	if _, ok := descriptor.WorkerConfigForm.DefaultValues["max_shard_transfers"]; ok {
		t.Fatal("max_shard_transfers default would collapse unset into 0")
	}
	if _, ok := descriptor.WorkerDefaultValues["max_shard_transfers"]; ok {
		t.Fatal("worker default max_shard_transfers would collapse unset into 0")
	}
	if prev != "placement" {
		t.Fatalf("replica_placement section = %q, want placement next to free_slots_deadband", prev)
	}
	for _, section := range descriptor.WorkerConfigForm.Sections {
		if section != nil && section.SectionId == "thresholds" {
			for _, field := range section.Fields {
				if field != nil && field.Name == "free_slots_deadband" {
					t.Fatal("free_slots_deadband must not sit under Detection Thresholds")
				}
			}
		}
	}
}

func TestErasureCodingDescriptorOmitsLocalExecutionFields(t *testing.T) {
	descriptor := NewErasureCodingHandler(nil, "").Descriptor()
	if descriptor == nil || descriptor.WorkerConfigForm == nil {
		t.Fatalf("expected worker config form in descriptor")
	}
	if workerConfigFormHasField(descriptor.WorkerConfigForm, "working_dir") {
		t.Fatalf("unexpected working_dir in erasure coding worker config form")
	}
	if workerConfigFormHasField(descriptor.WorkerConfigForm, "cleanup_source") {
		t.Fatalf("unexpected cleanup_source in erasure coding worker config form")
	}
}

func TestApplyErasureCodingExecutionDefaultsForcesLocalFields(t *testing.T) {
	baseWorkingDir := "/var/lib/seaweedfs-worker"
	params := &worker_pb.TaskParams{
		TaskId:   "ec-test",
		VolumeId: 100,
		TaskParams: &worker_pb.TaskParams_ErasureCodingParams{
			ErasureCodingParams: &worker_pb.ErasureCodingTaskParams{
				DataShards:    ecstorage.DataShardsCount,
				ParityShards:  ecstorage.ParityShardsCount,
				WorkingDir:    "/tmp/custom-from-job",
				CleanupSource: false,
			},
		},
	}

	applyErasureCodingExecutionDefaults(params, nil, baseWorkingDir)

	ecParams := params.GetErasureCodingParams()
	if ecParams == nil {
		t.Fatalf("expected erasure coding params")
	}
	if ecParams.WorkingDir != defaultErasureCodingWorkingDir(baseWorkingDir) {
		t.Fatalf("expected local working_dir %q, got %q", defaultErasureCodingWorkingDir(baseWorkingDir), ecParams.WorkingDir)
	}
	if !ecParams.CleanupSource {
		t.Fatalf("expected cleanup_source true")
	}
}

type noopDetectionSender struct{}

func (noopDetectionSender) SendProposals(*plugin_pb.DetectionProposals) error { return nil }
func (noopDetectionSender) SendComplete(*plugin_pb.DetectionComplete) error   { return nil }
func (noopDetectionSender) SendActivity(*plugin_pb.ActivityEvent) error       { return nil }

type noopExecutionSender struct{}

func (noopExecutionSender) SendProgress(*plugin_pb.JobProgressUpdate) error { return nil }
func (noopExecutionSender) SendCompleted(*plugin_pb.JobCompleted) error     { return nil }

type recordingDetectionSender struct {
	proposals *plugin_pb.DetectionProposals
	complete  *plugin_pb.DetectionComplete
	events    []*plugin_pb.ActivityEvent
}

func (r *recordingDetectionSender) SendProposals(proposals *plugin_pb.DetectionProposals) error {
	r.proposals = proposals
	return nil
}

func (r *recordingDetectionSender) SendComplete(complete *plugin_pb.DetectionComplete) error {
	r.complete = complete
	return nil
}

func (r *recordingDetectionSender) SendActivity(event *plugin_pb.ActivityEvent) error {
	if event != nil {
		r.events = append(r.events, event)
	}
	return nil
}

func workerConfigFormHasField(form *plugin_pb.ConfigForm, fieldName string) bool {
	if form == nil {
		return false
	}
	for _, section := range form.Sections {
		if section == nil {
			continue
		}
		for _, field := range section.Fields {
			if field != nil && field.Name == fieldName {
				return true
			}
		}
	}
	return false
}