Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

6.4 KB
Raw Blame
package plugin

import (
	"context"
	"testing"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
	"github.com/stretchr/testify/require"
)

func TestSchedulerCooldownUsesSavedPolicyAndHistory(t *testing.T) {
	p, err := New(Options{})
	require.NoError(t, err)
	defer p.Shutdown()
	const jobType = "vacuum"
	p.registry.UpsertFromHello(&plugin_pb.WorkerHello{
		WorkerId: "worker-a", Capabilities: []*plugin_pb.JobTypeCapability{{JobType: jobType, CanDetect: true}},
	})
	require.NoError(t, p.store.SaveDescriptor(jobType, &plugin_pb.JobTypeDescriptor{
		JobType: jobType, DetectionCooldownConfigField: "cooldown",
		WorkerDefaultValues: map[string]*plugin_pb.ConfigValue{"cooldown": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 36000}}},
	}))
	last := time.Now().UTC().Add(-time.Hour).Truncate(time.Second)
	require.NoError(t, p.store.AppendRunRecord(jobType, &JobRunRecord{Outcome: RunOutcomeSuccess, CompletedAt: &last}))
	p.markJobTypeInFlight(jobType)
	for _, seconds := range []int64{36000, 600, 0, 7200} {
		require.NoError(t, p.SaveJobTypeConfig(&plugin_pb.PersistedJobTypeConfig{
			JobType: jobType, AdminRuntime: &plugin_pb.AdminRuntimeConfig{Enabled: true, DetectionIntervalMinutes: 10},
			WorkerConfigValues: map[string]*plugin_pb.ConfigValue{"cooldown": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: seconds}}},
		}))
		for _, status := range []SchedulerStatus{p.GetLaneSchedulerStatus(LaneDefault), p.GetSchedulerStatus()} {
			require.Len(t, status.JobTypes, 1)
			job := status.JobTypes[0]
			require.Empty(t, job.CooldownError)
			require.Equal(t, seconds, job.CooldownSeconds)
			require.Equal(t, last, *job.LastSuccessfulRunAt)
			if seconds == 0 {
				require.Nil(t, job.DetectionEligibleAt)
			} else {
				require.Equal(t, last.Add(time.Duration(seconds)*time.Second), *job.DetectionEligibleAt)
			}
			var cooldown, inFlight bool
			for _, wait := range status.Waiting {
				cooldown = cooldown || wait.Reason == "cooldown"
				inFlight = inFlight || wait.Reason == "detection_in_flight"
			}
			require.Equal(t, seconds > 3600, cooldown)
			require.True(t, inFlight, "cooldown must not hide another blocker")
		}
	}
	require.NoError(t, p.store.SaveJobTypeConfig(&plugin_pb.PersistedJobTypeConfig{JobType: jobType}))
	job := SchedulerJobTypeStatus{JobType: jobType}
	p.populateSchedulerCooldown(&job)
	require.Equal(t, int64(36000), job.CooldownSeconds, "missing saved value uses descriptor default")
}
func TestGetSchedulerStatusIncludesInProcessJobs(t *testing.T) {
	pluginSvc, err := New(Options{})
	if err != nil {
		t.Fatalf("New: %v", err)
	}
	defer pluginSvc.Shutdown()

	pluginSvc.trackExecutionStart("req-1", "worker-a", &plugin_pb.JobSpec{
		JobId:   "job-1",
		JobType: "vacuum",
	}, 1)

	status := pluginSvc.GetSchedulerStatus()
	if len(status.InProcessJobs) != 1 {
		t.Fatalf("expected one in-process job, got %d", len(status.InProcessJobs))
	}
	if status.InProcessJobs[0].JobID != "job-1" {
		t.Fatalf("unexpected job id: %s", status.InProcessJobs[0].JobID)
	}
}

func TestGetSchedulerStatusIncludesLastDetectionCount(t *testing.T) {
	pluginSvc, err := New(Options{})
	if err != nil {
		t.Fatalf("New: %v", err)
	}
	defer pluginSvc.Shutdown()

	const jobType = "vacuum"
	pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
		WorkerId: "worker-a",
		Capabilities: []*plugin_pb.JobTypeCapability{
			{JobType: jobType, CanDetect: true},
		},
	})

	pluginSvc.recordSchedulerDetectionSuccess(jobType, 3)

	status := pluginSvc.GetSchedulerStatus()
	found := false
	for _, jt := range status.JobTypes {
		if jt.JobType != jobType {
			continue
		}
		found = true
		if jt.LastDetectedCount != 3 {
			t.Fatalf("unexpected last detected count: got=%d want=3", jt.LastDetectedCount)
		}
		if jt.LastDetectedAt == nil {
			t.Fatalf("expected last detected at to be set")
		}
	}
	if !found {
		t.Fatalf("expected job type status for %s", jobType)
	}
}

func TestGetLaneSchedulerStatusShowsActiveConcurrentLaneWork(t *testing.T) {
	clusterContextStarted := make(chan struct{})
	releaseClusterContext := make(chan struct{})

	// Create the Plugin without a ClusterContextProvider so no background
	// scheduler goroutines are started; they would race with the direct
	// runJobTypeIteration call below.
	pluginSvc, err := New(Options{})
	if err != nil {
		t.Fatalf("New: %v", err)
	}
	defer pluginSvc.Shutdown()

	// Set the provider after construction so runJobTypeIteration can use it.
	pluginSvc.clusterContextProvider = func(context.Context) (*plugin_pb.ClusterContext, error) {
		close(clusterContextStarted)
		<-releaseClusterContext
		return nil, context.Canceled
	}

	const jobType = "s3_lifecycle"
	err = pluginSvc.SaveJobTypeConfig(&plugin_pb.PersistedJobTypeConfig{
		JobType: jobType,
		AdminRuntime: &plugin_pb.AdminRuntimeConfig{
			Enabled:                  true,
			DetectionIntervalMinutes: 30,
			DetectionTimeoutSeconds:  15,
		},
	})
	if err != nil {
		t.Fatalf("SaveJobTypeConfig: %v", err)
	}

	policy, enabled, err := pluginSvc.loadSchedulerPolicy(jobType)
	if err != nil {
		t.Fatalf("loadSchedulerPolicy: %v", err)
	}
	if !enabled {
		t.Fatalf("expected enabled policy")
	}

	done := make(chan struct{})
	go func() {
		defer close(done)
		pluginSvc.runJobTypeIteration(jobType, policy)
	}()

	select {
	case <-clusterContextStarted:
	case <-time.After(time.Second):
		t.Fatalf("timed out waiting for job type iteration to start")
	}

	var laneStatus SchedulerStatus
	var aggregateStatus SchedulerStatus
	deadline := time.Now().Add(time.Second)
	for time.Now().Before(deadline) {
		laneStatus = pluginSvc.GetLaneSchedulerStatus(LaneLifecycle)
		aggregateStatus = pluginSvc.GetSchedulerStatus()
		if laneStatus.CurrentJobType == jobType && laneStatus.CurrentPhase == "detecting" &&
			aggregateStatus.CurrentJobType == jobType && aggregateStatus.CurrentPhase == "detecting" {
			break
		}
		time.Sleep(10 * time.Millisecond)
	}

	if laneStatus.CurrentJobType != jobType || laneStatus.CurrentPhase != "detecting" {
		t.Fatalf("unexpected lane status while work is active: job=%q phase=%q", laneStatus.CurrentJobType, laneStatus.CurrentPhase)
	}
	if aggregateStatus.CurrentJobType != jobType || aggregateStatus.CurrentPhase != "detecting" {
		t.Fatalf("unexpected aggregate status while work is active: job=%q phase=%q", aggregateStatus.CurrentJobType, aggregateStatus.CurrentPhase)
	}

	close(releaseClusterContext)
	select {
	case <-done:
	case <-time.After(time.Second):
		t.Fatalf("timed out waiting for job type iteration to finish")
	}
}