Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

18.0 KB
Raw Blame
package plugin

import (
	"fmt"
	"sort"
	"strings"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
)

type SchedulerStatus struct {
	Now                  time.Time                `json:"now"`
	Lane                 string                   `json:"lane,omitempty"`
	SchedulerTickSeconds int                      `json:"scheduler_tick_seconds"`
	IdleSleepSeconds     int                      `json:"idle_sleep_seconds,omitempty"`
	NextDetectionAt      *time.Time               `json:"next_detection_at,omitempty"`
	CurrentJobType       string                   `json:"current_job_type,omitempty"`
	CurrentPhase         string                   `json:"current_phase,omitempty"`
	LastIterationHadJobs bool                     `json:"last_iteration_had_jobs,omitempty"`
	LastIterationDoneAt  *time.Time               `json:"last_iteration_done_at,omitempty"`
	Waiting              []SchedulerWaitingStatus `json:"waiting,omitempty"`
	InProcessJobs        []SchedulerJobStatus     `json:"in_process_jobs,omitempty"`
	JobTypes             []SchedulerJobTypeStatus `json:"job_types,omitempty"`
}

type SchedulerWaitingStatus struct {
	Reason  string                 `json:"reason"`
	JobType string                 `json:"job_type,omitempty"`
	Since   *time.Time             `json:"since,omitempty"`
	Until   *time.Time             `json:"until,omitempty"`
	Details map[string]interface{} `json:"details,omitempty"`
}

type SchedulerJobStatus struct {
	JobID           string     `json:"job_id"`
	JobType         string     `json:"job_type"`
	State           string     `json:"state"`
	Stage           string     `json:"stage,omitempty"`
	WorkerID        string     `json:"worker_id,omitempty"`
	Message         string     `json:"message,omitempty"`
	Progress        float64    `json:"progress,omitempty"`
	CreatedAt       *time.Time `json:"created_at,omitempty"`
	UpdatedAt       *time.Time `json:"updated_at,omitempty"`
	DurationSeconds float64    `json:"duration_seconds,omitempty"`
}

type SchedulerJobTypeStatus struct {
	JobType                  string     `json:"job_type"`
	Enabled                  bool       `json:"enabled"`
	DetectionInFlight        bool       `json:"detection_in_flight"`
	NextDetectionAt          *time.Time `json:"next_detection_at,omitempty"`
	DetectionIntervalMinutes int32      `json:"detection_interval_minutes,omitempty"`
	LastDetectedAt           *time.Time `json:"last_detected_at,omitempty"`
	LastDetectedCount        int        `json:"last_detected_count,omitempty"`
	LastDetectionError       string     `json:"last_detection_error,omitempty"`
	LastDetectionSkipped     string     `json:"last_detection_skipped,omitempty"`
	CooldownSeconds          int64      `json:"cooldown_seconds,omitempty"`
	LastSuccessfulRunAt      *time.Time `json:"last_successful_run_at,omitempty"`
	DetectionEligibleAt      *time.Time `json:"detection_eligible_at,omitempty"`
	CooldownError            string     `json:"cooldown_error,omitempty"`
}

func (r *Plugin) populateSchedulerCooldown(status *SchedulerJobTypeStatus) {
	descriptor, err := r.store.LoadDescriptor(status.JobType)
	if err != nil {
		status.CooldownError = "Cannot load cooldown policy"
		return
	}
	field := descriptor.GetDetectionCooldownConfigField()
	if field == "" {
		return
	}
	_, _, values, err := r.loadJobTypeConfigPayload(status.JobType)
	if err != nil {
		status.CooldownError = "Cannot load cooldown configuration"
		return
	}
	value := values[field]
	if value == nil {
		value = descriptor.GetWorkerDefaultValues()[field]
	}
	if value == nil {
		value = descriptor.GetWorkerConfigForm().GetDefaultValues()[field]
	}
	if value == nil {
		status.CooldownError = "Cooldown configuration is missing"
		return
	}
	if _, ok := value.Kind.(*plugin_pb.ConfigValue_Int64Value); !ok {
		status.CooldownError = "Cooldown must be an integer number of seconds"
		return
	}
	seconds := value.GetInt64Value()
	if seconds < 0 || seconds > int64((1<<63-1)/time.Second) {
		status.CooldownError = "Cooldown is outside the supported range"
		return
	}
	status.CooldownSeconds = seconds
	history, err := r.store.LoadRunHistory(status.JobType)
	if err != nil {
		status.CooldownError = "Cannot load successful job history"
		return
	}
	if history == nil {
		return
	}
	for _, run := range history.SuccessfulRuns {
		if run.CompletedAt != nil && (status.LastSuccessfulRunAt == nil || run.CompletedAt.After(*status.LastSuccessfulRunAt)) {
			at := *run.CompletedAt
			status.LastSuccessfulRunAt = &at
		}
	}
	if status.LastSuccessfulRunAt != nil && seconds > 0 {
		at := status.LastSuccessfulRunAt.Add(time.Duration(seconds) * time.Second)
		status.DetectionEligibleAt = &at
	}
}

type schedulerDetectionInfo struct {
	lastDetectedAt    time.Time
	lastDetectedCount int
	lastErrorAt       time.Time
	lastError         string
	lastSkippedAt     time.Time
	lastSkippedReason string
}

type schedulerRunInfo struct {
	lastRunStartedAt   time.Time
	lastRunCompletedAt time.Time
	lastRunStatus      string
}

type schedulerLoopState struct {
	currentJobType         string
	currentPhase           string
	lastIterationHadJobs   bool
	lastIterationCompleted time.Time
}

func (r *Plugin) recordSchedulerDetectionSuccess(jobType string, count int) {
	if r == nil {
		return
	}
	r.schedulerDetectionMu.Lock()
	defer r.schedulerDetectionMu.Unlock()
	info := r.schedulerDetection[jobType]
	if info == nil {
		info = &schedulerDetectionInfo{}
		r.schedulerDetection[jobType] = info
	}
	info.lastDetectedAt = time.Now().UTC()
	info.lastDetectedCount = count
	info.lastError = ""
	info.lastSkippedReason = ""
}

func (r *Plugin) recordSchedulerDetectionError(jobType string, err error) {
	if r == nil {
		return
	}
	if err == nil {
		return
	}
	r.schedulerDetectionMu.Lock()
	defer r.schedulerDetectionMu.Unlock()
	info := r.schedulerDetection[jobType]
	if info == nil {
		info = &schedulerDetectionInfo{}
		r.schedulerDetection[jobType] = info
	}
	info.lastErrorAt = time.Now().UTC()
	info.lastError = err.Error()
}

func (r *Plugin) recordSchedulerDetectionSkip(jobType string, reason string) {
	if r == nil {
		return
	}
	if strings.TrimSpace(reason) == "" {
		return
	}
	r.schedulerDetectionMu.Lock()
	defer r.schedulerDetectionMu.Unlock()
	info := r.schedulerDetection[jobType]
	if info == nil {
		info = &schedulerDetectionInfo{}
		r.schedulerDetection[jobType] = info
	}
	info.lastSkippedAt = time.Now().UTC()
	info.lastSkippedReason = reason
}

func (r *Plugin) snapshotSchedulerDetection(jobType string) schedulerDetectionInfo {
	if r == nil {
		return schedulerDetectionInfo{}
	}
	r.schedulerDetectionMu.Lock()
	defer r.schedulerDetectionMu.Unlock()
	info := r.schedulerDetection[jobType]
	if info == nil {
		return schedulerDetectionInfo{}
	}
	return *info
}

func (r *Plugin) recordSchedulerRunStart(jobType string) {
	if r == nil {
		return
	}
	r.schedulerRunMu.Lock()
	defer r.schedulerRunMu.Unlock()
	info := r.schedulerRun[jobType]
	if info == nil {
		info = &schedulerRunInfo{}
		r.schedulerRun[jobType] = info
	}
	info.lastRunStartedAt = time.Now().UTC()
	info.lastRunStatus = ""
}

func (r *Plugin) recordSchedulerRunComplete(jobType, status string) {
	if r == nil {
		return
	}
	r.schedulerRunMu.Lock()
	defer r.schedulerRunMu.Unlock()
	info := r.schedulerRun[jobType]
	if info == nil {
		info = &schedulerRunInfo{}
		r.schedulerRun[jobType] = info
	}
	info.lastRunCompletedAt = time.Now().UTC()
	info.lastRunStatus = status
}

func (r *Plugin) snapshotSchedulerRun(jobType string) schedulerRunInfo {
	if r == nil {
		return schedulerRunInfo{}
	}
	r.schedulerRunMu.Lock()
	defer r.schedulerRunMu.Unlock()
	info := r.schedulerRun[jobType]
	if info == nil {
		return schedulerRunInfo{}
	}
	return *info
}

func (r *Plugin) setSchedulerLoopState(jobType, phase string) {
	if r == nil {
		return
	}
	r.schedulerLoopMu.Lock()
	r.schedulerLoopState.currentJobType = jobType
	r.schedulerLoopState.currentPhase = phase
	r.schedulerLoopMu.Unlock()
}

// setSchedulerLoopStateForJobType keeps the aggregate scheduler state and the
// owning lane state in sync while a specific job type is active.
func (r *Plugin) setSchedulerLoopStateForJobType(jobType, phase string) {
	if r == nil {
		return
	}
	r.setSchedulerLoopState(jobType, phase)
	if jobType == "" {
		return
	}
	if ls := r.lanes[JobTypeLane(jobType)]; ls != nil {
		r.setLaneLoopState(ls, jobType, phase)
	}
}

func (r *Plugin) snapshotSchedulerLoopState() schedulerLoopState {
	if r == nil {
		return schedulerLoopState{}
	}
	r.schedulerLoopMu.Lock()
	defer r.schedulerLoopMu.Unlock()
	return r.schedulerLoopState
}

// aggregateLaneLoopStates merges per-lane loop states into a single
// schedulerLoopState for the aggregate GetSchedulerStatus API. It picks
// the most recent iteration completion, any currently-active job type,
// and a phase that reflects whether any lane is actively working.
func (r *Plugin) aggregateLaneLoopStates() schedulerLoopState {
	if r == nil || len(r.lanes) == 0 {
		return r.snapshotSchedulerLoopState()
	}

	var agg schedulerLoopState
	for _, ls := range r.lanes {
		snap := r.snapshotLaneLoopState(ls)
		if snap.lastIterationCompleted.After(agg.lastIterationCompleted) {
			agg.lastIterationCompleted = snap.lastIterationCompleted
		}
		if snap.lastIterationHadJobs {
			agg.lastIterationHadJobs = true
		}
		// Prefer showing an active phase over idle/sleeping.
		if snap.currentJobType != "" {
			agg.currentJobType = snap.currentJobType
			agg.currentPhase = snap.currentPhase
		}
	}
	// If no lane is actively processing, show the most interesting phase.
	if agg.currentPhase == "" {
		for _, ls := range r.lanes {
			snap := r.snapshotLaneLoopState(ls)
			if snap.currentPhase != "" {
				agg.currentPhase = snap.currentPhase
				break
			}
		}
	}
	return agg
}

// --- Per-lane loop state helpers ---

func (r *Plugin) setLaneLoopState(ls *schedulerLaneState, jobType, phase string) {
	if r == nil || ls == nil {
		return
	}
	ls.loopMu.Lock()
	ls.loop.currentJobType = jobType
	ls.loop.currentPhase = phase
	ls.loopMu.Unlock()
}

func (r *Plugin) recordLaneIterationComplete(ls *schedulerLaneState, hadJobs bool) {
	if r == nil || ls == nil {
		return
	}
	ls.loopMu.Lock()
	ls.loop.lastIterationHadJobs = hadJobs
	ls.loop.lastIterationCompleted = time.Now().UTC()
	ls.loopMu.Unlock()
}

func (r *Plugin) snapshotLaneLoopState(ls *schedulerLaneState) schedulerLoopState {
	if r == nil || ls == nil {
		return schedulerLoopState{}
	}
	ls.loopMu.Lock()
	defer ls.loopMu.Unlock()
	return ls.loop
}

// GetLaneSchedulerStatus returns scheduler status scoped to a single lane.
func (r *Plugin) GetLaneSchedulerStatus(lane SchedulerLane) SchedulerStatus {
	ls := r.lanes[lane]
	if ls == nil {
		return SchedulerStatus{Now: time.Now().UTC()}
	}
	now := time.Now().UTC()
	loopState := r.snapshotLaneLoopState(ls)
	idleSleep := LaneIdleSleep(lane)
	allInProcess := r.listInProcessJobs(now)
	laneInProcess := make([]SchedulerJobStatus, 0, len(allInProcess))
	for _, job := range allInProcess {
		if JobTypeLane(job.JobType) == lane {
			laneInProcess = append(laneInProcess, job)
		}
	}

	status := SchedulerStatus{
		Now:                  now,
		Lane:                 string(lane),
		SchedulerTickSeconds: int(secondsFromDuration(r.schedulerTick)),
		InProcessJobs:        laneInProcess,
		IdleSleepSeconds:     int(idleSleep / time.Second),
		CurrentJobType:       loopState.currentJobType,
		CurrentPhase:         loopState.currentPhase,
		LastIterationHadJobs: loopState.lastIterationHadJobs,
	}
	nextDetectionAt := r.earliestLaneDetectionAt(lane)
	if nextDetectionAt.IsZero() && loopState.currentPhase == "sleeping" && !loopState.lastIterationCompleted.IsZero() {
		nextDetectionAt = loopState.lastIterationCompleted.Add(idleSleep)
	}
	if !nextDetectionAt.IsZero() {
		at := nextDetectionAt
		status.NextDetectionAt = &at
	}
	if !loopState.lastIterationCompleted.IsZero() {
		at := loopState.lastIterationCompleted
		status.LastIterationDoneAt = &at
	}

	states, err := r.ListSchedulerStates()
	if err != nil {
		return status
	}

	waiting := make([]SchedulerWaitingStatus, 0)
	jobTypes := make([]SchedulerJobTypeStatus, 0)

	for _, state := range states {
		if JobTypeLane(state.JobType) != lane {
			continue
		}
		jobType := state.JobType
		info := r.snapshotSchedulerDetection(jobType)

		jobStatus := SchedulerJobTypeStatus{
			JobType:                  jobType,
			Enabled:                  state.Enabled,
			DetectionInFlight:        state.DetectionInFlight,
			NextDetectionAt:          state.NextDetectionAt,
			DetectionIntervalMinutes: state.DetectionIntervalMinutes,
		}
		r.populateSchedulerCooldown(&jobStatus)
		if !info.lastDetectedAt.IsZero() {
			jobStatus.LastDetectedAt = timeToPtr(info.lastDetectedAt)
			jobStatus.LastDetectedCount = info.lastDetectedCount
		}
		if info.lastError != "" {
			jobStatus.LastDetectionError = info.lastError
		}
		if info.lastSkippedReason != "" {
			jobStatus.LastDetectionSkipped = info.lastSkippedReason
		}
		jobTypes = append(jobTypes, jobStatus)

		if jobStatus.Enabled && jobStatus.DetectionEligibleAt != nil && now.Before(*jobStatus.DetectionEligibleAt) {
			waiting = append(waiting, SchedulerWaitingStatus{
				Reason: "cooldown", JobType: jobType, Since: jobStatus.LastSuccessfulRunAt, Until: jobStatus.DetectionEligibleAt,
				Details: map[string]interface{}{"message": fmt.Sprintf("Cooldown after successful job (%s)", time.Duration(jobStatus.CooldownSeconds)*time.Second)},
			})
		}
		if state.DetectionInFlight {
			waiting = append(waiting, SchedulerWaitingStatus{
				Reason:  "detection_in_flight",
				JobType: jobType,
			})
		} else if state.Enabled && state.NextDetectionAt != nil && now.Before(*state.NextDetectionAt) {
			waiting = append(waiting, SchedulerWaitingStatus{
				Reason:  "next_detection_at",
				JobType: jobType,
				Until:   state.NextDetectionAt,
			})
		}
	}

	sort.Slice(jobTypes, func(i, j int) bool {
		return jobTypes[i].JobType < jobTypes[j].JobType
	})

	status.Waiting = waiting
	status.JobTypes = jobTypes
	return status
}

func (r *Plugin) GetSchedulerStatus() SchedulerStatus {
	now := time.Now().UTC()

	// Aggregate loop state across all lanes instead of reading the
	// legacy single-loop state which is no longer updated.
	aggregated := r.aggregateLaneLoopStates()

	status := SchedulerStatus{
		Now:                  now,
		SchedulerTickSeconds: int(secondsFromDuration(r.schedulerTick)),
		InProcessJobs:        r.listInProcessJobs(now),
		IdleSleepSeconds:     int(defaultSchedulerIdleSleep / time.Second),
		CurrentJobType:       aggregated.currentJobType,
		CurrentPhase:         aggregated.currentPhase,
		LastIterationHadJobs: aggregated.lastIterationHadJobs,
	}
	nextDetectionAt := r.earliestNextDetectionAt()
	if nextDetectionAt.IsZero() && aggregated.currentPhase == "sleeping" && !aggregated.lastIterationCompleted.IsZero() {
		nextDetectionAt = aggregated.lastIterationCompleted.Add(defaultSchedulerIdleSleep)
	}
	if !nextDetectionAt.IsZero() {
		at := nextDetectionAt
		status.NextDetectionAt = &at
	}
	if !aggregated.lastIterationCompleted.IsZero() {
		at := aggregated.lastIterationCompleted
		status.LastIterationDoneAt = &at
	}

	states, err := r.ListSchedulerStates()
	if err != nil {
		return status
	}

	waiting := make([]SchedulerWaitingStatus, 0)
	jobTypes := make([]SchedulerJobTypeStatus, 0, len(states))

	for _, state := range states {
		jobType := state.JobType
		info := r.snapshotSchedulerDetection(jobType)

		jobStatus := SchedulerJobTypeStatus{
			JobType:                  jobType,
			Enabled:                  state.Enabled,
			DetectionInFlight:        state.DetectionInFlight,
			NextDetectionAt:          state.NextDetectionAt,
			DetectionIntervalMinutes: state.DetectionIntervalMinutes,
		}
		r.populateSchedulerCooldown(&jobStatus)
		if !info.lastDetectedAt.IsZero() {
			jobStatus.LastDetectedAt = timeToPtr(info.lastDetectedAt)
			jobStatus.LastDetectedCount = info.lastDetectedCount
		}
		if info.lastError != "" {
			jobStatus.LastDetectionError = info.lastError
		}
		if info.lastSkippedReason != "" {
			jobStatus.LastDetectionSkipped = info.lastSkippedReason
		}
		jobTypes = append(jobTypes, jobStatus)

		if jobStatus.Enabled && jobStatus.DetectionEligibleAt != nil && now.Before(*jobStatus.DetectionEligibleAt) {
			waiting = append(waiting, SchedulerWaitingStatus{
				Reason: "cooldown", JobType: jobType, Since: jobStatus.LastSuccessfulRunAt, Until: jobStatus.DetectionEligibleAt,
				Details: map[string]interface{}{"message": fmt.Sprintf("Cooldown after successful job (%s)", time.Duration(jobStatus.CooldownSeconds)*time.Second)},
			})
		}
		if state.DetectionInFlight {
			waiting = append(waiting, SchedulerWaitingStatus{
				Reason:  "detection_in_flight",
				JobType: jobType,
			})
		} else if state.Enabled && state.NextDetectionAt != nil && now.Before(*state.NextDetectionAt) {
			waiting = append(waiting, SchedulerWaitingStatus{
				Reason:  "next_detection_at",
				JobType: jobType,
				Until:   state.NextDetectionAt,
			})
		}
	}

	sort.Slice(jobTypes, func(i, j int) bool {
		return jobTypes[i].JobType < jobTypes[j].JobType
	})

	status.Waiting = waiting
	status.JobTypes = jobTypes
	return status
}

func (r *Plugin) listInProcessJobs(now time.Time) []SchedulerJobStatus {
	active := make([]SchedulerJobStatus, 0)
	if r == nil {
		return active
	}

	r.jobsMu.RLock()
	for _, job := range r.jobs {
		if job == nil {
			continue
		}
		if !isActiveTrackedJobState(job.State) {
			continue
		}
		start := timeToPtr(now)
		if job.CreatedAt != nil && !job.CreatedAt.IsZero() {
			start = job.CreatedAt
		} else if job.UpdatedAt != nil && !job.UpdatedAt.IsZero() {
			start = job.UpdatedAt
		}
		durationSeconds := 0.0
		if start != nil {
			durationSeconds = now.Sub(*start).Seconds()
		}
		active = append(active, SchedulerJobStatus{
			JobID:           job.JobID,
			JobType:         job.JobType,
			State:           strings.ToLower(job.State),
			Stage:           job.Stage,
			WorkerID:        job.WorkerID,
			Message:         job.Message,
			Progress:        job.Progress,
			CreatedAt:       job.CreatedAt,
			UpdatedAt:       job.UpdatedAt,
			DurationSeconds: durationSeconds,
		})
	}
	r.jobsMu.RUnlock()

	sort.Slice(active, func(i, j int) bool {
		if active[i].DurationSeconds != active[j].DurationSeconds {
			return active[i].DurationSeconds > active[j].DurationSeconds
		}
		return active[i].JobID < active[j].JobID
	})

	return active
}