SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package plugin
import (
"fmt"
"sort"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
)
type SchedulerStatus struct {
Now time.Time `json:"now"`
Lane string `json:"lane,omitempty"`
SchedulerTickSeconds int `json:"scheduler_tick_seconds"`
IdleSleepSeconds int `json:"idle_sleep_seconds,omitempty"`
NextDetectionAt *time.Time `json:"next_detection_at,omitempty"`
CurrentJobType string `json:"current_job_type,omitempty"`
CurrentPhase string `json:"current_phase,omitempty"`
LastIterationHadJobs bool `json:"last_iteration_had_jobs,omitempty"`
LastIterationDoneAt *time.Time `json:"last_iteration_done_at,omitempty"`
Waiting []SchedulerWaitingStatus `json:"waiting,omitempty"`
InProcessJobs []SchedulerJobStatus `json:"in_process_jobs,omitempty"`
JobTypes []SchedulerJobTypeStatus `json:"job_types,omitempty"`
}
type SchedulerWaitingStatus struct {
Reason string `json:"reason"`
JobType string `json:"job_type,omitempty"`
Since *time.Time `json:"since,omitempty"`
Until *time.Time `json:"until,omitempty"`
Details map[string]interface{} `json:"details,omitempty"`
}
type SchedulerJobStatus struct {
JobID string `json:"job_id"`
JobType string `json:"job_type"`
State string `json:"state"`
Stage string `json:"stage,omitempty"`
WorkerID string `json:"worker_id,omitempty"`
Message string `json:"message,omitempty"`
Progress float64 `json:"progress,omitempty"`
CreatedAt *time.Time `json:"created_at,omitempty"`
UpdatedAt *time.Time `json:"updated_at,omitempty"`
DurationSeconds float64 `json:"duration_seconds,omitempty"`
}
type SchedulerJobTypeStatus struct {
JobType string `json:"job_type"`
Enabled bool `json:"enabled"`
DetectionInFlight bool `json:"detection_in_flight"`
NextDetectionAt *time.Time `json:"next_detection_at,omitempty"`
DetectionIntervalMinutes int32 `json:"detection_interval_minutes,omitempty"`
LastDetectedAt *time.Time `json:"last_detected_at,omitempty"`
LastDetectedCount int `json:"last_detected_count,omitempty"`
LastDetectionError string `json:"last_detection_error,omitempty"`
LastDetectionSkipped string `json:"last_detection_skipped,omitempty"`
CooldownSeconds int64 `json:"cooldown_seconds,omitempty"`
LastSuccessfulRunAt *time.Time `json:"last_successful_run_at,omitempty"`
DetectionEligibleAt *time.Time `json:"detection_eligible_at,omitempty"`
CooldownError string `json:"cooldown_error,omitempty"`
}
func (r *Plugin) populateSchedulerCooldown(status *SchedulerJobTypeStatus) {
descriptor, err := r.store.LoadDescriptor(status.JobType)
if err != nil {
status.CooldownError = "Cannot load cooldown policy"
return
}
field := descriptor.GetDetectionCooldownConfigField()
if field == "" {
return
}
_, _, values, err := r.loadJobTypeConfigPayload(status.JobType)
if err != nil {
status.CooldownError = "Cannot load cooldown configuration"
return
}
value := values[field]
if value == nil {
value = descriptor.GetWorkerDefaultValues()[field]
}
if value == nil {
value = descriptor.GetWorkerConfigForm().GetDefaultValues()[field]
}
if value == nil {
status.CooldownError = "Cooldown configuration is missing"
return
}
if _, ok := value.Kind.(*plugin_pb.ConfigValue_Int64Value); !ok {
status.CooldownError = "Cooldown must be an integer number of seconds"
return
}
seconds := value.GetInt64Value()
if seconds < 0 || seconds > int64((1<<63-1)/time.Second) {
status.CooldownError = "Cooldown is outside the supported range"
return
}
status.CooldownSeconds = seconds
history, err := r.store.LoadRunHistory(status.JobType)
if err != nil {
status.CooldownError = "Cannot load successful job history"
return
}
if history == nil {
return
}
for _, run := range history.SuccessfulRuns {
if run.CompletedAt != nil && (status.LastSuccessfulRunAt == nil || run.CompletedAt.After(*status.LastSuccessfulRunAt)) {
at := *run.CompletedAt
status.LastSuccessfulRunAt = &at
}
}
if status.LastSuccessfulRunAt != nil && seconds > 0 {
at := status.LastSuccessfulRunAt.Add(time.Duration(seconds) * time.Second)
status.DetectionEligibleAt = &at
}
}
type schedulerDetectionInfo struct {
lastDetectedAt time.Time
lastDetectedCount int
lastErrorAt time.Time
lastError string
lastSkippedAt time.Time
lastSkippedReason string
}
type schedulerRunInfo struct {
lastRunStartedAt time.Time
lastRunCompletedAt time.Time
lastRunStatus string
}
type schedulerLoopState struct {
currentJobType string
currentPhase string
lastIterationHadJobs bool
lastIterationCompleted time.Time
}
func (r *Plugin) recordSchedulerDetectionSuccess(jobType string, count int) {
if r == nil {
return
}
r.schedulerDetectionMu.Lock()
defer r.schedulerDetectionMu.Unlock()
info := r.schedulerDetection[jobType]
if info == nil {
info = &schedulerDetectionInfo{}
r.schedulerDetection[jobType] = info
}
info.lastDetectedAt = time.Now().UTC()
info.lastDetectedCount = count
info.lastError = ""
info.lastSkippedReason = ""
}
func (r *Plugin) recordSchedulerDetectionError(jobType string, err error) {
if r == nil {
return
}
if err == nil {
return
}
r.schedulerDetectionMu.Lock()
defer r.schedulerDetectionMu.Unlock()
info := r.schedulerDetection[jobType]
if info == nil {
info = &schedulerDetectionInfo{}
r.schedulerDetection[jobType] = info
}
info.lastErrorAt = time.Now().UTC()
info.lastError = err.Error()
}
func (r *Plugin) recordSchedulerDetectionSkip(jobType string, reason string) {
if r == nil {
return
}
if strings.TrimSpace(reason) == "" {
return
}
r.schedulerDetectionMu.Lock()
defer r.schedulerDetectionMu.Unlock()
info := r.schedulerDetection[jobType]
if info == nil {
info = &schedulerDetectionInfo{}
r.schedulerDetection[jobType] = info
}
info.lastSkippedAt = time.Now().UTC()
info.lastSkippedReason = reason
}
func (r *Plugin) snapshotSchedulerDetection(jobType string) schedulerDetectionInfo {
if r == nil {
return schedulerDetectionInfo{}
}
r.schedulerDetectionMu.Lock()
defer r.schedulerDetectionMu.Unlock()
info := r.schedulerDetection[jobType]
if info == nil {
return schedulerDetectionInfo{}
}
return *info
}
func (r *Plugin) recordSchedulerRunStart(jobType string) {
if r == nil {
return
}
r.schedulerRunMu.Lock()
defer r.schedulerRunMu.Unlock()
info := r.schedulerRun[jobType]
if info == nil {
info = &schedulerRunInfo{}
r.schedulerRun[jobType] = info
}
info.lastRunStartedAt = time.Now().UTC()
info.lastRunStatus = ""
}
func (r *Plugin) recordSchedulerRunComplete(jobType, status string) {
if r == nil {
return
}
r.schedulerRunMu.Lock()
defer r.schedulerRunMu.Unlock()
info := r.schedulerRun[jobType]
if info == nil {
info = &schedulerRunInfo{}
r.schedulerRun[jobType] = info
}
info.lastRunCompletedAt = time.Now().UTC()
info.lastRunStatus = status
}
func (r *Plugin) snapshotSchedulerRun(jobType string) schedulerRunInfo {
if r == nil {
return schedulerRunInfo{}
}
r.schedulerRunMu.Lock()
defer r.schedulerRunMu.Unlock()
info := r.schedulerRun[jobType]
if info == nil {
return schedulerRunInfo{}
}
return *info
}
func (r *Plugin) setSchedulerLoopState(jobType, phase string) {
if r == nil {
return
}
r.schedulerLoopMu.Lock()
r.schedulerLoopState.currentJobType = jobType
r.schedulerLoopState.currentPhase = phase
r.schedulerLoopMu.Unlock()
}
// setSchedulerLoopStateForJobType keeps the aggregate scheduler state and the
// owning lane state in sync while a specific job type is active.
func (r *Plugin) setSchedulerLoopStateForJobType(jobType, phase string) {
if r == nil {
return
}
r.setSchedulerLoopState(jobType, phase)
if jobType == "" {
return
}
if ls := r.lanes[JobTypeLane(jobType)]; ls != nil {
r.setLaneLoopState(ls, jobType, phase)
}
}
func (r *Plugin) snapshotSchedulerLoopState() schedulerLoopState {
if r == nil {
return schedulerLoopState{}
}
r.schedulerLoopMu.Lock()
defer r.schedulerLoopMu.Unlock()
return r.schedulerLoopState
}
// aggregateLaneLoopStates merges per-lane loop states into a single
// schedulerLoopState for the aggregate GetSchedulerStatus API. It picks
// the most recent iteration completion, any currently-active job type,
// and a phase that reflects whether any lane is actively working.
func (r *Plugin) aggregateLaneLoopStates() schedulerLoopState {
if r == nil || len(r.lanes) == 0 {
return r.snapshotSchedulerLoopState()
}
var agg schedulerLoopState
for _, ls := range r.lanes {
snap := r.snapshotLaneLoopState(ls)
if snap.lastIterationCompleted.After(agg.lastIterationCompleted) {
agg.lastIterationCompleted = snap.lastIterationCompleted
}
if snap.lastIterationHadJobs {
agg.lastIterationHadJobs = true
}
// Prefer showing an active phase over idle/sleeping.
if snap.currentJobType != "" {
agg.currentJobType = snap.currentJobType
agg.currentPhase = snap.currentPhase
}
}
// If no lane is actively processing, show the most interesting phase.
if agg.currentPhase == "" {
for _, ls := range r.lanes {
snap := r.snapshotLaneLoopState(ls)
if snap.currentPhase != "" {
agg.currentPhase = snap.currentPhase
break
}
}
}
return agg
}
// --- Per-lane loop state helpers ---
func (r *Plugin) setLaneLoopState(ls *schedulerLaneState, jobType, phase string) {
if r == nil || ls == nil {
return
}
ls.loopMu.Lock()
ls.loop.currentJobType = jobType
ls.loop.currentPhase = phase
ls.loopMu.Unlock()
}
func (r *Plugin) recordLaneIterationComplete(ls *schedulerLaneState, hadJobs bool) {
if r == nil || ls == nil {
return
}
ls.loopMu.Lock()
ls.loop.lastIterationHadJobs = hadJobs
ls.loop.lastIterationCompleted = time.Now().UTC()
ls.loopMu.Unlock()
}
func (r *Plugin) snapshotLaneLoopState(ls *schedulerLaneState) schedulerLoopState {
if r == nil || ls == nil {
return schedulerLoopState{}
}
ls.loopMu.Lock()
defer ls.loopMu.Unlock()
return ls.loop
}
// GetLaneSchedulerStatus returns scheduler status scoped to a single lane.
func (r *Plugin) GetLaneSchedulerStatus(lane SchedulerLane) SchedulerStatus {
ls := r.lanes[lane]
if ls == nil {
return SchedulerStatus{Now: time.Now().UTC()}
}
now := time.Now().UTC()
loopState := r.snapshotLaneLoopState(ls)
idleSleep := LaneIdleSleep(lane)
allInProcess := r.listInProcessJobs(now)
laneInProcess := make([]SchedulerJobStatus, 0, len(allInProcess))
for _, job := range allInProcess {
if JobTypeLane(job.JobType) == lane {
laneInProcess = append(laneInProcess, job)
}
}
status := SchedulerStatus{
Now: now,
Lane: string(lane),
SchedulerTickSeconds: int(secondsFromDuration(r.schedulerTick)),
InProcessJobs: laneInProcess,
IdleSleepSeconds: int(idleSleep / time.Second),
CurrentJobType: loopState.currentJobType,
CurrentPhase: loopState.currentPhase,
LastIterationHadJobs: loopState.lastIterationHadJobs,
}
nextDetectionAt := r.earliestLaneDetectionAt(lane)
if nextDetectionAt.IsZero() && loopState.currentPhase == "sleeping" && !loopState.lastIterationCompleted.IsZero() {
nextDetectionAt = loopState.lastIterationCompleted.Add(idleSleep)
}
if !nextDetectionAt.IsZero() {
at := nextDetectionAt
status.NextDetectionAt = &at
}
if !loopState.lastIterationCompleted.IsZero() {
at := loopState.lastIterationCompleted
status.LastIterationDoneAt = &at
}
states, err := r.ListSchedulerStates()
if err != nil {
return status
}
waiting := make([]SchedulerWaitingStatus, 0)
jobTypes := make([]SchedulerJobTypeStatus, 0)
for _, state := range states {
if JobTypeLane(state.JobType) != lane {
continue
}
jobType := state.JobType
info := r.snapshotSchedulerDetection(jobType)
jobStatus := SchedulerJobTypeStatus{
JobType: jobType,
Enabled: state.Enabled,
DetectionInFlight: state.DetectionInFlight,
NextDetectionAt: state.NextDetectionAt,
DetectionIntervalMinutes: state.DetectionIntervalMinutes,
}
r.populateSchedulerCooldown(&jobStatus)
if !info.lastDetectedAt.IsZero() {
jobStatus.LastDetectedAt = timeToPtr(info.lastDetectedAt)
jobStatus.LastDetectedCount = info.lastDetectedCount
}
if info.lastError != "" {
jobStatus.LastDetectionError = info.lastError
}
if info.lastSkippedReason != "" {
jobStatus.LastDetectionSkipped = info.lastSkippedReason
}
jobTypes = append(jobTypes, jobStatus)
if jobStatus.Enabled && jobStatus.DetectionEligibleAt != nil && now.Before(*jobStatus.DetectionEligibleAt) {
waiting = append(waiting, SchedulerWaitingStatus{
Reason: "cooldown", JobType: jobType, Since: jobStatus.LastSuccessfulRunAt, Until: jobStatus.DetectionEligibleAt,
Details: map[string]interface{}{"message": fmt.Sprintf("Cooldown after successful job (%s)", time.Duration(jobStatus.CooldownSeconds)*time.Second)},
})
}
if state.DetectionInFlight {
waiting = append(waiting, SchedulerWaitingStatus{
Reason: "detection_in_flight",
JobType: jobType,
})
} else if state.Enabled && state.NextDetectionAt != nil && now.Before(*state.NextDetectionAt) {
waiting = append(waiting, SchedulerWaitingStatus{
Reason: "next_detection_at",
JobType: jobType,
Until: state.NextDetectionAt,
})
}
}
sort.Slice(jobTypes, func(i, j int) bool {
return jobTypes[i].JobType < jobTypes[j].JobType
})
status.Waiting = waiting
status.JobTypes = jobTypes
return status
}
func (r *Plugin) GetSchedulerStatus() SchedulerStatus {
now := time.Now().UTC()
// Aggregate loop state across all lanes instead of reading the
// legacy single-loop state which is no longer updated.
aggregated := r.aggregateLaneLoopStates()
status := SchedulerStatus{
Now: now,
SchedulerTickSeconds: int(secondsFromDuration(r.schedulerTick)),
InProcessJobs: r.listInProcessJobs(now),
IdleSleepSeconds: int(defaultSchedulerIdleSleep / time.Second),
CurrentJobType: aggregated.currentJobType,
CurrentPhase: aggregated.currentPhase,
LastIterationHadJobs: aggregated.lastIterationHadJobs,
}
nextDetectionAt := r.earliestNextDetectionAt()
if nextDetectionAt.IsZero() && aggregated.currentPhase == "sleeping" && !aggregated.lastIterationCompleted.IsZero() {
nextDetectionAt = aggregated.lastIterationCompleted.Add(defaultSchedulerIdleSleep)
}
if !nextDetectionAt.IsZero() {
at := nextDetectionAt
status.NextDetectionAt = &at
}
if !aggregated.lastIterationCompleted.IsZero() {
at := aggregated.lastIterationCompleted
status.LastIterationDoneAt = &at
}
states, err := r.ListSchedulerStates()
if err != nil {
return status
}
waiting := make([]SchedulerWaitingStatus, 0)
jobTypes := make([]SchedulerJobTypeStatus, 0, len(states))
for _, state := range states {
jobType := state.JobType
info := r.snapshotSchedulerDetection(jobType)
jobStatus := SchedulerJobTypeStatus{
JobType: jobType,
Enabled: state.Enabled,
DetectionInFlight: state.DetectionInFlight,
NextDetectionAt: state.NextDetectionAt,
DetectionIntervalMinutes: state.DetectionIntervalMinutes,
}
r.populateSchedulerCooldown(&jobStatus)
if !info.lastDetectedAt.IsZero() {
jobStatus.LastDetectedAt = timeToPtr(info.lastDetectedAt)
jobStatus.LastDetectedCount = info.lastDetectedCount
}
if info.lastError != "" {
jobStatus.LastDetectionError = info.lastError
}
if info.lastSkippedReason != "" {
jobStatus.LastDetectionSkipped = info.lastSkippedReason
}
jobTypes = append(jobTypes, jobStatus)
if jobStatus.Enabled && jobStatus.DetectionEligibleAt != nil && now.Before(*jobStatus.DetectionEligibleAt) {
waiting = append(waiting, SchedulerWaitingStatus{
Reason: "cooldown", JobType: jobType, Since: jobStatus.LastSuccessfulRunAt, Until: jobStatus.DetectionEligibleAt,
Details: map[string]interface{}{"message": fmt.Sprintf("Cooldown after successful job (%s)", time.Duration(jobStatus.CooldownSeconds)*time.Second)},
})
}
if state.DetectionInFlight {
waiting = append(waiting, SchedulerWaitingStatus{
Reason: "detection_in_flight",
JobType: jobType,
})
} else if state.Enabled && state.NextDetectionAt != nil && now.Before(*state.NextDetectionAt) {
waiting = append(waiting, SchedulerWaitingStatus{
Reason: "next_detection_at",
JobType: jobType,
Until: state.NextDetectionAt,
})
}
}
sort.Slice(jobTypes, func(i, j int) bool {
return jobTypes[i].JobType < jobTypes[j].JobType
})
status.Waiting = waiting
status.JobTypes = jobTypes
return status
}
func (r *Plugin) listInProcessJobs(now time.Time) []SchedulerJobStatus {
active := make([]SchedulerJobStatus, 0)
if r == nil {
return active
}
r.jobsMu.RLock()
for _, job := range r.jobs {
if job == nil {
continue
}
if !isActiveTrackedJobState(job.State) {
continue
}
start := timeToPtr(now)
if job.CreatedAt != nil && !job.CreatedAt.IsZero() {
start = job.CreatedAt
} else if job.UpdatedAt != nil && !job.UpdatedAt.IsZero() {
start = job.UpdatedAt
}
durationSeconds := 0.0
if start != nil {
durationSeconds = now.Sub(*start).Seconds()
}
active = append(active, SchedulerJobStatus{
JobID: job.JobID,
JobType: job.JobType,
State: strings.ToLower(job.State),
Stage: job.Stage,
WorkerID: job.WorkerID,
Message: job.Message,
Progress: job.Progress,
CreatedAt: job.CreatedAt,
UpdatedAt: job.UpdatedAt,
DurationSeconds: durationSeconds,
})
}
r.jobsMu.RUnlock()
sort.Slice(active, func(i, j int) bool {
if active[i].DurationSeconds != active[j].DurationSeconds {
return active[i].DurationSeconds > active[j].DurationSeconds
}
return active[i].JobID < active[j].JobID
})
return active
}