Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix(scheduler): skip overlapping cron job runs
Kyle Mendell51cf6bb6a104
diff --git a/backend/pkg/scheduler/scheduler.go b/backend/pkg/scheduler/scheduler.goindex 647574960..de206d1ba 100644--- a/backend/pkg/scheduler/scheduler.go+++ b/backend/pkg/scheduler/scheduler.go@@ -5,6 +5,7 @@ import ("log/slog""maps""slices"+ "sync""time""emperror.dev/errors"@@ -29,6 +30,7 @@ type schedulerStateInternal struct {runners map[string]*actors.RunnerentryIDs map[string]cron.EntryIDschedules map[string]string+ runLocks map[string]*sync.Mutexstopping bool}@@ -72,6 +74,7 @@ func NewJobScheduler(ctx context.Context, runtime *actors.Runtime, location *timrunners: make(map[string]*actors.Runner),entryIDs: make(map[string]cron.EntryID),schedules: make(map[string]string),+ runLocks: make(map[string]*sync.Mutex),}state, err := actors.NewState(ctx, runtime, "scheduler", "control-plane", 3, initial, func(value schedulerStateInternal) schedulerStateInternal {value.jobs = slices.Clone(value.jobs)@@ -81,6 +84,7 @@ func NewJobScheduler(ctx context.Context, runtime *actors.Runtime, location *timvalue.runners = maps.Clone(value.runners)value.entryIDs = maps.Clone(value.entryIDs)value.schedules = maps.Clone(value.schedules)+ value.runLocks = maps.Clone(value.runLocks)return value})if err != nil {@@ -345,7 +349,12 @@ func (js *jobSchedulerInternal) upsertJobInternal(ctx context.Context, state *scdelete(state.entryIDs, jobName)}if shouldSchedule {- entryID, nextRun = js.addCronEntryInternal(job, schedule, parsedSchedule)+ runLock, ok := state.runLocks[jobName]+ if !ok {+ runLock = &sync.Mutex{}+ state.runLocks[jobName] = runLock+ }+ entryID, nextRun = js.addCronEntryInternal(job, schedule, parsedSchedule, runLock)state.entryIDs[jobName] = entryID}state.jobsByID[jobName] = job@@ -360,8 +369,13 @@ func (js *jobSchedulerInternal) upsertJobInternal(ctx context.Context, state *screturn nil}-func (js *jobSchedulerInternal) addCronEntryInternal(job schedulertypes.Job, schedule string, parsedSchedule cron.Schedule) (cron.EntryID, *time.Time) {+func (js *jobSchedulerInternal) addCronEntryInternal(job schedulertypes.Job, schedule string, parsedSchedule cron.Schedule, runLock *sync.Mutex) (cron.EntryID, *time.Time) {entryID := js.cron.Schedule(parsedSchedule, cron.FuncJob(func() {+ if !runLock.TryLock() {+ slog.WarnContext(js.context, "Job skipped; previous run still in progress", "name", job.Name(), "schedule", schedule)+ return+ }+ defer runLock.Unlock()defer utils.RecoverToError(nil, "scheduled job", "name", job.Name(), "schedule", schedule)slog.InfoContext(js.context, "Job starting", "name", job.Name(), "schedule", schedule)job.Run(js.context)