Skip to content

Modern Docker Management, Designed for Everyone

fix(jobs): recover oversized legacy job histories on startup (#4270)

Kyle Mendellf2666e3712fa
diff --git a/Justfile b/Justfile
index 6c589bc9c..8a8d1c0c2 100644
--- a/Justfile
+++ b/Justfile
@@ -478,6 +478,42 @@ _deps-update-frontend:
_deps-update-backend:
cd backend && go get -u ./... && go mod tidy
+# Update direct Go dependencies in all modules, then sync the workspace (requires jq).
+[group('deps')]
+_deps-update-go *args:
+ #!/usr/bin/env bash
+ set -euo pipefail
+ version=upgrade
+ set -- {{ args }}
+ case "$#:$*" in
+ 0:) ;;
+ 1:--patch) version=patch ;;
+ *) echo 'Usage: just deps go update [--patch]' >&2; exit 1 ;;
+ esac
+ command -v jq >/dev/null || { echo 'jq is required.' >&2; exit 1; }
+ for module in {{ modules }}; do
+ (
+ cd "$module"
+ export GOWORK=off
+ echo "Updating direct dependencies in $module..."
+ direct=$(go list -mod=mod -m -json all | jq -r '
+ select(.Main != true and .Indirect != true)
+ | select(.Replace == null or .Replace.Version != null)
+ | .Path')
+ dependencies=()
+ while IFS= read -r dependency; do
+ if [ -n "$dependency" ]; then
+ dependencies+=("$dependency@$version")
+ fi
+ done <<<"$direct"
+ if [ "${#dependencies[@]}" -gt 0 ]; then
+ go get "${dependencies[@]}"
+ fi
+ go mod tidy
+ )
+ go work sync
+ done
+
# Update pnpm version via corepack
[group('deps')]
_deps-update-pnpm:
@@ -494,10 +530,20 @@ _deps-dedupe-node:
[group('deps')]
_deps-dedupe-all: _deps-dedupe-node
-# Deps targets. Valid: "install [frontend|tests|backend|cli|types|go|node|all]", "update [frontend|backend|pnpm|all]", "dedupe [node|go|all]"
+# Deps targets. Also supports "go update [--patch]" for direct Go dependency updates.
[group('deps')]
-deps action="update" target="all":
- @just "_deps-{{ action }}-{{ target }}"
+deps action="update" target="all" *args:
+ #!/usr/bin/env bash
+ set -euo pipefail
+ if [ "{{ action }}" = "go" ]; then
+ if [ "{{ target }}" != "update" ]; then
+ echo 'Usage: just deps go update [--patch]' >&2
+ exit 1
+ fi
+ just _deps-update-go {{ args }}
+ else
+ just "_deps-{{ action }}-{{ target }}" {{ args }}
+ fi
# -----------------------------------------------------------------------------
# Code generation and docs
diff --git a/backend/internal/bootstrap/bootstrap_test.go b/backend/internal/bootstrap/bootstrap_test.go
index 63754d350..a3589b9a5 100644
--- a/backend/internal/bootstrap/bootstrap_test.go
+++ b/backend/internal/bootstrap/bootstrap_test.go
@@ -434,7 +434,10 @@ func TestJobSchedulerStopCancelsItsPrivateContextInternal(t *testing.T) {
}
require.NoError(t, jobScheduler.RegisterBusWatcher(watcher, false))
- lifecycle.RequireStart()
+ startCtx, cancelStart := context.WithTimeout(appCtx, 5*time.Second)
+ defer cancelStart()
+ require.NoError(t, lifecycle.Start(startCtx))
+ cancelStart()
select {
case run := <-executed:
require.Equal(t, "gitops-sync:overdue", run.JobID)
diff --git a/backend/internal/bootstrap/jobs_bootstrap.go b/backend/internal/bootstrap/jobs_bootstrap.go
index b88bd2ceb..91235dcad 100644
--- a/backend/internal/bootstrap/jobs_bootstrap.go
+++ b/backend/internal/bootstrap/jobs_bootstrap.go
@@ -42,16 +42,22 @@ func newJobScheduler(appCtx context.Context, lc fx.Lifecycle, cfg *config.Config
var repairDone chan struct{}
var resumeDone chan struct{}
lc.Append(fx.Hook{
- OnStart: func(context.Context) error {
+ OnStart: func(ctx context.Context) error {
+ started := false
+ defer func() {
+ if !started {
+ cancelScheduler()
+ }
+ }()
slog.InfoContext(appCtx, "Starting scheduler")
- if err := jobService.Coordinator().Start(schedulerCtx); err != nil {
- return err
+ if err := jobService.Coordinator().Start(ctx, schedulerCtx); err != nil {
+ return fmt.Errorf("initialize scheduler: %w", err)
}
- if err := reconcileCoordinatorStartupInternal(schedulerCtx, jobService, backupEngine, updaterService, volumes, systemBackups, gitopsSync); err != nil {
- return err
+ if err := reconcileCoordinatorStartupInternal(ctx, jobService, backupEngine, updaterService, volumes, systemBackups, gitopsSync); err != nil {
+ return fmt.Errorf("reconcile scheduler startup: %w", err)
}
if gitopsSync != nil {
- gitopsSync.RegisterAutoSyncJobsOnStartup(schedulerCtx)
+ gitopsSync.RegisterAutoSyncJobsOnStartup(ctx)
gitopsSync.SubscribeProjectFileChanges(schedulerCtx)
}
if imageUpdateWatcher != nil {
@@ -60,8 +66,8 @@ func newJobScheduler(appCtx context.Context, lc fx.Lifecycle, cfg *config.Config
return err
}
}
- if err := jobScheduler.StartScheduler(); err != nil {
- return err
+ if err := jobScheduler.StartScheduler(ctx); err != nil {
+ return fmt.Errorf("start scheduler: %w", err)
}
jobService.Coordinator().Activate()
if backupEngine != nil {
@@ -72,14 +78,18 @@ func newJobScheduler(appCtx context.Context, lc fx.Lifecycle, cfg *config.Config
}()
}
if analytics != nil {
- if _, err := jobService.Coordinator().Submit(schedulerCtx, schedulertypes.Request{JobID: analytics.Name(), EnvironmentID: "0", Trigger: "startup"}); err != nil {
- return err
+ if _, err := jobService.Coordinator().Submit(ctx, schedulertypes.Request{JobID: analytics.Name(), EnvironmentID: "0", Trigger: "startup"}); err != nil {
+ return fmt.Errorf("submit startup job %q: %w", analytics.Name(), err)
}
}
if !cfg.AgentMode && systemUpgrade != nil {
resumeDone = make(chan struct{})
go func() { defer close(resumeDone); systemUpgrade.ResumeUpdateAllOnStartup(schedulerCtx) }()
}
+ if err := ctx.Err(); err != nil {
+ return fmt.Errorf("complete scheduler startup: %w", err)
+ }
+ started = true
return nil
},
OnStop: func(ctx context.Context) error {
diff --git a/backend/internal/job/activity_test.go b/backend/internal/job/activity_test.go
index 41646f999..b95cb451a 100644
--- a/backend/internal/job/activity_test.go
+++ b/backend/internal/job/activity_test.go
@@ -139,12 +139,16 @@ func TestJobActivityRestartRepairsFailedProjection(t *testing.T) {
}))
require.NoError(t, db.AutoMigrate(&activity.Activity{}, &activity.ActivityMessage{}))
restarted := NewJobService(db, nil, &config.Config{}, svc.runs, nil, nil, activities)
- require.NoError(t, restarted.runs.Start(ctx))
+ require.NoError(t, restarted.runs.Start(ctx, ctx))
t.Cleanup(func() {
stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
require.NoError(t, restarted.runs.Stop(stopCtx))
})
+ require.Eventually(t, func() bool {
+ detail, getErr := activities.GetActivityDetail(ctx, "0", run.ActivityID, 10)
+ return getErr == nil && detail.Activity.Status == activitytypes.StatusFailed
+ }, 5*time.Second, 10*time.Millisecond)
detail, err := activities.GetActivityDetail(ctx, "0", run.ActivityID, 10)
require.NoError(t, err)
require.Equal(t, activitytypes.StatusFailed, detail.Activity.Status)
diff --git a/backend/pkg/scheduler/actor_test.go b/backend/pkg/scheduler/actor_test.go
index 0a81156db..05445140c 100644
--- a/backend/pkg/scheduler/actor_test.go
+++ b/backend/pkg/scheduler/actor_test.go
@@ -45,7 +45,7 @@ func newTestCoordinatorInternal(t testing.TB, ctx context.Context, location *tim
require.NoError(t, coordinator.Register(runtime))
require.NoError(t, runtime.Start(t.Context(), ctx, nil))
t.Cleanup(func() { require.NoError(t, runtime.Stop(context.Background())) })
- require.NoError(t, coordinator.Start(ctx))
+ require.NoError(t, coordinator.Start(t.Context(), ctx))
coordinator.Activate()
t.Cleanup(func() { require.NoError(t, coordinator.Stop(context.Background())) })
return coordinator, runtime
diff --git a/backend/pkg/scheduler/image_update_watcher_test.go b/backend/pkg/scheduler/image_update_watcher_test.go
index 5938a1b1f..b983ca6cb 100644
--- a/backend/pkg/scheduler/image_update_watcher_test.go
+++ b/backend/pkg/scheduler/image_update_watcher_test.go
@@ -579,7 +579,7 @@ func TestImageUpdateWatcher_ScheduledPollTriggersScanWithoutEvents(t *testing.T)
record, err := watcher.coordinator.ScheduleState(t.Context(), watcher.Name())
return err == nil && record.Schedule == settings.schedule && !record.NextRun.IsZero()
}, time.Second, 5*time.Millisecond)
- require.NoError(t, jobScheduler.StartScheduler())
+ require.NoError(t, jobScheduler.StartScheduler(t.Context()))
record, err := watcher.coordinator.ScheduleState(t.Context(), watcher.Name())
require.NoError(t, err)
require.Equal(t, settings.schedule, record.Schedule)
diff --git a/backend/pkg/scheduler/job_schedule_integration_test.go b/backend/pkg/scheduler/job_schedule_integration_test.go
index adc4a5695..1e6dd22b2 100644
--- a/backend/pkg/scheduler/job_schedule_integration_test.go
+++ b/backend/pkg/scheduler/job_schedule_integration_test.go
@@ -28,7 +28,7 @@ func TestDeprecatedImagePollingSchedulePersistsWithoutRuntimeJob(t *testing.T) {
jobService := job.NewJobService(db, settingsService, appConfig, jobScheduler.coordinator, nil, nil, nil)
jobService.SetScheduler(lifecycleCtx, jobScheduler)
- require.NoError(t, jobScheduler.StartScheduler())
+ require.NoError(t, jobScheduler.StartScheduler(t.Context()))
_, ok := jobScheduler.GetJobRuntimeState("image-polling")
require.False(t, ok)
@@ -67,7 +67,7 @@ func TestDeprecatedImagePollingSchedulePersistsWithoutRuntimeJob(t *testing.T) {
restartScheduler := newJobSchedulerForTestInternal(t, restartLifecycleCtx, appConfig.GetLocation())
restartJobService := job.NewJobService(restartDB, restartSettingsService, appConfig, restartScheduler.coordinator, nil, nil, nil)
restartJobService.SetScheduler(restartLifecycleCtx, restartScheduler)
- require.NoError(t, restartScheduler.StartScheduler())
+ require.NoError(t, restartScheduler.StartScheduler(t.Context()))
t.Cleanup(func() {
cancelRestartLifecycle()
waitForSchedulerStopInternal(restartScheduler)
diff --git a/backend/pkg/scheduler/runs/activity.go b/backend/pkg/scheduler/runs/activity.go
index 94a99c615..d5a69fa22 100644
--- a/backend/pkg/scheduler/runs/activity.go
+++ b/backend/pkg/scheduler/runs/activity.go
@@ -60,8 +60,6 @@ func (q *Coordinator) retryActivitySyncInternal(ctx context.Context) {
if q.observer == nil {
return
}
- q.observerMu.Lock()
- defer q.observerMu.Unlock()
cursor := ""
for {
page, err := q.service.ListStates(ctx, coordinatorTypeInternal, &actor.ListStatesOpts{IncludeData: true, After: cursor, Limit: 100})
@@ -79,7 +77,9 @@ func (q *Coordinator) retryActivitySyncInternal(ctx context.Context) {
if ctx.Err() != nil {
return
}
+ q.observerMu.Lock()
q.syncActivityInternal(ctx, scheduler.Run{ID: runID, EnvironmentID: state.Record.EnvironmentID, JobID: state.Record.JobID})
+ q.observerMu.Unlock()
}
}
cursor = page.AfterID()
diff --git a/backend/pkg/scheduler/runs/activity_test.go b/backend/pkg/scheduler/runs/activity_test.go
index 52fa7dd55..c0786d895 100644
--- a/backend/pkg/scheduler/runs/activity_test.go
+++ b/backend/pkg/scheduler/runs/activity_test.go
@@ -12,11 +12,17 @@ import (
)
type activityObserverFakeInternal struct {
- unavailable atomic.Bool
- observed atomic.Int32
+ unavailable atomic.Bool
+ observed atomic.Int32
+ existingOnly bool
}
-func (o *activityObserverFakeInternal) ActivityID(run st.Run) string { return run.ID }
+func (o *activityObserverFakeInternal) ActivityID(run st.Run) string {
+ if o.existingOnly {
+ return run.ActivityID
+ }
+ return run.ID
+}
func (o *activityObserverFakeInternal) SyncRunActivity(context.Context, st.Run) error {
if o.unavailable.Load() {
@@ -39,7 +45,7 @@ func TestActivityFailureNeverReplaysAcceptedExecution(t *testing.T) {
run, err := q.Submit(t.Context(), st.Request{JobID: "test-job", Trigger: "manual"})
require.NoError(t, err)
require.Equal(t, run.ID, run.ActivityID)
- require.NoError(t, q.Start(t.Context()))
+ require.NoError(t, q.Start(t.Context(), t.Context()))
q.Activate()
t.Cleanup(func() {
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
diff --git a/backend/pkg/scheduler/runs/actors.go b/backend/pkg/scheduler/runs/actors.go
index d3b226a82..018a8e5f1 100644
--- a/backend/pkg/scheduler/runs/actors.go
+++ b/backend/pkg/scheduler/runs/actors.go
@@ -357,42 +357,42 @@ func (q *Coordinator) repairInternal(ctx context.Context) error {
func (q *Coordinator) importLegacyInternal(ctx context.Context) error {
entries, err := q.store.ListByPrefix(ctx, queuePrefixInternal)
if err != nil {
- return err
+ return fmt.Errorf("list legacy job records: %w", err)
}
for _, entry := range entries {
marker := "francis-import/" + entry.Key
_, done, err := q.store.Get(ctx, marker)
if err != nil {
- return err
+ return fmt.Errorf("import legacy job record %q: %w", entry.Key, err)
}
if done {
continue
}
var record st.QueueRecord
if err := json.Unmarshal([]byte(entry.Value), &record); err != nil {
- return fmt.Errorf("decode legacy job record: %w", err)
+ return fmt.Errorf("decode legacy job record %q: %w", entry.Key, err)
}
id := kit.SHA256Hex(record.EnvironmentID + "\x00" + record.JobID)
if entry.Key != queuePrefixInternal+id {
- return errors.New("invalid legacy job key")
+ return fmt.Errorf("invalid legacy job key %q for job %q in environment %q", entry.Key, record.JobID, record.EnvironmentID)
}
sourceHash := kit.SHA256Hex(entry.Value)
command := st.CoordinatorImport{Record: record, SourceHash: sourceHash}
if _, err := q.service.Invoke(ctx, coordinatorTypeInternal, id, "import", command); err != nil {
- return err
+ return fmt.Errorf("import legacy job record %q: %w", entry.Key, err)
}
var imported st.CoordinatorState
if err := q.service.GetState(ctx, coordinatorTypeInternal, id, &imported); err != nil {
- return err
+ return fmt.Errorf("import legacy job record %q: %w", entry.Key, err)
}
if !imported.Imported || imported.ImportHash != sourceHash {
- return errors.New("legacy import verification failed")
+ return fmt.Errorf("legacy import verification failed for job %q in environment %q", record.JobID, record.EnvironmentID)
}
if imported.Record.EnvironmentID != record.EnvironmentID || imported.Record.JobID != record.JobID {
- return errors.New("legacy import identity mismatch")
+ return fmt.Errorf("legacy import identity mismatch for record %q", entry.Key)
}
if err := q.store.Set(ctx, marker, "complete"); err != nil {
- return err
+ return fmt.Errorf("import legacy job record %q: %w", entry.Key, err)
}
}
return nil
diff --git a/backend/pkg/scheduler/runs/actors_test.go b/backend/pkg/scheduler/runs/actors_test.go
index 9e1cbb508..3a12f0a6b 100644
--- a/backend/pkg/scheduler/runs/actors_test.go
+++ b/backend/pkg/scheduler/runs/actors_test.go
@@ -34,7 +34,18 @@ func TestLegacyImportSurvivesRestartAndDeduplicates(t *testing.T) {
sqlDB.SetMaxOpenConns(1)
t.Cleanup(func() { require.NoError(t, sqlDB.Close()) })
store := kv.NewKVService(&database.DB{DB: db})
- legacy := st.QueueRecord{JobID: "legacy", EnvironmentID: "0", Runs: []st.Run{{ID: uuid.New().String(), JobID: "legacy", EnvironmentID: "0", Trigger: "manual", Status: st.Succeeded, CreatedAt: time.Now().UTC()}}}
+ legacy := st.QueueRecord{JobID: "legacy", EnvironmentID: "0", Runs: []st.Run{{ID: uuid.New().String(), JobID: "legacy", EnvironmentID: "0", Trigger: "manual", Status: st.Succeeded, CreatedAt: time.Now().UTC(), UpdatedAt: time.Now().UTC()}}}
+ legacy.Runs[0].ActivityID = legacy.Runs[0].ID
+ for _, jobID := range []string{"gitops-sync:first", "gitops-sync:second"} {
+ record := st.QueueRecord{JobID: jobID, EnvironmentID: "0", Schedule: "@every 5m", NextRun: time.Now().UTC().Add(time.Hour)}
+ for i := range 1600 {
+ created := time.Now().UTC().Add(-time.Duration(1600-i) * 5 * time.Minute)
+ record.Runs = append(record.Runs, st.Run{ID: uuid.New().String(), ActivityID: uuid.New().String(), JobID: jobID, EnvironmentID: "0", Trigger: "scheduled", Status: st.NeedsAttention, CreatedAt: created, UpdatedAt: created})
+ }
+ data, marshalErr := json.Marshal(record)
+ require.NoError(t, marshalErr)
+ require.NoError(t, store.Set(t.Context(), queuePrefixInternal+kit.SHA256Hex("0\x00"+jobID), string(data)))
+ }
encoded, err := json.Marshal(legacy)
require.NoError(t, err)
key := queuePrefixInternal + kit.SHA256Hex("0\x00"+legacy.JobID)
@@ -46,7 +57,7 @@ func TestLegacyImportSurvivesRestartAndDeduplicates(t *testing.T) {
francistest.Start(t, first)
require.NoError(t, q.importLegacyInternal(t.Context()))
require.NoError(t, q.importLegacyInternal(t.Context()))
- observer := &activityObserverFakeInternal{}
+ observer := &activityObserverFakeInternal{existingOnly: true}
observer.unavailable.Store(true)
q.SetObserver(observer)
_, err = q.Submit(t.Context(), st.Request{JobID: legacy.JobID, RunID: legacy.Runs[0].ID, Trigger: "manual"})
@@ -56,13 +67,33 @@ func TestLegacyImportSurvivesRestartAndDeduplicates(t *testing.T) {
return nil
}))
require.True(t, q.hasPendingActivitySyncInternal(t.Context()))
+ q.SetExecutor(func(context.Context, st.Run) (st.Outcome, error) {
+ t.Error("startup must not execute jobs before activation")
+ return st.Outcome{Status: st.Failed}, nil
+ }, nil)
+ require.NoError(t, q.Start(t.Context(), t.Context()))
+ require.NoError(t, q.Stop(t.Context()))
require.NoError(t, first.Stop(t.Context()))
second := francistest.New(t, databaseURL)
restored := New(store, second.Service(), time.UTC)
+ restored.SetObserver(observer)
+ restored.SetExecutor(q.execute, nil)
require.NoError(t, restored.Register(second))
francistest.Start(t, second)
require.NoError(t, restored.importLegacyInternal(t.Context()))
+ require.NoError(t, restored.Start(t.Context(), t.Context()))
+ t.Cleanup(func() { require.NoError(t, restored.Stop(context.WithoutCancel(t.Context()))) })
+ for _, jobID := range []string{"gitops-sync:first", "gitops-sync:second"} {
+ history, listErr := restored.List(t.Context(), "0", jobID, 1, 100)
+ require.NoError(t, listErr)
+ require.Equal(t, 100, history.Total)
+ require.Equal(t, st.Failed, history.Runs[0].Status)
+ state, stateErr := restored.ScheduleState(t.Context(), jobID)
+ require.NoError(t, stateErr)
+ require.Equal(t, "@every 5m", state.Schedule)
+ require.True(t, state.NextRun.After(time.Now()))
+ }
duplicate, err := restored.Submit(t.Context(), st.Request{JobID: legacy.JobID, RunID: legacy.Runs[0].ID, Trigger: "manual"})
require.NoError(t, err)
require.Equal(t, st.Succeeded, duplicate.Status)
@@ -72,7 +103,6 @@ func TestLegacyImportSurvivesRestartAndDeduplicates(t *testing.T) {
require.Equal(t, 1, history.Total)
require.Equal(t, time.UTC, history.Runs[0].CreatedAt.Location())
observer.unavailable.Store(false)
- restored.SetObserver(observer)
restored.retryActivitySyncInternal(t.Context())
require.False(t, restored.hasPendingActivitySyncInternal(t.Context()))
require.Positive(t, observer.observed.Load())
@@ -134,7 +164,7 @@ func TestDuplicateDispatchAndCanceledDelivery(t *testing.T) {
_, err = q.Cancel(t.Context(), "0", run.JobID, run.ID)
require.NoError(t, err)
}
- require.NoError(t, q.Start(t.Context()))
+ require.NoError(t, q.Start(t.Context(), t.Context()))
q.Activate()
t.Cleanup(func() { require.NoError(t, q.Stop(context.WithoutCancel(t.Context()))) })
require.Eventually(t, func() bool {
@@ -183,7 +213,7 @@ func TestCapacityGroupsReserveHealthSlots(t *testing.T) {
_, err := q.Submit(t.Context(), st.Request{JobID: job, Trigger: "manual"})
require.NoError(t, err)
}
- require.NoError(t, q.Start(t.Context()))
+ require.NoError(t, q.Start(t.Context(), t.Context()))
q.Activate()
t.Cleanup(func() {
once.Do(func() { close(release) })
@@ -281,7 +311,7 @@ func TestLostDispatchAcknowledgementRetainsOneExecution(t *testing.T) {
}, nil)
run, err := q.Submit(t.Context(), st.Request{JobID: "lost-ack"})
require.NoError(t, err)
- require.NoError(t, q.Start(t.Context()))
+ require.NoError(t, q.Start(t.Context(), t.Context()))
q.Activate()
t.Cleanup(func() {
once.Do(func() { close(finish) })
diff --git a/backend/pkg/scheduler/runs/coordinator.go b/backend/pkg/scheduler/runs/coordinator.go
index b25e3dd21..bc51f7ee4 100644
--- a/backend/pkg/scheduler/runs/coordinator.go
+++ b/backend/pkg/scheduler/runs/coordinator.go
@@ -197,37 +197,62 @@ func (q *Coordinator) Checkpoint(ctx context.Context, jobID, schedule string, ne
return err
}
-// Start validates persisted state and starts dispatching on ctx. Repeated calls
+// Start validates persisted state on ctx and starts dispatching on appCtx. Repeated calls
// are harmless, but a stopped queue cannot be restarted; create a new instance.
-func (q *Coordinator) Start(ctx context.Context) error {
+func (q *Coordinator) Start(ctx, appCtx context.Context) error {
q.mu.Lock()
defer q.mu.Unlock()
if q.started {
return nil
}
+ if err := ctx.Err(); err != nil {
+ return fmt.Errorf("initialize job coordinator: %w", err)
+ }
+ if q.stopping {
+ return errors.New("job queue is stopping")
+ }
if q.execute == nil {
return errors.New("job executor unavailable")
}
if err := q.importLegacyInternal(ctx); err != nil {
- return err
+ return fmt.Errorf("import legacy job history: %w", err)
}
// Fail startup if persisted state cannot be read. Never replace it with empty state.
records, err := q.Records(ctx)
if err != nil {
- return err
+ return fmt.Errorf("read startup job history: %w", err)
}
for _, record := range records {
+ changed := false
for _, run := range record.Runs {
- if err := q.UpdateRun(ctx, run, func(current *st.Run) error {
- normalizeLegacyRunInternal(current)
- return nil
- }); err != nil {
- return err
+ if normalizeLegacyRunInternal(&run) {
+ changed = true
+ break
}
}
+ if !changed {
+ continue
+ }
+ if err := q.mutateInternal(ctx, record.EnvironmentID, record.JobID, func(current *st.QueueRecord) error {
+ for i := range current.Runs {
+ normalizeLegacyRunInternal(&current.Runs[i])
+ }
+ return nil
+ }); err != nil {
+ return fmt.Errorf("normalize startup job %q in environment %q: %w", record.JobID, record.EnvironmentID, err)
+ }
+ }
+ if err := q.pruneInternal(ctx); err != nil {
+ return fmt.Errorf("prune startup job history: %w", err)
+ }
+ if err := ctx.Err(); err != nil {
+ return fmt.Errorf("initialize job coordinator: %w", err)
+ }
+ if err := appCtx.Err(); err != nil {
+ return fmt.Errorf("start job dispatcher: %w", err)
}
- runCtx, cancel := context.WithCancel(ctx)
+ runCtx, cancel := context.WithCancel(appCtx)
q.cancel = cancel
q.lifecycleCtx = runCtx
q.started = true
@@ -260,12 +285,17 @@ func (q *Coordinator) Stop(ctx context.Context) error {
func (q *Coordinator) dispatchInternal(ctx context.Context) {
defer close(q.done)
- nextRetention := time.Now().Add(time.Hour)
+ initialRetention := true
+ nextRetention := time.Now()
for ctx.Err() == nil {
q.retryActivitySyncInternal(ctx)
- if !time.Now().Before(nextRetention) {
- if err := q.pruneInternal(ctx); err != nil && ctx.Err() == nil {
- slog.ErrorContext(ctx, "Job retention failed", "error", err)
+ if initialRetention || !time.Now().Before(nextRetention) {
+ if err := q.pruneInternal(ctx); err != nil {
+ if ctx.Err() == nil {
+ slog.ErrorContext(ctx, "Job retention failed", "error", err)
+ }
+ } else {
+ initialRetention = q.hasPendingActivitySyncInternal(ctx)
}
nextRetention = time.Now().Add(time.Hour)
}
diff --git a/backend/pkg/scheduler/runs/coordinator_store.go b/backend/pkg/scheduler/runs/coordinator_store.go
index f8474a0d8..62c4939e4 100644
--- a/backend/pkg/scheduler/runs/coordinator_store.go
+++ b/backend/pkg/scheduler/runs/coordinator_store.go
@@ -5,6 +5,7 @@ import (
"context"
"encoding/json/v2"
"errors"
+ "fmt"
"slices"
"sort"
"time"
@@ -92,16 +93,43 @@ func snapshotRunsInternal(record st.QueueRecord) (map[string][]byte, map[string]
}
func preserveUnsyncedRunsInternal(state *st.CoordinatorState, previous []st.Run) {
- retained := make(map[string]bool, len(state.Record.Runs))
+ retained := make(map[string]st.Run, len(state.Record.Runs))
for _, run := range state.Record.Runs {
- retained[run.ID] = true
+ retained[run.ID] = run
}
+ restore := false
for _, run := range previous {
- if _, pending := state.PendingActivitySync[run.ID]; pending && !retained[run.ID] {
- state.Record.Runs = append(state.Record.Runs, run)
+ _, exists := retained[run.ID]
+ if _, pending := state.PendingActivitySync[run.ID]; pending && !exists {
+ // Activity projection must not extend completed run retention.
+ if run.Status.Terminal() {
+ if _, archived := state.Record.Receipts[run.ID]; !archived {
+ delete(state.PendingActivitySync, run.ID)
+ }
+ continue
+ }
+ retained[run.ID] = run
delete(state.Record.Receipts, run.ID)
+ restore = true
+ }
+ }
+ if !restore {
+ return
+ }
+ // Keep history chronological when pending activity updates prevent pruning.
+ runs := make([]st.Run, 0, len(retained))
+ for _, run := range previous {
+ if current, exists := retained[run.ID]; exists {
+ runs = append(runs, current)
+ delete(retained, run.ID)
}
}
+ for _, run := range state.Record.Runs {
+ if _, exists := retained[run.ID]; exists {
+ runs = append(runs, run)
+ }
+ }
+ state.Record.Runs = runs
}
func pruneDispatchesInternal(state *st.CoordinatorState) {
@@ -123,12 +151,12 @@ func (q *Coordinator) Records(ctx context.Context) ([]st.QueueRecord, error) {
for {
page, err := q.service.ListStates(ctx, coordinatorTypeInternal, &actor.ListStatesOpts{IncludeData: true, After: cursor, Limit: 100})
if err != nil {
- return nil, err
+ return nil, fmt.Errorf("list job coordinator states: %w", err)
}
for _, entry := range page.States {
var state st.CoordinatorState
if err := entry.Data.Decode(&state); err != nil {
- return nil, err
+ return nil, fmt.Errorf("decode job coordinator %q: %w", entry.ActorID, err)
}
normalizeRecordTimesInternal(&state.Record)
q.runtime.NameActor(entry.ActorID, state.Record.JobID+"@"+state.Record.EnvironmentID)
@@ -234,17 +262,21 @@ func (q *Coordinator) pruneInternal(ctx context.Context) error {
}
now := time.Now().UTC()
for _, record := range records {
+ if !pruneRunsInternal(&record, now) {
+ continue
+ }
if err := q.mutateInternal(ctx, record.EnvironmentID, record.JobID, func(current *st.QueueRecord) error {
pruneRunsInternal(current, now)
return nil
}); err != nil {
- return err
+ return fmt.Errorf("prune job %q in environment %q: %w", record.JobID, record.EnvironmentID, err)
}
}
return nil
}
-func pruneRunsInternal(record *st.QueueRecord, now time.Time) {
+func pruneRunsInternal(record *st.QueueRecord, now time.Time) bool {
+ changed := false
terminal := 0
for index, run := range slices.Backward(record.Runs) {
if !run.Status.Terminal() || ((run.RemoteDeliveryAttempted || run.RemoteAccepted) && !run.RemoteSettled) {
@@ -263,7 +295,9 @@ func pruneRunsInternal(record *st.QueueRecord, now time.Time) {
record.Receipts[run.ID] = run
}
record.Runs = append(record.Runs[:index], record.Runs[index+1:]...)
+ changed = true
}
+ return changed
}
// ErrRunNotFound indicates that neither a run nor its receipt exists.
@@ -361,10 +395,10 @@ func normalizeRunTimesInternal(run *st.Run) {
}
}
-func normalizeLegacyRunInternal(run *st.Run) {
+func normalizeLegacyRunInternal(run *st.Run) bool {
automaticResolution := run.Status == st.Canceled && run.Resolution != nil && run.Resolution.ResolvedBy == common.SystemUser.Username
if run.Status != st.NeedsAttention && !automaticResolution {
- return
+ return false
}
run.Owner = ""
run.NextAttempt = nil
@@ -379,6 +413,7 @@ func normalizeLegacyRunInternal(run *st.Run) {
}
}
run.UpdatedAt = time.Now().UTC()
+ return true
}
func normalizeTimePointerInternal(value *time.Time) {
diff --git a/backend/pkg/scheduler/runs/resolution_test.go b/backend/pkg/scheduler/runs/resolution_test.go
index e4d262c3b..d0f13cef8 100644
--- a/backend/pkg/scheduler/runs/resolution_test.go
+++ b/backend/pkg/scheduler/runs/resolution_test.go
@@ -97,7 +97,14 @@ func TestResolvePreservesEvidenceWithoutBlockingSchedule(t *testing.T) {
t.Error("startup must not repeat legacy work")
return st.Outcome{Status: st.Failed}, nil
}, nil)
- require.NoError(t, q.Start(ctx))
+ startCtx, cancelStart := context.WithCancel(ctx)
+ cancelStart()
+ require.ErrorIs(t, q.Start(startCtx, ctx), context.Canceled)
+ startCtx, cancelStart = context.WithCancel(ctx)
+ defer cancelStart()
+ require.NoError(t, q.Start(startCtx, ctx))
+ cancelStart()
+ require.NoError(t, q.lifecycleCtx.Err())
t.Cleanup(func() { require.NoError(t, q.Stop(context.WithoutCancel(ctx))) })
localLegacy, err = q.Get(ctx, "0", localLegacy.JobID, localLegacy.ID)
require.NoError(t, err)
diff --git a/backend/pkg/scheduler/scheduler.go b/backend/pkg/scheduler/scheduler.go
index 75ef453a6..a92dd1f62 100644
--- a/backend/pkg/scheduler/scheduler.go
+++ b/backend/pkg/scheduler/scheduler.go
@@ -117,7 +117,7 @@ func (js *jobSchedulerInternal) GetJobRuntimeState(id string) (st.JobRuntimeStat
return state, true
}
-func (js *jobSchedulerInternal) StartScheduler() error {
+func (js *jobSchedulerInternal) StartScheduler(ctx context.Context) error {
js.mu.Lock()
if js.stopping {
js.mu.Unlock()
@@ -125,7 +125,7 @@ func (js *jobSchedulerInternal) StartScheduler() error {
}
js.started = true
js.mu.Unlock()
- records, err := js.coordinator.Records(js.context)
+ records, err := js.coordinator.Records(ctx)
if err != nil {
return err
}
@@ -134,14 +134,14 @@ func (js *jobSchedulerInternal) StartScheduler() error {
_, watching := js.watchers[record.JobID]
js.mu.RUnlock()
if record.EnvironmentID == "0" && record.Schedule != "" && !js.HasJob(record.JobID) && !watching {
- if err := js.coordinator.Checkpoint(js.context, record.JobID, "", time.Time{}); err != nil {
+ if err := js.coordinator.Checkpoint(ctx, record.JobID, "", time.Time{}); err != nil {
return err
}
}
}
var schedulingErr error
for _, job := range js.ListRegisteredJobs() {
- if err := js.installInternal(js.context, job); err != nil {
+ if err := js.installInternal(ctx, job); err != nil {
schedulingErr = errors.Join(schedulingErr, fmt.Errorf("%s: %w", "schedule "+job.Name(), err))
}
}
diff --git a/backend/pkg/scheduler/scheduler_test.go b/backend/pkg/scheduler/scheduler_test.go
index 026c901b2..78edd03cc 100644
--- a/backend/pkg/scheduler/scheduler_test.go
+++ b/backend/pkg/scheduler/scheduler_test.go
@@ -106,7 +106,7 @@ func TestJobScheduler_StartScheduler_SkipsDisabledConditionalJobs(t *testing.T)
}
require.NoError(t, js.RegisterJob(job))
- require.NoError(t, js.StartScheduler())
+ require.NoError(t, js.StartScheduler(t.Context()))
state, ok := js.GetJobRuntimeState(job.Name())
require.True(t, ok)
@@ -123,7 +123,7 @@ func TestJobScheduler_StartScheduler_ReportsInvalidJobSchedule(t *testing.T) {
require.NoError(t, js.RegisterJob(invalid))
require.NoError(t, js.RegisterJob(valid))
- require.Error(t, js.StartScheduler())
+ require.Error(t, js.StartScheduler(t.Context()))
invalidState, ok := js.GetJobRuntimeState(invalid.Name())
require.True(t, ok)
@@ -145,7 +145,7 @@ func TestJobScheduler_StopWaitsForBusWatchers(t *testing.T) {
}
require.NoError(t, js.RegisterBusWatcher(watcher, true))
- require.NoError(t, js.StartScheduler())
+ require.NoError(t, js.StartScheduler(t.Context()))
select {
case <-watcher.started:
@@ -214,7 +214,7 @@ func TestJobScheduler_RegisterBusWatcherManualRunOption(t *testing.T) {
func TestJobScheduler_RescheduleJob_RemovesEntryWhenDisabled(t *testing.T) {
js := newJobSchedulerForTestInternal(t, context.Background(), nil)
- require.NoError(t, js.StartScheduler())
+ require.NoError(t, js.StartScheduler(t.Context()))
enabled := true
job := &conditionalTestSchedulerJob{
@@ -243,7 +243,7 @@ func TestJobScheduler_RescheduleJob_RemovesEntryWhenDisabled(t *testing.T) {
func TestJobScheduler_RescheduleJob_AddsEntryWhenEnabled(t *testing.T) {
js := newJobSchedulerForTestInternal(t, context.Background(), nil)
- require.NoError(t, js.StartScheduler())
+ require.NoError(t, js.StartScheduler(t.Context()))
enabled := false
job := &conditionalTestSchedulerJob{
@@ -293,7 +293,7 @@ func TestJobScheduler_StartScheduler_SchedulesNonConditionalJobs(t *testing.T) {
}
require.NoError(t, js.RegisterJob(job))
- require.NoError(t, js.StartScheduler())
+ require.NoError(t, js.StartScheduler(t.Context()))
state, ok := js.GetJobRuntimeState(job.Name())
require.True(t, ok)
@@ -330,7 +330,7 @@ func TestJobScheduler_RescheduleJob_UsesProvidedContext(t *testing.T) {
}
require.NoError(t, js.RescheduleJob(runCtx, job))
- require.NoError(t, js.StartScheduler())
+ require.NoError(t, js.StartScheduler(t.Context()))
select {
case err := <-runErrCh:
@@ -363,7 +363,7 @@ func TestJobScheduler_RescheduleJob_UsesLifecycleContextForShutdown(t *testing.T
}
require.NoError(t, js.RescheduleJob(lifecycleCtx, job))
- require.NoError(t, js.StartScheduler())
+ require.NoError(t, js.StartScheduler(t.Context()))
select {
case <-startedCh:
@@ -382,7 +382,7 @@ func TestJobScheduler_RescheduleJob_UsesLifecycleContextForShutdown(t *testing.T
func TestJobScheduler_AddJob_UpsertReplacesEntryWithoutLeaking(t *testing.T) {
js := newJobSchedulerForTestInternal(t, context.Background(), nil)
- require.NoError(t, js.StartScheduler())
+ require.NoError(t, js.StartScheduler(t.Context()))
job := &testSchedulerJob{name: "dyn-upsert", schedule: "*/5 * * * * *"}
require.NoError(t, js.AddJob(context.Background(), job))
@@ -409,7 +409,7 @@ func TestJobScheduler_AddJob_UpsertReplacesEntryWithoutLeaking(t *testing.T) {
func TestJobScheduler_AddJob_InvalidRescheduleKeepsExistingEntry(t *testing.T) {
js := newJobSchedulerForTestInternal(t, context.Background(), nil)
- require.NoError(t, js.StartScheduler())
+ require.NoError(t, js.StartScheduler(t.Context()))
job := &testSchedulerJob{name: "dyn-invalid-reschedule", schedule: "*/5 * * * * *"}
require.NoError(t, js.AddJob(context.Background(), job))
@@ -434,7 +434,7 @@ func TestJobScheduler_AddJob_InvalidRescheduleKeepsExistingEntry(t *testing.T) {
func TestJobScheduler_RemoveJob_RemovesEntryAndIsNoopWhenAbsent(t *testing.T) {
js := newJobSchedulerForTestInternal(t, context.Background(), nil)
- require.NoError(t, js.StartScheduler())
+ require.NoError(t, js.StartScheduler(t.Context()))
// Removing an unknown job must be a safe no-op (e.g. deleting a sync that never
// had auto-sync enabled).
@@ -454,7 +454,7 @@ func TestJobScheduler_RemoveJob_RemovesEntryAndIsNoopWhenAbsent(t *testing.T) {
func TestJobScheduler_AddJob_GenericJobWithoutShouldRunIsScheduled(t *testing.T) {
js := newJobSchedulerForTestInternal(t, context.Background(), nil)
- require.NoError(t, js.StartScheduler())
+ require.NoError(t, js.StartScheduler(t.Context()))
job := &schedulertypes.GenericJob{
JobName: "generic-dyn",
@@ -488,7 +488,7 @@ func TestJobScheduler_StopWaitsForCanceledJobToFinish(t *testing.T) {
})
},
}))
- require.NoError(t, js.StartScheduler())
+ require.NoError(t, js.StartScheduler(t.Context()))
t.Cleanup(func() {
cancelLifecycle()
select {
diff --git a/types/scheduler/job.go b/types/scheduler/job.go
index 0401a65f1..e668e8aa0 100644
--- a/types/scheduler/job.go
+++ b/types/scheduler/job.go
@@ -71,7 +71,7 @@ type JobScheduler interface {
RegisterJob(job Job) error
RegisterBusWatcher(watcher BusWatcher, canRunManually bool) error
- StartScheduler() error
+ StartScheduler(ctx context.Context) error
GetLocation() *time.Location
Stop(ctx context.Context) error
}