Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix(jobs): recover oversized legacy job histories on startup (#4270)
Kyle Mendellf2666e3712fa
diff --git a/Justfile b/Justfileindex 6c589bc9c..8a8d1c0c2 100644--- a/Justfile+++ b/Justfile@@ -478,6 +478,42 @@ _deps-update-frontend:_deps-update-backend:cd backend && go get -u ./... && go mod tidy+# Update direct Go dependencies in all modules, then sync the workspace (requires jq).+[group('deps')]+_deps-update-go *args:+ #!/usr/bin/env bash+ set -euo pipefail+ version=upgrade+ set -- {{ args }}+ case "$#:$*" in+ 0:) ;;+ 1:--patch) version=patch ;;+ *) echo 'Usage: just deps go update [--patch]' >&2; exit 1 ;;+ esac+ command -v jq >/dev/null || { echo 'jq is required.' >&2; exit 1; }+ for module in {{ modules }}; do+ (+ cd "$module"+ export GOWORK=off+ echo "Updating direct dependencies in $module..."+ direct=$(go list -mod=mod -m -json all | jq -r '+ select(.Main != true and .Indirect != true)+ | select(.Replace == null or .Replace.Version != null)+ | .Path')+ dependencies=()+ while IFS= read -r dependency; do+ if [ -n "$dependency" ]; then+ dependencies+=("$dependency@$version")+ fi+ done <<<"$direct"+ if [ "${#dependencies[@]}" -gt 0 ]; then+ go get "${dependencies[@]}"+ fi+ go mod tidy+ )+ go work sync+ done+# Update pnpm version via corepack[group('deps')]_deps-update-pnpm:@@ -494,10 +530,20 @@ _deps-dedupe-node:[group('deps')]_deps-dedupe-all: _deps-dedupe-node-# Deps targets. Valid: "install [frontend|tests|backend|cli|types|go|node|all]", "update [frontend|backend|pnpm|all]", "dedupe [node|go|all]"+# Deps targets. Also supports "go update [--patch]" for direct Go dependency updates.[group('deps')]-deps action="update" target="all":- @just "_deps-{{ action }}-{{ target }}"+deps action="update" target="all" *args:+ #!/usr/bin/env bash+ set -euo pipefail+ if [ "{{ action }}" = "go" ]; then+ if [ "{{ target }}" != "update" ]; then+ echo 'Usage: just deps go update [--patch]' >&2+ exit 1+ fi+ just _deps-update-go {{ args }}+ else+ just "_deps-{{ action }}-{{ target }}" {{ args }}+ fi# -----------------------------------------------------------------------------# Code generation and docsdiff --git a/backend/internal/bootstrap/bootstrap_test.go b/backend/internal/bootstrap/bootstrap_test.goindex 63754d350..a3589b9a5 100644--- a/backend/internal/bootstrap/bootstrap_test.go+++ b/backend/internal/bootstrap/bootstrap_test.go@@ -434,7 +434,10 @@ func TestJobSchedulerStopCancelsItsPrivateContextInternal(t *testing.T) {}require.NoError(t, jobScheduler.RegisterBusWatcher(watcher, false))- lifecycle.RequireStart()+ startCtx, cancelStart := context.WithTimeout(appCtx, 5*time.Second)+ defer cancelStart()+ require.NoError(t, lifecycle.Start(startCtx))+ cancelStart()select {case run := <-executed:require.Equal(t, "gitops-sync:overdue", run.JobID)diff --git a/backend/internal/bootstrap/jobs_bootstrap.go b/backend/internal/bootstrap/jobs_bootstrap.goindex b88bd2ceb..91235dcad 100644--- a/backend/internal/bootstrap/jobs_bootstrap.go+++ b/backend/internal/bootstrap/jobs_bootstrap.go@@ -42,16 +42,22 @@ func newJobScheduler(appCtx context.Context, lc fx.Lifecycle, cfg *config.Configvar repairDone chan struct{}var resumeDone chan struct{}lc.Append(fx.Hook{- OnStart: func(context.Context) error {+ OnStart: func(ctx context.Context) error {+ started := false+ defer func() {+ if !started {+ cancelScheduler()+ }+ }()slog.InfoContext(appCtx, "Starting scheduler")- if err := jobService.Coordinator().Start(schedulerCtx); err != nil {- return err+ if err := jobService.Coordinator().Start(ctx, schedulerCtx); err != nil {+ return fmt.Errorf("initialize scheduler: %w", err)}- if err := reconcileCoordinatorStartupInternal(schedulerCtx, jobService, backupEngine, updaterService, volumes, systemBackups, gitopsSync); err != nil {- return err+ if err := reconcileCoordinatorStartupInternal(ctx, jobService, backupEngine, updaterService, volumes, systemBackups, gitopsSync); err != nil {+ return fmt.Errorf("reconcile scheduler startup: %w", err)}if gitopsSync != nil {- gitopsSync.RegisterAutoSyncJobsOnStartup(schedulerCtx)+ gitopsSync.RegisterAutoSyncJobsOnStartup(ctx)gitopsSync.SubscribeProjectFileChanges(schedulerCtx)}if imageUpdateWatcher != nil {@@ -60,8 +66,8 @@ func newJobScheduler(appCtx context.Context, lc fx.Lifecycle, cfg *config.Configreturn err}}- if err := jobScheduler.StartScheduler(); err != nil {- return err+ if err := jobScheduler.StartScheduler(ctx); err != nil {+ return fmt.Errorf("start scheduler: %w", err)}jobService.Coordinator().Activate()if backupEngine != nil {@@ -72,14 +78,18 @@ func newJobScheduler(appCtx context.Context, lc fx.Lifecycle, cfg *config.Config}()}if analytics != nil {- if _, err := jobService.Coordinator().Submit(schedulerCtx, schedulertypes.Request{JobID: analytics.Name(), EnvironmentID: "0", Trigger: "startup"}); err != nil {- return err+ if _, err := jobService.Coordinator().Submit(ctx, schedulertypes.Request{JobID: analytics.Name(), EnvironmentID: "0", Trigger: "startup"}); err != nil {+ return fmt.Errorf("submit startup job %q: %w", analytics.Name(), err)}}if !cfg.AgentMode && systemUpgrade != nil {resumeDone = make(chan struct{})go func() { defer close(resumeDone); systemUpgrade.ResumeUpdateAllOnStartup(schedulerCtx) }()}+ if err := ctx.Err(); err != nil {+ return fmt.Errorf("complete scheduler startup: %w", err)+ }+ started = truereturn nil},OnStop: func(ctx context.Context) error {diff --git a/backend/internal/job/activity_test.go b/backend/internal/job/activity_test.goindex 41646f999..b95cb451a 100644--- a/backend/internal/job/activity_test.go+++ b/backend/internal/job/activity_test.go@@ -139,12 +139,16 @@ func TestJobActivityRestartRepairsFailedProjection(t *testing.T) {}))require.NoError(t, db.AutoMigrate(&activity.Activity{}, &activity.ActivityMessage{}))restarted := NewJobService(db, nil, &config.Config{}, svc.runs, nil, nil, activities)- require.NoError(t, restarted.runs.Start(ctx))+ require.NoError(t, restarted.runs.Start(ctx, ctx))t.Cleanup(func() {stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)defer cancel()require.NoError(t, restarted.runs.Stop(stopCtx))})+ require.Eventually(t, func() bool {+ detail, getErr := activities.GetActivityDetail(ctx, "0", run.ActivityID, 10)+ return getErr == nil && detail.Activity.Status == activitytypes.StatusFailed+ }, 5*time.Second, 10*time.Millisecond)detail, err := activities.GetActivityDetail(ctx, "0", run.ActivityID, 10)require.NoError(t, err)require.Equal(t, activitytypes.StatusFailed, detail.Activity.Status)diff --git a/backend/pkg/scheduler/actor_test.go b/backend/pkg/scheduler/actor_test.goindex 0a81156db..05445140c 100644--- a/backend/pkg/scheduler/actor_test.go+++ b/backend/pkg/scheduler/actor_test.go@@ -45,7 +45,7 @@ func newTestCoordinatorInternal(t testing.TB, ctx context.Context, location *timrequire.NoError(t, coordinator.Register(runtime))require.NoError(t, runtime.Start(t.Context(), ctx, nil))t.Cleanup(func() { require.NoError(t, runtime.Stop(context.Background())) })- require.NoError(t, coordinator.Start(ctx))+ require.NoError(t, coordinator.Start(t.Context(), ctx))coordinator.Activate()t.Cleanup(func() { require.NoError(t, coordinator.Stop(context.Background())) })return coordinator, runtimediff --git a/backend/pkg/scheduler/image_update_watcher_test.go b/backend/pkg/scheduler/image_update_watcher_test.goindex 5938a1b1f..b983ca6cb 100644--- a/backend/pkg/scheduler/image_update_watcher_test.go+++ b/backend/pkg/scheduler/image_update_watcher_test.go@@ -579,7 +579,7 @@ func TestImageUpdateWatcher_ScheduledPollTriggersScanWithoutEvents(t *testing.T)record, err := watcher.coordinator.ScheduleState(t.Context(), watcher.Name())return err == nil && record.Schedule == settings.schedule && !record.NextRun.IsZero()}, time.Second, 5*time.Millisecond)- require.NoError(t, jobScheduler.StartScheduler())+ require.NoError(t, jobScheduler.StartScheduler(t.Context()))record, err := watcher.coordinator.ScheduleState(t.Context(), watcher.Name())require.NoError(t, err)require.Equal(t, settings.schedule, record.Schedule)diff --git a/backend/pkg/scheduler/job_schedule_integration_test.go b/backend/pkg/scheduler/job_schedule_integration_test.goindex adc4a5695..1e6dd22b2 100644--- a/backend/pkg/scheduler/job_schedule_integration_test.go+++ b/backend/pkg/scheduler/job_schedule_integration_test.go@@ -28,7 +28,7 @@ func TestDeprecatedImagePollingSchedulePersistsWithoutRuntimeJob(t *testing.T) {jobService := job.NewJobService(db, settingsService, appConfig, jobScheduler.coordinator, nil, nil, nil)jobService.SetScheduler(lifecycleCtx, jobScheduler)- require.NoError(t, jobScheduler.StartScheduler())+ require.NoError(t, jobScheduler.StartScheduler(t.Context()))_, ok := jobScheduler.GetJobRuntimeState("image-polling")require.False(t, ok)@@ -67,7 +67,7 @@ func TestDeprecatedImagePollingSchedulePersistsWithoutRuntimeJob(t *testing.T) {restartScheduler := newJobSchedulerForTestInternal(t, restartLifecycleCtx, appConfig.GetLocation())restartJobService := job.NewJobService(restartDB, restartSettingsService, appConfig, restartScheduler.coordinator, nil, nil, nil)restartJobService.SetScheduler(restartLifecycleCtx, restartScheduler)- require.NoError(t, restartScheduler.StartScheduler())+ require.NoError(t, restartScheduler.StartScheduler(t.Context()))t.Cleanup(func() {cancelRestartLifecycle()waitForSchedulerStopInternal(restartScheduler)diff --git a/backend/pkg/scheduler/runs/activity.go b/backend/pkg/scheduler/runs/activity.goindex 94a99c615..d5a69fa22 100644--- a/backend/pkg/scheduler/runs/activity.go+++ b/backend/pkg/scheduler/runs/activity.go@@ -60,8 +60,6 @@ func (q *Coordinator) retryActivitySyncInternal(ctx context.Context) {if q.observer == nil {return}- q.observerMu.Lock()- defer q.observerMu.Unlock()cursor := ""for {page, err := q.service.ListStates(ctx, coordinatorTypeInternal, &actor.ListStatesOpts{IncludeData: true, After: cursor, Limit: 100})@@ -79,7 +77,9 @@ func (q *Coordinator) retryActivitySyncInternal(ctx context.Context) {if ctx.Err() != nil {return}+ q.observerMu.Lock()q.syncActivityInternal(ctx, scheduler.Run{ID: runID, EnvironmentID: state.Record.EnvironmentID, JobID: state.Record.JobID})+ q.observerMu.Unlock()}}cursor = page.AfterID()diff --git a/backend/pkg/scheduler/runs/activity_test.go b/backend/pkg/scheduler/runs/activity_test.goindex 52fa7dd55..c0786d895 100644--- a/backend/pkg/scheduler/runs/activity_test.go+++ b/backend/pkg/scheduler/runs/activity_test.go@@ -12,11 +12,17 @@ import ()type activityObserverFakeInternal struct {- unavailable atomic.Bool- observed atomic.Int32+ unavailable atomic.Bool+ observed atomic.Int32+ existingOnly bool}-func (o *activityObserverFakeInternal) ActivityID(run st.Run) string { return run.ID }+func (o *activityObserverFakeInternal) ActivityID(run st.Run) string {+ if o.existingOnly {+ return run.ActivityID+ }+ return run.ID+}func (o *activityObserverFakeInternal) SyncRunActivity(context.Context, st.Run) error {if o.unavailable.Load() {@@ -39,7 +45,7 @@ func TestActivityFailureNeverReplaysAcceptedExecution(t *testing.T) {run, err := q.Submit(t.Context(), st.Request{JobID: "test-job", Trigger: "manual"})require.NoError(t, err)require.Equal(t, run.ID, run.ActivityID)- require.NoError(t, q.Start(t.Context()))+ require.NoError(t, q.Start(t.Context(), t.Context()))q.Activate()t.Cleanup(func() {ctx, cancel := context.WithTimeout(context.Background(), time.Second)diff --git a/backend/pkg/scheduler/runs/actors.go b/backend/pkg/scheduler/runs/actors.goindex d3b226a82..018a8e5f1 100644--- a/backend/pkg/scheduler/runs/actors.go+++ b/backend/pkg/scheduler/runs/actors.go@@ -357,42 +357,42 @@ func (q *Coordinator) repairInternal(ctx context.Context) error {func (q *Coordinator) importLegacyInternal(ctx context.Context) error {entries, err := q.store.ListByPrefix(ctx, queuePrefixInternal)if err != nil {- return err+ return fmt.Errorf("list legacy job records: %w", err)}for _, entry := range entries {marker := "francis-import/" + entry.Key_, done, err := q.store.Get(ctx, marker)if err != nil {- return err+ return fmt.Errorf("import legacy job record %q: %w", entry.Key, err)}if done {continue}var record st.QueueRecordif err := json.Unmarshal([]byte(entry.Value), &record); err != nil {- return fmt.Errorf("decode legacy job record: %w", err)+ return fmt.Errorf("decode legacy job record %q: %w", entry.Key, err)}id := kit.SHA256Hex(record.EnvironmentID + "\x00" + record.JobID)if entry.Key != queuePrefixInternal+id {- return errors.New("invalid legacy job key")+ return fmt.Errorf("invalid legacy job key %q for job %q in environment %q", entry.Key, record.JobID, record.EnvironmentID)}sourceHash := kit.SHA256Hex(entry.Value)command := st.CoordinatorImport{Record: record, SourceHash: sourceHash}if _, err := q.service.Invoke(ctx, coordinatorTypeInternal, id, "import", command); err != nil {- return err+ return fmt.Errorf("import legacy job record %q: %w", entry.Key, err)}var imported st.CoordinatorStateif err := q.service.GetState(ctx, coordinatorTypeInternal, id, &imported); err != nil {- return err+ return fmt.Errorf("import legacy job record %q: %w", entry.Key, err)}if !imported.Imported || imported.ImportHash != sourceHash {- return errors.New("legacy import verification failed")+ return fmt.Errorf("legacy import verification failed for job %q in environment %q", record.JobID, record.EnvironmentID)}if imported.Record.EnvironmentID != record.EnvironmentID || imported.Record.JobID != record.JobID {- return errors.New("legacy import identity mismatch")+ return fmt.Errorf("legacy import identity mismatch for record %q", entry.Key)}if err := q.store.Set(ctx, marker, "complete"); err != nil {- return err+ return fmt.Errorf("import legacy job record %q: %w", entry.Key, err)}}return nildiff --git a/backend/pkg/scheduler/runs/actors_test.go b/backend/pkg/scheduler/runs/actors_test.goindex 9e1cbb508..3a12f0a6b 100644--- a/backend/pkg/scheduler/runs/actors_test.go+++ b/backend/pkg/scheduler/runs/actors_test.go@@ -34,7 +34,18 @@ func TestLegacyImportSurvivesRestartAndDeduplicates(t *testing.T) {sqlDB.SetMaxOpenConns(1)t.Cleanup(func() { require.NoError(t, sqlDB.Close()) })store := kv.NewKVService(&database.DB{DB: db})- legacy := st.QueueRecord{JobID: "legacy", EnvironmentID: "0", Runs: []st.Run{{ID: uuid.New().String(), JobID: "legacy", EnvironmentID: "0", Trigger: "manual", Status: st.Succeeded, CreatedAt: time.Now().UTC()}}}+ legacy := st.QueueRecord{JobID: "legacy", EnvironmentID: "0", Runs: []st.Run{{ID: uuid.New().String(), JobID: "legacy", EnvironmentID: "0", Trigger: "manual", Status: st.Succeeded, CreatedAt: time.Now().UTC(), UpdatedAt: time.Now().UTC()}}}+ legacy.Runs[0].ActivityID = legacy.Runs[0].ID+ for _, jobID := range []string{"gitops-sync:first", "gitops-sync:second"} {+ record := st.QueueRecord{JobID: jobID, EnvironmentID: "0", Schedule: "@every 5m", NextRun: time.Now().UTC().Add(time.Hour)}+ for i := range 1600 {+ created := time.Now().UTC().Add(-time.Duration(1600-i) * 5 * time.Minute)+ record.Runs = append(record.Runs, st.Run{ID: uuid.New().String(), ActivityID: uuid.New().String(), JobID: jobID, EnvironmentID: "0", Trigger: "scheduled", Status: st.NeedsAttention, CreatedAt: created, UpdatedAt: created})+ }+ data, marshalErr := json.Marshal(record)+ require.NoError(t, marshalErr)+ require.NoError(t, store.Set(t.Context(), queuePrefixInternal+kit.SHA256Hex("0\x00"+jobID), string(data)))+ }encoded, err := json.Marshal(legacy)require.NoError(t, err)key := queuePrefixInternal + kit.SHA256Hex("0\x00"+legacy.JobID)@@ -46,7 +57,7 @@ func TestLegacyImportSurvivesRestartAndDeduplicates(t *testing.T) {francistest.Start(t, first)require.NoError(t, q.importLegacyInternal(t.Context()))require.NoError(t, q.importLegacyInternal(t.Context()))- observer := &activityObserverFakeInternal{}+ observer := &activityObserverFakeInternal{existingOnly: true}observer.unavailable.Store(true)q.SetObserver(observer)_, err = q.Submit(t.Context(), st.Request{JobID: legacy.JobID, RunID: legacy.Runs[0].ID, Trigger: "manual"})@@ -56,13 +67,33 @@ func TestLegacyImportSurvivesRestartAndDeduplicates(t *testing.T) {return nil}))require.True(t, q.hasPendingActivitySyncInternal(t.Context()))+ q.SetExecutor(func(context.Context, st.Run) (st.Outcome, error) {+ t.Error("startup must not execute jobs before activation")+ return st.Outcome{Status: st.Failed}, nil+ }, nil)+ require.NoError(t, q.Start(t.Context(), t.Context()))+ require.NoError(t, q.Stop(t.Context()))require.NoError(t, first.Stop(t.Context()))second := francistest.New(t, databaseURL)restored := New(store, second.Service(), time.UTC)+ restored.SetObserver(observer)+ restored.SetExecutor(q.execute, nil)require.NoError(t, restored.Register(second))francistest.Start(t, second)require.NoError(t, restored.importLegacyInternal(t.Context()))+ require.NoError(t, restored.Start(t.Context(), t.Context()))+ t.Cleanup(func() { require.NoError(t, restored.Stop(context.WithoutCancel(t.Context()))) })+ for _, jobID := range []string{"gitops-sync:first", "gitops-sync:second"} {+ history, listErr := restored.List(t.Context(), "0", jobID, 1, 100)+ require.NoError(t, listErr)+ require.Equal(t, 100, history.Total)+ require.Equal(t, st.Failed, history.Runs[0].Status)+ state, stateErr := restored.ScheduleState(t.Context(), jobID)+ require.NoError(t, stateErr)+ require.Equal(t, "@every 5m", state.Schedule)+ require.True(t, state.NextRun.After(time.Now()))+ }duplicate, err := restored.Submit(t.Context(), st.Request{JobID: legacy.JobID, RunID: legacy.Runs[0].ID, Trigger: "manual"})require.NoError(t, err)require.Equal(t, st.Succeeded, duplicate.Status)@@ -72,7 +103,6 @@ func TestLegacyImportSurvivesRestartAndDeduplicates(t *testing.T) {require.Equal(t, 1, history.Total)require.Equal(t, time.UTC, history.Runs[0].CreatedAt.Location())observer.unavailable.Store(false)- restored.SetObserver(observer)restored.retryActivitySyncInternal(t.Context())require.False(t, restored.hasPendingActivitySyncInternal(t.Context()))require.Positive(t, observer.observed.Load())@@ -134,7 +164,7 @@ func TestDuplicateDispatchAndCanceledDelivery(t *testing.T) {_, err = q.Cancel(t.Context(), "0", run.JobID, run.ID)require.NoError(t, err)}- require.NoError(t, q.Start(t.Context()))+ require.NoError(t, q.Start(t.Context(), t.Context()))q.Activate()t.Cleanup(func() { require.NoError(t, q.Stop(context.WithoutCancel(t.Context()))) })require.Eventually(t, func() bool {@@ -183,7 +213,7 @@ func TestCapacityGroupsReserveHealthSlots(t *testing.T) {_, err := q.Submit(t.Context(), st.Request{JobID: job, Trigger: "manual"})require.NoError(t, err)}- require.NoError(t, q.Start(t.Context()))+ require.NoError(t, q.Start(t.Context(), t.Context()))q.Activate()t.Cleanup(func() {once.Do(func() { close(release) })@@ -281,7 +311,7 @@ func TestLostDispatchAcknowledgementRetainsOneExecution(t *testing.T) {}, nil)run, err := q.Submit(t.Context(), st.Request{JobID: "lost-ack"})require.NoError(t, err)- require.NoError(t, q.Start(t.Context()))+ require.NoError(t, q.Start(t.Context(), t.Context()))q.Activate()t.Cleanup(func() {once.Do(func() { close(finish) })diff --git a/backend/pkg/scheduler/runs/coordinator.go b/backend/pkg/scheduler/runs/coordinator.goindex b25e3dd21..bc51f7ee4 100644--- a/backend/pkg/scheduler/runs/coordinator.go+++ b/backend/pkg/scheduler/runs/coordinator.go@@ -197,37 +197,62 @@ func (q *Coordinator) Checkpoint(ctx context.Context, jobID, schedule string, nereturn err}-// Start validates persisted state and starts dispatching on ctx. Repeated calls+// Start validates persisted state on ctx and starts dispatching on appCtx. Repeated calls// are harmless, but a stopped queue cannot be restarted; create a new instance.-func (q *Coordinator) Start(ctx context.Context) error {+func (q *Coordinator) Start(ctx, appCtx context.Context) error {q.mu.Lock()defer q.mu.Unlock()if q.started {return nil}+ if err := ctx.Err(); err != nil {+ return fmt.Errorf("initialize job coordinator: %w", err)+ }+ if q.stopping {+ return errors.New("job queue is stopping")+ }if q.execute == nil {return errors.New("job executor unavailable")}if err := q.importLegacyInternal(ctx); err != nil {- return err+ return fmt.Errorf("import legacy job history: %w", err)}// Fail startup if persisted state cannot be read. Never replace it with empty state.records, err := q.Records(ctx)if err != nil {- return err+ return fmt.Errorf("read startup job history: %w", err)}for _, record := range records {+ changed := falsefor _, run := range record.Runs {- if err := q.UpdateRun(ctx, run, func(current *st.Run) error {- normalizeLegacyRunInternal(current)- return nil- }); err != nil {- return err+ if normalizeLegacyRunInternal(&run) {+ changed = true+ break}}+ if !changed {+ continue+ }+ if err := q.mutateInternal(ctx, record.EnvironmentID, record.JobID, func(current *st.QueueRecord) error {+ for i := range current.Runs {+ normalizeLegacyRunInternal(¤t.Runs[i])+ }+ return nil+ }); err != nil {+ return fmt.Errorf("normalize startup job %q in environment %q: %w", record.JobID, record.EnvironmentID, err)+ }+ }+ if err := q.pruneInternal(ctx); err != nil {+ return fmt.Errorf("prune startup job history: %w", err)+ }+ if err := ctx.Err(); err != nil {+ return fmt.Errorf("initialize job coordinator: %w", err)+ }+ if err := appCtx.Err(); err != nil {+ return fmt.Errorf("start job dispatcher: %w", err)}- runCtx, cancel := context.WithCancel(ctx)+ runCtx, cancel := context.WithCancel(appCtx)q.cancel = cancelq.lifecycleCtx = runCtxq.started = true@@ -260,12 +285,17 @@ func (q *Coordinator) Stop(ctx context.Context) error {func (q *Coordinator) dispatchInternal(ctx context.Context) {defer close(q.done)- nextRetention := time.Now().Add(time.Hour)+ initialRetention := true+ nextRetention := time.Now()for ctx.Err() == nil {q.retryActivitySyncInternal(ctx)- if !time.Now().Before(nextRetention) {- if err := q.pruneInternal(ctx); err != nil && ctx.Err() == nil {- slog.ErrorContext(ctx, "Job retention failed", "error", err)+ if initialRetention || !time.Now().Before(nextRetention) {+ if err := q.pruneInternal(ctx); err != nil {+ if ctx.Err() == nil {+ slog.ErrorContext(ctx, "Job retention failed", "error", err)+ }+ } else {+ initialRetention = q.hasPendingActivitySyncInternal(ctx)}nextRetention = time.Now().Add(time.Hour)}diff --git a/backend/pkg/scheduler/runs/coordinator_store.go b/backend/pkg/scheduler/runs/coordinator_store.goindex f8474a0d8..62c4939e4 100644--- a/backend/pkg/scheduler/runs/coordinator_store.go+++ b/backend/pkg/scheduler/runs/coordinator_store.go@@ -5,6 +5,7 @@ import ("context""encoding/json/v2""errors"+ "fmt""slices""sort""time"@@ -92,16 +93,43 @@ func snapshotRunsInternal(record st.QueueRecord) (map[string][]byte, map[string]}func preserveUnsyncedRunsInternal(state *st.CoordinatorState, previous []st.Run) {- retained := make(map[string]bool, len(state.Record.Runs))+ retained := make(map[string]st.Run, len(state.Record.Runs))for _, run := range state.Record.Runs {- retained[run.ID] = true+ retained[run.ID] = run}+ restore := falsefor _, run := range previous {- if _, pending := state.PendingActivitySync[run.ID]; pending && !retained[run.ID] {- state.Record.Runs = append(state.Record.Runs, run)+ _, exists := retained[run.ID]+ if _, pending := state.PendingActivitySync[run.ID]; pending && !exists {+ // Activity projection must not extend completed run retention.+ if run.Status.Terminal() {+ if _, archived := state.Record.Receipts[run.ID]; !archived {+ delete(state.PendingActivitySync, run.ID)+ }+ continue+ }+ retained[run.ID] = rundelete(state.Record.Receipts, run.ID)+ restore = true+ }+ }+ if !restore {+ return+ }+ // Keep history chronological when pending activity updates prevent pruning.+ runs := make([]st.Run, 0, len(retained))+ for _, run := range previous {+ if current, exists := retained[run.ID]; exists {+ runs = append(runs, current)+ delete(retained, run.ID)}}+ for _, run := range state.Record.Runs {+ if _, exists := retained[run.ID]; exists {+ runs = append(runs, run)+ }+ }+ state.Record.Runs = runs}func pruneDispatchesInternal(state *st.CoordinatorState) {@@ -123,12 +151,12 @@ func (q *Coordinator) Records(ctx context.Context) ([]st.QueueRecord, error) {for {page, err := q.service.ListStates(ctx, coordinatorTypeInternal, &actor.ListStatesOpts{IncludeData: true, After: cursor, Limit: 100})if err != nil {- return nil, err+ return nil, fmt.Errorf("list job coordinator states: %w", err)}for _, entry := range page.States {var state st.CoordinatorStateif err := entry.Data.Decode(&state); err != nil {- return nil, err+ return nil, fmt.Errorf("decode job coordinator %q: %w", entry.ActorID, err)}normalizeRecordTimesInternal(&state.Record)q.runtime.NameActor(entry.ActorID, state.Record.JobID+"@"+state.Record.EnvironmentID)@@ -234,17 +262,21 @@ func (q *Coordinator) pruneInternal(ctx context.Context) error {}now := time.Now().UTC()for _, record := range records {+ if !pruneRunsInternal(&record, now) {+ continue+ }if err := q.mutateInternal(ctx, record.EnvironmentID, record.JobID, func(current *st.QueueRecord) error {pruneRunsInternal(current, now)return nil}); err != nil {- return err+ return fmt.Errorf("prune job %q in environment %q: %w", record.JobID, record.EnvironmentID, err)}}return nil}-func pruneRunsInternal(record *st.QueueRecord, now time.Time) {+func pruneRunsInternal(record *st.QueueRecord, now time.Time) bool {+ changed := falseterminal := 0for index, run := range slices.Backward(record.Runs) {if !run.Status.Terminal() || ((run.RemoteDeliveryAttempted || run.RemoteAccepted) && !run.RemoteSettled) {@@ -263,7 +295,9 @@ func pruneRunsInternal(record *st.QueueRecord, now time.Time) {record.Receipts[run.ID] = run}record.Runs = append(record.Runs[:index], record.Runs[index+1:]...)+ changed = true}+ return changed}// ErrRunNotFound indicates that neither a run nor its receipt exists.@@ -361,10 +395,10 @@ func normalizeRunTimesInternal(run *st.Run) {}}-func normalizeLegacyRunInternal(run *st.Run) {+func normalizeLegacyRunInternal(run *st.Run) bool {automaticResolution := run.Status == st.Canceled && run.Resolution != nil && run.Resolution.ResolvedBy == common.SystemUser.Usernameif run.Status != st.NeedsAttention && !automaticResolution {- return+ return false}run.Owner = ""run.NextAttempt = nil@@ -379,6 +413,7 @@ func normalizeLegacyRunInternal(run *st.Run) {}}run.UpdatedAt = time.Now().UTC()+ return true}func normalizeTimePointerInternal(value *time.Time) {diff --git a/backend/pkg/scheduler/runs/resolution_test.go b/backend/pkg/scheduler/runs/resolution_test.goindex e4d262c3b..d0f13cef8 100644--- a/backend/pkg/scheduler/runs/resolution_test.go+++ b/backend/pkg/scheduler/runs/resolution_test.go@@ -97,7 +97,14 @@ func TestResolvePreservesEvidenceWithoutBlockingSchedule(t *testing.T) {t.Error("startup must not repeat legacy work")return st.Outcome{Status: st.Failed}, nil}, nil)- require.NoError(t, q.Start(ctx))+ startCtx, cancelStart := context.WithCancel(ctx)+ cancelStart()+ require.ErrorIs(t, q.Start(startCtx, ctx), context.Canceled)+ startCtx, cancelStart = context.WithCancel(ctx)+ defer cancelStart()+ require.NoError(t, q.Start(startCtx, ctx))+ cancelStart()+ require.NoError(t, q.lifecycleCtx.Err())t.Cleanup(func() { require.NoError(t, q.Stop(context.WithoutCancel(ctx))) })localLegacy, err = q.Get(ctx, "0", localLegacy.JobID, localLegacy.ID)require.NoError(t, err)diff --git a/backend/pkg/scheduler/scheduler.go b/backend/pkg/scheduler/scheduler.goindex 75ef453a6..a92dd1f62 100644--- a/backend/pkg/scheduler/scheduler.go+++ b/backend/pkg/scheduler/scheduler.go@@ -117,7 +117,7 @@ func (js *jobSchedulerInternal) GetJobRuntimeState(id string) (st.JobRuntimeStatreturn state, true}-func (js *jobSchedulerInternal) StartScheduler() error {+func (js *jobSchedulerInternal) StartScheduler(ctx context.Context) error {js.mu.Lock()if js.stopping {js.mu.Unlock()@@ -125,7 +125,7 @@ func (js *jobSchedulerInternal) StartScheduler() error {}js.started = truejs.mu.Unlock()- records, err := js.coordinator.Records(js.context)+ records, err := js.coordinator.Records(ctx)if err != nil {return err}@@ -134,14 +134,14 @@ func (js *jobSchedulerInternal) StartScheduler() error {_, watching := js.watchers[record.JobID]js.mu.RUnlock()if record.EnvironmentID == "0" && record.Schedule != "" && !js.HasJob(record.JobID) && !watching {- if err := js.coordinator.Checkpoint(js.context, record.JobID, "", time.Time{}); err != nil {+ if err := js.coordinator.Checkpoint(ctx, record.JobID, "", time.Time{}); err != nil {return err}}}var schedulingErr errorfor _, job := range js.ListRegisteredJobs() {- if err := js.installInternal(js.context, job); err != nil {+ if err := js.installInternal(ctx, job); err != nil {schedulingErr = errors.Join(schedulingErr, fmt.Errorf("%s: %w", "schedule "+job.Name(), err))}}diff --git a/backend/pkg/scheduler/scheduler_test.go b/backend/pkg/scheduler/scheduler_test.goindex 026c901b2..78edd03cc 100644--- a/backend/pkg/scheduler/scheduler_test.go+++ b/backend/pkg/scheduler/scheduler_test.go@@ -106,7 +106,7 @@ func TestJobScheduler_StartScheduler_SkipsDisabledConditionalJobs(t *testing.T)}require.NoError(t, js.RegisterJob(job))- require.NoError(t, js.StartScheduler())+ require.NoError(t, js.StartScheduler(t.Context()))state, ok := js.GetJobRuntimeState(job.Name())require.True(t, ok)@@ -123,7 +123,7 @@ func TestJobScheduler_StartScheduler_ReportsInvalidJobSchedule(t *testing.T) {require.NoError(t, js.RegisterJob(invalid))require.NoError(t, js.RegisterJob(valid))- require.Error(t, js.StartScheduler())+ require.Error(t, js.StartScheduler(t.Context()))invalidState, ok := js.GetJobRuntimeState(invalid.Name())require.True(t, ok)@@ -145,7 +145,7 @@ func TestJobScheduler_StopWaitsForBusWatchers(t *testing.T) {}require.NoError(t, js.RegisterBusWatcher(watcher, true))- require.NoError(t, js.StartScheduler())+ require.NoError(t, js.StartScheduler(t.Context()))select {case <-watcher.started:@@ -214,7 +214,7 @@ func TestJobScheduler_RegisterBusWatcherManualRunOption(t *testing.T) {func TestJobScheduler_RescheduleJob_RemovesEntryWhenDisabled(t *testing.T) {js := newJobSchedulerForTestInternal(t, context.Background(), nil)- require.NoError(t, js.StartScheduler())+ require.NoError(t, js.StartScheduler(t.Context()))enabled := truejob := &conditionalTestSchedulerJob{@@ -243,7 +243,7 @@ func TestJobScheduler_RescheduleJob_RemovesEntryWhenDisabled(t *testing.T) {func TestJobScheduler_RescheduleJob_AddsEntryWhenEnabled(t *testing.T) {js := newJobSchedulerForTestInternal(t, context.Background(), nil)- require.NoError(t, js.StartScheduler())+ require.NoError(t, js.StartScheduler(t.Context()))enabled := falsejob := &conditionalTestSchedulerJob{@@ -293,7 +293,7 @@ func TestJobScheduler_StartScheduler_SchedulesNonConditionalJobs(t *testing.T) {}require.NoError(t, js.RegisterJob(job))- require.NoError(t, js.StartScheduler())+ require.NoError(t, js.StartScheduler(t.Context()))state, ok := js.GetJobRuntimeState(job.Name())require.True(t, ok)@@ -330,7 +330,7 @@ func TestJobScheduler_RescheduleJob_UsesProvidedContext(t *testing.T) {}require.NoError(t, js.RescheduleJob(runCtx, job))- require.NoError(t, js.StartScheduler())+ require.NoError(t, js.StartScheduler(t.Context()))select {case err := <-runErrCh:@@ -363,7 +363,7 @@ func TestJobScheduler_RescheduleJob_UsesLifecycleContextForShutdown(t *testing.T}require.NoError(t, js.RescheduleJob(lifecycleCtx, job))- require.NoError(t, js.StartScheduler())+ require.NoError(t, js.StartScheduler(t.Context()))select {case <-startedCh:@@ -382,7 +382,7 @@ func TestJobScheduler_RescheduleJob_UsesLifecycleContextForShutdown(t *testing.Tfunc TestJobScheduler_AddJob_UpsertReplacesEntryWithoutLeaking(t *testing.T) {js := newJobSchedulerForTestInternal(t, context.Background(), nil)- require.NoError(t, js.StartScheduler())+ require.NoError(t, js.StartScheduler(t.Context()))job := &testSchedulerJob{name: "dyn-upsert", schedule: "*/5 * * * * *"}require.NoError(t, js.AddJob(context.Background(), job))@@ -409,7 +409,7 @@ func TestJobScheduler_AddJob_UpsertReplacesEntryWithoutLeaking(t *testing.T) {func TestJobScheduler_AddJob_InvalidRescheduleKeepsExistingEntry(t *testing.T) {js := newJobSchedulerForTestInternal(t, context.Background(), nil)- require.NoError(t, js.StartScheduler())+ require.NoError(t, js.StartScheduler(t.Context()))job := &testSchedulerJob{name: "dyn-invalid-reschedule", schedule: "*/5 * * * * *"}require.NoError(t, js.AddJob(context.Background(), job))@@ -434,7 +434,7 @@ func TestJobScheduler_AddJob_InvalidRescheduleKeepsExistingEntry(t *testing.T) {func TestJobScheduler_RemoveJob_RemovesEntryAndIsNoopWhenAbsent(t *testing.T) {js := newJobSchedulerForTestInternal(t, context.Background(), nil)- require.NoError(t, js.StartScheduler())+ require.NoError(t, js.StartScheduler(t.Context()))// Removing an unknown job must be a safe no-op (e.g. deleting a sync that never// had auto-sync enabled).@@ -454,7 +454,7 @@ func TestJobScheduler_RemoveJob_RemovesEntryAndIsNoopWhenAbsent(t *testing.T) {func TestJobScheduler_AddJob_GenericJobWithoutShouldRunIsScheduled(t *testing.T) {js := newJobSchedulerForTestInternal(t, context.Background(), nil)- require.NoError(t, js.StartScheduler())+ require.NoError(t, js.StartScheduler(t.Context()))job := &schedulertypes.GenericJob{JobName: "generic-dyn",@@ -488,7 +488,7 @@ func TestJobScheduler_StopWaitsForCanceledJobToFinish(t *testing.T) {})},}))- require.NoError(t, js.StartScheduler())+ require.NoError(t, js.StartScheduler(t.Context()))t.Cleanup(func() {cancelLifecycle()select {diff --git a/types/scheduler/job.go b/types/scheduler/job.goindex 0401a65f1..e668e8aa0 100644--- a/types/scheduler/job.go+++ b/types/scheduler/job.go@@ -71,7 +71,7 @@ type JobScheduler interface {RegisterJob(job Job) errorRegisterBusWatcher(watcher BusWatcher, canRunManually bool) error- StartScheduler() error+ StartScheduler(ctx context.Context) errorGetLocation() *time.LocationStop(ctx context.Context) error}