Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix: recover scheduled runs whose finished result was never recorded (#4458)
Kyle Mendell01e67e9f787c
diff --git a/backend/internal/job/service.go b/backend/internal/job/service.goindex 0ac2d8f07..0bed39de9 100644--- a/backend/internal/job/service.go+++ b/backend/internal/job/service.go@@ -665,7 +665,7 @@ func (s *JobService) reconcileRun(ctx context.Context, run scheduler.Run) (schedif err := s.authorizeRun(ctx, run); err != nil {return scheduler.Outcome{Status: scheduler.Failed}, err}- if run.EnvironmentID != "0" || safeJob(run.JobID) {+ if run.EnvironmentID != "0" {return s.executeRun(ctx, run)}if job, ok := s.scheduler.GetJob(run.JobID); ok {@@ -673,6 +673,9 @@ func (s *JobService) reconcileRun(ctx context.Context, run scheduler.Run) (schedreturn reconciler.Reconcile(s.runs.ExecutionContext(ctx, run, ""), run)}}+ if safeJob(run.JobID) {+ return s.executeRun(ctx, run)+ }var activityIDs []stringfor _, target := range run.Outcome.Targets {if run.JobID != "auto-update" && target.Status != scheduler.Succeeded && target.Status != scheduler.Skipped && target.ActivityID != "" {diff --git a/backend/pkg/scheduler/flow/flow.go b/backend/pkg/scheduler/flow/flow.goindex f9c46e845..8d07e185c 100644--- a/backend/pkg/scheduler/flow/flow.go+++ b/backend/pkg/scheduler/flow/flow.go@@ -318,7 +318,8 @@ func (e *Engine) Wait(ctx context.Context, wf *Workflow, instanceID string) (sch}// Resume attaches to the run's active instance, or reads the result of one that finished before the run saved it.-// It returns ErrNotResumable when the instance is gone or belongs to a version this host no longer serves.+// When the instance is gone or belongs to a version this host no longer serves, a recorded non-failed result is+// returned; otherwise it returns ErrNotResumable so the job's fallback can run.func (e *Engine) Resume(ctx context.Context, wf *Workflow, run scheduler.Run) (scheduler.Outcome, error) {target, ok := activeTarget(run, wf.def.Name, true)if !ok {@@ -330,6 +331,13 @@ func (e *Engine) Resume(ctx context.Context, wf *Workflow, run scheduler.Run) (sreturn scheduler.Outcome{}, fmt.Errorf("read %s instance: %w", wf.def.Name, err)}if err != nil || status.Version != wf.def.Version {+ if target.Status.Terminal() && target.Status != scheduler.Failed {+ return scheduler.Outcome{+ Status: target.Status,+ Message: target.Message,+ ActivityID: target.ActivityID,+ }, nil+ }if !target.Status.Terminal() {e.supersede(ctx, wf, target, "Workflow instance could not be resumed")}diff --git a/backend/pkg/scheduler/runs/actors.go b/backend/pkg/scheduler/runs/actors.goindex d07b3acd7..1f6d33a41 100644--- a/backend/pkg/scheduler/runs/actors.go+++ b/backend/pkg/scheduler/runs/actors.go@@ -25,6 +25,8 @@ const (coordinatorTypeInternal = "job-coordinator"executorTypeInternal = "job-executor"healthExecutorTypeInternal = "job-health-executor"+ orphanedDispatchAge = 15 * time.Minute+ outcomePersistGrace = 30 * time.Second)type coordinatorActorInternal struct {@@ -159,7 +161,7 @@ func (a *coordinatorActorInternal) flushInternal(ctx context.Context, state *st.return a.q.service.SetAlarm(ctx, coordinatorTypeInternal, a.id, name, actor.AlarmProperties{DueTime: retryAt})}if selected.Status == st.Running {- active, err := a.reconcileDispatchInternal(ctx, state, *selected)+ active, err := a.reconcileDispatch(ctx, state, *selected)if err != nil {return err}@@ -167,16 +169,21 @@ func (a *coordinatorActorInternal) flushInternal(ctx context.Context, state *st.return nil}}- return a.dispatchRunInternal(ctx, state, *selected, selected.AttemptCount+1)+ return a.dispatchRun(ctx, state, *selected, selected.AttemptCount+1)}-// reconcileDispatchInternal recovers a lost job ID before considering redelivery.-func (a *coordinatorActorInternal) reconcileDispatchInternal(ctx context.Context, state *st.CoordinatorState, run st.Run) (bool, error) {+// reconcileDispatch recovers a lost job ID before considering redelivery.+func (a *coordinatorActorInternal) reconcileDispatch(ctx context.Context, state *st.CoordinatorState, run st.Run) (bool, error) {key := fmt.Sprintf("%s-%d", run.ID, run.AttemptCount)previous, exists := state.Dispatches[key]if !exists {return false, nil}+ actorType := kit.Ternary(+ strings.HasPrefix(run.JobID, "environment-health") || run.JobID == "docker-client-refresh" || run.JobID == "activity-sweep",+ healthExecutorTypeInternal,+ executorTypeInternal,+ )if previous.JobID == "" {command := st.ExecutionCommand{EnvironmentID: run.EnvironmentID,@@ -184,10 +191,6 @@ func (a *coordinatorActorInternal) reconcileDispatchInternal(ctx context.ContextRunID: run.ID,Attempt: run.AttemptCount,}- actorType := executorTypeInternal- if strings.HasPrefix(run.JobID, "environment-health") || run.JobID == "docker-client-refresh" || run.JobID == "activity-sweep" {- actorType = healthExecutorTypeInternal- }id, _, err := a.q.service.Dispatch(ctx, actorType, a.id, "execute", command, actor.WithIdempotencyKey(key))if err != nil {return false, err@@ -206,22 +209,30 @@ func (a *coordinatorActorInternal) reconcileDispatchInternal(ctx context.Contextif err != nil {return false, err}- return !job.Status.IsTerminal(), nil+ if job.Status != actor.JobStatusPending || time.Since(job.DueTime) < orphanedDispatchAge {+ return !job.Status.IsTerminal(), nil+ }+ slog.WarnContext(ctx, "Deleting orphaned job dispatch", "runId", run.ID, "jobId", job.JobID, "dueTime", job.DueTime)+ if deleteErr := a.q.service.DeleteJob(ctx, actorType, a.id, job.JobID, actor.WithLiveJobsOnly()); deleteErr != nil && !errors.Is(deleteErr, actor.ErrJobNotFound) {+ return false, deleteErr+ }+ return false, nil}-func (a *coordinatorActorInternal) dispatchRunInternal(ctx context.Context, state *st.CoordinatorState, run st.Run, attempt int) error {+func (a *coordinatorActorInternal) dispatchRun(ctx context.Context, state *st.CoordinatorState, run st.Run, attempt int) error {key := fmt.Sprintf("%s-%d", run.ID, attempt)if state.Dispatches == nil {state.Dispatches = make(map[string]st.DispatchIntent)}- actorType := executorTypeInternal- if strings.HasPrefix(run.JobID, "environment-health") || run.JobID == "docker-client-refresh" || run.JobID == "activity-sweep" {- actorType = healthExecutorTypeInternal- }+ actorType := kit.Ternary(+ strings.HasPrefix(run.JobID, "environment-health") || run.JobID == "docker-client-refresh" || run.JobID == "activity-sweep",+ healthExecutorTypeInternal,+ executorTypeInternal,+ )intent := state.Dispatches[key]if intent.JobID != "" {job, err := a.q.service.GetJob(ctx, intent.JobID)- if err == nil && !job.Status.IsTerminal() {+ if err == nil && !job.Status.IsTerminal() && (job.Status != actor.JobStatusPending || time.Since(job.DueTime) < orphanedDispatchAge) {return nil}if err != nil && !errors.Is(err, actor.ErrJobNotFound) {@@ -452,14 +463,18 @@ func (a *executorActorInternal) Job(ctx context.Context, _ string, data actor.Enif runErr != nil && outcome.Message == "" {outcome.Message = runErr.Error()}- if ctx.Err() != nil {- return ctx.Err()- }- q.persistOutcomeInternal(ctx, claim.Run, outcome)- if ctx.Err() != nil {+ if runErr != nil {+ if ctx.Err() != nil {+ return ctx.Err()+ }+ q.persistOutcome(ctx, claim.Run, outcome)return ctx.Err()}- return nil+ persistCtx, cancel := context.WithCancel(context.WithoutCancel(ctx))+ defer cancel()+ defer context.AfterFunc(ctx, func() { time.AfterFunc(outcomePersistGrace, cancel) })()+ q.persistOutcome(persistCtx, claim.Run, outcome)+ return ctx.Err()}func (q *Coordinator) claimExecutionInternal(ctx context.Context, command st.ExecutionCommand) (st.ExecutionClaim, error) {diff --git a/backend/pkg/scheduler/runs/actors_test.go b/backend/pkg/scheduler/runs/actors_test.goindex 6265fda00..d3f406091 100644--- a/backend/pkg/scheduler/runs/actors_test.go+++ b/backend/pkg/scheduler/runs/actors_test.go@@ -312,7 +312,7 @@ func TestAdmissionAndCompletionRejectStaleTokens(t *testing.T) {require.NoError(t, err)require.NoError(t, q.UpdateRun(t.Context(), run, func(current *st.Run) error { current.Status = st.Running; current.Owner = "new-token"; return nil }))run.Owner = "stale-token"- q.persistOutcomeInternal(t.Context(), run, st.Outcome{Status: st.Succeeded})+ q.persistOutcome(t.Context(), run, st.Outcome{Status: st.Succeeded})persisted, err := q.Get(t.Context(), "0", run.JobID, run.ID)require.NoError(t, err)require.Equal(t, st.Running, persisted.Status)diff --git a/backend/pkg/scheduler/runs/coordinator.go b/backend/pkg/scheduler/runs/coordinator.goindex 65ee97444..1242a90b7 100644--- a/backend/pkg/scheduler/runs/coordinator.go+++ b/backend/pkg/scheduler/runs/coordinator.go@@ -336,11 +336,14 @@ func (q *Coordinator) dispatchInternal(ctx context.Context) {}}-func (q *Coordinator) persistOutcomeInternal(ctx context.Context, run st.Run, outcome st.Outcome) {+func (q *Coordinator) persistOutcome(ctx context.Context, run st.Run, outcome st.Outcome) {// Keep the execution slot until the terminal result has been durably recorded.for {+ var conflictOwner string+ var conflictStatus st.RunStatuserr := q.UpdateRun(ctx, run, func(current *st.Run) error {if current.Owner != run.Owner || current.Status != st.Running {+ conflictOwner, conflictStatus = current.Owner, current.Statusreturn ErrRunConflict}now := time.Now().UTC()@@ -374,7 +377,16 @@ func (q *Coordinator) persistOutcomeInternal(ctx context.Context, run st.Run, ou}return nil})- if err == nil || errors.Is(err, ErrRunConflict) {+ if errors.Is(err, ErrRunConflict) {+ slog.WarnContext(ctx, "Discarding job result after ownership changed",+ "runId", run.ID,+ "expectedOwner", run.Owner,+ "currentOwner", conflictOwner,+ "currentStatus", conflictStatus,+ "outcomeStatus", outcome.Status)+ return+ }+ if err == nil {return}slog.ErrorContext(ctx, "Unable to persist job result; retaining execution guard", "runId", run.ID, "error", err)diff --git a/backend/pkg/utils/s3/s3.go b/backend/pkg/utils/s3/s3.goindex 0cc80e254..ce8c6470a 100644--- a/backend/pkg/utils/s3/s3.go+++ b/backend/pkg/utils/s3/s3.go@@ -300,9 +300,9 @@ type gcsHttpSigner struct {wrapped awss3.HTTPSignerV4}-func (s *gcsHttpSigner) SignHTTP(ctx context.Context, credentials aws.Credentials, req *http.Request, payloadHash string, service string, region string, signingTime time.Time, optFns ...func(*v4.SignerOptions)) error {+func (s *gcsHttpSigner) SignHTTP(ctx context.Context, creds aws.Credentials, req *http.Request, payloadHash, service, region string, signingTime time.Time, optFns ...func(*v4.SignerOptions)) error {req.Header.Del("Accept-Encoding")- return s.wrapped.SignHTTP(ctx, credentials, req, payloadHash, service, region, signingTime, optFns...)+ return s.wrapped.SignHTTP(ctx, creds, req, payloadHash, service, region, signingTime, optFns...)}func newClientInternal(ctx context.Context, configuration Configuration) (*awss3.Client, error) {