Skip to content

Modern Docker Management, Designed for Everyone

fix: recover scheduled runs whose finished result was never recorded (#4458)

Kyle Mendell01e67e9f787c
diff --git a/backend/internal/job/service.go b/backend/internal/job/service.go
index 0ac2d8f07..0bed39de9 100644
--- a/backend/internal/job/service.go
+++ b/backend/internal/job/service.go
@@ -665,7 +665,7 @@ func (s *JobService) reconcileRun(ctx context.Context, run scheduler.Run) (sched
if err := s.authorizeRun(ctx, run); err != nil {
return scheduler.Outcome{Status: scheduler.Failed}, err
}
- if run.EnvironmentID != "0" || safeJob(run.JobID) {
+ if run.EnvironmentID != "0" {
return s.executeRun(ctx, run)
}
if job, ok := s.scheduler.GetJob(run.JobID); ok {
@@ -673,6 +673,9 @@ func (s *JobService) reconcileRun(ctx context.Context, run scheduler.Run) (sched
return reconciler.Reconcile(s.runs.ExecutionContext(ctx, run, ""), run)
}
}
+ if safeJob(run.JobID) {
+ return s.executeRun(ctx, run)
+ }
var activityIDs []string
for _, target := range run.Outcome.Targets {
if run.JobID != "auto-update" && target.Status != scheduler.Succeeded && target.Status != scheduler.Skipped && target.ActivityID != "" {
diff --git a/backend/pkg/scheduler/flow/flow.go b/backend/pkg/scheduler/flow/flow.go
index f9c46e845..8d07e185c 100644
--- a/backend/pkg/scheduler/flow/flow.go
+++ b/backend/pkg/scheduler/flow/flow.go
@@ -318,7 +318,8 @@ func (e *Engine) Wait(ctx context.Context, wf *Workflow, instanceID string) (sch
}
// Resume attaches to the run's active instance, or reads the result of one that finished before the run saved it.
-// It returns ErrNotResumable when the instance is gone or belongs to a version this host no longer serves.
+// When the instance is gone or belongs to a version this host no longer serves, a recorded non-failed result is
+// returned; otherwise it returns ErrNotResumable so the job's fallback can run.
func (e *Engine) Resume(ctx context.Context, wf *Workflow, run scheduler.Run) (scheduler.Outcome, error) {
target, ok := activeTarget(run, wf.def.Name, true)
if !ok {
@@ -330,6 +331,13 @@ func (e *Engine) Resume(ctx context.Context, wf *Workflow, run scheduler.Run) (s
return scheduler.Outcome{}, fmt.Errorf("read %s instance: %w", wf.def.Name, err)
}
if err != nil || status.Version != wf.def.Version {
+ if target.Status.Terminal() && target.Status != scheduler.Failed {
+ return scheduler.Outcome{
+ Status: target.Status,
+ Message: target.Message,
+ ActivityID: target.ActivityID,
+ }, nil
+ }
if !target.Status.Terminal() {
e.supersede(ctx, wf, target, "Workflow instance could not be resumed")
}
diff --git a/backend/pkg/scheduler/runs/actors.go b/backend/pkg/scheduler/runs/actors.go
index d07b3acd7..1f6d33a41 100644
--- a/backend/pkg/scheduler/runs/actors.go
+++ b/backend/pkg/scheduler/runs/actors.go
@@ -25,6 +25,8 @@ const (
coordinatorTypeInternal = "job-coordinator"
executorTypeInternal = "job-executor"
healthExecutorTypeInternal = "job-health-executor"
+ orphanedDispatchAge = 15 * time.Minute
+ outcomePersistGrace = 30 * time.Second
)
type coordinatorActorInternal struct {
@@ -159,7 +161,7 @@ func (a *coordinatorActorInternal) flushInternal(ctx context.Context, state *st.
return a.q.service.SetAlarm(ctx, coordinatorTypeInternal, a.id, name, actor.AlarmProperties{DueTime: retryAt})
}
if selected.Status == st.Running {
- active, err := a.reconcileDispatchInternal(ctx, state, *selected)
+ active, err := a.reconcileDispatch(ctx, state, *selected)
if err != nil {
return err
}
@@ -167,16 +169,21 @@ func (a *coordinatorActorInternal) flushInternal(ctx context.Context, state *st.
return nil
}
}
- return a.dispatchRunInternal(ctx, state, *selected, selected.AttemptCount+1)
+ return a.dispatchRun(ctx, state, *selected, selected.AttemptCount+1)
}
-// reconcileDispatchInternal recovers a lost job ID before considering redelivery.
-func (a *coordinatorActorInternal) reconcileDispatchInternal(ctx context.Context, state *st.CoordinatorState, run st.Run) (bool, error) {
+// reconcileDispatch recovers a lost job ID before considering redelivery.
+func (a *coordinatorActorInternal) reconcileDispatch(ctx context.Context, state *st.CoordinatorState, run st.Run) (bool, error) {
key := fmt.Sprintf("%s-%d", run.ID, run.AttemptCount)
previous, exists := state.Dispatches[key]
if !exists {
return false, nil
}
+ actorType := kit.Ternary(
+ strings.HasPrefix(run.JobID, "environment-health") || run.JobID == "docker-client-refresh" || run.JobID == "activity-sweep",
+ healthExecutorTypeInternal,
+ executorTypeInternal,
+ )
if previous.JobID == "" {
command := st.ExecutionCommand{
EnvironmentID: run.EnvironmentID,
@@ -184,10 +191,6 @@ func (a *coordinatorActorInternal) reconcileDispatchInternal(ctx context.Context
RunID: run.ID,
Attempt: run.AttemptCount,
}
- actorType := executorTypeInternal
- if strings.HasPrefix(run.JobID, "environment-health") || run.JobID == "docker-client-refresh" || run.JobID == "activity-sweep" {
- actorType = healthExecutorTypeInternal
- }
id, _, err := a.q.service.Dispatch(ctx, actorType, a.id, "execute", command, actor.WithIdempotencyKey(key))
if err != nil {
return false, err
@@ -206,22 +209,30 @@ func (a *coordinatorActorInternal) reconcileDispatchInternal(ctx context.Context
if err != nil {
return false, err
}
- return !job.Status.IsTerminal(), nil
+ if job.Status != actor.JobStatusPending || time.Since(job.DueTime) < orphanedDispatchAge {
+ return !job.Status.IsTerminal(), nil
+ }
+ slog.WarnContext(ctx, "Deleting orphaned job dispatch", "runId", run.ID, "jobId", job.JobID, "dueTime", job.DueTime)
+ if deleteErr := a.q.service.DeleteJob(ctx, actorType, a.id, job.JobID, actor.WithLiveJobsOnly()); deleteErr != nil && !errors.Is(deleteErr, actor.ErrJobNotFound) {
+ return false, deleteErr
+ }
+ return false, nil
}
-func (a *coordinatorActorInternal) dispatchRunInternal(ctx context.Context, state *st.CoordinatorState, run st.Run, attempt int) error {
+func (a *coordinatorActorInternal) dispatchRun(ctx context.Context, state *st.CoordinatorState, run st.Run, attempt int) error {
key := fmt.Sprintf("%s-%d", run.ID, attempt)
if state.Dispatches == nil {
state.Dispatches = make(map[string]st.DispatchIntent)
}
- actorType := executorTypeInternal
- if strings.HasPrefix(run.JobID, "environment-health") || run.JobID == "docker-client-refresh" || run.JobID == "activity-sweep" {
- actorType = healthExecutorTypeInternal
- }
+ actorType := kit.Ternary(
+ strings.HasPrefix(run.JobID, "environment-health") || run.JobID == "docker-client-refresh" || run.JobID == "activity-sweep",
+ healthExecutorTypeInternal,
+ executorTypeInternal,
+ )
intent := state.Dispatches[key]
if intent.JobID != "" {
job, err := a.q.service.GetJob(ctx, intent.JobID)
- if err == nil && !job.Status.IsTerminal() {
+ if err == nil && !job.Status.IsTerminal() && (job.Status != actor.JobStatusPending || time.Since(job.DueTime) < orphanedDispatchAge) {
return nil
}
if err != nil && !errors.Is(err, actor.ErrJobNotFound) {
@@ -452,14 +463,18 @@ func (a *executorActorInternal) Job(ctx context.Context, _ string, data actor.En
if runErr != nil && outcome.Message == "" {
outcome.Message = runErr.Error()
}
- if ctx.Err() != nil {
- return ctx.Err()
- }
- q.persistOutcomeInternal(ctx, claim.Run, outcome)
- if ctx.Err() != nil {
+ if runErr != nil {
+ if ctx.Err() != nil {
+ return ctx.Err()
+ }
+ q.persistOutcome(ctx, claim.Run, outcome)
return ctx.Err()
}
- return nil
+ persistCtx, cancel := context.WithCancel(context.WithoutCancel(ctx))
+ defer cancel()
+ defer context.AfterFunc(ctx, func() { time.AfterFunc(outcomePersistGrace, cancel) })()
+ q.persistOutcome(persistCtx, claim.Run, outcome)
+ return ctx.Err()
}
func (q *Coordinator) claimExecutionInternal(ctx context.Context, command st.ExecutionCommand) (st.ExecutionClaim, error) {
diff --git a/backend/pkg/scheduler/runs/actors_test.go b/backend/pkg/scheduler/runs/actors_test.go
index 6265fda00..d3f406091 100644
--- a/backend/pkg/scheduler/runs/actors_test.go
+++ b/backend/pkg/scheduler/runs/actors_test.go
@@ -312,7 +312,7 @@ func TestAdmissionAndCompletionRejectStaleTokens(t *testing.T) {
require.NoError(t, err)
require.NoError(t, q.UpdateRun(t.Context(), run, func(current *st.Run) error { current.Status = st.Running; current.Owner = "new-token"; return nil }))
run.Owner = "stale-token"
- q.persistOutcomeInternal(t.Context(), run, st.Outcome{Status: st.Succeeded})
+ q.persistOutcome(t.Context(), run, st.Outcome{Status: st.Succeeded})
persisted, err := q.Get(t.Context(), "0", run.JobID, run.ID)
require.NoError(t, err)
require.Equal(t, st.Running, persisted.Status)
diff --git a/backend/pkg/scheduler/runs/coordinator.go b/backend/pkg/scheduler/runs/coordinator.go
index 65ee97444..1242a90b7 100644
--- a/backend/pkg/scheduler/runs/coordinator.go
+++ b/backend/pkg/scheduler/runs/coordinator.go
@@ -336,11 +336,14 @@ func (q *Coordinator) dispatchInternal(ctx context.Context) {
}
}
-func (q *Coordinator) persistOutcomeInternal(ctx context.Context, run st.Run, outcome st.Outcome) {
+func (q *Coordinator) persistOutcome(ctx context.Context, run st.Run, outcome st.Outcome) {
// Keep the execution slot until the terminal result has been durably recorded.
for {
+ var conflictOwner string
+ var conflictStatus st.RunStatus
err := q.UpdateRun(ctx, run, func(current *st.Run) error {
if current.Owner != run.Owner || current.Status != st.Running {
+ conflictOwner, conflictStatus = current.Owner, current.Status
return ErrRunConflict
}
now := time.Now().UTC()
@@ -374,7 +377,16 @@ func (q *Coordinator) persistOutcomeInternal(ctx context.Context, run st.Run, ou
}
return nil
})
- if err == nil || errors.Is(err, ErrRunConflict) {
+ if errors.Is(err, ErrRunConflict) {
+ slog.WarnContext(ctx, "Discarding job result after ownership changed",
+ "runId", run.ID,
+ "expectedOwner", run.Owner,
+ "currentOwner", conflictOwner,
+ "currentStatus", conflictStatus,
+ "outcomeStatus", outcome.Status)
+ return
+ }
+ if err == nil {
return
}
slog.ErrorContext(ctx, "Unable to persist job result; retaining execution guard", "runId", run.ID, "error", err)
diff --git a/backend/pkg/utils/s3/s3.go b/backend/pkg/utils/s3/s3.go
index 0cc80e254..ce8c6470a 100644
--- a/backend/pkg/utils/s3/s3.go
+++ b/backend/pkg/utils/s3/s3.go
@@ -300,9 +300,9 @@ type gcsHttpSigner struct {
wrapped awss3.HTTPSignerV4
}
-func (s *gcsHttpSigner) SignHTTP(ctx context.Context, credentials aws.Credentials, req *http.Request, payloadHash string, service string, region string, signingTime time.Time, optFns ...func(*v4.SignerOptions)) error {
+func (s *gcsHttpSigner) SignHTTP(ctx context.Context, creds aws.Credentials, req *http.Request, payloadHash, service, region string, signingTime time.Time, optFns ...func(*v4.SignerOptions)) error {
req.Header.Del("Accept-Encoding")
- return s.wrapped.SignHTTP(ctx, credentials, req, payloadHash, service, region, signingTime, optFns...)
+ return s.wrapped.SignHTTP(ctx, creds, req, payloadHash, service, region, signingTime, optFns...)
}
func newClientInternal(ctx context.Context, configuration Configuration) (*awss3.Client, error) {