Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix(jobs): remove manual review and fix Francis scheduling (#4268)
Kyle Mendelle96f588293f0
diff --git a/backend/internal/activity/job.go b/backend/internal/activity/job.goindex b3892134a..164441df3 100644--- a/backend/internal/activity/job.go+++ b/backend/internal/activity/job.go@@ -7,6 +7,7 @@ import ("fmt""time"+ "github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/internal/database"activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"st "github.com/getarcaneapp/arcane/types/v2/scheduler"@@ -102,7 +103,9 @@ func jobActivityInternal(run st.Run, name string) Activity {}if run.Resolution != nil {model.Metadata["resolution"] = run.Resolution- model.LatestMessage = "Resolved after review; future scheduled runs may proceed"+ if run.Resolution.ResolvedBy != common.SystemUser.Username {+ model.LatestMessage = "Resolved after review"+ }}if isTerminalActivityStatusInternal(status) {model.EndedAt = new(run.UpdatedAt)diff --git a/backend/internal/bootstrap/bootstrap_test.go b/backend/internal/bootstrap/bootstrap_test.goindex 77c553557..63754d350 100644--- a/backend/internal/bootstrap/bootstrap_test.go+++ b/backend/internal/bootstrap/bootstrap_test.go@@ -16,6 +16,8 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/container""github.com/getarcaneapp/arcane/backend/v2/internal/database"+ "github.com/getarcaneapp/arcane/backend/v2/internal/environment"+ "github.com/getarcaneapp/arcane/backend/v2/internal/gitops""github.com/getarcaneapp/arcane/backend/v2/internal/job""github.com/getarcaneapp/arcane/backend/v2/internal/kv""github.com/getarcaneapp/arcane/backend/v2/internal/project"@@ -26,6 +28,7 @@ import (francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"tunnelpb "github.com/getarcaneapp/arcane/backend/v2/proto/tunnel/v1"+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler""github.com/labstack/echo/v5""github.com/libtnb/sqlite""github.com/stretchr/testify/assert"@@ -396,17 +399,35 @@ func TestJobSchedulerStopCancelsItsPrivateContextInternal(t *testing.T) {runtime := francistest.New(t)db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})require.NoError(t, err)- require.NoError(t, db.AutoMigrate(&kv.KVEntry{}))+ require.NoError(t, db.AutoMigrate(&kv.KVEntry{}, &environment.Environment{}, &project.GitOpsSync{}))sqlDB, err := db.DB()require.NoError(t, err)sqlDB.SetMaxOpenConns(1)t.Cleanup(func() { _ = sqlDB.Close() })coordinator := runs.New(kv.NewKVService(&database.DB{DB: db}), runtime.Service(), time.UTC)require.NoError(t, coordinator.Register(runtime))- francistest.Start(t, runtime)+ admission := runs.NewAdmission(runtime.Service(), t.Name())+ require.NoError(t, admission.Register(runtime))+ lifecycle.Append(fx.Hook{+ OnStart: func(ctx context.Context) error { return runtime.Start(ctx, appCtx, nil) },+ OnStop: runtime.Stop,+ })+ t.Cleanup(func() { lifecycle.RequireStop() })jobService := job.NewJobService(&database.DB{DB: db}, nil, &config.Config{}, coordinator, nil, nil, nil)- jobScheduler, err := newJobScheduler(appCtx, lifecycle, &config.Config{}, coordinator, nil, nil, nil, nil, jobService, nil, nil, nil, nil, nil)+ gitopsSync := gitops.NewGitOpsSyncService(&database.DB{DB: db}, nil, nil, nil, nil, nil)+ require.NoError(t, db.Create(&environment.Environment{ID: "0", Name: "Local", Enabled: true}).Error)+ require.NoError(t, db.Create(&project.GitOpsSync{ID: "overdue", EnvironmentID: "0", AutoSync: true, SyncInterval: 1}).Error)+ jobScheduler, err := newJobScheduler(appCtx, lifecycle, &config.Config{}, coordinator, admission, nil, nil, nil, jobService, nil, nil, nil, nil, gitopsSync)require.NoError(t, err)+ require.NoError(t, registerDynamicJobs(dynamicJobsParams{+ AppCtx: appCtx, Config: &config.Config{}, Scheduler: jobScheduler,+ GitOpsSync: gitopsSync, Admission: admission,+ }))+ executed := make(chan schedulertypes.Run, 1)+ coordinator.SetExecutor(func(_ context.Context, run schedulertypes.Run) (schedulertypes.Outcome, error) {+ executed <- run+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil+ }, nil)watcher := &blockingBusWatcherInternal{started: make(chan struct{}),stopped: make(chan struct{}),@@ -415,6 +436,14 @@ func TestJobSchedulerStopCancelsItsPrivateContextInternal(t *testing.T) {lifecycle.RequireStart()select {+ case run := <-executed:+ require.Equal(t, "gitops-sync:overdue", run.JobID)+ require.Equal(t, "startup", run.Trigger)+ require.True(t, jobScheduler.HasJob(run.JobID))+ case <-time.After(5 * time.Second):+ require.FailNow(t, "overdue GitOps startup job did not execute through Francis")+ }+ select {case <-watcher.started:case <-time.After(time.Second):require.FailNow(t, "watcher did not start")diff --git a/backend/internal/bootstrap/jobs_bootstrap.go b/backend/internal/bootstrap/jobs_bootstrap.goindex 89fcf7e72..b88bd2ceb 100644--- a/backend/internal/bootstrap/jobs_bootstrap.go+++ b/backend/internal/bootstrap/jobs_bootstrap.go@@ -50,6 +50,10 @@ func newJobScheduler(appCtx context.Context, lc fx.Lifecycle, cfg *config.Configif err := reconcileCoordinatorStartupInternal(schedulerCtx, jobService, backupEngine, updaterService, volumes, systemBackups, gitopsSync); err != nil {return err}+ if gitopsSync != nil {+ gitopsSync.RegisterAutoSyncJobsOnStartup(schedulerCtx)+ gitopsSync.SubscribeProjectFileChanges(schedulerCtx)+ }if imageUpdateWatcher != nil {imageUpdateWatcher.SetCoordinator(jobService.Coordinator())if err := jobScheduler.RegisterBusWatcher(imageUpdateWatcher, true); err != nil {@@ -343,13 +347,11 @@ func registerDynamicJobs(params dynamicJobsParams) error {}params.SystemBackup.RegisterBackupJobOnStartup(params.AppCtx)}- // GitOps: one job per auto-sync-enabled sync (runs on manager and agents).+ // GitOps startup submissions wait for Francis in the scheduler start hook.if params.GitOpsSync != nil {if err := params.GitOpsSync.SetScheduler(params.AppCtx, params.Scheduler, params.Admission); err != nil {return err}- params.GitOpsSync.RegisterAutoSyncJobsOnStartup(params.AppCtx)- params.GitOpsSync.SubscribeProjectFileChanges(params.AppCtx)}// Environment health: one job per enabled environment (manager only). The Jobsdiff --git a/backend/internal/job/activity.go b/backend/internal/job/activity.goindex ab7464183..307832bc5 100644--- a/backend/internal/job/activity.go+++ b/backend/internal/job/activity.go@@ -51,7 +51,7 @@ func (s *JobService) SyncRunActivity(ctx context.Context, run st.Run) error {if metadata, found := meta.GetJobMetadata(current.JobID); found {name = metadata.Name}- return s.activity.SyncJobRun(ctx, current, name)+ return s.activity.SyncJobRun(ctx, projectRunOutcomeInternal(current), name)}func (s *JobService) ReconcileStartupActivities(ctx context.Context, extraProtectedIDs ...string) error {diff --git a/backend/internal/job/activity_test.go b/backend/internal/job/activity_test.goindex c75152eb1..41646f999 100644--- a/backend/internal/job/activity_test.go+++ b/backend/internal/job/activity_test.go@@ -47,7 +47,7 @@ func TestJobActivityLifecycle(t *testing.T) {duplicate, err := svc.runs.Submit(ctx, st.Request{JobID: run.JobID, RunID: run.ID, Trigger: "scheduled"})require.NoError(t, err)require.Equal(t, run.ActivityID, duplicate.ActivityID)- for _, status := range []st.RunStatus{st.Running, st.Waiting, st.NeedsAttention} {+ for _, status := range []st.RunStatus{st.Running, st.Waiting, st.Failed} {require.NoError(t, svc.runs.UpdateRun(ctx, run, func(current *st.Run) error {current.Status = statuscurrent.UpdatedAt = time.Now().UTC()@@ -55,7 +55,22 @@ func TestJobActivityLifecycle(t *testing.T) {return nil}))}- check(activitytypes.StatusFailed, st.NeedsAttention)+ check(activitytypes.StatusFailed, st.Failed)+ failedDetail, err := activities.GetActivityDetail(ctx, "0", run.ActivityID, 10)+ require.NoError(t, err)+ require.NotNil(t, failedDetail.Activity.Error)+ require.Equal(t, "operation detail", *failedDetail.Activity.Error)+ later, err := svc.runs.Submit(ctx, st.Request{JobID: run.JobID, Trigger: "scheduled"})+ require.NoError(t, err)+ require.NoError(t, svc.runs.UpdateRun(ctx, later, func(current *st.Run) error {+ current.Status = st.Succeeded+ current.UpdatedAt = time.Now().UTC()+ return nil+ }))+ failedDetail, err = activities.GetActivityDetail(ctx, "0", run.ActivityID, 10)+ require.NoError(t, err)+ require.Equal(t, activitytypes.StatusFailed, failedDetail.Activity.Status)+ require.Equal(t, "operation detail", *failedDetail.Activity.Error)_, err = svc.runs.Retry(ctx, "0", run.JobID, run.ID)require.NoError(t, err)check(activitytypes.StatusQueued, st.Queued)@@ -72,7 +87,7 @@ func TestJobActivityLifecycle(t *testing.T) {check(activitytypes.StatusCancelled, st.Canceled)var count int64require.NoError(t, db.Model(&activity.Activity{}).Count(&count).Error)- require.Equal(t, int64(1), count)+ require.Equal(t, int64(2), count)}func TestJobActivityVisibilityAndGrouping(t *testing.T) {@@ -136,6 +151,9 @@ func TestJobActivityRestartRepairsFailedProjection(t *testing.T) {require.Contains(t, detail.Activity.LatestMessage, "interrupted")persisted, err := restarted.runs.Get(ctx, "0", run.JobID, run.ID)require.NoError(t, err)- require.Zero(t, persisted.AttemptCount, "repair must not execute blocked work")+ require.Zero(t, persisted.AttemptCount, "repair must not execute interrupted work")+ require.Equal(t, st.Failed, persisted.Status)+ require.Equal(t, "interrupted before applying updates", persisted.Outcome.Message)+ require.NotNil(t, persisted.FinishedAt)require.Equal(t, run.ActivityID, persisted.ActivityID)}diff --git a/backend/internal/job/execution.go b/backend/internal/job/execution.goindex 50f7e96f3..add6a7e28 100644--- a/backend/internal/job/execution.go+++ b/backend/internal/job/execution.go@@ -104,7 +104,7 @@ func (s *JobService) authorizeRunInternal(ctx context.Context, run st.Run) errorfunc (s *JobService) executeRunInternal(ctx context.Context, run st.Run) (st.Outcome, error) {if err := s.authorizeRunInternal(ctx, run); err != nil {- return st.Outcome{Status: st.NeedsAttention}, err+ return st.Outcome{Status: st.Failed}, err}if run.EnvironmentID != "0" {return s.deliverRemoteInternal(ctx, run)@@ -127,7 +127,7 @@ func (s *JobService) executeRunInternal(ctx context.Context, run st.Run) (st.Outerr := s.scheduler.RunBusWatcherNow(ctx, run.JobID)return classifyOutcomeInternal(run.JobID, st.Outcome{}, err)}- unavailableStatus := kit.Ternary(run.JobID == "auto-update" && run.AttemptCount > 1, st.NeedsAttention, st.Canceled)+ unavailableStatus := kit.Ternary(run.JobID == "auto-update" && run.AttemptCount > 1, st.Failed, st.Canceled)job, ok := s.scheduler.GetJob(run.JobID)if !ok {return st.Outcome{Status: unavailableStatus, Message: "Job or target no longer exists"}, nil@@ -141,25 +141,25 @@ func (s *JobService) executeRunInternal(ctx context.Context, run st.Run) (st.Outfunc classifyOutcomeInternal(jobID string, outcome st.Outcome, err error) (st.Outcome, error) {if resultErr, ok := errors.AsType[*st.OutcomeError](err); ok {- return resultErr.Outcome, err+ outcome = resultErr.Outcome}- if err == nil {- outcome.Status = cmp.Or(outcome.Status, st.Succeeded)- return outcome, nil- }- if outcome.Status == st.Waiting || outcome.Status == st.Retrying {+ switch outcome.Status {+ case st.NeedsAttention:+ outcome.Status = st.Failedreturn outcome, err- }- if outcome.Status == st.NeedsAttention || outcome.Status == st.Partial || outcome.Status == st.Canceled || outcome.Status == st.Skipped {+ case st.Waiting, st.Retrying, st.Failed, st.Partial, st.Canceled, st.Skipped:return outcome, err+ case st.Queued, st.Running, st.Succeeded:+ // Classify these and unspecified outcomes below.}+var networkErr net.Errorswitch {+ case err == nil:+ outcome.Status = cmp.Or(outcome.Status, st.Succeeded)case safeJobInternal(jobID) && (errors.As(err, &networkErr) || errors.Is(err, context.DeadlineExceeded)):outcome.Status = st.Retrying- case !safeJobInternal(jobID):- outcome.Status = st.NeedsAttention- case outcome.Status == "":+ case outcome.Status == "" || outcome.Status == st.Succeeded:outcome.Status = st.Failed}@@ -176,7 +176,7 @@ func safeJobInternal(jobID string) bool {func (s *JobService) reconcileRunInternal(ctx context.Context, run st.Run) (st.Outcome, error) {if err := s.authorizeRunInternal(ctx, run); err != nil {- return st.Outcome{Status: st.NeedsAttention}, err+ return st.Outcome{Status: st.Failed}, err}if run.EnvironmentID != "0" {return s.executeRunInternal(ctx, run)@@ -200,14 +200,14 @@ func (s *JobService) reconcileRunInternal(ctx context.Context, run st.Run) (st.Ocontinue}if target.ActivityID == "" {- return st.Outcome{Status: st.NeedsAttention, Message: "Interrupted operation has no confirmed outcome", Targets: run.Outcome.Targets}, nil+ return st.Outcome{Status: st.Failed, Message: "Interrupted operation has no confirmed outcome", Targets: run.Outcome.Targets}, nil}var record activity.Activityif err := s.db.WithContext(ctx).First(&record, "id = ?", target.ActivityID).Error; err != nil {- return st.Outcome{Status: st.NeedsAttention}, err+ return st.Outcome{Status: st.Failed}, err}if record.Status != activitytypes.StatusSuccess {- return st.Outcome{Status: st.NeedsAttention, Message: "Interrupted operation requires review", Targets: run.Outcome.Targets}, nil+ return st.Outcome{Status: st.Failed, Message: "Interrupted operation has no confirmed completion", Targets: run.Outcome.Targets}, nil}run.Outcome.Targets[index].Status = st.Succeeded}@@ -220,7 +220,7 @@ func (s *JobService) reconcileRunInternal(ctx context.Context, run st.Run) (st.Oreturn outcome, err}}- return st.Outcome{Status: st.NeedsAttention, Message: "Interrupted operation requires review before retrying", Targets: run.Outcome.Targets}, nil+ return st.Outcome{Status: st.Failed, Message: "Interrupted operation has no confirmed completion", Targets: run.Outcome.Targets}, nil}func (s *JobService) runContextInternal(ctx context.Context, run st.Run) context.Context {diff --git a/backend/internal/job/remote.go b/backend/internal/job/remote.goindex 2da9e5f45..bda5cd2dd 100644--- a/backend/internal/job/remote.go+++ b/backend/internal/job/remote.go@@ -9,6 +9,7 @@ import ("sort""time"+ "github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/pkg/remenv""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs""github.com/getarcaneapp/arcane/types/v2/jobschedule"@@ -22,7 +23,7 @@ func remoteFailureInternal(err error) (st.Outcome, error) {rejected := errors.As(err, &status) && status.StatusCode >= 400 && status.StatusCode < 500 &&status.StatusCode != http.StatusRequestTimeout && status.StatusCode != http.StatusTooManyRequestsif rejected {- return st.Outcome{Status: st.NeedsAttention, Message: "Remote agent rejected the request; check configuration and permissions"}, err+ return st.Outcome{Status: st.Failed, Message: "Remote agent rejected the request; check configuration and permissions"}, err}return st.Outcome{Status: st.Waiting, Message: "Waiting for environment"}, err}@@ -30,10 +31,10 @@ func remoteFailureInternal(err error) (st.Outcome, error) {func (s *JobService) deliverRemoteInternal(ctx context.Context, run st.Run) (st.Outcome, error) {env, err := s.environment.GetEnvironmentByID(ctx, run.EnvironmentID)if err != nil {- return st.Outcome{Status: st.NeedsAttention}, err+ return st.Outcome{Status: st.Failed}, err}if !env.Enabled {- return st.Outcome{Status: st.NeedsAttention, Message: "Environment is disabled"}, nil+ return st.Outcome{Status: st.Failed, Message: "Environment is disabled"}, nil}basePath := "/api/environments/0/jobs/" + url.PathEscape(run.JobID)runPath := basePath + "/runs/" + url.PathEscape(run.ID)@@ -46,7 +47,7 @@ func (s *JobService) deliverRemoteInternal(ctx context.Context, run st.Run) (st.return remoteFailureInternal(err)}if !catalog.DurableRuns {- return st.Outcome{Status: st.NeedsAttention, Message: "Upgrade required: agent does not support durable job runs"}, nil+ return st.Outcome{Status: st.Failed, Message: "Upgrade required: agent does not support durable job runs"}, nil}var remoteRun st.Runerr = s.environment.ProxyJSONRequest(ctx, run.EnvironmentID, http.MethodGet, runPath, nil, &remoteRun)@@ -61,19 +62,19 @@ func (s *JobService) deliverRemoteInternal(ctx context.Context, run st.Run) (st.return st.Outcome{Status: st.Canceled, Message: err.Error()}, nil}if errors.Is(err, errRemoteReceiptMissingInternal) {- return st.Outcome{Status: st.NeedsAttention, Message: err.Error()}, nil+ return st.Outcome{Status: st.Failed, Message: err.Error()}, nil}return remoteFailureInternal(err)}}if remoteRun.ID != run.ID || remoteRun.JobID != run.JobID {- return st.Outcome{Status: st.NeedsAttention, Message: "Agent returned an inconsistent run identity"}, nil+ return st.Outcome{Status: st.Failed, Message: "Agent returned an inconsistent run identity"}, nil}if run.RemoteRetryRequested && (remoteRun.Status.Terminal() || remoteRun.Status == st.NeedsAttention) && remoteRun.AttemptCount <= run.RemoteAttemptCount {remoteRun, err = s.retryDeliveryInternal(ctx, run, runPath)if err != nil {if errors.Is(err, errRemoteRetryUncertainInternal) {- return st.Outcome{Status: st.NeedsAttention, Message: err.Error()}, nil+ return st.Outcome{Status: st.Failed, Message: err.Error()}, nil}return remoteFailureInternal(err)}@@ -82,6 +83,16 @@ func (s *JobService) deliverRemoteInternal(ctx context.Context, run st.Run) (st.}func (s *JobService) confirmRemoteRunInternal(ctx context.Context, run, remoteRun st.Run, runPath string) (st.Outcome, error) {+ if remoteRun.Status == st.NeedsAttention || (remoteRun.Status == st.Canceled && remoteRun.Resolution != nil && remoteRun.Resolution.ResolvedBy == common.SystemUser.Username) {+ settled, err := s.resolveAgentReviewInternal(ctx, run.EnvironmentID, run.JobID, run.ID, common.SystemUser.Username)+ if err != nil {+ return st.Outcome{Status: st.Waiting, Message: "Waiting for the agent to acknowledge the failed run", Targets: remoteRun.Outcome.Targets, ActivityID: remoteRun.Outcome.ActivityID}, err+ }+ remoteRun = settled+ if remoteRun.Status == st.Canceled && remoteRun.Resolution != nil && remoteRun.Resolution.ResolvedBy == common.SystemUser.Username {+ remoteRun.Status = st.Failed+ }+ }now := time.Now().UTC()outcome := remoteRun.Outcomeoutcome.Status = remoteRun.Status@@ -92,17 +103,21 @@ func (s *JobService) confirmRemoteRunInternal(ctx context.Context, run, remoteRucurrent.RemoteOutcome = &outcomecurrent.RemoteAttemptCount = remoteRun.AttemptCountcurrent.RemoteRetryRequested = false+ current.RemoteSettled = remoteRun.RemoteSettled+ current.Resolution = remoteRun.Resolution+ current.StartedAt = remoteRun.StartedAt+ current.FinishedAt = remoteRun.FinishedAtcurrent.Outcome = outcomereturn nil}); err != nil {return st.Outcome{Status: st.Retrying}, err}if outcome.Status.Terminal() {+ if remoteRun.RemoteSettled {+ return outcome, nil+ }return s.acknowledgeRemoteInternal(ctx, run, runPath, outcome)}- if outcome.Status == st.NeedsAttention {- return outcome, nil- }return st.Outcome{Status: st.Waiting, Message: "Waiting for the agent to complete this run", Targets: outcome.Targets, ActivityID: outcome.ActivityID}, nil}@@ -111,10 +126,19 @@ func (s *JobService) acknowledgeRemoteInternal(ctx context.Context, run st.Run,if err := s.environment.ProxyJSONRequest(ctx, run.EnvironmentID, http.MethodPost, path+"/ack", nil, &response); err != nil {return st.Outcome{Status: st.Waiting, Message: "Remote operation completed; waiting for delivery acknowledgement", Targets: outcome.Targets, ActivityID: outcome.ActivityID}, err}- if response.ID != run.ID || !response.RemoteSettled {+ status := response.Status+ if status == st.Canceled && response.Resolution != nil && response.Resolution.ResolvedBy == common.SystemUser.Username {+ status = st.Failed+ }+ if response.ID != run.ID || response.JobID != run.JobID || response.EnvironmentID != "0" || status != outcome.Status || !response.RemoteSettled {return st.Outcome{Status: st.Waiting, Message: "Remote operation completed; acknowledgement not confirmed"}, nil}- if err := s.runs.UpdateRun(ctx, run, func(current *st.Run) error { current.RemoteSettled = true; return nil }); err != nil {+ if err := s.runs.UpdateRun(ctx, run, func(current *st.Run) error {+ current.RemoteSettled = true+ current.StartedAt = response.StartedAt+ current.FinishedAt = response.FinishedAt+ return nil+ }); err != nil {return st.Outcome{Status: st.Retrying}, err}return outcome, nil@@ -131,6 +155,11 @@ func (s *JobService) ListRemoteJobs(ctx context.Context, environmentID string) (if remoteErr == nil {catalog.ObservedAt = time.Now().UTC()catalog.Offline = false+ if catalog.DurableRuns {+ if err := s.reconcileLegacyRemoteCatalogInternal(ctx, environmentID, catalog.Jobs); err != nil {+ return nil, err+ }+ }raw, err := json.Marshal(catalog)if err != nil {return nil, err@@ -180,6 +209,16 @@ func (s *JobService) RetryRemoteRun(ctx context.Context, environmentID, jobID, rif err := s.authorizeRunInternal(ctx, run); err != nil {return run, err}+ if run.Resolution != nil && run.Resolution.ResolvedBy == common.SystemUser.Username {+ var remote st.Run+ path := "/api/environments/0/jobs/" + url.PathEscape(jobID) + "/runs/" + url.PathEscape(runID)+ if err := s.environment.ProxyJSONRequest(ctx, environmentID, http.MethodGet, path, nil, &remote); err != nil {+ return run, err+ }+ if remote.ID != runID || remote.JobID != jobID || remote.EnvironmentID != "0" || remote.Status != st.Failed {+ return run, errors.New("upgrade the agent before retrying a legacy settled run")+ }+ }err = s.runs.UpdateRun(ctx, run, func(current *st.Run) error {if current.Status != st.Failed && current.Status != st.Partial && current.Status != st.NeedsAttention {return errors.New("run is not eligible for retry")diff --git a/backend/internal/job/remote_history.go b/backend/internal/job/remote_history.goindex 0a60fb4d7..f3345e1e0 100644--- a/backend/internal/job/remote_history.go+++ b/backend/internal/job/remote_history.go@@ -43,7 +43,7 @@ func (s *JobService) ListRuns(ctx context.Context, environmentID, jobID string,}runs := make([]st.Run, 0, len(merged))for _, run := range merged {- runs = append(runs, run)+ runs = append(runs, projectRunOutcomeInternal(run))}sort.Slice(runs, func(i, j int) bool {if runs[i].CreatedAt.Equal(runs[j].CreatedAt) {@@ -86,7 +86,7 @@ func (s *JobService) remoteHistoryInternal(ctx context.Context, environmentID, jfunc (s *JobService) GetRun(ctx context.Context, environmentID, jobID, runID string) (st.Run, error) {run, err := s.runs.Get(ctx, environmentID, jobID, runID)if environmentID == "0" || !errors.Is(err, runs.ErrRunNotFound) {- return run, err+ return projectRunOutcomeInternal(run), err}path := "/api/environments/0/jobs/" + url.PathEscape(jobID) + "/runs/" + url.PathEscape(runID)if err := s.environment.ProxyJSONRequest(ctx, environmentID, http.MethodGet, path, nil, &run); err != nil {@@ -99,7 +99,7 @@ func (s *JobService) GetRun(ctx context.Context, environmentID, jobID, runID strif run.ActivityID != "" {run.ActivityEnvironmentID = environmentID}- return run, nil+ return projectRunOutcomeInternal(run), nil}// CancelRun cancels manager admission or explicitly forwards an agent-owned cancellation.diff --git a/backend/internal/job/remote_resolution.go b/backend/internal/job/remote_resolution.goindex e48e9c16b..f7523a073 100644--- a/backend/internal/job/remote_resolution.go+++ b/backend/internal/job/remote_resolution.go@@ -8,10 +8,41 @@ import ("net/url""time"+ "github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"+ "github.com/getarcaneapp/arcane/types/v2/jobschedule"st "github.com/getarcaneapp/arcane/types/v2/scheduler")+// TODO(v3): remove adoption of legacy agent-owned runs.+func (s *JobService) reconcileLegacyRemoteCatalogInternal(ctx context.Context, environmentID string, jobs []jobschedule.JobStatus) error {+ for _, job := range jobs {+ for _, remote := range []*st.Run{job.CurrentRun, job.LastRun} {+ if remote == nil || remote.Status != st.NeedsAttention {+ continue+ }+ if remote.JobID != job.ID || remote.EnvironmentID != "0" {+ return errors.New("agent returned an inconsistent run identity")+ }+ _, err := s.runs.Get(ctx, environmentID, job.ID, remote.ID)+ if err == nil {+ continue+ }+ if !errors.Is(err, runs.ErrRunNotFound) {+ return err+ }+ _, err = s.runs.Submit(ctx, st.Request{RunID: remote.ID, JobID: job.ID, EnvironmentID: environmentID, Trigger: "recovery", ObservedAgentRun: remote})+ if err != nil {+ return err+ }+ }+ if err := s.reconcileLegacyRemoteCatalogInternal(ctx, environmentID, job.Children); err != nil {+ return err+ }+ }+ return nil+}+func (s *JobService) resolveRemoteRunInternal(ctx context.Context, environmentID, jobID, runID, actor string) (st.Run, error) {local, localErr := s.runs.Get(ctx, environmentID, jobID, runID)if localErr != nil && !errors.Is(localErr, runs.ErrRunNotFound) {@@ -57,7 +88,8 @@ func (s *JobService) resolveRemoteRunInternal(ctx context.Context, environmentIDreturn s.runs.Resolve(ctx, environmentID, jobID, runID, acknowledged.Resolution.ResolvedBy)}-// resolveAgentReviewInternal confirms ownership, records review, and settles delivery.+// resolveAgentReviewInternal settles legacy inactive runs on their owning agent.+// TODO(v3): remove the legacy resolution protocol.func (s *JobService) resolveAgentReviewInternal(ctx context.Context, environmentID, jobID, runID, actor string) (st.Run, error) {env, err := s.environment.GetEnvironmentByID(ctx, environmentID)if err != nil {@@ -75,10 +107,7 @@ func (s *JobService) resolveAgentReviewInternal(ctx context.Context, environmentif remote.ID != runID || remote.JobID != jobID || remote.EnvironmentID != "0" {return st.Run{}, errors.New("agent returned an inconsistent run identity")}- if remote.Status != st.Canceled || remote.Resolution == nil {- if remote.Status != st.NeedsAttention {- return st.Run{}, errors.New("agent run must need attention before review resolution")- }+ if remote.Status == st.NeedsAttention {body, err := json.Marshal(map[string]string{"resolvedBy": actor})if err != nil {return st.Run{}, err@@ -86,15 +115,21 @@ func (s *JobService) resolveAgentReviewInternal(ctx context.Context, environmentif err := s.environment.ProxyJSONRequest(ctx, environmentID, http.MethodPost, path+"/resolve", body, &remote); err != nil {return st.Run{}, err}+ } else if actor != common.SystemUser.Username {+ if remote.Status != st.Canceled || remote.Resolution == nil {+ return st.Run{}, errors.New("agent run must need attention before review resolution")+ }}- if remote.ID != runID || remote.JobID != jobID || remote.EnvironmentID != "0" || remote.Status != st.Canceled || remote.Resolution == nil {+ if remote.ID != runID || remote.JobID != jobID || remote.EnvironmentID != "0" || !remote.Status.Terminal() ||+ (actor != common.SystemUser.Username && (remote.Status != st.Canceled || remote.Resolution == nil)) {return st.Run{}, errors.New("agent resolution was not confirmed")}var acknowledged st.Runif err := s.environment.ProxyJSONRequest(ctx, environmentID, http.MethodPost, path+"/ack", nil, &acknowledged); err != nil {return st.Run{}, err}- if acknowledged.ID != runID || acknowledged.JobID != jobID || acknowledged.EnvironmentID != "0" || acknowledged.Status != st.Canceled || acknowledged.Resolution == nil || !acknowledged.RemoteSettled {+ if acknowledged.ID != runID || acknowledged.JobID != jobID || acknowledged.EnvironmentID != "0" || acknowledged.Status != remote.Status || !acknowledged.RemoteSettled ||+ (remote.Resolution != nil && (acknowledged.Resolution == nil || acknowledged.Resolution.ResolvedBy != remote.Resolution.ResolvedBy)) {return st.Run{}, errors.New("agent resolution acknowledgement was not confirmed")}return acknowledged, nildiff --git a/backend/internal/job/remote_resolution_test.go b/backend/internal/job/remote_resolution_test.goindex 59cf768fd..7cd8e15d2 100644--- a/backend/internal/job/remote_resolution_test.go+++ b/backend/internal/job/remote_resolution_test.go@@ -9,67 +9,172 @@ import ("testing""time"+ "github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/internal/environment""github.com/getarcaneapp/arcane/backend/v2/internal/middleware""github.com/getarcaneapp/arcane/backend/v2/pkg/authz"+ "github.com/getarcaneapp/arcane/types/v2/jobschedule"st "github.com/getarcaneapp/arcane/types/v2/scheduler""github.com/stretchr/testify/assert""github.com/stretchr/testify/require")func TestResolveRemoteRunConfirmsOwnerAfterLostResponse(t *testing.T) {- db := setupSettingsTestDBInternal(t)- require.NoError(t, db.AutoMigrate(&environment.Environment{}))- q := newJobCoordinatorForTestInternal(t, db)- local, err := q.Submit(t.Context(), st.Request{JobID: "auto-update", EnvironmentID: "remote"})- require.NoError(t, err)- require.NoError(t, q.UpdateRun(t.Context(), local, func(run *st.Run) error {- run.Status = st.NeedsAttention- run.RemoteAccepted = true- run.RemoteDeliveryAttempted = true- run.Outcome = st.Outcome{Status: st.NeedsAttention, Message: "original failure"}- return nil- }))- remote := st.Run{ID: local.ID, JobID: local.JobID, EnvironmentID: "0", Status: st.Running}- mutations := 0- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {- switch {- case r.Method == http.MethodGet:- case strings.HasSuffix(r.URL.Path, "/resolve"):- mutations++- remote.Status = st.Canceled- remote.Resolution = &st.RunResolution{ResolvedBy: "operator", ResolvedAt: time.Now().UTC(), Reason: "resolved_after_review"}- http.Error(w, "lost response", http.StatusServiceUnavailable)- return- case strings.HasSuffix(r.URL.Path, "/ack"):- remote.RemoteSettled = true- default:- http.Error(w, "unexpected request", http.StatusBadRequest)- return+ for _, automatic := range []bool{false, true} {+ name := "legacy operator"+ if automatic {+ name = "automatic settlement"}- raw, marshalErr := json.Marshal(remote)- assert.NoError(t, marshalErr)- w.Header().Set("Content-Type", "application/json")- _, writeErr := w.Write(raw)- assert.NoError(t, writeErr)- }))- defer server.Close()- require.NoError(t, db.Create(&environment.Environment{ID: "remote", Name: "agent", ApiUrl: server.URL, Enabled: true}).Error)- svc := &JobService{runs: q, environment: environment.NewEnvironmentService(db, server.Client(), nil, nil, nil, nil)}- ctx := context.WithValue(t.Context(), middleware.ContextKeyUserPermissions, authz.EnvironmentPermissionSet("remote"))- _, err = svc.ResolveRun(ctx, "remote", local.JobID, local.ID, "operator")- require.ErrorContains(t, err, "agent run must need attention")- require.Zero(t, mutations)- remote.Status = st.NeedsAttention- _, err = svc.ResolveRun(ctx, "remote", local.JobID, local.ID, "operator")- require.Error(t, err)- unchanged, err := q.Get(t.Context(), "remote", local.JobID, local.ID)- require.NoError(t, err)- require.Equal(t, st.NeedsAttention, unchanged.Status)- resolved, err := svc.ResolveRun(ctx, "remote", local.JobID, local.ID, "operator")- require.NoError(t, err)- require.Equal(t, st.Canceled, resolved.Status)- require.True(t, resolved.RemoteSettled)- require.Equal(t, "original failure", resolved.Outcome.Message)- require.Equal(t, 1, mutations)+ t.Run(name, func(t *testing.T) {+ db := setupSettingsTestDBInternal(t)+ require.NoError(t, db.AutoMigrate(&environment.Environment{}))+ q := newJobCoordinatorForTestInternal(t, db)+ local, err := q.Submit(t.Context(), st.Request{JobID: "auto-update", EnvironmentID: "remote", Trigger: "scheduled"})+ require.NoError(t, err)+ require.NoError(t, q.UpdateRun(t.Context(), local, func(run *st.Run) error {+ run.Status = st.NeedsAttention+ if automatic {+ run.Status = st.Running+ }+ run.RemoteAccepted = true+ run.RemoteDeliveryAttempted = true+ run.Outcome = st.Outcome{Status: st.NeedsAttention, Message: "original failure"}+ return nil+ }))+ remote := st.Run{ID: local.ID, JobID: local.JobID, EnvironmentID: "0", Status: st.Running, Outcome: st.Outcome{Status: st.NeedsAttention, Message: "original failure", Targets: []st.TargetOutcome{{ID: "completed", Status: st.Succeeded}}}}+ mutations, acknowledgements := 0, 0+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch {+ case r.Method == http.MethodGet && r.URL.Query().Has("page"):+ raw, marshalErr := json.Marshal(st.RunList{Runs: []st.Run{remote}, Total: 1})+ assert.NoError(t, marshalErr)+ w.Header().Set("Content-Type", "application/json")+ _, writeErr := w.Write(raw)+ assert.NoError(t, writeErr)+ return+ case r.Method == http.MethodGet:+ case strings.HasSuffix(r.URL.Path, "/resolve"):+ mutations+++ var body struct {+ ResolvedBy string `json:"resolvedBy"`+ }+ assert.NoError(t, json.UnmarshalRead(r.Body, &body))+ remote.Status = st.Canceled+ remote.Resolution = &st.RunResolution{ResolvedBy: body.ResolvedBy, ResolvedAt: time.Now().UTC(), Reason: "resolved_after_review"}+ http.Error(w, "lost response", http.StatusServiceUnavailable)+ return+ case strings.HasSuffix(r.URL.Path, "/ack"):+ acknowledgements+++ remote.RemoteSettled = true+ if automatic && acknowledgements == 1 {+ http.Error(w, "lost acknowledgement", http.StatusServiceUnavailable)+ return+ }+ default:+ http.Error(w, "unexpected request", http.StatusBadRequest)+ return+ }+ raw, marshalErr := json.Marshal(remote)+ assert.NoError(t, marshalErr)+ w.Header().Set("Content-Type", "application/json")+ _, writeErr := w.Write(raw)+ assert.NoError(t, writeErr)+ }))+ defer server.Close()+ require.NoError(t, db.Create(&environment.Environment{ID: "remote", Name: "agent", ApiUrl: server.URL, Enabled: true}).Error)+ svc := &JobService{runs: q, environment: environment.NewEnvironmentService(db, server.Client(), nil, nil, nil, nil)}+ ctx := context.WithValue(t.Context(), middleware.ContextKeyUserPermissions, authz.EnvironmentPermissionSet("remote"))+ path := "/api/environments/0/jobs/auto-update/runs/" + local.ID+ if !automatic {+ _, err = svc.ResolveRun(ctx, "remote", local.JobID, local.ID, "operator")+ require.ErrorContains(t, err, "agent run must need attention")+ } else {+ outcome, confirmErr := svc.confirmRemoteRunInternal(ctx, local, remote, path)+ require.NoError(t, confirmErr)+ require.Equal(t, st.Waiting, outcome.Status)+ }+ require.Zero(t, mutations)+ remote.Status = st.NeedsAttention+ if !automatic {+ _, err = svc.ResolveRun(ctx, "remote", local.JobID, local.ID, "operator")+ require.Error(t, err)+ unchanged, getErr := q.Get(t.Context(), "remote", local.JobID, local.ID)+ require.NoError(t, getErr)+ require.Equal(t, st.NeedsAttention, unchanged.Status)+ resolved, resolveErr := svc.ResolveRun(ctx, "remote", local.JobID, local.ID, "operator")+ require.NoError(t, resolveErr)+ require.Equal(t, st.Canceled, resolved.Status)+ require.True(t, resolved.RemoteSettled)+ require.Equal(t, "original failure", resolved.Outcome.Message)+ } else {+ outcome, confirmErr := svc.confirmRemoteRunInternal(ctx, local, remote, path)+ require.Error(t, confirmErr)+ require.Equal(t, st.Waiting, outcome.Status)+ outcome, confirmErr = svc.confirmRemoteRunInternal(ctx, local, remote, path)+ require.Error(t, confirmErr)+ require.Equal(t, st.Waiting, outcome.Status)+ outcome, confirmErr = svc.confirmRemoteRunInternal(ctx, local, remote, path)+ require.NoError(t, confirmErr)+ require.Equal(t, st.Failed, outcome.Status)+ require.Equal(t, "original failure", outcome.Message)+ require.Equal(t, st.Succeeded, outcome.Targets[0].Status)+ stored, getErr := q.Get(t.Context(), "remote", local.JobID, local.ID)+ require.NoError(t, getErr)+ require.True(t, stored.RemoteSettled)+ require.Equal(t, st.Failed, stored.RemoteOutcome.Status)+ require.Equal(t, common.SystemUser.Username, stored.Resolution.ResolvedBy)+ require.NoError(t, q.UpdateRun(ctx, stored, func(current *st.Run) error {+ current.Status = st.Failed+ return nil+ }))+ _, retryErr := svc.RetryRemoteRun(ctx, "remote", local.JobID, local.ID)+ require.ErrorContains(t, retryErr, "upgrade the agent")+ remote.Status = st.Failed+ retried, retryErr := svc.RetryRemoteRun(ctx, "remote", local.JobID, local.ID)+ require.NoError(t, retryErr)+ require.Equal(t, st.Queued, retried.Status)+ require.Equal(t, local.ID, retried.ID)++ // Adopt agent-owned history without coalescing its explicit identity.+ agentOwned := remote+ agentOwned.ID = "d7a9a3c7-d364-4c36-831a-381249e5727e"+ agentOwned.Status = st.NeedsAttention+ agentOwned.CreatedAt = time.Now().UTC().Add(-time.Hour)+ agentOwned.FinishedAt = new(agentOwned.CreatedAt.Add(time.Minute))+ agentOwned.ActivityID = "agent-summary"+ agentOwned.RemoteSettled = false+ catalog := []jobschedule.JobStatus{{ID: "updates", Children: []jobschedule.JobStatus{{ID: agentOwned.JobID, LastRun: &agentOwned}}}}+ require.NoError(t, svc.reconcileLegacyRemoteCatalogInternal(ctx, "remote", catalog))+ require.NoError(t, svc.reconcileLegacyRemoteCatalogInternal(ctx, "remote", catalog))+ adopted, getErr := q.Get(ctx, "remote", agentOwned.JobID, agentOwned.ID)+ require.NoError(t, getErr)+ require.True(t, adopted.RemoteAccepted)+ require.True(t, adopted.RemoteDeliveryAttempted)+ require.Equal(t, agentOwned.CreatedAt, adopted.CreatedAt)+ require.Equal(t, st.Queued, adopted.Status)+ require.Equal(t, agentOwned.FinishedAt, adopted.FinishedAt)+ require.Equal(t, agentOwned.ActivityID, adopted.ActivityID)+ require.Equal(t, "remote", adopted.ActivityEnvironmentID)+ status := jobschedule.JobStatus{LastRun: &agentOwned}+ applyRunStatusInternal(&status, []st.Run{adopted})+ require.Nil(t, status.CurrentRun)+ require.Equal(t, st.Failed, status.LastRun.Status)+ require.Equal(t, "original failure", status.LastError)+ remote = agentOwned+ merged, listErr := svc.ListRuns(ctx, "remote", agentOwned.JobID, 1, 20)+ require.NoError(t, listErr)+ detail, detailErr := svc.GetRun(ctx, "remote", agentOwned.JobID, agentOwned.ID)+ require.NoError(t, detailErr)+ require.Equal(t, st.Failed, detail.Status)+ require.Equal(t, "original failure", detail.Outcome.Message)+ require.Equal(t, agentOwned.ActivityID, detail.ActivityID)+ require.Equal(t, "remote", detail.ActivityEnvironmentID)+ require.Contains(t, merged.Runs, detail)+ history, listErr := q.List(ctx, "remote", agentOwned.JobID, 1, 20)+ require.NoError(t, listErr)+ require.Equal(t, 2, history.Total)+ }+ require.Equal(t, 1, mutations)+ })+ }}diff --git a/backend/internal/job/resolution.go b/backend/internal/job/resolution.goindex a06bb8007..ba33acdf1 100644--- a/backend/internal/job/resolution.go+++ b/backend/internal/job/resolution.go@@ -11,7 +11,8 @@ import (st "github.com/getarcaneapp/arcane/types/v2/scheduler")-// ResolveRun records operator review and releases the schedule's blocked run.+// ResolveRun records legacy operator review.+// TODO(v3): remove this deprecated mixed-version compatibility contract.func (h *JobSchedulesHandler) ResolveRun(ctx context.Context, input *jobschedule.ResolveRunInput) (*jobschedule.RunOutput, error) {actor, _ := middleware.GetUserIDFromContext(ctx)permissions, _ := middleware.PermissionsFromContext(ctx)@@ -29,6 +30,7 @@ func (h *JobSchedulesHandler) ResolveRun(ctx context.Context, input *jobschedule}// ResolveRun authorizes the current operator independently of the original requester.+// TODO(v3): remove this deprecated mixed-version compatibility contract.func (s *JobService) ResolveRun(ctx context.Context, environmentID, jobID, runID, actor string) (st.Run, error) {permissions, _ := middleware.PermissionsFromContext(ctx)if !permissions.Allows(authz.PermJobsManage, environmentID) {diff --git a/backend/internal/job/run_routes.go b/backend/internal/job/run_routes.goindex 737b35c40..4dd213639 100644--- a/backend/internal/job/run_routes.go+++ b/backend/internal/job/run_routes.go@@ -21,7 +21,7 @@ func (h *JobSchedulesHandler) registerRunRoutesInternal(api huma.API) {middleware.RegisterWithPermission(api, huma.Operation{OperationID: "get-job-run", Method: http.MethodGet, Path: base + "/runs/{runId}", Summary: "Get job run", Security: handlerutil.DefaultOperationSecurity()}, authz.PermJobsManage, h.GetRun)middleware.RegisterWithPermission(api, huma.Operation{OperationID: "retry-job-run", Method: http.MethodPost, Path: base + "/runs/{runId}/retry", Summary: "Retry job run", Security: handlerutil.DefaultOperationSecurity()}, authz.PermJobsManage, h.RetryRun)middleware.RegisterWithPermission(api, huma.Operation{OperationID: "cancel-job-run", Method: http.MethodPost, Path: base + "/runs/{runId}/cancel", Summary: "Cancel pending job run", Security: handlerutil.DefaultOperationSecurity()}, authz.PermJobsManage, h.CancelRun)- middleware.RegisterWithPermission(api, huma.Operation{OperationID: "resolve-job-run", Method: http.MethodPost, Path: base + "/runs/{runId}/resolve", Summary: "Resolve job run after review", Security: handlerutil.DefaultOperationSecurity()}, authz.PermJobsManage, h.ResolveRun)+ middleware.RegisterWithPermission(api, huma.Operation{OperationID: "resolve-job-run", Method: http.MethodPost, Path: base + "/runs/{runId}/resolve", Summary: "Resolve legacy job run", Description: "Deprecated compatibility endpoint for mixed-version upgrades. Removed in v3.", Deprecated: true, Security: handlerutil.DefaultOperationSecurity()}, authz.PermJobsManage, h.ResolveRun)middleware.RegisterWithPermission(api, huma.Operation{OperationID: "ack-job-run", Method: http.MethodPost, Path: base + "/runs/{runId}/ack", Summary: "Acknowledge remote run completion", Security: handlerutil.DefaultOperationSecurity()}, authz.PermJobsManage, h.AcknowledgeRun)middleware.RegisterWithPermission(api, huma.Operation{OperationID: "restart-job-worker", Method: http.MethodPost, Path: base + "/restart", Summary: "Restart continuous job worker", Security: handlerutil.DefaultOperationSecurity()}, authz.PermJobsManage, h.RestartWorker)}diff --git a/backend/internal/job/runtime_status.go b/backend/internal/job/runtime_status.goindex 919d6dc63..bddb7bd85 100644--- a/backend/internal/job/runtime_status.go+++ b/backend/internal/job/runtime_status.go@@ -7,6 +7,7 @@ import ("strings""time"+ "github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/types/v2/jobschedule"st "github.com/getarcaneapp/arcane/types/v2/scheduler")@@ -132,26 +133,52 @@ func dynamicJobStatusInternal(ctx context.Context, runtime st.JobScheduler, job}func applyRunStatusInternal(status *jobschedule.JobStatus, runs []st.Run) {- var errorAt time.Time- if status.LastError != "" {- for _, run := range []*st.Run{status.CurrentRun, status.LastRun} {- if run != nil && run.UpdatedAt.After(errorAt) {- errorAt = run.UpdatedAt+ latest := make(map[string]st.Run, len(runs)+2)+ for _, run := range []*st.Run{status.CurrentRun, status.LastRun} {+ if run != nil {+ if previous, found := latest[run.ID]; !found || !run.UpdatedAt.Before(previous.UpdatedAt) {+ latest[run.ID] = *run}}}- if current := status.CurrentRun; current != nil && current.Status == st.NeedsAttention {- status.CurrentRun = nil- updateLatestRunInternal(status, *current)- }for _, run := range runs {+ if previous, found := latest[run.ID]; !found || !run.UpdatedAt.Before(previous.UpdatedAt) {+ latest[run.ID] = run+ }+ }+ status.CurrentRun = nil+ status.LastRun = nil+ status.LastError = ""+ for _, run := range latest {+ run = projectRunOutcomeInternal(run)updateLatestRunInternal(status, run)updateLastSuccessInternal(status, run)- if run.UpdatedAt.After(errorAt) && run.Outcome.Message != "" && runHasErrorInternal(run.Status) {- status.LastError = run.Outcome.Message- errorAt = run.UpdatedAt+ }+ run := status.CurrentRun+ if last := status.LastRun; last != nil {+ if run == nil || runExecutionTimeInternal(*last).After(runExecutionTimeInternal(*run)) {+ run = last}}+ if run != nil && runHasErrorInternal(run.Status) {+ status.LastError = run.Outcome.Message+ }+}++func projectRunOutcomeInternal(run st.Run) st.Run {+ // Delivery reconciliation must not hide a failure already reported by the agent.+ if run.RemoteOutcome != nil && run.RemoteOutcome.Status == st.NeedsAttention && !run.RemoteRetryRequested {+ run.Status = st.Failed+ run.Outcome = *run.RemoteOutcome+ }+ automaticResolution := run.Status == st.Canceled && run.Resolution != nil && run.Resolution.ResolvedBy == common.SystemUser.Username+ if run.Status == st.NeedsAttention || automaticResolution {+ run.Status = st.Failed+ }+ if run.Status == st.Failed {+ run.Outcome.Status = st.Failed+ }+ return run}func updateLatestRunInternal(status *jobschedule.JobStatus, run st.Run) {@@ -161,20 +188,33 @@ func updateLatestRunInternal(status *jobschedule.JobStatus, run st.Run) {}return}- current := status.CurrentRun- if current != nil && current.ID == run.ID && !run.UpdatedAt.Before(current.UpdatedAt) && (run.Status == st.NeedsAttention || !run.RemoteAccepted || run.RemoteSettled) {- status.CurrentRun = nil- }- if status.LastRun == nil || run.UpdatedAt.After(status.LastRun.UpdatedAt) {+ if status.LastRun == nil || runExecutionTimeInternal(run).After(runExecutionTimeInternal(*status.LastRun)) ||+ (runExecutionTimeInternal(run).Equal(runExecutionTimeInternal(*status.LastRun)) && run.ID > status.LastRun.ID) {status.LastRun = new(run)}}+func runExecutionTimeInternal(run st.Run) time.Time {+ startedAt := run.CreatedAt+ if run.StartedAt != nil && run.StartedAt.After(startedAt) {+ startedAt = *run.StartedAt+ }+ // Manager delivery attempts poll the agent; they are not job executions.+ if run.EnvironmentID == "0" || !run.RemoteAccepted {+ for _, attempt := range run.Attempts {+ if attempt.StartedAt.After(startedAt) {+ startedAt = attempt.StartedAt+ }+ }+ }+ return startedAt+}+func updateLastSuccessInternal(status *jobschedule.JobStatus, run st.Run) {if run.Status != st.Succeeded {return}- succeededAt := run.UpdatedAt+ succeededAt := runExecutionTimeInternal(run)if run.FinishedAt != nil {succeededAt = *run.FinishedAt}@@ -185,9 +225,9 @@ func updateLastSuccessInternal(status *jobschedule.JobStatus, run st.Run) {func runHasErrorInternal(status st.RunStatus) bool {switch status {- case st.Failed, st.Partial, st.NeedsAttention, st.Retrying, st.Waiting:+ case st.Failed, st.Partial, st.NeedsAttention, st.Retrying:return true- case st.Queued, st.Running, st.Succeeded, st.Skipped, st.Canceled:+ case st.Queued, st.Running, st.Waiting, st.Succeeded, st.Skipped, st.Canceled:return falsedefault:return false@@ -203,7 +243,10 @@ func preferCurrentRunInternal(candidate, current st.Run) bool {if candidatePriority != currentPriority {return candidatePriority < currentPriority}- return candidate.CreatedAt.Before(current.CreatedAt)+ if candidate.CreatedAt.Equal(current.CreatedAt) {+ return candidate.ID > current.ID+ }+ return candidate.CreatedAt.After(current.CreatedAt)}func currentRunPriorityInternal(status st.RunStatus) int {diff --git a/backend/internal/job/service_test.go b/backend/internal/job/service_test.goindex b3356f246..89cf1b2d8 100644--- a/backend/internal/job/service_test.go+++ b/backend/internal/job/service_test.go@@ -153,6 +153,76 @@ func TestJobService_ListJobs_UsesRuntimeScheduleAndNextRun(t *testing.T) {autoUpdateJob := findJobStatusByIDInternal(t, jobs.Jobs, "auto-update")require.Equal(t, "0 0 8 * * *", autoUpdateJob.Schedule)require.Equal(t, nextRun, *autoUpdateJob.NextRun)++ now := time.Now().UTC()+ failed := schedulertypes.Run{+ ID: "failed", EnvironmentID: "0", Status: schedulertypes.Failed, CreatedAt: now.Add(-time.Hour), UpdatedAt: now,+ Outcome: schedulertypes.Outcome{Status: schedulertypes.Failed, Message: "clone failed"},+ }+ succeeded := schedulertypes.Run{ID: "succeeded", EnvironmentID: "0", Status: schedulertypes.Succeeded, CreatedAt: now.Add(-time.Minute), UpdatedAt: now.Add(-time.Minute), FinishedAt: new(now.Add(-time.Minute))}+ for _, history := range [][]schedulertypes.Run{{failed, succeeded}, {succeeded, failed}} {+ status := jobschedule.JobStatus{LastRun: new(failed), LastError: failed.Outcome.Message}+ applyRunStatusInternal(&status, history)+ require.Equal(t, succeeded.ID, status.LastRun.ID)+ require.Empty(t, status.LastError)+ require.Equal(t, "clone failed", history[0].Outcome.Message+history[1].Outcome.Message)+ }+ // An acknowledgement updates an old failure's revision, not its execution time.+ failed.UpdatedAt = now.Add(time.Hour)+ status := jobschedule.JobStatus{LastRun: new(succeeded)}+ applyRunStatusInternal(&status, []schedulertypes.Run{failed})+ require.Equal(t, succeeded.ID, status.LastRun.ID)+ require.Empty(t, status.LastError)+ // A real retry of the older run becomes the latest execution.+ failed.StartedAt = new(now.Add(time.Second))+ failed.Attempts = []schedulertypes.Attempt{{Number: 2, StartedAt: *failed.StartedAt}}+ applyRunStatusInternal(&status, []schedulertypes.Run{failed})+ require.Equal(t, failed.ID, status.LastRun.ID)+ require.Equal(t, "clone failed", status.LastError)+ // Merge the same run's terminal revision before choosing an active run.+ status.CurrentRun = new(succeeded)+ status.CurrentRun.Status = schedulertypes.Running+ succeeded.UpdatedAt = now.Add(2 * time.Second)+ applyRunStatusInternal(&status, []schedulertypes.Run{succeeded})+ require.Nil(t, status.CurrentRun)+ failed.Status = schedulertypes.Running+ failed.UpdatedAt = now.Add(2 * time.Hour)+ applyRunStatusInternal(&status, []schedulertypes.Run{failed})+ require.Equal(t, failed.ID, status.CurrentRun.ID)+ require.Empty(t, status.LastError)++ // A later manual failure stays visible while older work waits to retry.+ retrying := failed+ retrying.Status = schedulertypes.Retrying+ retrying.NextAttempt = new(now.Add(time.Hour))+ manualFailure := failed+ manualFailure.ID = "manual-failure"+ manualFailure.Status = schedulertypes.Failed+ manualFailure.CreatedAt = now.Add(time.Minute)+ manualFailure.StartedAt = nil+ manualFailure.Attempts = nil+ manualFailure.Outcome.Message = "latest manual failure"+ succeeded.CreatedAt = now.Add(2 * time.Minute)+ for _, history := range [][]schedulertypes.Run{{retrying, manualFailure}, {manualFailure, retrying}} {+ status = jobschedule.JobStatus{}+ applyRunStatusInternal(&status, history)+ require.Equal(t, retrying.ID, status.CurrentRun.ID)+ require.Equal(t, manualFailure.ID, status.LastRun.ID)+ require.Equal(t, manualFailure.Outcome.Message, status.LastError)+ applyRunStatusInternal(&status, []schedulertypes.Run{succeeded})+ require.Empty(t, status.LastError)+ }++ cloneErr := errors.New("failed to clone repository: network is unreachable")+ outcome, runErr := classifyOutcomeInternal("gitops-sync:project", schedulertypes.Outcome{}, cloneErr)+ require.ErrorIs(t, runErr, cloneErr)+ require.Equal(t, schedulertypes.Failed, outcome.Status)+ outcome, runErr = classifyOutcomeInternal("environment-health:0", schedulertypes.Outcome{}, context.DeadlineExceeded)+ require.ErrorIs(t, runErr, context.DeadlineExceeded)+ require.Equal(t, schedulertypes.Retrying, outcome.Status)+ outcome, runErr = classifyOutcomeInternal("auto-update", schedulertypes.Outcome{Status: schedulertypes.Partial}, cloneErr)+ require.ErrorIs(t, runErr, cloneErr)+ require.Equal(t, schedulertypes.Partial, outcome.Status)}func TestJobService_ListJobs_ImageUpdateWatcherIsContinuousAndRespectsEnabled(t *testing.T) {diff --git a/backend/internal/updater/recovery.go b/backend/internal/updater/recovery.goindex 383feda5f..afbf3068d 100644--- a/backend/internal/updater/recovery.go+++ b/backend/internal/updater/recovery.go@@ -437,13 +437,13 @@ func (s *UpdaterService) ReconcilePending(ctx context.Context, run schedulertypefor _, target := range run.Outcome.Targets {if target.ID == "auto-update" && len(target.RecoveryData) > 0 {if err := json.Unmarshal(target.RecoveryData, &plan); err != nil {- return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention}, err+ return schedulertypes.Outcome{Status: schedulertypes.Failed}, err}found = true}}if !found {- return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: "Interrupted update has no frozen target plan", Targets: run.Outcome.Targets}, nil+ return schedulertypes.Outcome{Status: schedulertypes.Failed, Message: "Interrupted update has no frozen target plan", Targets: run.Outcome.Targets}, nil}remaining := frozenUpdatePlanInternal{}unresolved := false@@ -471,14 +471,14 @@ func (s *UpdaterService) ReconcilePending(ctx context.Context, run schedulertypeif len(remaining.Records) > 0 {result, err := s.ApplyPending(context.WithValue(ctx, frozenPendingKeyInternal{}, &remaining), arcaneupdater.Options{})if err != nil || result == nil || result.Failed > 0 {- return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: "Frozen update recovery requires review"}, err+ return schedulertypes.Outcome{Status: schedulertypes.Failed, Message: "Frozen update recovery could not confirm completion"}, err}}status := schedulertypes.Succeededmessage := "Frozen update targets confirmed"if unresolved {- status = schedulertypes.NeedsAttention- message = "Unconfirmed update effects require review"+ status = schedulertypes.Failed+ message = "Some update effects could not be confirmed"}return schedulertypes.Outcome{Status: status, Message: message}, nil}diff --git a/backend/pkg/scheduler/auto_heal_job.go b/backend/pkg/scheduler/auto_heal_job.goindex 88bdcd052..fb272462f 100644--- a/backend/pkg/scheduler/auto_heal_job.go+++ b/backend/pkg/scheduler/auto_heal_job.go@@ -544,7 +544,7 @@ func (j *AutoHealJob) Reconcile(ctx context.Context, previous schedulertypes.Runreturn schedulertypes.Outcome{}, err}if !admitted {- return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: "A container healing run already owns admission"}, nil+ return schedulertypes.Outcome{Status: schedulertypes.Failed, Message: "A container healing run already owns admission"}, nil}defer lease.Release(ctx)var plan autoHealPlanInternal@@ -568,7 +568,7 @@ func (j *AutoHealJob) Reconcile(ctx context.Context, previous schedulertypes.Runreturn schedulertypes.Outcome{}, err}if message != "" {- return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: message, Targets: previous.Outcome.Targets}, nil+ return schedulertypes.Outcome{Status: schedulertypes.Failed, Message: message, Targets: previous.Outcome.Targets}, nil}if !planned {return schedulertypes.Outcome{Status: schedulertypes.Succeeded, Targets: previous.Outcome.Targets}, nil@@ -621,7 +621,7 @@ func (j *AutoHealJob) resumeHealingPlanInternal(ctx context.Context, dockerClien}previous.Outcome.Targets = append(previous.Outcome.Targets, target)if err != nil {- outcome.Status = schedulertypes.NeedsAttention+ outcome.Status = schedulertypes.Failedoutcome.Message = err.Error()break}diff --git a/backend/pkg/scheduler/auto_update_job.go b/backend/pkg/scheduler/auto_update_job.goindex dbff8a6aa..bb5938153 100644--- a/backend/pkg/scheduler/auto_update_job.go+++ b/backend/pkg/scheduler/auto_update_job.go@@ -64,7 +64,7 @@ func (j *AutoUpdateJob) Run(ctx context.Context) (schedulertypes.Outcome, error)options, priorOutcome, unresolvedTargets = autoUpdateRetryOptionsInternal(previous)if priorOutcome.Status != "" {priorOutcome.ActivityID = cmp.Or(priorOutcome.ActivityID, previous.Outcome.ActivityID)- if priorOutcome.Status == schedulertypes.NeedsAttention && previous.Outcome.Message != "" {+ if priorOutcome.Status == schedulertypes.Failed && previous.Outcome.Message != "" {priorOutcome.Message += ": " + previous.Outcome.Message}return priorOutcome, nil@@ -75,7 +75,7 @@ func (j *AutoUpdateJob) Run(ctx context.Context) (schedulertypes.Outcome, error)pollingEnabled := j.settingsService.GetBoolSetting(ctx, "pollingEnabled", true)if !enabled || !pollingEnabled {if len(options.ResourceIds) > 0 {- return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: "Auto-update is disabled; review this run before resolving it or retrying"}, nil+ return schedulertypes.Outcome{Status: schedulertypes.Failed, Message: "Auto-update is disabled; the remaining targets were not updated"}, nil}slog.DebugContext(ctx, "auto-update disabled or polling disabled; skipping run","autoUpdate", enabled, "pollingEnabled", pollingEnabled)@@ -89,7 +89,7 @@ func (j *AutoUpdateJob) Run(ctx context.Context) (schedulertypes.Outcome, error)}if !admitted {if len(options.ResourceIds) > 0 {- return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: "Another update is active; review this run before retrying"}, nil+ return schedulertypes.Outcome{Status: schedulertypes.Failed, Message: "Another update is active; the remaining targets were not updated"}, nil}slog.WarnContext(ctx, "auto-update run still in progress; skipping overlapping run")return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil@@ -106,7 +106,7 @@ func (j *AutoUpdateJob) Run(ctx context.Context) (schedulertypes.Outcome, error)}}if result == nil {- return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: "Updater returned no operation result"}, nil+ return schedulertypes.Outcome{Status: schedulertypes.Failed, Message: "Updater returned no operation result"}, nil}return autoUpdateOutcomeInternal(ctx, result, unresolvedTargets, err)@@ -123,7 +123,7 @@ func (j *AutoUpdateJob) Reconcile(ctx context.Context, previous schedulertypes.R}confirmed := confirmedAutoUpdateInternal(previous)if confirmed.Status != schedulertypes.Succeeded {- confirmed.Message = "Interrupted auto-update has no confirmed full-batch completion; review this run's results"+ confirmed.Message = "Interrupted auto-update has no confirmed full-batch completion"if previous.Outcome.Message != "" {confirmed.Message += ": " + previous.Outcome.Message}@@ -160,12 +160,12 @@ func autoUpdateRetryOptionsInternal(previous schedulertypes.Run) (updatertypes.O}if hasResourceTargets && len(options.ResourceIds) == 0 {if unresolvedTargets {- return options, schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: "Remaining image or unconfirmed targets need review before retrying", Targets: previous.Outcome.Targets}, true+ return options, schedulertypes.Outcome{Status: schedulertypes.Failed, Message: "Remaining targets have no confirmed completion or safe retry", Targets: previous.Outcome.Targets}, true}- return options, schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: "Target results do not confirm full-batch completion; review and resolve this run", ActivityID: previous.Outcome.ActivityID, Targets: previous.Outcome.Targets}, true+ return options, schedulertypes.Outcome{Status: schedulertypes.Failed, Message: "Target results do not confirm full-batch completion", ActivityID: previous.Outcome.ActivityID, Targets: previous.Outcome.Targets}, true}if !hasResourceTargets {- return options, schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: "The previous update has no confirmed retry targets; review this run's results", ActivityID: previous.Outcome.ActivityID, Targets: previous.Outcome.Targets}, true+ return options, schedulertypes.Outcome{Status: schedulertypes.Failed, Message: "The previous update has no confirmed retry targets", ActivityID: previous.Outcome.ActivityID, Targets: previous.Outcome.Targets}, true}if len(options.ResourceIds) > 0 {options.Type = "container"@@ -177,7 +177,7 @@ func autoUpdateRetryOptionsInternal(previous schedulertypes.Run) (updatertypes.Ofunc (j *AutoUpdateJob) ValidateRetry(_ context.Context, previous schedulertypes.Run) error {options, outcome, _ := autoUpdateRetryOptionsInternal(previous)if outcome.Status != "" || len(options.ResourceIds) == 0 {- return errors.New("auto-update has no safely retryable failed containers; review and resolve the run")+ return errors.New("auto-update has no safely retryable failed containers")}return nil}@@ -203,14 +203,14 @@ func autoUpdateOutcomeInternal(ctx context.Context, result *updatertypes.Result,}if result.Failed > 0 || err != nil || unresolvedTargets {outcome.Status = schedulertypes.Partial- outcome.Message = "Some updates failed; review target outcomes before retrying"+ outcome.Message = "Some updates failed"if err != nil {outcome.Message += ": " + err.Error()}}if unresolvedTargets {- outcome.Status = schedulertypes.NeedsAttention- outcome.Message = "Unconfirmed targets remain; review this run's results"+ outcome.Status = schedulertypes.Failed+ outcome.Message = "Some target outcomes could not be confirmed"}if outcomeErr, ok := errors.AsType[*schedulertypes.OutcomeError](err); ok {outcome.Status = outcomeErr.Outcome.Status@@ -224,13 +224,13 @@ func confirmedAutoUpdateInternal(previous schedulertypes.Run) schedulertypes.Outconfirmed := jobcontext.ConfirmedTarget(previous, "auto-update")for _, target := range previous.Outcome.Targets {if target.ID == "auto-update" && target.ResourceType != "update-batch" {- confirmed.Status = schedulertypes.NeedsAttention+ confirmed.Status = schedulertypes.Failed}}if confirmed.Status == schedulertypes.Succeeded {for _, target := range previous.Outcome.Targets {if target.Status != schedulertypes.Succeeded && target.Status != schedulertypes.Skipped {- confirmed.Status = schedulertypes.NeedsAttention+ confirmed.Status = schedulertypes.Failedbreak}}diff --git a/backend/pkg/scheduler/auto_update_job_test.go b/backend/pkg/scheduler/auto_update_job_test.goindex 73861840d..20f77e8eb 100644--- a/backend/pkg/scheduler/auto_update_job_test.go+++ b/backend/pkg/scheduler/auto_update_job_test.go@@ -85,11 +85,11 @@ func TestAutoUpdateJob_OverlappingRunIsSkippedInternal(t *testing.T) {status schedulertypes.RunStatusunresolved bool}{- {name: "missing progress", unresolved: true, status: schedulertypes.NeedsAttention},- {name: "image pull alone", unresolved: true, targets: []schedulertypes.TargetOutcome{{ID: "image", ResourceType: "image", Status: schedulertypes.Succeeded}}, status: schedulertypes.NeedsAttention},+ {name: "missing progress", unresolved: true, status: schedulertypes.Failed},+ {name: "image pull alone", unresolved: true, targets: []schedulertypes.TargetOutcome{{ID: "image", ResourceType: "image", Status: schedulertypes.Succeeded}}, status: schedulertypes.Failed},{name: "failed containers only", unresolved: true, targets: []schedulertypes.TargetOutcome{{ID: "done", ResourceType: "container", Status: schedulertypes.Succeeded}, {ID: "failed", ResourceType: "container", Status: schedulertypes.Failed}, {ID: "image", ResourceType: "image", Status: schedulertypes.Failed}}, retryIDs: []string{"failed"}},{name: "completed failed batch retry", targets: []schedulertypes.TargetOutcome{{ID: "auto-update", ResourceType: "update-batch", Status: schedulertypes.Partial}, {ID: "failed", ResourceType: "container", Status: schedulertypes.Failed}}, retryIDs: []string{"failed"}},- {name: "scoped retry cannot confirm original batch", targets: []schedulertypes.TargetOutcome{{ID: "auto-update", ResourceType: "update-retry", Status: schedulertypes.Succeeded}, {ID: "done", ResourceType: "container", Status: schedulertypes.Succeeded}}, status: schedulertypes.NeedsAttention, unresolved: true},+ {name: "scoped retry cannot confirm original batch", targets: []schedulertypes.TargetOutcome{{ID: "auto-update", ResourceType: "update-retry", Status: schedulertypes.Succeeded}, {ID: "done", ResourceType: "container", Status: schedulertypes.Succeeded}}, status: schedulertypes.Failed, unresolved: true},{name: "completed batch", targets: []schedulertypes.TargetOutcome{{ID: "auto-update", ResourceType: "update-batch", Status: schedulertypes.Succeeded}}, status: schedulertypes.Succeeded},} {t.Run(tc.name, func(t *testing.T) {@@ -108,7 +108,7 @@ func TestAutoUpdateJob_OverlappingRunIsSkippedInternal(t *testing.T) {if tc.status == schedulertypes.Succeeded {require.Equal(t, schedulertypes.Succeeded, reconciled.Status)} else {- require.Equal(t, schedulertypes.NeedsAttention, reconciled.Status)+ require.Equal(t, schedulertypes.Failed, reconciled.Status)}})}@@ -117,6 +117,6 @@ func TestAutoUpdateJob_OverlappingRunIsSkippedInternal(t *testing.T) {retryCtx := jobcontext.WithExecution(ctx, schedulertypes.Run{AttemptCount: 2, Outcome: schedulertypes.Outcome{Targets: []schedulertypes.TargetOutcome{{ID: "failed", ResourceType: "container", Status: schedulertypes.Failed}}}}, nil)outcome, err := job.Run(retryCtx)require.NoError(t, err)- require.Equal(t, schedulertypes.NeedsAttention, outcome.Status)+ require.Equal(t, schedulertypes.Failed, outcome.Status)require.Equal(t, int32(2), applier.calls.Load())}diff --git a/backend/pkg/scheduler/image_update_watcher_test.go b/backend/pkg/scheduler/image_update_watcher_test.goindex c5d8dc028..5938a1b1f 100644--- a/backend/pkg/scheduler/image_update_watcher_test.go+++ b/backend/pkg/scheduler/image_update_watcher_test.go@@ -569,10 +569,34 @@ func TestImageUpdateWatcher_ScheduledPollTriggersScanWithoutEvents(t *testing.T)scanner := &imageUpdateScannerFakeInternal{}settings := &pollingSettingReaderFakeInternal{enabled: true, schedule: "* * * * * *"}watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), nil)- startImageUpdateWatcherForTestInternal(t, watcher)+ jobScheduler, err := NewJobScheduler(t.Context(), watcher.coordinator, time.UTC)+ require.NoError(t, err)+ t.Cleanup(func() { require.NoError(t, jobScheduler.Stop(context.WithoutCancel(t.Context()))) })+ require.NoError(t, jobScheduler.RegisterBusWatcher(watcher, true))require.Eventually(t, func() bool { return scanner.countInternal() == 1 }, time.Second, 5*time.Millisecond)- require.Eventually(t, func() bool { return scanner.countInternal() >= 2 }, 3*time.Second, 10*time.Millisecond)+ require.Eventually(t, func() bool {+ record, err := watcher.coordinator.ScheduleState(t.Context(), watcher.Name())+ return err == nil && record.Schedule == settings.schedule && !record.NextRun.IsZero()+ }, time.Second, 5*time.Millisecond)+ require.NoError(t, jobScheduler.StartScheduler())+ record, err := watcher.coordinator.ScheduleState(t.Context(), watcher.Name())+ require.NoError(t, err)+ require.Equal(t, settings.schedule, record.Schedule)+ require.False(t, record.NextRun.IsZero())+ require.Eventually(t, func() bool { return scanner.countInternal() >= 3 }, 5*time.Second, 10*time.Millisecond)+ require.Eventually(t, func() bool {+ history, err := watcher.coordinator.List(t.Context(), "0", watcher.Name(), 1, 10)+ if err != nil {+ return false+ }+ for _, run := range history.Runs {+ if run.Trigger == "scheduled" && run.Status == schedulertypes.Succeeded {+ return true+ }+ }+ return false+ }, time.Second, 5*time.Millisecond)}func TestImageUpdateWatcher_ClosedEventSubscriptionKeepsWatcherRunning(t *testing.T) {diff --git a/backend/pkg/scheduler/jobcontext/context.go b/backend/pkg/scheduler/jobcontext/context.goindex 1390341df..38fc7bb86 100644--- a/backend/pkg/scheduler/jobcontext/context.go+++ b/backend/pkg/scheduler/jobcontext/context.go@@ -40,5 +40,5 @@ func ConfirmedTarget(run schedulertypes.Run, targetID string) schedulertypes.Outreturn schedulertypes.Outcome{Status: schedulertypes.Succeeded, ActivityID: target.ActivityID, Targets: run.Outcome.Targets}}}- return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: "The interrupted operation has no confirmed completion", Targets: run.Outcome.Targets}+ return schedulertypes.Outcome{Status: schedulertypes.Failed, Message: "The interrupted operation has no confirmed completion", Targets: run.Outcome.Targets}}diff --git a/backend/pkg/scheduler/runs/actors.go b/backend/pkg/scheduler/runs/actors.goindex 9470297e8..d3b226a82 100644--- a/backend/pkg/scheduler/runs/actors.go+++ b/backend/pkg/scheduler/runs/actors.go@@ -34,6 +34,7 @@ type executorActorInternal struct {}func (q *Coordinator) Register(runtime *francis.Runtime) error {+ q.runtime = runtimeif err := runtime.RegisterActor(coordinatorTypeInternal, func(id string, _ *actor.Service) actor.Actor {return &coordinatorActorInternal{id: id, q: q}}); err != nil {@@ -77,7 +78,7 @@ func (a *coordinatorActorInternal) Invoke(ctx context.Context, method string, da}// Acceptance is persisted; dispatch failures are retried.if err := a.flushInternal(ctx, &next); err != nil {- slog.WarnContext(ctx, "Job dispatch requires reconciliation", "actorId", a.id, "error", err)+ slog.WarnContext(ctx, "Job dispatch requires reconciliation", "jobId", next.Record.JobID, "environmentId", next.Record.EnvironmentID, "error", err)a.q.signalInternal()}return true, nil@@ -307,7 +308,7 @@ func (a *coordinatorActorInternal) Alarm(ctx context.Context, name string, data}pending := falsefor _, run := range record.Runs {- if run.Trigger == "manual" || run.Trigger == "remote" {+ if separateTriggerInternal(run.Trigger) {continue}if run.Status.Terminal() || run.Status == st.NeedsAttention {@@ -437,6 +438,10 @@ func (a *executorActorInternal) Job(ctx context.Context, _ string, data actor.Enoutcome.Status = st.Succeeded}}+ // Domain recovery may still report uncertainty; job runs finish as failures.+ if outcome.Status == st.NeedsAttention {+ outcome.Status = st.Failed+ }if runErr != nil && outcome.Message == "" {outcome.Message = runErr.Error()}@@ -447,9 +452,6 @@ func (a *executorActorInternal) Job(ctx context.Context, _ string, data actor.Enif ctx.Err() != nil {return ctx.Err()}- if outcome.Status == st.NeedsAttention {- return actor.ErrJobPermanentFailure- }return nil}@@ -481,7 +483,10 @@ func (q *Coordinator) claimExecutionInternal(ctx context.Context, command st.Execurrent.Status = st.Runningcurrent.Owner = q.owner + "/" + uuid.New().String()current.UpdatedAt = now- current.StartedAt = &now+ // Manager claims track delivery; accepted agent runs keep execution timestamps.+ if current.EnvironmentID == "0" || !current.RemoteAccepted {+ current.StartedAt = &now+ }current.NextAttempt = nilcurrent.AttemptCount = command.Attemptcurrent.Attempts = append(current.Attempts, st.Attempt{Number: current.AttemptCount, StartedAt: now})diff --git a/backend/pkg/scheduler/runs/actors_test.go b/backend/pkg/scheduler/runs/actors_test.goindex 9fcffe92e..9e1cbb508 100644--- a/backend/pkg/scheduler/runs/actors_test.go+++ b/backend/pkg/scheduler/runs/actors_test.go@@ -117,7 +117,7 @@ func TestDuplicateDispatchAndCanceledDelivery(t *testing.T) {var executed atomic.Int32q.SetExecutor(func(context.Context, st.Run) (st.Outcome, error) {executed.Add(1)- return st.Outcome{Status: st.Succeeded}, nil+ return st.Outcome{Status: st.NeedsAttention, Message: "Completion could not be confirmed", Targets: []st.TargetOutcome{{ID: "target", Status: st.NeedsAttention}}}, nil}, nil)run, err := q.Submit(t.Context(), st.Request{JobID: "dispatch", Trigger: "manual"})require.NoError(t, err)@@ -148,7 +148,10 @@ func TestDuplicateDispatchAndCanceledDelivery(t *testing.T) {require.Zero(t, executed.Load())require.Zero(t, persisted.AttemptCount)} else {- require.Equal(t, st.Succeeded, persisted.Status)+ require.Equal(t, st.Failed, persisted.Status)+ require.Equal(t, "Completion could not be confirmed", persisted.Outcome.Message)+ require.Equal(t, st.NeedsAttention, persisted.Outcome.Targets[0].Status)+ require.NotNil(t, persisted.FinishedAt)require.Equal(t, int32(1), executed.Load())require.Equal(t, 1, persisted.AttemptCount)}diff --git a/backend/pkg/scheduler/runs/coordinator.go b/backend/pkg/scheduler/runs/coordinator.goindex d5e485d32..b25e3dd21 100644--- a/backend/pkg/scheduler/runs/coordinator.go+++ b/backend/pkg/scheduler/runs/coordinator.go@@ -16,6 +16,7 @@ import ("uuid""github.com/getarcaneapp/arcane/backend/v2/internal/kv"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis"st "github.com/getarcaneapp/arcane/types/v2/scheduler""github.com/italypaleale/francis/actor"kit "go.getarcane.app/kit/pkg"@@ -25,6 +26,7 @@ import (type Coordinator struct {store *kv.KVServiceservice *actor.Service+ runtime *francis.Runtimeowner stringmu sync.Mutexcheckpointed map[string]bool@@ -64,18 +66,49 @@ func (q *Coordinator) Submit(ctx context.Context, request st.Request) (st.Run, ereturn st.Run{}, errors.New("invalid run ID")}now := time.Now().UTC()- result := st.Run{ID: request.RunID, JobID: request.JobID, EnvironmentID: request.EnvironmentID, Trigger: request.Trigger, RequestedBy: request.RequestedBy, RequestedWithKey: request.RequestedWithKey, RemoteAccepted: request.Trigger == "remote", Status: st.Queued, CreatedAt: now, UpdatedAt: now}+ observed := st.Run{CreatedAt: now}+ var remoteOutcome *st.Outcome+ if agentRun := request.ObservedAgentRun; agentRun != nil {+ switch {+ case request.EnvironmentID == "0":+ return st.Run{}, errors.New("observed agent run requires a remote environment")+ case agentRun.EnvironmentID != "0":+ return st.Run{}, errors.New("observed agent run must belong to the agent's local environment")+ case agentRun.ID != request.RunID || agentRun.JobID != request.JobID:+ return st.Run{}, errors.New("observed agent run does not match the requested run and job")+ case agentRun.Status != st.NeedsAttention:+ return st.Run{}, errors.New("observed agent run is not awaiting legacy resolution")+ }+ observed = *agentRun+ remoteOutcome = new(observed.Outcome)+ remoteOutcome.Status = observed.Status+ }+ result := st.Run{+ ID: request.RunID,+ JobID: request.JobID,+ EnvironmentID: request.EnvironmentID,+ Trigger: request.Trigger,+ RequestedBy: request.RequestedBy,+ RequestedWithKey: request.RequestedWithKey,+ Status: st.Queued,+ CreatedAt: observed.CreatedAt,+ UpdatedAt: now,+ StartedAt: observed.StartedAt,+ FinishedAt: observed.FinishedAt,+ ActivityID: observed.ActivityID,+ Outcome: observed.Outcome,+ RemoteAccepted: request.Trigger == "remote" || request.ObservedAgentRun != nil,+ RemoteDeliveryAttempted: request.ObservedAgentRun != nil,+ RemoteAttemptCount: observed.AttemptCount,+ RemoteOutcome: remoteOutcome,+ }q.associateActivityInternal(&result)err := q.mutateInternal(ctx, request.EnvironmentID, request.JobID, func(record *st.QueueRecord) error {if request.Trigger == "scheduled" || request.Trigger == "recovery" {record.LastEnqueuedAt = now}for _, existing := range record.Runs {- if existing.ID == request.RunID {- result = existing- return nil- }- if request.Trigger != "manual" && request.Trigger != "remote" && existing.Trigger != "manual" && existing.Trigger != "remote" && !existing.Status.Terminal() && existing.Status != st.Running && existing.Status != st.NeedsAttention {+ if existing.ID == request.RunID || coalescesPendingInternal(request, existing) {result = existingreturn nil}@@ -102,6 +135,24 @@ func (q *Coordinator) signalInternal() {}}+// Manual and remote work never joins another run. Scheduled and recovery work+// joins an existing run only while it is still waiting to start.+func coalescesPendingInternal(request st.Request, existing st.Run) bool {+ if request.ObservedAgentRun != nil || separateTriggerInternal(request.Trigger) || separateTriggerInternal(existing.Trigger) {+ return false+ }+ return existing.Status == st.Queued || existing.Status == st.Waiting || existing.Status == st.Retrying+}++func separateTriggerInternal(trigger string) bool {+ switch trigger {+ case "manual", "remote":+ return true+ default:+ return false+ }+}+// Checkpoint records the next due occurrence atomically with overdue recovery.func (q *Coordinator) Checkpoint(ctx context.Context, jobID, schedule string, next time.Time) error {q.mu.Lock()@@ -167,10 +218,11 @@ func (q *Coordinator) Start(ctx context.Context) error {}for _, record := range records {for _, run := range record.Runs {- if q.observer != nil {- if err := q.UpdateRun(ctx, run, func(*st.Run) error { return nil }); err != nil {- return err- }+ if err := q.UpdateRun(ctx, run, func(current *st.Run) error {+ normalizeLegacyRunInternal(current)+ return nil+ }); err != nil {+ return err}}}@@ -270,7 +322,7 @@ func (q *Coordinator) persistOutcomeInternal(ctx context.Context, run st.Run, ou} else {current.NextAttempt = nil}- if outcome.Status.Terminal() {+ if outcome.Status.Terminal() && (current.EnvironmentID == "0" || current.FinishedAt == nil) {current.FinishedAt = &now}if len(current.Attempts) > 0 {@@ -297,7 +349,7 @@ func (q *Coordinator) persistOutcomeInternal(ctx context.Context, run st.Run, oufunc (q *Coordinator) invokeInternal(ctx context.Context, run st.Run, reconcile bool) (outcome st.Outcome, err error) {defer func() {if recovered := recover(); recovered != nil {- outcome = st.Outcome{Status: st.NeedsAttention, Message: "Job panicked; inspect logs before retrying"}+ outcome = st.Outcome{Status: st.Failed, Message: "Job panicked; inspect logs before retrying"}err = fmt.Errorf("job panic: %v", recovered)slog.ErrorContext(ctx, "Job panicked", "runId", run.ID, "error", err)}@@ -423,7 +475,8 @@ func mergeTargetProgressInternal(previous, outcome st.Outcome) st.Outcome {func (q *Coordinator) Active() bool { return q.enabled.Load() }-// Resolve releases reviewed, inactive work without discarding its execution evidence.+// Resolve releases legacy reviewed work without discarding execution evidence.+// TODO(v3): remove this deprecated mixed-version compatibility contract.func (q *Coordinator) Resolve(ctx context.Context, environmentID, jobID, runID, resolvedBy string) (st.Run, error) {run, err := q.Get(ctx, environmentID, jobID, runID)if err != nil {diff --git a/backend/pkg/scheduler/runs/coordinator_store.go b/backend/pkg/scheduler/runs/coordinator_store.goindex a31670e2c..f8474a0d8 100644--- a/backend/pkg/scheduler/runs/coordinator_store.go+++ b/backend/pkg/scheduler/runs/coordinator_store.go@@ -10,6 +10,7 @@ import ("time""uuid"+ "github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/internal/kv"st "github.com/getarcaneapp/arcane/types/v2/scheduler""github.com/italypaleale/francis/actor"@@ -20,6 +21,7 @@ const queuePrefixInternal = "jobs."func (q *Coordinator) mutateInternal(ctx context.Context, environmentID, jobID string, change func(*st.QueueRecord) error) error {id := kit.SHA256Hex(environmentID + "\x00" + jobID)+ q.runtime.NameActor(id, jobID+"@"+environmentID)for range 20 {var state st.CoordinatorStateerr := q.service.GetState(ctx, coordinatorTypeInternal, id, &state)@@ -129,6 +131,7 @@ func (q *Coordinator) Records(ctx context.Context) ([]st.QueueRecord, error) {return nil, err}normalizeRecordTimesInternal(&state.Record)+ q.runtime.NameActor(entry.ActorID, state.Record.JobID+"@"+state.Record.EnvironmentID)records = append(records, state.Record)}cursor = page.AfterID()@@ -251,7 +254,7 @@ func pruneRunsInternal(record *st.QueueRecord, now time.Time) {if terminal <= 100 && now.Sub(run.UpdatedAt) < 7*24*time.Hour {continue}- if run.Trigger == "manual" || run.Trigger == "remote" {+ if separateTriggerInternal(run.Trigger) {if record.Receipts == nil {record.Receipts = make(map[string]st.Run)}@@ -358,6 +361,26 @@ func normalizeRunTimesInternal(run *st.Run) {}}+func normalizeLegacyRunInternal(run *st.Run) {+ automaticResolution := run.Status == st.Canceled && run.Resolution != nil && run.Resolution.ResolvedBy == common.SystemUser.Username+ if run.Status != st.NeedsAttention && !automaticResolution {+ return+ }+ run.Owner = ""+ run.NextAttempt = nil+ if run.EnvironmentID != "0" && (run.RemoteDeliveryAttempted || run.RemoteAccepted) && !run.RemoteSettled {+ run.Status = st.Waiting+ run.FinishedAt = nil+ } else {+ run.Status = st.Failed+ run.Outcome.Status = st.Failed+ if run.FinishedAt == nil {+ run.FinishedAt = new(run.UpdatedAt)+ }+ }+ run.UpdatedAt = time.Now().UTC()+}+func normalizeTimePointerInternal(value *time.Time) {if value != nil {*value = value.UTC()diff --git a/backend/pkg/scheduler/runs/resolution_test.go b/backend/pkg/scheduler/runs/resolution_test.goindex 27d6fd87c..e4d262c3b 100644--- a/backend/pkg/scheduler/runs/resolution_test.go+++ b/backend/pkg/scheduler/runs/resolution_test.go@@ -78,6 +78,43 @@ func TestResolvePreservesEvidenceWithoutBlockingSchedule(t *testing.T) {require.Equal(t, resolved.Outcome, duplicate.Outcome)require.Equal(t, resolved.Resolution.ResolvedBy, duplicate.Resolution.ResolvedBy)require.True(t, resolved.Resolution.ResolvedAt.Equal(duplicate.Resolution.ResolvedAt))++ // Startup closes legacy local failures and reconciles unsettled remote work.+ localLegacy, err := q.Submit(ctx, st.Request{JobID: "gitops-sync:project", Trigger: "scheduled"})+ require.NoError(t, err)+ remoteLegacy, err := q.Submit(ctx, st.Request{JobID: "gitops-sync:project", EnvironmentID: "remote", Trigger: "scheduled"})+ require.NoError(t, err)+ for _, legacy := range []st.Run{localLegacy, remoteLegacy} {+ require.NoError(t, q.UpdateRun(ctx, legacy, func(current *st.Run) error {+ current.Status = st.NeedsAttention+ current.Outcome = evidence+ current.Attempts = []st.Attempt{{Number: 1, Outcome: evidence}}+ current.RemoteDeliveryAttempted = current.EnvironmentID != "0"+ return nil+ }))+ }+ q.SetExecutor(func(context.Context, st.Run) (st.Outcome, error) {+ t.Error("startup must not repeat legacy work")+ return st.Outcome{Status: st.Failed}, nil+ }, nil)+ require.NoError(t, q.Start(ctx))+ t.Cleanup(func() { require.NoError(t, q.Stop(context.WithoutCancel(ctx))) })+ localLegacy, err = q.Get(ctx, "0", localLegacy.JobID, localLegacy.ID)+ require.NoError(t, err)+ require.Equal(t, st.Failed, localLegacy.Status)+ require.Equal(t, evidence.Message, localLegacy.Outcome.Message)+ require.Equal(t, evidence.Targets, localLegacy.Outcome.Targets)+ require.Equal(t, evidence, localLegacy.Attempts[0].Outcome)+ require.NotNil(t, localLegacy.FinishedAt)+ remoteLegacy, err = q.Get(ctx, "remote", remoteLegacy.JobID, remoteLegacy.ID)+ require.NoError(t, err)+ require.Equal(t, st.Waiting, remoteLegacy.Status)+ require.Equal(t, evidence, remoteLegacy.Outcome)+ require.False(t, remoteLegacy.RemoteSettled)+ require.Nil(t, remoteLegacy.FinishedAt)+ future, err := q.Submit(ctx, st.Request{JobID: localLegacy.JobID, Trigger: "scheduled"})+ require.NoError(t, err)+ require.NotEqual(t, localLegacy.ID, future.ID)}func TestResolveRejectsUnsafeStates(t *testing.T) {diff --git a/backend/pkg/scheduler/scheduler.go b/backend/pkg/scheduler/scheduler.goindex ea0e92329..75ef453a6 100644--- a/backend/pkg/scheduler/scheduler.go+++ b/backend/pkg/scheduler/scheduler.go@@ -130,7 +130,10 @@ func (js *jobSchedulerInternal) StartScheduler() error {return err}for _, record := range records {- if record.EnvironmentID == "0" && record.Schedule != "" && !js.HasJob(record.JobID) {+ js.mu.RLock()+ _, watching := js.watchers[record.JobID]+ js.mu.RUnlock()+ if record.EnvironmentID == "0" && record.Schedule != "" && !js.HasJob(record.JobID) && !watching {if err := js.coordinator.Checkpoint(js.context, record.JobID, "", time.Time{}); err != nil {return err}diff --git a/backend/pkg/scheduler/scheduler_test.go b/backend/pkg/scheduler/scheduler_test.goindex e40be8d0c..026c901b2 100644--- a/backend/pkg/scheduler/scheduler_test.go+++ b/backend/pkg/scheduler/scheduler_test.go@@ -270,10 +270,26 @@ func TestJobScheduler_RescheduleJob_AddsEntryWhenEnabled(t *testing.T) {func TestJobScheduler_StartScheduler_SchedulesNonConditionalJobs(t *testing.T) {js := newJobSchedulerForTestInternal(t, context.Background(), nil)+ watcher := &testBusWatcherInternal{+ name: "image-polling",+ started: make(chan struct{}),+ stopped: make(chan struct{}),+ }+ require.NoError(t, js.RegisterBusWatcher(watcher, true))+ nextPolling := time.Now().UTC().Add(time.Hour)+ require.NoError(t, js.coordinator.Checkpoint(t.Context(), watcher.Name(), "0 0 * * * *", nextPolling))+ require.NoError(t, js.coordinator.Checkpoint(t.Context(), "removed-job", "0 0 * * * *", nextPolling))+ executed := make(chan struct{}, 1)job := &testSchedulerJob{name: "test-non-conditional-startup",schedule: "*/1 * * * * *",+ run: func(context.Context) {+ select {+ case executed <- struct{}{}:+ default:+ }+ },}require.NoError(t, js.RegisterJob(job))@@ -283,6 +299,19 @@ func TestJobScheduler_StartScheduler_SchedulesNonConditionalJobs(t *testing.T) {require.True(t, ok)require.True(t, state.Scheduled)requireScheduledJobInternal(t, js, job.Name())+ polling, err := js.coordinator.ScheduleState(t.Context(), watcher.Name())+ require.NoError(t, err)+ require.Equal(t, "0 0 * * * *", polling.Schedule)+ require.True(t, nextPolling.Equal(polling.NextRun))+ removed, err := js.coordinator.ScheduleState(t.Context(), "removed-job")+ require.NoError(t, err)+ require.Empty(t, removed.Schedule)+ require.True(t, removed.NextRun.IsZero())+ select {+ case <-executed:+ case <-time.After(5 * time.Second):+ require.FailNow(t, "scheduled job did not execute through Francis")+ }}func TestJobScheduler_RescheduleJob_UsesProvidedContext(t *testing.T) {diff --git a/backend/pkg/utils/francis/logging.go b/backend/pkg/utils/francis/logging.gonew file mode 100644index 000000000..0fb39c0ef--- /dev/null+++ b/backend/pkg/utils/francis/logging.go@@ -0,0 +1,97 @@+package francis++import (+ "context"+ "log/slog"+ "slices"+ "strings"+ "sync"+)++// NameActor changes only the log display; durable actor IDs remain unchanged.+func (r *Runtime) NameActor(actorID, name string) {+ r.actorNames.Store(actorID, name)+}++type actorLogHandlerInternal struct {+ slog.Handler++ names *sync.Map+ steps []actorLogStepInternal+}++type actorLogStepInternal struct {+ attrs []slog.Attr+ group string+}++func (h *actorLogHandlerInternal) Handle(ctx context.Context, record slog.Record) error {+ handler := h.Handler+ // Resolve at emission, including loggers created before restored jobs are loaded.+ for _, step := range h.steps {+ if step.group != "" {+ handler = handler.WithGroup(step.group)+ continue+ }+ attrs := make([]slog.Attr, len(step.attrs))+ for i, attr := range step.attrs {+ attrs[i] = h.nameAttrInternal(attr)+ }+ handler = handler.WithAttrs(attrs)+ }+ named := slog.NewRecord(record.Time, record.Level, record.Message, record.PC)+ record.Attrs(func(attr slog.Attr) bool {+ named.AddAttrs(h.nameAttrInternal(attr))+ return true+ })+ return handler.Handle(ctx, named)+}++func (h *actorLogHandlerInternal) WithAttrs(attrs []slog.Attr) slog.Handler {+ clone := *h+ clone.steps = append(slices.Clone(h.steps), actorLogStepInternal{attrs: slices.Clone(attrs)})+ return &clone+}++func (h *actorLogHandlerInternal) WithGroup(name string) slog.Handler {+ if name == "" {+ return h+ }+ clone := *h+ clone.steps = append(slices.Clone(h.steps), actorLogStepInternal{group: name})+ return &clone+}++func (h *actorLogHandlerInternal) nameAttrInternal(attr slog.Attr) slog.Attr {+ attr.Value = attr.Value.Resolve()+ if attr.Value.Kind() == slog.KindGroup {+ attrs := slices.Clone(attr.Value.Group())+ for i := range attrs {+ attrs[i] = h.nameAttrInternal(attrs[i])+ }+ attr.Value = slog.GroupValue(attrs...)+ return attr+ }+ if (attr.Key != "id" && attr.Key != "actorRef") || attr.Value.Kind() != slog.KindString {+ return attr+ }+ actorType, rest, ok := strings.Cut(attr.Value.String(), "/")+ if !ok {+ return attr+ }+ actorID, suffix, hasSuffix := strings.Cut(rest, "/")+ name, found := h.names.Load(actorID)+ if !found {+ return attr+ }+ displayName, ok := name.(string)+ if !ok {+ return attr+ }+ value := actorType + "/" + displayName+ if hasSuffix {+ value += "/" + suffix+ }+ attr.Value = slog.StringValue(value)+ return attr+}diff --git a/backend/pkg/utils/francis/runtime.go b/backend/pkg/utils/francis/runtime.goindex f5ddbf924..a49585f5b 100644--- a/backend/pkg/utils/francis/runtime.go+++ b/backend/pkg/utils/francis/runtime.go@@ -37,6 +37,7 @@ type Runtime struct {started boolrunError erroraddress string+ actorNames sync.Map}type registrationInternal struct {@@ -62,12 +63,15 @@ func New(databaseURL, encryptionKey, instanceID, port string, options ...local.Hservice: &actor.Service{}, ready: make(chan struct{}), done: make(chan struct{}), address: address,options: []local.HostOption{local.WithAddress(address), providerOption,- local.WithLogger(slog.Default().With("scope", "actor-host")),local.WithMaxHosts(1), local.WithHostHealthCheckDeadline(90 * time.Second),local.WithShutdownGracePeriod(10 * time.Second), local.WithAlarmsPollInterval(time.Second),local.WithAlarmsFetchAheadInterval(30 * time.Second), local.WithAlarmsLeaseDuration(180 * time.Second),},}+ runtime.options = append(runtime.options, local.WithLogger(slog.New(&actorLogHandlerInternal{+ Handler: slog.Default().With("scope", "actor-host").Handler(),+ names: &runtime.actorNames,+ })))if encryptionKey != "" && instanceID != "" {if err := runtime.ConfigureIdentity(encryptionKey, instanceID); err != nil {return nil, errdiff --git a/backend/pkg/utils/francis/runtime_test.go b/backend/pkg/utils/francis/runtime_test.goindex 41bdea34a..3f1d94ceb 100644--- a/backend/pkg/utils/francis/runtime_test.go+++ b/backend/pkg/utils/francis/runtime_test.go@@ -1,7 +1,9 @@package francisimport (+ "bytes""context"+ "log/slog""net""os""path/filepath"@@ -16,6 +18,23 @@ import (func TestSQLiteHostRestartPersistence(t *testing.T) {databaseURL := "file:" + filepath.ToSlash(filepath.Join(t.TempDir(), "actors.db"))testRestartPersistenceInternal(t, databaseURL)++ var output bytes.Buffer+ runtime := &Runtime{}+ logger := slog.New(&actorLogHandlerInternal{+ Handler: slog.NewTextHandler(&output, nil),+ names: &runtime.actorNames,+ })+ bound := logger.With("id", "job-executor/durable-hash/run-1").WithGroup("alarm")+ runtime.NameActor("durable-hash", "image-polling@0")+ bound.WarnContext(t.Context(), "Lease was lost", "actorRef", "job-coordinator/durable-hash")+ require.Contains(t, output.String(), "id=job-executor/image-polling@0/run-1")+ require.Contains(t, output.String(), "alarm.actorRef=job-coordinator/image-polling@0")+ output.Reset()+ logger.InfoContext(t.Context(), "unknown", "id", "other/unknown/run-1", "hostId", "durable-hash", slog.Group("nested", "actorRef", "job-health-executor/durable-hash"))+ require.Contains(t, output.String(), "id=other/unknown/run-1")+ require.Contains(t, output.String(), "hostId=durable-hash")+ require.Contains(t, output.String(), "nested.actorRef=job-health-executor/image-polling@0")}func TestPostgresHostRestartPersistence(t *testing.T) {@@ -35,6 +54,7 @@ func testRestartPersistenceInternal(t *testing.T, databaseURL string) {first, err := New(databaseURL, "test-encryption-key", "test-instance", port)require.NoError(t, err)service := first.Service()+ first.NameActor(actorID, "durable@0")require.NoError(t, first.RegisterActor("test", func(string, *actor.Service) actor.Actor { return struct{}{} }))require.NoError(t, first.Start(ctx, ctx, func(err error) { t.Errorf("unexpected host failure: %v", err) }))t.Cleanup(func() { require.NoError(t, first.Stop(context.WithoutCancel(ctx))) })@@ -48,6 +68,7 @@ func testRestartPersistenceInternal(t *testing.T, databaseURL string) {second, err := New(databaseURL, "test-encryption-key", "test-instance", port)require.NoError(t, err)+ second.NameActor(actorID, "durable@0")require.NoError(t, second.RegisterActor("test", func(string, *actor.Service) actor.Actor { return struct{}{} }))require.NoError(t, second.Start(ctx, ctx, nil))t.Cleanup(func() { require.NoError(t, second.Stop(context.WithoutCancel(ctx))) })diff --git a/frontend/src/lib/components/job-card/job-card.svelte b/frontend/src/lib/components/job-card/job-card.svelteindex cefe72771..22967324a 100644--- a/frontend/src/lib/components/job-card/job-card.svelte+++ b/frontend/src/lib/components/job-card/job-card.svelte@@ -96,7 +96,7 @@id: 'worker-health',visible: job.workerHealth,label: m.jobs_worker_health(),- value: jobStatusLabel(job.workerHealth?.status ?? '')+ value: jobStatusLabel(job.workerHealth?.status ?? '', true)}].filter((item) => item.visible));@@ -107,7 +107,7 @@{id: 'last',visible: job.lastError,- text: `${m.jobs_last_error()}: ${job.lastError}`,+ text: job.lastError,class: 'text-xs break-words text-destructive'},{@@ -189,6 +189,7 @@</div>{#each errors as error (error.id)}<p class={error.class}>{error.text}</p>{/each}+ {#if run?.status === 'waiting' && run.outcome.message}<p class="text-xs text-muted-foreground">{run.outcome.message}</p>{/if}{#if job.workerHealth?.nextRetry}<p class="text-xs text-muted-foreground">{m.jobs_next_retry()}: {formatDateTimeShort(job.workerHealth.nextRetry)}</p>{/if}diff --git a/frontend/src/lib/components/job-card/job-run-history.svelte b/frontend/src/lib/components/job-card/job-run-history.svelteindex f6fd79a73..db6901173 100644--- a/frontend/src/lib/components/job-card/job-run-history.svelte+++ b/frontend/src/lib/components/job-card/job-run-history.svelte@@ -1,5 +1,4 @@<script lang="ts">- import { openConfirmDialog } from '#lib/components/confirm-dialog/index.js';import IfPermitted from '#lib/components/if-permitted.svelte';import { queryKeys } from '#lib/query/query-keys.js';import { createQuery, createMutation } from '@tanstack/svelte-query';@@ -32,7 +31,7 @@refetchInterval: 5000}));const action = createMutation(() => ({- mutationFn: ({ runId, action }: { runId: string; action: 'retry' | 'cancel' | 'resolve' }) =>+ mutationFn: ({ runId, action }: { runId: string; action: 'retry' | 'cancel' }) =>jobScheduleService.updateRun(jobId, runId, action, environmentId),onSuccess: () => {void runs.refetch();@@ -87,16 +86,6 @@}].filter((item) => item.visible));- function resolveRun(runId: string) {- openConfirmDialog({- title: m.jobs_resolve_run(),- message: m.jobs_resolve_description(),- confirm: {- label: m.jobs_resolve_run(),- action: () => action.mutateAsync({ runId, action: 'resolve' }).then(() => undefined)- }- });- }</script>{#snippet outcomeMessage(message: string | undefined)}@@ -170,8 +159,7 @@{#each metadata as item (item.id)}<p class={item.class}>{item.label ? `${item.label}: ` : ''}{item.value}</p>{/each}{@render outcomeActivity(run.activityId, m.jobs_activity_summary(), run.activityEnvironmentId ?? environmentId)}{@render outcomeActivity(run.outcome.activityId)}- {#if run.status === 'needs_attention'}<p>{m.jobs_attention_schedule_blocked()}</p>{/if}- {#if run.resolution}+ {#if run.resolution && run.resolution.resolvedBy !== 'System'}<p>{m.jobs_resolution_details({ user: run.resolution.resolvedBy, date: formatDateTimeShort(run.resolution.resolvedAt) })}</p>@@ -196,11 +184,6 @@{/if}{#if action.error}<p class="text-destructive">{action.error.message}</p>{/if}<IfPermitted perm="jobs:manage" envId={environmentId}>- {#if run.status === 'needs_attention'}- <Button variant="outline" disabled={action.isPending} onclick={() => resolveRun(run.id)}- >{m.jobs_resolve_run()}</Button- >- {/if}{#each availableActions as item (item.id)}<Buttonvariant={item.variant}diff --git a/frontend/src/lib/components/job-card/job-status.ts b/frontend/src/lib/components/job-card/job-status.tsindex bded58d6b..2089ecda4 100644--- a/frontend/src/lib/components/job-card/job-status.ts+++ b/frontend/src/lib/components/job-card/job-status.ts@@ -1,6 +1,6 @@import type { JobStatus } from '#lib/types/settings.js';import { m } from '#lib/paraglide/messages.js';-export function jobStatusLabel(status: string): string {+export function jobStatusLabel(status: string, isWorker = false): string {const labels: Record<string, () => string> = {queued: m.jobs_status_queued,waiting: m.jobs_status_waiting,@@ -10,7 +10,7 @@ export function jobStatusLabel(status: string): string {partial: m.jobs_status_partial,skipped: m.jobs_status_skipped,failed: m.jobs_status_failed,- needs_attention: m.jobs_status_needs_attention,+ needs_attention: isWorker ? m.jobs_status_needs_attention : m.jobs_status_failed,canceled: m.jobs_status_canceled,healthy: m.jobs_status_healthy,degraded: m.jobs_status_degraded,diff --git a/frontend/src/lib/services/job-schedule-service.ts b/frontend/src/lib/services/job-schedule-service.tsindex bbdf980f7..5ef3c77f7 100644--- a/frontend/src/lib/services/job-schedule-service.ts+++ b/frontend/src/lib/services/job-schedule-service.ts@@ -28,7 +28,7 @@ class JobScheduleService extends BaseAPIService {this.api.get(`/environments/${environmentId}/jobs/${encodeURIComponent(jobId)}/runs/${encodeURIComponent(runId)}`));}- async updateRun(jobId: string, runId: string, action: 'retry' | 'cancel' | 'resolve', environmentId: string): Promise<JobRun> {+ async updateRun(jobId: string, runId: string, action: 'retry' | 'cancel', environmentId: string): Promise<JobRun> {return this.handleResponse(this.api.post(`/environments/${environmentId}/jobs/${encodeURIComponent(jobId)}/runs/${encodeURIComponent(runId)}/${action}`diff --git a/types/jobschedule/runs.go b/types/jobschedule/runs.goindex 8e114eb9d..c46bad1f7 100644--- a/types/jobschedule/runs.go+++ b/types/jobschedule/runs.go@@ -23,6 +23,7 @@ type ()// ResolveRunInput accepts the original operator identity only from trusted agent transport.+// TODO(v3): remove this deprecated mixed-version compatibility contract.type ResolveRunInput struct {RunInputdiff --git a/types/scheduler/job.go b/types/scheduler/job.goindex f5fa02da5..0401a65f1 100644--- a/types/scheduler/job.go+++ b/types/scheduler/job.go@@ -113,5 +113,5 @@ func (g *GenericJob) Reconcile(ctx context.Context, previous Run) (Outcome, erroif g.ReconcileFn != nil {return g.ReconcileFn(ctx, previous)}- return Outcome{Status: NeedsAttention, Message: "The interrupted operation has no confirmed result; review its domain records before retrying", Targets: previous.Outcome.Targets}, nil+ return Outcome{Status: Failed, Message: "The interrupted operation has no confirmed completion", Targets: previous.Outcome.Targets}, nil}diff --git a/types/scheduler/run.go b/types/scheduler/run.goindex f4d91e70c..7ec541ed5 100644--- a/types/scheduler/run.go+++ b/types/scheduler/run.go@@ -10,14 +10,15 @@ import (type RunStatus stringconst (- Queued RunStatus = "queued"- Waiting RunStatus = "waiting"- Running RunStatus = "running"- Retrying RunStatus = "retrying"- Succeeded RunStatus = "succeeded"- Partial RunStatus = "partial"- Skipped RunStatus = "skipped"- Failed RunStatus = "failed"+ Queued RunStatus = "queued"+ Waiting RunStatus = "waiting"+ Running RunStatus = "running"+ Retrying RunStatus = "retrying"+ Succeeded RunStatus = "succeeded"+ Partial RunStatus = "partial"+ Skipped RunStatus = "skipped"+ Failed RunStatus = "failed"+ // NeedsAttention preserves domain recovery evidence and legacy job outcomes.NeedsAttention RunStatus = "needs_attention"Canceled RunStatus = "canceled")@@ -43,6 +44,9 @@ type Outcome struct {}type Request struct {+ // ObservedAgentRun adopts existing agent work without another submission.+ // TODO(v3): remove legacy-agent adoption.+ ObservedAgentRun *Run `json:"-"`RunID string `json:"runId,omitempty"`JobID string `json:"jobId"`EnvironmentID string `json:"environmentId"`@@ -58,6 +62,8 @@ type Attempt struct {Outcome Outcome `json:"outcome"`}+// RunResolution preserves legacy resolution history.+// TODO(v3): remove this deprecated mixed-version compatibility contract.type RunResolution struct {ResolvedBy string `json:"resolvedBy"`ResolvedAt time.Time `json:"resolvedAt"`