Skip to content

Modern Docker Management, Designed for Everyone

fix(jobs): remove manual review and fix Francis scheduling (#4268)

Kyle Mendelle96f588293f0
diff --git a/backend/internal/activity/job.go b/backend/internal/activity/job.go
index b3892134a..164441df3 100644
--- a/backend/internal/activity/job.go
+++ b/backend/internal/activity/job.go
@@ -7,6 +7,7 @@ import (
"fmt"
"time"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
st "github.com/getarcaneapp/arcane/types/v2/scheduler"
@@ -102,7 +103,9 @@ func jobActivityInternal(run st.Run, name string) Activity {
}
if run.Resolution != nil {
model.Metadata["resolution"] = run.Resolution
- model.LatestMessage = "Resolved after review; future scheduled runs may proceed"
+ if run.Resolution.ResolvedBy != common.SystemUser.Username {
+ model.LatestMessage = "Resolved after review"
+ }
}
if isTerminalActivityStatusInternal(status) {
model.EndedAt = new(run.UpdatedAt)
diff --git a/backend/internal/bootstrap/bootstrap_test.go b/backend/internal/bootstrap/bootstrap_test.go
index 77c553557..63754d350 100644
--- a/backend/internal/bootstrap/bootstrap_test.go
+++ b/backend/internal/bootstrap/bootstrap_test.go
@@ -16,6 +16,8 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/container"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/environment"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/gitops"
"github.com/getarcaneapp/arcane/backend/v2/internal/job"
"github.com/getarcaneapp/arcane/backend/v2/internal/kv"
"github.com/getarcaneapp/arcane/backend/v2/internal/project"
@@ -26,6 +28,7 @@ import (
francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
tunnelpb "github.com/getarcaneapp/arcane/backend/v2/proto/tunnel/v1"
+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
"github.com/labstack/echo/v5"
"github.com/libtnb/sqlite"
"github.com/stretchr/testify/assert"
@@ -396,17 +399,35 @@ func TestJobSchedulerStopCancelsItsPrivateContextInternal(t *testing.T) {
runtime := francistest.New(t)
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
- require.NoError(t, db.AutoMigrate(&kv.KVEntry{}))
+ require.NoError(t, db.AutoMigrate(&kv.KVEntry{}, &environment.Environment{}, &project.GitOpsSync{}))
sqlDB, err := db.DB()
require.NoError(t, err)
sqlDB.SetMaxOpenConns(1)
t.Cleanup(func() { _ = sqlDB.Close() })
coordinator := runs.New(kv.NewKVService(&database.DB{DB: db}), runtime.Service(), time.UTC)
require.NoError(t, coordinator.Register(runtime))
- francistest.Start(t, runtime)
+ admission := runs.NewAdmission(runtime.Service(), t.Name())
+ require.NoError(t, admission.Register(runtime))
+ lifecycle.Append(fx.Hook{
+ OnStart: func(ctx context.Context) error { return runtime.Start(ctx, appCtx, nil) },
+ OnStop: runtime.Stop,
+ })
+ t.Cleanup(func() { lifecycle.RequireStop() })
jobService := job.NewJobService(&database.DB{DB: db}, nil, &config.Config{}, coordinator, nil, nil, nil)
- jobScheduler, err := newJobScheduler(appCtx, lifecycle, &config.Config{}, coordinator, nil, nil, nil, nil, jobService, nil, nil, nil, nil, nil)
+ gitopsSync := gitops.NewGitOpsSyncService(&database.DB{DB: db}, nil, nil, nil, nil, nil)
+ require.NoError(t, db.Create(&environment.Environment{ID: "0", Name: "Local", Enabled: true}).Error)
+ require.NoError(t, db.Create(&project.GitOpsSync{ID: "overdue", EnvironmentID: "0", AutoSync: true, SyncInterval: 1}).Error)
+ jobScheduler, err := newJobScheduler(appCtx, lifecycle, &config.Config{}, coordinator, admission, nil, nil, nil, jobService, nil, nil, nil, nil, gitopsSync)
require.NoError(t, err)
+ require.NoError(t, registerDynamicJobs(dynamicJobsParams{
+ AppCtx: appCtx, Config: &config.Config{}, Scheduler: jobScheduler,
+ GitOpsSync: gitopsSync, Admission: admission,
+ }))
+ executed := make(chan schedulertypes.Run, 1)
+ coordinator.SetExecutor(func(_ context.Context, run schedulertypes.Run) (schedulertypes.Outcome, error) {
+ executed <- run
+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil
+ }, nil)
watcher := &blockingBusWatcherInternal{
started: make(chan struct{}),
stopped: make(chan struct{}),
@@ -415,6 +436,14 @@ func TestJobSchedulerStopCancelsItsPrivateContextInternal(t *testing.T) {
lifecycle.RequireStart()
select {
+ case run := <-executed:
+ require.Equal(t, "gitops-sync:overdue", run.JobID)
+ require.Equal(t, "startup", run.Trigger)
+ require.True(t, jobScheduler.HasJob(run.JobID))
+ case <-time.After(5 * time.Second):
+ require.FailNow(t, "overdue GitOps startup job did not execute through Francis")
+ }
+ select {
case <-watcher.started:
case <-time.After(time.Second):
require.FailNow(t, "watcher did not start")
diff --git a/backend/internal/bootstrap/jobs_bootstrap.go b/backend/internal/bootstrap/jobs_bootstrap.go
index 89fcf7e72..b88bd2ceb 100644
--- a/backend/internal/bootstrap/jobs_bootstrap.go
+++ b/backend/internal/bootstrap/jobs_bootstrap.go
@@ -50,6 +50,10 @@ func newJobScheduler(appCtx context.Context, lc fx.Lifecycle, cfg *config.Config
if err := reconcileCoordinatorStartupInternal(schedulerCtx, jobService, backupEngine, updaterService, volumes, systemBackups, gitopsSync); err != nil {
return err
}
+ if gitopsSync != nil {
+ gitopsSync.RegisterAutoSyncJobsOnStartup(schedulerCtx)
+ gitopsSync.SubscribeProjectFileChanges(schedulerCtx)
+ }
if imageUpdateWatcher != nil {
imageUpdateWatcher.SetCoordinator(jobService.Coordinator())
if err := jobScheduler.RegisterBusWatcher(imageUpdateWatcher, true); err != nil {
@@ -343,13 +347,11 @@ func registerDynamicJobs(params dynamicJobsParams) error {
}
params.SystemBackup.RegisterBackupJobOnStartup(params.AppCtx)
}
- // GitOps: one job per auto-sync-enabled sync (runs on manager and agents).
+ // GitOps startup submissions wait for Francis in the scheduler start hook.
if params.GitOpsSync != nil {
if err := params.GitOpsSync.SetScheduler(params.AppCtx, params.Scheduler, params.Admission); err != nil {
return err
}
- params.GitOpsSync.RegisterAutoSyncJobsOnStartup(params.AppCtx)
- params.GitOpsSync.SubscribeProjectFileChanges(params.AppCtx)
}
// Environment health: one job per enabled environment (manager only). The Jobs
diff --git a/backend/internal/job/activity.go b/backend/internal/job/activity.go
index ab7464183..307832bc5 100644
--- a/backend/internal/job/activity.go
+++ b/backend/internal/job/activity.go
@@ -51,7 +51,7 @@ func (s *JobService) SyncRunActivity(ctx context.Context, run st.Run) error {
if metadata, found := meta.GetJobMetadata(current.JobID); found {
name = metadata.Name
}
- return s.activity.SyncJobRun(ctx, current, name)
+ return s.activity.SyncJobRun(ctx, projectRunOutcomeInternal(current), name)
}
func (s *JobService) ReconcileStartupActivities(ctx context.Context, extraProtectedIDs ...string) error {
diff --git a/backend/internal/job/activity_test.go b/backend/internal/job/activity_test.go
index c75152eb1..41646f999 100644
--- a/backend/internal/job/activity_test.go
+++ b/backend/internal/job/activity_test.go
@@ -47,7 +47,7 @@ func TestJobActivityLifecycle(t *testing.T) {
duplicate, err := svc.runs.Submit(ctx, st.Request{JobID: run.JobID, RunID: run.ID, Trigger: "scheduled"})
require.NoError(t, err)
require.Equal(t, run.ActivityID, duplicate.ActivityID)
- for _, status := range []st.RunStatus{st.Running, st.Waiting, st.NeedsAttention} {
+ for _, status := range []st.RunStatus{st.Running, st.Waiting, st.Failed} {
require.NoError(t, svc.runs.UpdateRun(ctx, run, func(current *st.Run) error {
current.Status = status
current.UpdatedAt = time.Now().UTC()
@@ -55,7 +55,22 @@ func TestJobActivityLifecycle(t *testing.T) {
return nil
}))
}
- check(activitytypes.StatusFailed, st.NeedsAttention)
+ check(activitytypes.StatusFailed, st.Failed)
+ failedDetail, err := activities.GetActivityDetail(ctx, "0", run.ActivityID, 10)
+ require.NoError(t, err)
+ require.NotNil(t, failedDetail.Activity.Error)
+ require.Equal(t, "operation detail", *failedDetail.Activity.Error)
+ later, err := svc.runs.Submit(ctx, st.Request{JobID: run.JobID, Trigger: "scheduled"})
+ require.NoError(t, err)
+ require.NoError(t, svc.runs.UpdateRun(ctx, later, func(current *st.Run) error {
+ current.Status = st.Succeeded
+ current.UpdatedAt = time.Now().UTC()
+ return nil
+ }))
+ failedDetail, err = activities.GetActivityDetail(ctx, "0", run.ActivityID, 10)
+ require.NoError(t, err)
+ require.Equal(t, activitytypes.StatusFailed, failedDetail.Activity.Status)
+ require.Equal(t, "operation detail", *failedDetail.Activity.Error)
_, err = svc.runs.Retry(ctx, "0", run.JobID, run.ID)
require.NoError(t, err)
check(activitytypes.StatusQueued, st.Queued)
@@ -72,7 +87,7 @@ func TestJobActivityLifecycle(t *testing.T) {
check(activitytypes.StatusCancelled, st.Canceled)
var count int64
require.NoError(t, db.Model(&activity.Activity{}).Count(&count).Error)
- require.Equal(t, int64(1), count)
+ require.Equal(t, int64(2), count)
}
func TestJobActivityVisibilityAndGrouping(t *testing.T) {
@@ -136,6 +151,9 @@ func TestJobActivityRestartRepairsFailedProjection(t *testing.T) {
require.Contains(t, detail.Activity.LatestMessage, "interrupted")
persisted, err := restarted.runs.Get(ctx, "0", run.JobID, run.ID)
require.NoError(t, err)
- require.Zero(t, persisted.AttemptCount, "repair must not execute blocked work")
+ require.Zero(t, persisted.AttemptCount, "repair must not execute interrupted work")
+ require.Equal(t, st.Failed, persisted.Status)
+ require.Equal(t, "interrupted before applying updates", persisted.Outcome.Message)
+ require.NotNil(t, persisted.FinishedAt)
require.Equal(t, run.ActivityID, persisted.ActivityID)
}
diff --git a/backend/internal/job/execution.go b/backend/internal/job/execution.go
index 50f7e96f3..add6a7e28 100644
--- a/backend/internal/job/execution.go
+++ b/backend/internal/job/execution.go
@@ -104,7 +104,7 @@ func (s *JobService) authorizeRunInternal(ctx context.Context, run st.Run) error
func (s *JobService) executeRunInternal(ctx context.Context, run st.Run) (st.Outcome, error) {
if err := s.authorizeRunInternal(ctx, run); err != nil {
- return st.Outcome{Status: st.NeedsAttention}, err
+ return st.Outcome{Status: st.Failed}, err
}
if run.EnvironmentID != "0" {
return s.deliverRemoteInternal(ctx, run)
@@ -127,7 +127,7 @@ func (s *JobService) executeRunInternal(ctx context.Context, run st.Run) (st.Out
err := s.scheduler.RunBusWatcherNow(ctx, run.JobID)
return classifyOutcomeInternal(run.JobID, st.Outcome{}, err)
}
- unavailableStatus := kit.Ternary(run.JobID == "auto-update" && run.AttemptCount > 1, st.NeedsAttention, st.Canceled)
+ unavailableStatus := kit.Ternary(run.JobID == "auto-update" && run.AttemptCount > 1, st.Failed, st.Canceled)
job, ok := s.scheduler.GetJob(run.JobID)
if !ok {
return st.Outcome{Status: unavailableStatus, Message: "Job or target no longer exists"}, nil
@@ -141,25 +141,25 @@ func (s *JobService) executeRunInternal(ctx context.Context, run st.Run) (st.Out
func classifyOutcomeInternal(jobID string, outcome st.Outcome, err error) (st.Outcome, error) {
if resultErr, ok := errors.AsType[*st.OutcomeError](err); ok {
- return resultErr.Outcome, err
+ outcome = resultErr.Outcome
}
- if err == nil {
- outcome.Status = cmp.Or(outcome.Status, st.Succeeded)
- return outcome, nil
- }
- if outcome.Status == st.Waiting || outcome.Status == st.Retrying {
+ switch outcome.Status {
+ case st.NeedsAttention:
+ outcome.Status = st.Failed
return outcome, err
- }
- if outcome.Status == st.NeedsAttention || outcome.Status == st.Partial || outcome.Status == st.Canceled || outcome.Status == st.Skipped {
+ case st.Waiting, st.Retrying, st.Failed, st.Partial, st.Canceled, st.Skipped:
return outcome, err
+ case st.Queued, st.Running, st.Succeeded:
+ // Classify these and unspecified outcomes below.
}
+
var networkErr net.Error
switch {
+ case err == nil:
+ outcome.Status = cmp.Or(outcome.Status, st.Succeeded)
case safeJobInternal(jobID) && (errors.As(err, &networkErr) || errors.Is(err, context.DeadlineExceeded)):
outcome.Status = st.Retrying
- case !safeJobInternal(jobID):
- outcome.Status = st.NeedsAttention
- case outcome.Status == "":
+ case outcome.Status == "" || outcome.Status == st.Succeeded:
outcome.Status = st.Failed
}
@@ -176,7 +176,7 @@ func safeJobInternal(jobID string) bool {
func (s *JobService) reconcileRunInternal(ctx context.Context, run st.Run) (st.Outcome, error) {
if err := s.authorizeRunInternal(ctx, run); err != nil {
- return st.Outcome{Status: st.NeedsAttention}, err
+ return st.Outcome{Status: st.Failed}, err
}
if run.EnvironmentID != "0" {
return s.executeRunInternal(ctx, run)
@@ -200,14 +200,14 @@ func (s *JobService) reconcileRunInternal(ctx context.Context, run st.Run) (st.O
continue
}
if target.ActivityID == "" {
- return st.Outcome{Status: st.NeedsAttention, Message: "Interrupted operation has no confirmed outcome", Targets: run.Outcome.Targets}, nil
+ return st.Outcome{Status: st.Failed, Message: "Interrupted operation has no confirmed outcome", Targets: run.Outcome.Targets}, nil
}
var record activity.Activity
if err := s.db.WithContext(ctx).First(&record, "id = ?", target.ActivityID).Error; err != nil {
- return st.Outcome{Status: st.NeedsAttention}, err
+ return st.Outcome{Status: st.Failed}, err
}
if record.Status != activitytypes.StatusSuccess {
- return st.Outcome{Status: st.NeedsAttention, Message: "Interrupted operation requires review", Targets: run.Outcome.Targets}, nil
+ return st.Outcome{Status: st.Failed, Message: "Interrupted operation has no confirmed completion", Targets: run.Outcome.Targets}, nil
}
run.Outcome.Targets[index].Status = st.Succeeded
}
@@ -220,7 +220,7 @@ func (s *JobService) reconcileRunInternal(ctx context.Context, run st.Run) (st.O
return outcome, err
}
}
- return st.Outcome{Status: st.NeedsAttention, Message: "Interrupted operation requires review before retrying", Targets: run.Outcome.Targets}, nil
+ return st.Outcome{Status: st.Failed, Message: "Interrupted operation has no confirmed completion", Targets: run.Outcome.Targets}, nil
}
func (s *JobService) runContextInternal(ctx context.Context, run st.Run) context.Context {
diff --git a/backend/internal/job/remote.go b/backend/internal/job/remote.go
index 2da9e5f45..bda5cd2dd 100644
--- a/backend/internal/job/remote.go
+++ b/backend/internal/job/remote.go
@@ -9,6 +9,7 @@ import (
"sort"
"time"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/pkg/remenv"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
"github.com/getarcaneapp/arcane/types/v2/jobschedule"
@@ -22,7 +23,7 @@ func remoteFailureInternal(err error) (st.Outcome, error) {
rejected := errors.As(err, &status) && status.StatusCode >= 400 && status.StatusCode < 500 &&
status.StatusCode != http.StatusRequestTimeout && status.StatusCode != http.StatusTooManyRequests
if rejected {
- return st.Outcome{Status: st.NeedsAttention, Message: "Remote agent rejected the request; check configuration and permissions"}, err
+ return st.Outcome{Status: st.Failed, Message: "Remote agent rejected the request; check configuration and permissions"}, err
}
return st.Outcome{Status: st.Waiting, Message: "Waiting for environment"}, err
}
@@ -30,10 +31,10 @@ func remoteFailureInternal(err error) (st.Outcome, error) {
func (s *JobService) deliverRemoteInternal(ctx context.Context, run st.Run) (st.Outcome, error) {
env, err := s.environment.GetEnvironmentByID(ctx, run.EnvironmentID)
if err != nil {
- return st.Outcome{Status: st.NeedsAttention}, err
+ return st.Outcome{Status: st.Failed}, err
}
if !env.Enabled {
- return st.Outcome{Status: st.NeedsAttention, Message: "Environment is disabled"}, nil
+ return st.Outcome{Status: st.Failed, Message: "Environment is disabled"}, nil
}
basePath := "/api/environments/0/jobs/" + url.PathEscape(run.JobID)
runPath := basePath + "/runs/" + url.PathEscape(run.ID)
@@ -46,7 +47,7 @@ func (s *JobService) deliverRemoteInternal(ctx context.Context, run st.Run) (st.
return remoteFailureInternal(err)
}
if !catalog.DurableRuns {
- return st.Outcome{Status: st.NeedsAttention, Message: "Upgrade required: agent does not support durable job runs"}, nil
+ return st.Outcome{Status: st.Failed, Message: "Upgrade required: agent does not support durable job runs"}, nil
}
var remoteRun st.Run
err = s.environment.ProxyJSONRequest(ctx, run.EnvironmentID, http.MethodGet, runPath, nil, &remoteRun)
@@ -61,19 +62,19 @@ func (s *JobService) deliverRemoteInternal(ctx context.Context, run st.Run) (st.
return st.Outcome{Status: st.Canceled, Message: err.Error()}, nil
}
if errors.Is(err, errRemoteReceiptMissingInternal) {
- return st.Outcome{Status: st.NeedsAttention, Message: err.Error()}, nil
+ return st.Outcome{Status: st.Failed, Message: err.Error()}, nil
}
return remoteFailureInternal(err)
}
}
if remoteRun.ID != run.ID || remoteRun.JobID != run.JobID {
- return st.Outcome{Status: st.NeedsAttention, Message: "Agent returned an inconsistent run identity"}, nil
+ return st.Outcome{Status: st.Failed, Message: "Agent returned an inconsistent run identity"}, nil
}
if run.RemoteRetryRequested && (remoteRun.Status.Terminal() || remoteRun.Status == st.NeedsAttention) && remoteRun.AttemptCount <= run.RemoteAttemptCount {
remoteRun, err = s.retryDeliveryInternal(ctx, run, runPath)
if err != nil {
if errors.Is(err, errRemoteRetryUncertainInternal) {
- return st.Outcome{Status: st.NeedsAttention, Message: err.Error()}, nil
+ return st.Outcome{Status: st.Failed, Message: err.Error()}, nil
}
return remoteFailureInternal(err)
}
@@ -82,6 +83,16 @@ func (s *JobService) deliverRemoteInternal(ctx context.Context, run st.Run) (st.
}
func (s *JobService) confirmRemoteRunInternal(ctx context.Context, run, remoteRun st.Run, runPath string) (st.Outcome, error) {
+ if remoteRun.Status == st.NeedsAttention || (remoteRun.Status == st.Canceled && remoteRun.Resolution != nil && remoteRun.Resolution.ResolvedBy == common.SystemUser.Username) {
+ settled, err := s.resolveAgentReviewInternal(ctx, run.EnvironmentID, run.JobID, run.ID, common.SystemUser.Username)
+ if err != nil {
+ return st.Outcome{Status: st.Waiting, Message: "Waiting for the agent to acknowledge the failed run", Targets: remoteRun.Outcome.Targets, ActivityID: remoteRun.Outcome.ActivityID}, err
+ }
+ remoteRun = settled
+ if remoteRun.Status == st.Canceled && remoteRun.Resolution != nil && remoteRun.Resolution.ResolvedBy == common.SystemUser.Username {
+ remoteRun.Status = st.Failed
+ }
+ }
now := time.Now().UTC()
outcome := remoteRun.Outcome
outcome.Status = remoteRun.Status
@@ -92,17 +103,21 @@ func (s *JobService) confirmRemoteRunInternal(ctx context.Context, run, remoteRu
current.RemoteOutcome = &outcome
current.RemoteAttemptCount = remoteRun.AttemptCount
current.RemoteRetryRequested = false
+ current.RemoteSettled = remoteRun.RemoteSettled
+ current.Resolution = remoteRun.Resolution
+ current.StartedAt = remoteRun.StartedAt
+ current.FinishedAt = remoteRun.FinishedAt
current.Outcome = outcome
return nil
}); err != nil {
return st.Outcome{Status: st.Retrying}, err
}
if outcome.Status.Terminal() {
+ if remoteRun.RemoteSettled {
+ return outcome, nil
+ }
return s.acknowledgeRemoteInternal(ctx, run, runPath, outcome)
}
- if outcome.Status == st.NeedsAttention {
- return outcome, nil
- }
return st.Outcome{Status: st.Waiting, Message: "Waiting for the agent to complete this run", Targets: outcome.Targets, ActivityID: outcome.ActivityID}, nil
}
@@ -111,10 +126,19 @@ func (s *JobService) acknowledgeRemoteInternal(ctx context.Context, run st.Run,
if err := s.environment.ProxyJSONRequest(ctx, run.EnvironmentID, http.MethodPost, path+"/ack", nil, &response); err != nil {
return st.Outcome{Status: st.Waiting, Message: "Remote operation completed; waiting for delivery acknowledgement", Targets: outcome.Targets, ActivityID: outcome.ActivityID}, err
}
- if response.ID != run.ID || !response.RemoteSettled {
+ status := response.Status
+ if status == st.Canceled && response.Resolution != nil && response.Resolution.ResolvedBy == common.SystemUser.Username {
+ status = st.Failed
+ }
+ if response.ID != run.ID || response.JobID != run.JobID || response.EnvironmentID != "0" || status != outcome.Status || !response.RemoteSettled {
return st.Outcome{Status: st.Waiting, Message: "Remote operation completed; acknowledgement not confirmed"}, nil
}
- if err := s.runs.UpdateRun(ctx, run, func(current *st.Run) error { current.RemoteSettled = true; return nil }); err != nil {
+ if err := s.runs.UpdateRun(ctx, run, func(current *st.Run) error {
+ current.RemoteSettled = true
+ current.StartedAt = response.StartedAt
+ current.FinishedAt = response.FinishedAt
+ return nil
+ }); err != nil {
return st.Outcome{Status: st.Retrying}, err
}
return outcome, nil
@@ -131,6 +155,11 @@ func (s *JobService) ListRemoteJobs(ctx context.Context, environmentID string) (
if remoteErr == nil {
catalog.ObservedAt = time.Now().UTC()
catalog.Offline = false
+ if catalog.DurableRuns {
+ if err := s.reconcileLegacyRemoteCatalogInternal(ctx, environmentID, catalog.Jobs); err != nil {
+ return nil, err
+ }
+ }
raw, err := json.Marshal(catalog)
if err != nil {
return nil, err
@@ -180,6 +209,16 @@ func (s *JobService) RetryRemoteRun(ctx context.Context, environmentID, jobID, r
if err := s.authorizeRunInternal(ctx, run); err != nil {
return run, err
}
+ if run.Resolution != nil && run.Resolution.ResolvedBy == common.SystemUser.Username {
+ var remote st.Run
+ path := "/api/environments/0/jobs/" + url.PathEscape(jobID) + "/runs/" + url.PathEscape(runID)
+ if err := s.environment.ProxyJSONRequest(ctx, environmentID, http.MethodGet, path, nil, &remote); err != nil {
+ return run, err
+ }
+ if remote.ID != runID || remote.JobID != jobID || remote.EnvironmentID != "0" || remote.Status != st.Failed {
+ return run, errors.New("upgrade the agent before retrying a legacy settled run")
+ }
+ }
err = s.runs.UpdateRun(ctx, run, func(current *st.Run) error {
if current.Status != st.Failed && current.Status != st.Partial && current.Status != st.NeedsAttention {
return errors.New("run is not eligible for retry")
diff --git a/backend/internal/job/remote_history.go b/backend/internal/job/remote_history.go
index 0a60fb4d7..f3345e1e0 100644
--- a/backend/internal/job/remote_history.go
+++ b/backend/internal/job/remote_history.go
@@ -43,7 +43,7 @@ func (s *JobService) ListRuns(ctx context.Context, environmentID, jobID string,
}
runs := make([]st.Run, 0, len(merged))
for _, run := range merged {
- runs = append(runs, run)
+ runs = append(runs, projectRunOutcomeInternal(run))
}
sort.Slice(runs, func(i, j int) bool {
if runs[i].CreatedAt.Equal(runs[j].CreatedAt) {
@@ -86,7 +86,7 @@ func (s *JobService) remoteHistoryInternal(ctx context.Context, environmentID, j
func (s *JobService) GetRun(ctx context.Context, environmentID, jobID, runID string) (st.Run, error) {
run, err := s.runs.Get(ctx, environmentID, jobID, runID)
if environmentID == "0" || !errors.Is(err, runs.ErrRunNotFound) {
- return run, err
+ return projectRunOutcomeInternal(run), err
}
path := "/api/environments/0/jobs/" + url.PathEscape(jobID) + "/runs/" + url.PathEscape(runID)
if err := s.environment.ProxyJSONRequest(ctx, environmentID, http.MethodGet, path, nil, &run); err != nil {
@@ -99,7 +99,7 @@ func (s *JobService) GetRun(ctx context.Context, environmentID, jobID, runID str
if run.ActivityID != "" {
run.ActivityEnvironmentID = environmentID
}
- return run, nil
+ return projectRunOutcomeInternal(run), nil
}
// CancelRun cancels manager admission or explicitly forwards an agent-owned cancellation.
diff --git a/backend/internal/job/remote_resolution.go b/backend/internal/job/remote_resolution.go
index e48e9c16b..f7523a073 100644
--- a/backend/internal/job/remote_resolution.go
+++ b/backend/internal/job/remote_resolution.go
@@ -8,10 +8,41 @@ import (
"net/url"
"time"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
+ "github.com/getarcaneapp/arcane/types/v2/jobschedule"
st "github.com/getarcaneapp/arcane/types/v2/scheduler"
)
+// TODO(v3): remove adoption of legacy agent-owned runs.
+func (s *JobService) reconcileLegacyRemoteCatalogInternal(ctx context.Context, environmentID string, jobs []jobschedule.JobStatus) error {
+ for _, job := range jobs {
+ for _, remote := range []*st.Run{job.CurrentRun, job.LastRun} {
+ if remote == nil || remote.Status != st.NeedsAttention {
+ continue
+ }
+ if remote.JobID != job.ID || remote.EnvironmentID != "0" {
+ return errors.New("agent returned an inconsistent run identity")
+ }
+ _, err := s.runs.Get(ctx, environmentID, job.ID, remote.ID)
+ if err == nil {
+ continue
+ }
+ if !errors.Is(err, runs.ErrRunNotFound) {
+ return err
+ }
+ _, err = s.runs.Submit(ctx, st.Request{RunID: remote.ID, JobID: job.ID, EnvironmentID: environmentID, Trigger: "recovery", ObservedAgentRun: remote})
+ if err != nil {
+ return err
+ }
+ }
+ if err := s.reconcileLegacyRemoteCatalogInternal(ctx, environmentID, job.Children); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
func (s *JobService) resolveRemoteRunInternal(ctx context.Context, environmentID, jobID, runID, actor string) (st.Run, error) {
local, localErr := s.runs.Get(ctx, environmentID, jobID, runID)
if localErr != nil && !errors.Is(localErr, runs.ErrRunNotFound) {
@@ -57,7 +88,8 @@ func (s *JobService) resolveRemoteRunInternal(ctx context.Context, environmentID
return s.runs.Resolve(ctx, environmentID, jobID, runID, acknowledged.Resolution.ResolvedBy)
}
-// resolveAgentReviewInternal confirms ownership, records review, and settles delivery.
+// resolveAgentReviewInternal settles legacy inactive runs on their owning agent.
+// TODO(v3): remove the legacy resolution protocol.
func (s *JobService) resolveAgentReviewInternal(ctx context.Context, environmentID, jobID, runID, actor string) (st.Run, error) {
env, err := s.environment.GetEnvironmentByID(ctx, environmentID)
if err != nil {
@@ -75,10 +107,7 @@ func (s *JobService) resolveAgentReviewInternal(ctx context.Context, environment
if remote.ID != runID || remote.JobID != jobID || remote.EnvironmentID != "0" {
return st.Run{}, errors.New("agent returned an inconsistent run identity")
}
- if remote.Status != st.Canceled || remote.Resolution == nil {
- if remote.Status != st.NeedsAttention {
- return st.Run{}, errors.New("agent run must need attention before review resolution")
- }
+ if remote.Status == st.NeedsAttention {
body, err := json.Marshal(map[string]string{"resolvedBy": actor})
if err != nil {
return st.Run{}, err
@@ -86,15 +115,21 @@ func (s *JobService) resolveAgentReviewInternal(ctx context.Context, environment
if err := s.environment.ProxyJSONRequest(ctx, environmentID, http.MethodPost, path+"/resolve", body, &remote); err != nil {
return st.Run{}, err
}
+ } else if actor != common.SystemUser.Username {
+ if remote.Status != st.Canceled || remote.Resolution == nil {
+ return st.Run{}, errors.New("agent run must need attention before review resolution")
+ }
}
- if remote.ID != runID || remote.JobID != jobID || remote.EnvironmentID != "0" || remote.Status != st.Canceled || remote.Resolution == nil {
+ if remote.ID != runID || remote.JobID != jobID || remote.EnvironmentID != "0" || !remote.Status.Terminal() ||
+ (actor != common.SystemUser.Username && (remote.Status != st.Canceled || remote.Resolution == nil)) {
return st.Run{}, errors.New("agent resolution was not confirmed")
}
var acknowledged st.Run
if err := s.environment.ProxyJSONRequest(ctx, environmentID, http.MethodPost, path+"/ack", nil, &acknowledged); err != nil {
return st.Run{}, err
}
- if acknowledged.ID != runID || acknowledged.JobID != jobID || acknowledged.EnvironmentID != "0" || acknowledged.Status != st.Canceled || acknowledged.Resolution == nil || !acknowledged.RemoteSettled {
+ if acknowledged.ID != runID || acknowledged.JobID != jobID || acknowledged.EnvironmentID != "0" || acknowledged.Status != remote.Status || !acknowledged.RemoteSettled ||
+ (remote.Resolution != nil && (acknowledged.Resolution == nil || acknowledged.Resolution.ResolvedBy != remote.Resolution.ResolvedBy)) {
return st.Run{}, errors.New("agent resolution acknowledgement was not confirmed")
}
return acknowledged, nil
diff --git a/backend/internal/job/remote_resolution_test.go b/backend/internal/job/remote_resolution_test.go
index 59cf768fd..7cd8e15d2 100644
--- a/backend/internal/job/remote_resolution_test.go
+++ b/backend/internal/job/remote_resolution_test.go
@@ -9,67 +9,172 @@ import (
"testing"
"time"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/environment"
"github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
"github.com/getarcaneapp/arcane/backend/v2/pkg/authz"
+ "github.com/getarcaneapp/arcane/types/v2/jobschedule"
st "github.com/getarcaneapp/arcane/types/v2/scheduler"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestResolveRemoteRunConfirmsOwnerAfterLostResponse(t *testing.T) {
- db := setupSettingsTestDBInternal(t)
- require.NoError(t, db.AutoMigrate(&environment.Environment{}))
- q := newJobCoordinatorForTestInternal(t, db)
- local, err := q.Submit(t.Context(), st.Request{JobID: "auto-update", EnvironmentID: "remote"})
- require.NoError(t, err)
- require.NoError(t, q.UpdateRun(t.Context(), local, func(run *st.Run) error {
- run.Status = st.NeedsAttention
- run.RemoteAccepted = true
- run.RemoteDeliveryAttempted = true
- run.Outcome = st.Outcome{Status: st.NeedsAttention, Message: "original failure"}
- return nil
- }))
- remote := st.Run{ID: local.ID, JobID: local.JobID, EnvironmentID: "0", Status: st.Running}
- mutations := 0
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- switch {
- case r.Method == http.MethodGet:
- case strings.HasSuffix(r.URL.Path, "/resolve"):
- mutations++
- remote.Status = st.Canceled
- remote.Resolution = &st.RunResolution{ResolvedBy: "operator", ResolvedAt: time.Now().UTC(), Reason: "resolved_after_review"}
- http.Error(w, "lost response", http.StatusServiceUnavailable)
- return
- case strings.HasSuffix(r.URL.Path, "/ack"):
- remote.RemoteSettled = true
- default:
- http.Error(w, "unexpected request", http.StatusBadRequest)
- return
+ for _, automatic := range []bool{false, true} {
+ name := "legacy operator"
+ if automatic {
+ name = "automatic settlement"
}
- raw, marshalErr := json.Marshal(remote)
- assert.NoError(t, marshalErr)
- w.Header().Set("Content-Type", "application/json")
- _, writeErr := w.Write(raw)
- assert.NoError(t, writeErr)
- }))
- defer server.Close()
- require.NoError(t, db.Create(&environment.Environment{ID: "remote", Name: "agent", ApiUrl: server.URL, Enabled: true}).Error)
- svc := &JobService{runs: q, environment: environment.NewEnvironmentService(db, server.Client(), nil, nil, nil, nil)}
- ctx := context.WithValue(t.Context(), middleware.ContextKeyUserPermissions, authz.EnvironmentPermissionSet("remote"))
- _, err = svc.ResolveRun(ctx, "remote", local.JobID, local.ID, "operator")
- require.ErrorContains(t, err, "agent run must need attention")
- require.Zero(t, mutations)
- remote.Status = st.NeedsAttention
- _, err = svc.ResolveRun(ctx, "remote", local.JobID, local.ID, "operator")
- require.Error(t, err)
- unchanged, err := q.Get(t.Context(), "remote", local.JobID, local.ID)
- require.NoError(t, err)
- require.Equal(t, st.NeedsAttention, unchanged.Status)
- resolved, err := svc.ResolveRun(ctx, "remote", local.JobID, local.ID, "operator")
- require.NoError(t, err)
- require.Equal(t, st.Canceled, resolved.Status)
- require.True(t, resolved.RemoteSettled)
- require.Equal(t, "original failure", resolved.Outcome.Message)
- require.Equal(t, 1, mutations)
+ t.Run(name, func(t *testing.T) {
+ db := setupSettingsTestDBInternal(t)
+ require.NoError(t, db.AutoMigrate(&environment.Environment{}))
+ q := newJobCoordinatorForTestInternal(t, db)
+ local, err := q.Submit(t.Context(), st.Request{JobID: "auto-update", EnvironmentID: "remote", Trigger: "scheduled"})
+ require.NoError(t, err)
+ require.NoError(t, q.UpdateRun(t.Context(), local, func(run *st.Run) error {
+ run.Status = st.NeedsAttention
+ if automatic {
+ run.Status = st.Running
+ }
+ run.RemoteAccepted = true
+ run.RemoteDeliveryAttempted = true
+ run.Outcome = st.Outcome{Status: st.NeedsAttention, Message: "original failure"}
+ return nil
+ }))
+ remote := st.Run{ID: local.ID, JobID: local.JobID, EnvironmentID: "0", Status: st.Running, Outcome: st.Outcome{Status: st.NeedsAttention, Message: "original failure", Targets: []st.TargetOutcome{{ID: "completed", Status: st.Succeeded}}}}
+ mutations, acknowledgements := 0, 0
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch {
+ case r.Method == http.MethodGet && r.URL.Query().Has("page"):
+ raw, marshalErr := json.Marshal(st.RunList{Runs: []st.Run{remote}, Total: 1})
+ assert.NoError(t, marshalErr)
+ w.Header().Set("Content-Type", "application/json")
+ _, writeErr := w.Write(raw)
+ assert.NoError(t, writeErr)
+ return
+ case r.Method == http.MethodGet:
+ case strings.HasSuffix(r.URL.Path, "/resolve"):
+ mutations++
+ var body struct {
+ ResolvedBy string `json:"resolvedBy"`
+ }
+ assert.NoError(t, json.UnmarshalRead(r.Body, &body))
+ remote.Status = st.Canceled
+ remote.Resolution = &st.RunResolution{ResolvedBy: body.ResolvedBy, ResolvedAt: time.Now().UTC(), Reason: "resolved_after_review"}
+ http.Error(w, "lost response", http.StatusServiceUnavailable)
+ return
+ case strings.HasSuffix(r.URL.Path, "/ack"):
+ acknowledgements++
+ remote.RemoteSettled = true
+ if automatic && acknowledgements == 1 {
+ http.Error(w, "lost acknowledgement", http.StatusServiceUnavailable)
+ return
+ }
+ default:
+ http.Error(w, "unexpected request", http.StatusBadRequest)
+ return
+ }
+ raw, marshalErr := json.Marshal(remote)
+ assert.NoError(t, marshalErr)
+ w.Header().Set("Content-Type", "application/json")
+ _, writeErr := w.Write(raw)
+ assert.NoError(t, writeErr)
+ }))
+ defer server.Close()
+ require.NoError(t, db.Create(&environment.Environment{ID: "remote", Name: "agent", ApiUrl: server.URL, Enabled: true}).Error)
+ svc := &JobService{runs: q, environment: environment.NewEnvironmentService(db, server.Client(), nil, nil, nil, nil)}
+ ctx := context.WithValue(t.Context(), middleware.ContextKeyUserPermissions, authz.EnvironmentPermissionSet("remote"))
+ path := "/api/environments/0/jobs/auto-update/runs/" + local.ID
+ if !automatic {
+ _, err = svc.ResolveRun(ctx, "remote", local.JobID, local.ID, "operator")
+ require.ErrorContains(t, err, "agent run must need attention")
+ } else {
+ outcome, confirmErr := svc.confirmRemoteRunInternal(ctx, local, remote, path)
+ require.NoError(t, confirmErr)
+ require.Equal(t, st.Waiting, outcome.Status)
+ }
+ require.Zero(t, mutations)
+ remote.Status = st.NeedsAttention
+ if !automatic {
+ _, err = svc.ResolveRun(ctx, "remote", local.JobID, local.ID, "operator")
+ require.Error(t, err)
+ unchanged, getErr := q.Get(t.Context(), "remote", local.JobID, local.ID)
+ require.NoError(t, getErr)
+ require.Equal(t, st.NeedsAttention, unchanged.Status)
+ resolved, resolveErr := svc.ResolveRun(ctx, "remote", local.JobID, local.ID, "operator")
+ require.NoError(t, resolveErr)
+ require.Equal(t, st.Canceled, resolved.Status)
+ require.True(t, resolved.RemoteSettled)
+ require.Equal(t, "original failure", resolved.Outcome.Message)
+ } else {
+ outcome, confirmErr := svc.confirmRemoteRunInternal(ctx, local, remote, path)
+ require.Error(t, confirmErr)
+ require.Equal(t, st.Waiting, outcome.Status)
+ outcome, confirmErr = svc.confirmRemoteRunInternal(ctx, local, remote, path)
+ require.Error(t, confirmErr)
+ require.Equal(t, st.Waiting, outcome.Status)
+ outcome, confirmErr = svc.confirmRemoteRunInternal(ctx, local, remote, path)
+ require.NoError(t, confirmErr)
+ require.Equal(t, st.Failed, outcome.Status)
+ require.Equal(t, "original failure", outcome.Message)
+ require.Equal(t, st.Succeeded, outcome.Targets[0].Status)
+ stored, getErr := q.Get(t.Context(), "remote", local.JobID, local.ID)
+ require.NoError(t, getErr)
+ require.True(t, stored.RemoteSettled)
+ require.Equal(t, st.Failed, stored.RemoteOutcome.Status)
+ require.Equal(t, common.SystemUser.Username, stored.Resolution.ResolvedBy)
+ require.NoError(t, q.UpdateRun(ctx, stored, func(current *st.Run) error {
+ current.Status = st.Failed
+ return nil
+ }))
+ _, retryErr := svc.RetryRemoteRun(ctx, "remote", local.JobID, local.ID)
+ require.ErrorContains(t, retryErr, "upgrade the agent")
+ remote.Status = st.Failed
+ retried, retryErr := svc.RetryRemoteRun(ctx, "remote", local.JobID, local.ID)
+ require.NoError(t, retryErr)
+ require.Equal(t, st.Queued, retried.Status)
+ require.Equal(t, local.ID, retried.ID)
+
+ // Adopt agent-owned history without coalescing its explicit identity.
+ agentOwned := remote
+ agentOwned.ID = "d7a9a3c7-d364-4c36-831a-381249e5727e"
+ agentOwned.Status = st.NeedsAttention
+ agentOwned.CreatedAt = time.Now().UTC().Add(-time.Hour)
+ agentOwned.FinishedAt = new(agentOwned.CreatedAt.Add(time.Minute))
+ agentOwned.ActivityID = "agent-summary"
+ agentOwned.RemoteSettled = false
+ catalog := []jobschedule.JobStatus{{ID: "updates", Children: []jobschedule.JobStatus{{ID: agentOwned.JobID, LastRun: &agentOwned}}}}
+ require.NoError(t, svc.reconcileLegacyRemoteCatalogInternal(ctx, "remote", catalog))
+ require.NoError(t, svc.reconcileLegacyRemoteCatalogInternal(ctx, "remote", catalog))
+ adopted, getErr := q.Get(ctx, "remote", agentOwned.JobID, agentOwned.ID)
+ require.NoError(t, getErr)
+ require.True(t, adopted.RemoteAccepted)
+ require.True(t, adopted.RemoteDeliveryAttempted)
+ require.Equal(t, agentOwned.CreatedAt, adopted.CreatedAt)
+ require.Equal(t, st.Queued, adopted.Status)
+ require.Equal(t, agentOwned.FinishedAt, adopted.FinishedAt)
+ require.Equal(t, agentOwned.ActivityID, adopted.ActivityID)
+ require.Equal(t, "remote", adopted.ActivityEnvironmentID)
+ status := jobschedule.JobStatus{LastRun: &agentOwned}
+ applyRunStatusInternal(&status, []st.Run{adopted})
+ require.Nil(t, status.CurrentRun)
+ require.Equal(t, st.Failed, status.LastRun.Status)
+ require.Equal(t, "original failure", status.LastError)
+ remote = agentOwned
+ merged, listErr := svc.ListRuns(ctx, "remote", agentOwned.JobID, 1, 20)
+ require.NoError(t, listErr)
+ detail, detailErr := svc.GetRun(ctx, "remote", agentOwned.JobID, agentOwned.ID)
+ require.NoError(t, detailErr)
+ require.Equal(t, st.Failed, detail.Status)
+ require.Equal(t, "original failure", detail.Outcome.Message)
+ require.Equal(t, agentOwned.ActivityID, detail.ActivityID)
+ require.Equal(t, "remote", detail.ActivityEnvironmentID)
+ require.Contains(t, merged.Runs, detail)
+ history, listErr := q.List(ctx, "remote", agentOwned.JobID, 1, 20)
+ require.NoError(t, listErr)
+ require.Equal(t, 2, history.Total)
+ }
+ require.Equal(t, 1, mutations)
+ })
+ }
}
diff --git a/backend/internal/job/resolution.go b/backend/internal/job/resolution.go
index a06bb8007..ba33acdf1 100644
--- a/backend/internal/job/resolution.go
+++ b/backend/internal/job/resolution.go
@@ -11,7 +11,8 @@ import (
st "github.com/getarcaneapp/arcane/types/v2/scheduler"
)
-// ResolveRun records operator review and releases the schedule's blocked run.
+// ResolveRun records legacy operator review.
+// TODO(v3): remove this deprecated mixed-version compatibility contract.
func (h *JobSchedulesHandler) ResolveRun(ctx context.Context, input *jobschedule.ResolveRunInput) (*jobschedule.RunOutput, error) {
actor, _ := middleware.GetUserIDFromContext(ctx)
permissions, _ := middleware.PermissionsFromContext(ctx)
@@ -29,6 +30,7 @@ func (h *JobSchedulesHandler) ResolveRun(ctx context.Context, input *jobschedule
}
// ResolveRun authorizes the current operator independently of the original requester.
+// TODO(v3): remove this deprecated mixed-version compatibility contract.
func (s *JobService) ResolveRun(ctx context.Context, environmentID, jobID, runID, actor string) (st.Run, error) {
permissions, _ := middleware.PermissionsFromContext(ctx)
if !permissions.Allows(authz.PermJobsManage, environmentID) {
diff --git a/backend/internal/job/run_routes.go b/backend/internal/job/run_routes.go
index 737b35c40..4dd213639 100644
--- a/backend/internal/job/run_routes.go
+++ b/backend/internal/job/run_routes.go
@@ -21,7 +21,7 @@ func (h *JobSchedulesHandler) registerRunRoutesInternal(api huma.API) {
middleware.RegisterWithPermission(api, huma.Operation{OperationID: "get-job-run", Method: http.MethodGet, Path: base + "/runs/{runId}", Summary: "Get job run", Security: handlerutil.DefaultOperationSecurity()}, authz.PermJobsManage, h.GetRun)
middleware.RegisterWithPermission(api, huma.Operation{OperationID: "retry-job-run", Method: http.MethodPost, Path: base + "/runs/{runId}/retry", Summary: "Retry job run", Security: handlerutil.DefaultOperationSecurity()}, authz.PermJobsManage, h.RetryRun)
middleware.RegisterWithPermission(api, huma.Operation{OperationID: "cancel-job-run", Method: http.MethodPost, Path: base + "/runs/{runId}/cancel", Summary: "Cancel pending job run", Security: handlerutil.DefaultOperationSecurity()}, authz.PermJobsManage, h.CancelRun)
- middleware.RegisterWithPermission(api, huma.Operation{OperationID: "resolve-job-run", Method: http.MethodPost, Path: base + "/runs/{runId}/resolve", Summary: "Resolve job run after review", Security: handlerutil.DefaultOperationSecurity()}, authz.PermJobsManage, h.ResolveRun)
+ middleware.RegisterWithPermission(api, huma.Operation{OperationID: "resolve-job-run", Method: http.MethodPost, Path: base + "/runs/{runId}/resolve", Summary: "Resolve legacy job run", Description: "Deprecated compatibility endpoint for mixed-version upgrades. Removed in v3.", Deprecated: true, Security: handlerutil.DefaultOperationSecurity()}, authz.PermJobsManage, h.ResolveRun)
middleware.RegisterWithPermission(api, huma.Operation{OperationID: "ack-job-run", Method: http.MethodPost, Path: base + "/runs/{runId}/ack", Summary: "Acknowledge remote run completion", Security: handlerutil.DefaultOperationSecurity()}, authz.PermJobsManage, h.AcknowledgeRun)
middleware.RegisterWithPermission(api, huma.Operation{OperationID: "restart-job-worker", Method: http.MethodPost, Path: base + "/restart", Summary: "Restart continuous job worker", Security: handlerutil.DefaultOperationSecurity()}, authz.PermJobsManage, h.RestartWorker)
}
diff --git a/backend/internal/job/runtime_status.go b/backend/internal/job/runtime_status.go
index 919d6dc63..bddb7bd85 100644
--- a/backend/internal/job/runtime_status.go
+++ b/backend/internal/job/runtime_status.go
@@ -7,6 +7,7 @@ import (
"strings"
"time"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/types/v2/jobschedule"
st "github.com/getarcaneapp/arcane/types/v2/scheduler"
)
@@ -132,26 +133,52 @@ func dynamicJobStatusInternal(ctx context.Context, runtime st.JobScheduler, job
}
func applyRunStatusInternal(status *jobschedule.JobStatus, runs []st.Run) {
- var errorAt time.Time
- if status.LastError != "" {
- for _, run := range []*st.Run{status.CurrentRun, status.LastRun} {
- if run != nil && run.UpdatedAt.After(errorAt) {
- errorAt = run.UpdatedAt
+ latest := make(map[string]st.Run, len(runs)+2)
+ for _, run := range []*st.Run{status.CurrentRun, status.LastRun} {
+ if run != nil {
+ if previous, found := latest[run.ID]; !found || !run.UpdatedAt.Before(previous.UpdatedAt) {
+ latest[run.ID] = *run
}
}
}
- if current := status.CurrentRun; current != nil && current.Status == st.NeedsAttention {
- status.CurrentRun = nil
- updateLatestRunInternal(status, *current)
- }
for _, run := range runs {
+ if previous, found := latest[run.ID]; !found || !run.UpdatedAt.Before(previous.UpdatedAt) {
+ latest[run.ID] = run
+ }
+ }
+ status.CurrentRun = nil
+ status.LastRun = nil
+ status.LastError = ""
+ for _, run := range latest {
+ run = projectRunOutcomeInternal(run)
updateLatestRunInternal(status, run)
updateLastSuccessInternal(status, run)
- if run.UpdatedAt.After(errorAt) && run.Outcome.Message != "" && runHasErrorInternal(run.Status) {
- status.LastError = run.Outcome.Message
- errorAt = run.UpdatedAt
+ }
+ run := status.CurrentRun
+ if last := status.LastRun; last != nil {
+ if run == nil || runExecutionTimeInternal(*last).After(runExecutionTimeInternal(*run)) {
+ run = last
}
}
+ if run != nil && runHasErrorInternal(run.Status) {
+ status.LastError = run.Outcome.Message
+ }
+}
+
+func projectRunOutcomeInternal(run st.Run) st.Run {
+ // Delivery reconciliation must not hide a failure already reported by the agent.
+ if run.RemoteOutcome != nil && run.RemoteOutcome.Status == st.NeedsAttention && !run.RemoteRetryRequested {
+ run.Status = st.Failed
+ run.Outcome = *run.RemoteOutcome
+ }
+ automaticResolution := run.Status == st.Canceled && run.Resolution != nil && run.Resolution.ResolvedBy == common.SystemUser.Username
+ if run.Status == st.NeedsAttention || automaticResolution {
+ run.Status = st.Failed
+ }
+ if run.Status == st.Failed {
+ run.Outcome.Status = st.Failed
+ }
+ return run
}
func updateLatestRunInternal(status *jobschedule.JobStatus, run st.Run) {
@@ -161,20 +188,33 @@ func updateLatestRunInternal(status *jobschedule.JobStatus, run st.Run) {
}
return
}
- current := status.CurrentRun
- if current != nil && current.ID == run.ID && !run.UpdatedAt.Before(current.UpdatedAt) && (run.Status == st.NeedsAttention || !run.RemoteAccepted || run.RemoteSettled) {
- status.CurrentRun = nil
- }
- if status.LastRun == nil || run.UpdatedAt.After(status.LastRun.UpdatedAt) {
+ if status.LastRun == nil || runExecutionTimeInternal(run).After(runExecutionTimeInternal(*status.LastRun)) ||
+ (runExecutionTimeInternal(run).Equal(runExecutionTimeInternal(*status.LastRun)) && run.ID > status.LastRun.ID) {
status.LastRun = new(run)
}
}
+func runExecutionTimeInternal(run st.Run) time.Time {
+ startedAt := run.CreatedAt
+ if run.StartedAt != nil && run.StartedAt.After(startedAt) {
+ startedAt = *run.StartedAt
+ }
+ // Manager delivery attempts poll the agent; they are not job executions.
+ if run.EnvironmentID == "0" || !run.RemoteAccepted {
+ for _, attempt := range run.Attempts {
+ if attempt.StartedAt.After(startedAt) {
+ startedAt = attempt.StartedAt
+ }
+ }
+ }
+ return startedAt
+}
+
func updateLastSuccessInternal(status *jobschedule.JobStatus, run st.Run) {
if run.Status != st.Succeeded {
return
}
- succeededAt := run.UpdatedAt
+ succeededAt := runExecutionTimeInternal(run)
if run.FinishedAt != nil {
succeededAt = *run.FinishedAt
}
@@ -185,9 +225,9 @@ func updateLastSuccessInternal(status *jobschedule.JobStatus, run st.Run) {
func runHasErrorInternal(status st.RunStatus) bool {
switch status {
- case st.Failed, st.Partial, st.NeedsAttention, st.Retrying, st.Waiting:
+ case st.Failed, st.Partial, st.NeedsAttention, st.Retrying:
return true
- case st.Queued, st.Running, st.Succeeded, st.Skipped, st.Canceled:
+ case st.Queued, st.Running, st.Waiting, st.Succeeded, st.Skipped, st.Canceled:
return false
default:
return false
@@ -203,7 +243,10 @@ func preferCurrentRunInternal(candidate, current st.Run) bool {
if candidatePriority != currentPriority {
return candidatePriority < currentPriority
}
- return candidate.CreatedAt.Before(current.CreatedAt)
+ if candidate.CreatedAt.Equal(current.CreatedAt) {
+ return candidate.ID > current.ID
+ }
+ return candidate.CreatedAt.After(current.CreatedAt)
}
func currentRunPriorityInternal(status st.RunStatus) int {
diff --git a/backend/internal/job/service_test.go b/backend/internal/job/service_test.go
index b3356f246..89cf1b2d8 100644
--- a/backend/internal/job/service_test.go
+++ b/backend/internal/job/service_test.go
@@ -153,6 +153,76 @@ func TestJobService_ListJobs_UsesRuntimeScheduleAndNextRun(t *testing.T) {
autoUpdateJob := findJobStatusByIDInternal(t, jobs.Jobs, "auto-update")
require.Equal(t, "0 0 8 * * *", autoUpdateJob.Schedule)
require.Equal(t, nextRun, *autoUpdateJob.NextRun)
+
+ now := time.Now().UTC()
+ failed := schedulertypes.Run{
+ ID: "failed", EnvironmentID: "0", Status: schedulertypes.Failed, CreatedAt: now.Add(-time.Hour), UpdatedAt: now,
+ Outcome: schedulertypes.Outcome{Status: schedulertypes.Failed, Message: "clone failed"},
+ }
+ succeeded := schedulertypes.Run{ID: "succeeded", EnvironmentID: "0", Status: schedulertypes.Succeeded, CreatedAt: now.Add(-time.Minute), UpdatedAt: now.Add(-time.Minute), FinishedAt: new(now.Add(-time.Minute))}
+ for _, history := range [][]schedulertypes.Run{{failed, succeeded}, {succeeded, failed}} {
+ status := jobschedule.JobStatus{LastRun: new(failed), LastError: failed.Outcome.Message}
+ applyRunStatusInternal(&status, history)
+ require.Equal(t, succeeded.ID, status.LastRun.ID)
+ require.Empty(t, status.LastError)
+ require.Equal(t, "clone failed", history[0].Outcome.Message+history[1].Outcome.Message)
+ }
+ // An acknowledgement updates an old failure's revision, not its execution time.
+ failed.UpdatedAt = now.Add(time.Hour)
+ status := jobschedule.JobStatus{LastRun: new(succeeded)}
+ applyRunStatusInternal(&status, []schedulertypes.Run{failed})
+ require.Equal(t, succeeded.ID, status.LastRun.ID)
+ require.Empty(t, status.LastError)
+ // A real retry of the older run becomes the latest execution.
+ failed.StartedAt = new(now.Add(time.Second))
+ failed.Attempts = []schedulertypes.Attempt{{Number: 2, StartedAt: *failed.StartedAt}}
+ applyRunStatusInternal(&status, []schedulertypes.Run{failed})
+ require.Equal(t, failed.ID, status.LastRun.ID)
+ require.Equal(t, "clone failed", status.LastError)
+ // Merge the same run's terminal revision before choosing an active run.
+ status.CurrentRun = new(succeeded)
+ status.CurrentRun.Status = schedulertypes.Running
+ succeeded.UpdatedAt = now.Add(2 * time.Second)
+ applyRunStatusInternal(&status, []schedulertypes.Run{succeeded})
+ require.Nil(t, status.CurrentRun)
+ failed.Status = schedulertypes.Running
+ failed.UpdatedAt = now.Add(2 * time.Hour)
+ applyRunStatusInternal(&status, []schedulertypes.Run{failed})
+ require.Equal(t, failed.ID, status.CurrentRun.ID)
+ require.Empty(t, status.LastError)
+
+ // A later manual failure stays visible while older work waits to retry.
+ retrying := failed
+ retrying.Status = schedulertypes.Retrying
+ retrying.NextAttempt = new(now.Add(time.Hour))
+ manualFailure := failed
+ manualFailure.ID = "manual-failure"
+ manualFailure.Status = schedulertypes.Failed
+ manualFailure.CreatedAt = now.Add(time.Minute)
+ manualFailure.StartedAt = nil
+ manualFailure.Attempts = nil
+ manualFailure.Outcome.Message = "latest manual failure"
+ succeeded.CreatedAt = now.Add(2 * time.Minute)
+ for _, history := range [][]schedulertypes.Run{{retrying, manualFailure}, {manualFailure, retrying}} {
+ status = jobschedule.JobStatus{}
+ applyRunStatusInternal(&status, history)
+ require.Equal(t, retrying.ID, status.CurrentRun.ID)
+ require.Equal(t, manualFailure.ID, status.LastRun.ID)
+ require.Equal(t, manualFailure.Outcome.Message, status.LastError)
+ applyRunStatusInternal(&status, []schedulertypes.Run{succeeded})
+ require.Empty(t, status.LastError)
+ }
+
+ cloneErr := errors.New("failed to clone repository: network is unreachable")
+ outcome, runErr := classifyOutcomeInternal("gitops-sync:project", schedulertypes.Outcome{}, cloneErr)
+ require.ErrorIs(t, runErr, cloneErr)
+ require.Equal(t, schedulertypes.Failed, outcome.Status)
+ outcome, runErr = classifyOutcomeInternal("environment-health:0", schedulertypes.Outcome{}, context.DeadlineExceeded)
+ require.ErrorIs(t, runErr, context.DeadlineExceeded)
+ require.Equal(t, schedulertypes.Retrying, outcome.Status)
+ outcome, runErr = classifyOutcomeInternal("auto-update", schedulertypes.Outcome{Status: schedulertypes.Partial}, cloneErr)
+ require.ErrorIs(t, runErr, cloneErr)
+ require.Equal(t, schedulertypes.Partial, outcome.Status)
}
func TestJobService_ListJobs_ImageUpdateWatcherIsContinuousAndRespectsEnabled(t *testing.T) {
diff --git a/backend/internal/updater/recovery.go b/backend/internal/updater/recovery.go
index 383feda5f..afbf3068d 100644
--- a/backend/internal/updater/recovery.go
+++ b/backend/internal/updater/recovery.go
@@ -437,13 +437,13 @@ func (s *UpdaterService) ReconcilePending(ctx context.Context, run schedulertype
for _, target := range run.Outcome.Targets {
if target.ID == "auto-update" && len(target.RecoveryData) > 0 {
if err := json.Unmarshal(target.RecoveryData, &plan); err != nil {
- return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention}, err
+ return schedulertypes.Outcome{Status: schedulertypes.Failed}, err
}
found = true
}
}
if !found {
- return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: "Interrupted update has no frozen target plan", Targets: run.Outcome.Targets}, nil
+ return schedulertypes.Outcome{Status: schedulertypes.Failed, Message: "Interrupted update has no frozen target plan", Targets: run.Outcome.Targets}, nil
}
remaining := frozenUpdatePlanInternal{}
unresolved := false
@@ -471,14 +471,14 @@ func (s *UpdaterService) ReconcilePending(ctx context.Context, run schedulertype
if len(remaining.Records) > 0 {
result, err := s.ApplyPending(context.WithValue(ctx, frozenPendingKeyInternal{}, &remaining), arcaneupdater.Options{})
if err != nil || result == nil || result.Failed > 0 {
- return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: "Frozen update recovery requires review"}, err
+ return schedulertypes.Outcome{Status: schedulertypes.Failed, Message: "Frozen update recovery could not confirm completion"}, err
}
}
status := schedulertypes.Succeeded
message := "Frozen update targets confirmed"
if unresolved {
- status = schedulertypes.NeedsAttention
- message = "Unconfirmed update effects require review"
+ status = schedulertypes.Failed
+ message = "Some update effects could not be confirmed"
}
return schedulertypes.Outcome{Status: status, Message: message}, nil
}
diff --git a/backend/pkg/scheduler/auto_heal_job.go b/backend/pkg/scheduler/auto_heal_job.go
index 88bdcd052..fb272462f 100644
--- a/backend/pkg/scheduler/auto_heal_job.go
+++ b/backend/pkg/scheduler/auto_heal_job.go
@@ -544,7 +544,7 @@ func (j *AutoHealJob) Reconcile(ctx context.Context, previous schedulertypes.Run
return schedulertypes.Outcome{}, err
}
if !admitted {
- return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: "A container healing run already owns admission"}, nil
+ return schedulertypes.Outcome{Status: schedulertypes.Failed, Message: "A container healing run already owns admission"}, nil
}
defer lease.Release(ctx)
var plan autoHealPlanInternal
@@ -568,7 +568,7 @@ func (j *AutoHealJob) Reconcile(ctx context.Context, previous schedulertypes.Run
return schedulertypes.Outcome{}, err
}
if message != "" {
- return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: message, Targets: previous.Outcome.Targets}, nil
+ return schedulertypes.Outcome{Status: schedulertypes.Failed, Message: message, Targets: previous.Outcome.Targets}, nil
}
if !planned {
return schedulertypes.Outcome{Status: schedulertypes.Succeeded, Targets: previous.Outcome.Targets}, nil
@@ -621,7 +621,7 @@ func (j *AutoHealJob) resumeHealingPlanInternal(ctx context.Context, dockerClien
}
previous.Outcome.Targets = append(previous.Outcome.Targets, target)
if err != nil {
- outcome.Status = schedulertypes.NeedsAttention
+ outcome.Status = schedulertypes.Failed
outcome.Message = err.Error()
break
}
diff --git a/backend/pkg/scheduler/auto_update_job.go b/backend/pkg/scheduler/auto_update_job.go
index dbff8a6aa..bb5938153 100644
--- a/backend/pkg/scheduler/auto_update_job.go
+++ b/backend/pkg/scheduler/auto_update_job.go
@@ -64,7 +64,7 @@ func (j *AutoUpdateJob) Run(ctx context.Context) (schedulertypes.Outcome, error)
options, priorOutcome, unresolvedTargets = autoUpdateRetryOptionsInternal(previous)
if priorOutcome.Status != "" {
priorOutcome.ActivityID = cmp.Or(priorOutcome.ActivityID, previous.Outcome.ActivityID)
- if priorOutcome.Status == schedulertypes.NeedsAttention && previous.Outcome.Message != "" {
+ if priorOutcome.Status == schedulertypes.Failed && previous.Outcome.Message != "" {
priorOutcome.Message += ": " + previous.Outcome.Message
}
return priorOutcome, nil
@@ -75,7 +75,7 @@ func (j *AutoUpdateJob) Run(ctx context.Context) (schedulertypes.Outcome, error)
pollingEnabled := j.settingsService.GetBoolSetting(ctx, "pollingEnabled", true)
if !enabled || !pollingEnabled {
if len(options.ResourceIds) > 0 {
- return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: "Auto-update is disabled; review this run before resolving it or retrying"}, nil
+ return schedulertypes.Outcome{Status: schedulertypes.Failed, Message: "Auto-update is disabled; the remaining targets were not updated"}, nil
}
slog.DebugContext(ctx, "auto-update disabled or polling disabled; skipping run",
"autoUpdate", enabled, "pollingEnabled", pollingEnabled)
@@ -89,7 +89,7 @@ func (j *AutoUpdateJob) Run(ctx context.Context) (schedulertypes.Outcome, error)
}
if !admitted {
if len(options.ResourceIds) > 0 {
- return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: "Another update is active; review this run before retrying"}, nil
+ return schedulertypes.Outcome{Status: schedulertypes.Failed, Message: "Another update is active; the remaining targets were not updated"}, nil
}
slog.WarnContext(ctx, "auto-update run still in progress; skipping overlapping run")
return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil
@@ -106,7 +106,7 @@ func (j *AutoUpdateJob) Run(ctx context.Context) (schedulertypes.Outcome, error)
}
}
if result == nil {
- return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: "Updater returned no operation result"}, nil
+ return schedulertypes.Outcome{Status: schedulertypes.Failed, Message: "Updater returned no operation result"}, nil
}
return autoUpdateOutcomeInternal(ctx, result, unresolvedTargets, err)
@@ -123,7 +123,7 @@ func (j *AutoUpdateJob) Reconcile(ctx context.Context, previous schedulertypes.R
}
confirmed := confirmedAutoUpdateInternal(previous)
if confirmed.Status != schedulertypes.Succeeded {
- confirmed.Message = "Interrupted auto-update has no confirmed full-batch completion; review this run's results"
+ confirmed.Message = "Interrupted auto-update has no confirmed full-batch completion"
if previous.Outcome.Message != "" {
confirmed.Message += ": " + previous.Outcome.Message
}
@@ -160,12 +160,12 @@ func autoUpdateRetryOptionsInternal(previous schedulertypes.Run) (updatertypes.O
}
if hasResourceTargets && len(options.ResourceIds) == 0 {
if unresolvedTargets {
- return options, schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: "Remaining image or unconfirmed targets need review before retrying", Targets: previous.Outcome.Targets}, true
+ return options, schedulertypes.Outcome{Status: schedulertypes.Failed, Message: "Remaining targets have no confirmed completion or safe retry", Targets: previous.Outcome.Targets}, true
}
- return options, schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: "Target results do not confirm full-batch completion; review and resolve this run", ActivityID: previous.Outcome.ActivityID, Targets: previous.Outcome.Targets}, true
+ return options, schedulertypes.Outcome{Status: schedulertypes.Failed, Message: "Target results do not confirm full-batch completion", ActivityID: previous.Outcome.ActivityID, Targets: previous.Outcome.Targets}, true
}
if !hasResourceTargets {
- return options, schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: "The previous update has no confirmed retry targets; review this run's results", ActivityID: previous.Outcome.ActivityID, Targets: previous.Outcome.Targets}, true
+ return options, schedulertypes.Outcome{Status: schedulertypes.Failed, Message: "The previous update has no confirmed retry targets", ActivityID: previous.Outcome.ActivityID, Targets: previous.Outcome.Targets}, true
}
if len(options.ResourceIds) > 0 {
options.Type = "container"
@@ -177,7 +177,7 @@ func autoUpdateRetryOptionsInternal(previous schedulertypes.Run) (updatertypes.O
func (j *AutoUpdateJob) ValidateRetry(_ context.Context, previous schedulertypes.Run) error {
options, outcome, _ := autoUpdateRetryOptionsInternal(previous)
if outcome.Status != "" || len(options.ResourceIds) == 0 {
- return errors.New("auto-update has no safely retryable failed containers; review and resolve the run")
+ return errors.New("auto-update has no safely retryable failed containers")
}
return nil
}
@@ -203,14 +203,14 @@ func autoUpdateOutcomeInternal(ctx context.Context, result *updatertypes.Result,
}
if result.Failed > 0 || err != nil || unresolvedTargets {
outcome.Status = schedulertypes.Partial
- outcome.Message = "Some updates failed; review target outcomes before retrying"
+ outcome.Message = "Some updates failed"
if err != nil {
outcome.Message += ": " + err.Error()
}
}
if unresolvedTargets {
- outcome.Status = schedulertypes.NeedsAttention
- outcome.Message = "Unconfirmed targets remain; review this run's results"
+ outcome.Status = schedulertypes.Failed
+ outcome.Message = "Some target outcomes could not be confirmed"
}
if outcomeErr, ok := errors.AsType[*schedulertypes.OutcomeError](err); ok {
outcome.Status = outcomeErr.Outcome.Status
@@ -224,13 +224,13 @@ func confirmedAutoUpdateInternal(previous schedulertypes.Run) schedulertypes.Out
confirmed := jobcontext.ConfirmedTarget(previous, "auto-update")
for _, target := range previous.Outcome.Targets {
if target.ID == "auto-update" && target.ResourceType != "update-batch" {
- confirmed.Status = schedulertypes.NeedsAttention
+ confirmed.Status = schedulertypes.Failed
}
}
if confirmed.Status == schedulertypes.Succeeded {
for _, target := range previous.Outcome.Targets {
if target.Status != schedulertypes.Succeeded && target.Status != schedulertypes.Skipped {
- confirmed.Status = schedulertypes.NeedsAttention
+ confirmed.Status = schedulertypes.Failed
break
}
}
diff --git a/backend/pkg/scheduler/auto_update_job_test.go b/backend/pkg/scheduler/auto_update_job_test.go
index 73861840d..20f77e8eb 100644
--- a/backend/pkg/scheduler/auto_update_job_test.go
+++ b/backend/pkg/scheduler/auto_update_job_test.go
@@ -85,11 +85,11 @@ func TestAutoUpdateJob_OverlappingRunIsSkippedInternal(t *testing.T) {
status schedulertypes.RunStatus
unresolved bool
}{
- {name: "missing progress", unresolved: true, status: schedulertypes.NeedsAttention},
- {name: "image pull alone", unresolved: true, targets: []schedulertypes.TargetOutcome{{ID: "image", ResourceType: "image", Status: schedulertypes.Succeeded}}, status: schedulertypes.NeedsAttention},
+ {name: "missing progress", unresolved: true, status: schedulertypes.Failed},
+ {name: "image pull alone", unresolved: true, targets: []schedulertypes.TargetOutcome{{ID: "image", ResourceType: "image", Status: schedulertypes.Succeeded}}, status: schedulertypes.Failed},
{name: "failed containers only", unresolved: true, targets: []schedulertypes.TargetOutcome{{ID: "done", ResourceType: "container", Status: schedulertypes.Succeeded}, {ID: "failed", ResourceType: "container", Status: schedulertypes.Failed}, {ID: "image", ResourceType: "image", Status: schedulertypes.Failed}}, retryIDs: []string{"failed"}},
{name: "completed failed batch retry", targets: []schedulertypes.TargetOutcome{{ID: "auto-update", ResourceType: "update-batch", Status: schedulertypes.Partial}, {ID: "failed", ResourceType: "container", Status: schedulertypes.Failed}}, retryIDs: []string{"failed"}},
- {name: "scoped retry cannot confirm original batch", targets: []schedulertypes.TargetOutcome{{ID: "auto-update", ResourceType: "update-retry", Status: schedulertypes.Succeeded}, {ID: "done", ResourceType: "container", Status: schedulertypes.Succeeded}}, status: schedulertypes.NeedsAttention, unresolved: true},
+ {name: "scoped retry cannot confirm original batch", targets: []schedulertypes.TargetOutcome{{ID: "auto-update", ResourceType: "update-retry", Status: schedulertypes.Succeeded}, {ID: "done", ResourceType: "container", Status: schedulertypes.Succeeded}}, status: schedulertypes.Failed, unresolved: true},
{name: "completed batch", targets: []schedulertypes.TargetOutcome{{ID: "auto-update", ResourceType: "update-batch", Status: schedulertypes.Succeeded}}, status: schedulertypes.Succeeded},
} {
t.Run(tc.name, func(t *testing.T) {
@@ -108,7 +108,7 @@ func TestAutoUpdateJob_OverlappingRunIsSkippedInternal(t *testing.T) {
if tc.status == schedulertypes.Succeeded {
require.Equal(t, schedulertypes.Succeeded, reconciled.Status)
} else {
- require.Equal(t, schedulertypes.NeedsAttention, reconciled.Status)
+ require.Equal(t, schedulertypes.Failed, reconciled.Status)
}
})
}
@@ -117,6 +117,6 @@ func TestAutoUpdateJob_OverlappingRunIsSkippedInternal(t *testing.T) {
retryCtx := jobcontext.WithExecution(ctx, schedulertypes.Run{AttemptCount: 2, Outcome: schedulertypes.Outcome{Targets: []schedulertypes.TargetOutcome{{ID: "failed", ResourceType: "container", Status: schedulertypes.Failed}}}}, nil)
outcome, err := job.Run(retryCtx)
require.NoError(t, err)
- require.Equal(t, schedulertypes.NeedsAttention, outcome.Status)
+ require.Equal(t, schedulertypes.Failed, outcome.Status)
require.Equal(t, int32(2), applier.calls.Load())
}
diff --git a/backend/pkg/scheduler/image_update_watcher_test.go b/backend/pkg/scheduler/image_update_watcher_test.go
index c5d8dc028..5938a1b1f 100644
--- a/backend/pkg/scheduler/image_update_watcher_test.go
+++ b/backend/pkg/scheduler/image_update_watcher_test.go
@@ -569,10 +569,34 @@ func TestImageUpdateWatcher_ScheduledPollTriggersScanWithoutEvents(t *testing.T)
scanner := &imageUpdateScannerFakeInternal{}
settings := &pollingSettingReaderFakeInternal{enabled: true, schedule: "* * * * * *"}
watcher := newImageUpdateWatcherForTestInternal(t, scanner, settings, bus.NewDockerEventBus(), nil)
- startImageUpdateWatcherForTestInternal(t, watcher)
+ jobScheduler, err := NewJobScheduler(t.Context(), watcher.coordinator, time.UTC)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, jobScheduler.Stop(context.WithoutCancel(t.Context()))) })
+ require.NoError(t, jobScheduler.RegisterBusWatcher(watcher, true))
require.Eventually(t, func() bool { return scanner.countInternal() == 1 }, time.Second, 5*time.Millisecond)
- require.Eventually(t, func() bool { return scanner.countInternal() >= 2 }, 3*time.Second, 10*time.Millisecond)
+ require.Eventually(t, func() bool {
+ record, err := watcher.coordinator.ScheduleState(t.Context(), watcher.Name())
+ return err == nil && record.Schedule == settings.schedule && !record.NextRun.IsZero()
+ }, time.Second, 5*time.Millisecond)
+ require.NoError(t, jobScheduler.StartScheduler())
+ record, err := watcher.coordinator.ScheduleState(t.Context(), watcher.Name())
+ require.NoError(t, err)
+ require.Equal(t, settings.schedule, record.Schedule)
+ require.False(t, record.NextRun.IsZero())
+ require.Eventually(t, func() bool { return scanner.countInternal() >= 3 }, 5*time.Second, 10*time.Millisecond)
+ require.Eventually(t, func() bool {
+ history, err := watcher.coordinator.List(t.Context(), "0", watcher.Name(), 1, 10)
+ if err != nil {
+ return false
+ }
+ for _, run := range history.Runs {
+ if run.Trigger == "scheduled" && run.Status == schedulertypes.Succeeded {
+ return true
+ }
+ }
+ return false
+ }, time.Second, 5*time.Millisecond)
}
func TestImageUpdateWatcher_ClosedEventSubscriptionKeepsWatcherRunning(t *testing.T) {
diff --git a/backend/pkg/scheduler/jobcontext/context.go b/backend/pkg/scheduler/jobcontext/context.go
index 1390341df..38fc7bb86 100644
--- a/backend/pkg/scheduler/jobcontext/context.go
+++ b/backend/pkg/scheduler/jobcontext/context.go
@@ -40,5 +40,5 @@ func ConfirmedTarget(run schedulertypes.Run, targetID string) schedulertypes.Out
return schedulertypes.Outcome{Status: schedulertypes.Succeeded, ActivityID: target.ActivityID, Targets: run.Outcome.Targets}
}
}
- return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: "The interrupted operation has no confirmed completion", Targets: run.Outcome.Targets}
+ return schedulertypes.Outcome{Status: schedulertypes.Failed, Message: "The interrupted operation has no confirmed completion", Targets: run.Outcome.Targets}
}
diff --git a/backend/pkg/scheduler/runs/actors.go b/backend/pkg/scheduler/runs/actors.go
index 9470297e8..d3b226a82 100644
--- a/backend/pkg/scheduler/runs/actors.go
+++ b/backend/pkg/scheduler/runs/actors.go
@@ -34,6 +34,7 @@ type executorActorInternal struct {
}
func (q *Coordinator) Register(runtime *francis.Runtime) error {
+ q.runtime = runtime
if err := runtime.RegisterActor(coordinatorTypeInternal, func(id string, _ *actor.Service) actor.Actor {
return &coordinatorActorInternal{id: id, q: q}
}); err != nil {
@@ -77,7 +78,7 @@ func (a *coordinatorActorInternal) Invoke(ctx context.Context, method string, da
}
// Acceptance is persisted; dispatch failures are retried.
if err := a.flushInternal(ctx, &next); err != nil {
- slog.WarnContext(ctx, "Job dispatch requires reconciliation", "actorId", a.id, "error", err)
+ slog.WarnContext(ctx, "Job dispatch requires reconciliation", "jobId", next.Record.JobID, "environmentId", next.Record.EnvironmentID, "error", err)
a.q.signalInternal()
}
return true, nil
@@ -307,7 +308,7 @@ func (a *coordinatorActorInternal) Alarm(ctx context.Context, name string, data
}
pending := false
for _, run := range record.Runs {
- if run.Trigger == "manual" || run.Trigger == "remote" {
+ if separateTriggerInternal(run.Trigger) {
continue
}
if run.Status.Terminal() || run.Status == st.NeedsAttention {
@@ -437,6 +438,10 @@ func (a *executorActorInternal) Job(ctx context.Context, _ string, data actor.En
outcome.Status = st.Succeeded
}
}
+ // Domain recovery may still report uncertainty; job runs finish as failures.
+ if outcome.Status == st.NeedsAttention {
+ outcome.Status = st.Failed
+ }
if runErr != nil && outcome.Message == "" {
outcome.Message = runErr.Error()
}
@@ -447,9 +452,6 @@ func (a *executorActorInternal) Job(ctx context.Context, _ string, data actor.En
if ctx.Err() != nil {
return ctx.Err()
}
- if outcome.Status == st.NeedsAttention {
- return actor.ErrJobPermanentFailure
- }
return nil
}
@@ -481,7 +483,10 @@ func (q *Coordinator) claimExecutionInternal(ctx context.Context, command st.Exe
current.Status = st.Running
current.Owner = q.owner + "/" + uuid.New().String()
current.UpdatedAt = now
- current.StartedAt = &now
+ // Manager claims track delivery; accepted agent runs keep execution timestamps.
+ if current.EnvironmentID == "0" || !current.RemoteAccepted {
+ current.StartedAt = &now
+ }
current.NextAttempt = nil
current.AttemptCount = command.Attempt
current.Attempts = append(current.Attempts, st.Attempt{Number: current.AttemptCount, StartedAt: now})
diff --git a/backend/pkg/scheduler/runs/actors_test.go b/backend/pkg/scheduler/runs/actors_test.go
index 9fcffe92e..9e1cbb508 100644
--- a/backend/pkg/scheduler/runs/actors_test.go
+++ b/backend/pkg/scheduler/runs/actors_test.go
@@ -117,7 +117,7 @@ func TestDuplicateDispatchAndCanceledDelivery(t *testing.T) {
var executed atomic.Int32
q.SetExecutor(func(context.Context, st.Run) (st.Outcome, error) {
executed.Add(1)
- return st.Outcome{Status: st.Succeeded}, nil
+ return st.Outcome{Status: st.NeedsAttention, Message: "Completion could not be confirmed", Targets: []st.TargetOutcome{{ID: "target", Status: st.NeedsAttention}}}, nil
}, nil)
run, err := q.Submit(t.Context(), st.Request{JobID: "dispatch", Trigger: "manual"})
require.NoError(t, err)
@@ -148,7 +148,10 @@ func TestDuplicateDispatchAndCanceledDelivery(t *testing.T) {
require.Zero(t, executed.Load())
require.Zero(t, persisted.AttemptCount)
} else {
- require.Equal(t, st.Succeeded, persisted.Status)
+ require.Equal(t, st.Failed, persisted.Status)
+ require.Equal(t, "Completion could not be confirmed", persisted.Outcome.Message)
+ require.Equal(t, st.NeedsAttention, persisted.Outcome.Targets[0].Status)
+ require.NotNil(t, persisted.FinishedAt)
require.Equal(t, int32(1), executed.Load())
require.Equal(t, 1, persisted.AttemptCount)
}
diff --git a/backend/pkg/scheduler/runs/coordinator.go b/backend/pkg/scheduler/runs/coordinator.go
index d5e485d32..b25e3dd21 100644
--- a/backend/pkg/scheduler/runs/coordinator.go
+++ b/backend/pkg/scheduler/runs/coordinator.go
@@ -16,6 +16,7 @@ import (
"uuid"
"github.com/getarcaneapp/arcane/backend/v2/internal/kv"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis"
st "github.com/getarcaneapp/arcane/types/v2/scheduler"
"github.com/italypaleale/francis/actor"
kit "go.getarcane.app/kit/pkg"
@@ -25,6 +26,7 @@ import (
type Coordinator struct {
store *kv.KVService
service *actor.Service
+ runtime *francis.Runtime
owner string
mu sync.Mutex
checkpointed map[string]bool
@@ -64,18 +66,49 @@ func (q *Coordinator) Submit(ctx context.Context, request st.Request) (st.Run, e
return st.Run{}, errors.New("invalid run ID")
}
now := time.Now().UTC()
- result := st.Run{ID: request.RunID, JobID: request.JobID, EnvironmentID: request.EnvironmentID, Trigger: request.Trigger, RequestedBy: request.RequestedBy, RequestedWithKey: request.RequestedWithKey, RemoteAccepted: request.Trigger == "remote", Status: st.Queued, CreatedAt: now, UpdatedAt: now}
+ observed := st.Run{CreatedAt: now}
+ var remoteOutcome *st.Outcome
+ if agentRun := request.ObservedAgentRun; agentRun != nil {
+ switch {
+ case request.EnvironmentID == "0":
+ return st.Run{}, errors.New("observed agent run requires a remote environment")
+ case agentRun.EnvironmentID != "0":
+ return st.Run{}, errors.New("observed agent run must belong to the agent's local environment")
+ case agentRun.ID != request.RunID || agentRun.JobID != request.JobID:
+ return st.Run{}, errors.New("observed agent run does not match the requested run and job")
+ case agentRun.Status != st.NeedsAttention:
+ return st.Run{}, errors.New("observed agent run is not awaiting legacy resolution")
+ }
+ observed = *agentRun
+ remoteOutcome = new(observed.Outcome)
+ remoteOutcome.Status = observed.Status
+ }
+ result := st.Run{
+ ID: request.RunID,
+ JobID: request.JobID,
+ EnvironmentID: request.EnvironmentID,
+ Trigger: request.Trigger,
+ RequestedBy: request.RequestedBy,
+ RequestedWithKey: request.RequestedWithKey,
+ Status: st.Queued,
+ CreatedAt: observed.CreatedAt,
+ UpdatedAt: now,
+ StartedAt: observed.StartedAt,
+ FinishedAt: observed.FinishedAt,
+ ActivityID: observed.ActivityID,
+ Outcome: observed.Outcome,
+ RemoteAccepted: request.Trigger == "remote" || request.ObservedAgentRun != nil,
+ RemoteDeliveryAttempted: request.ObservedAgentRun != nil,
+ RemoteAttemptCount: observed.AttemptCount,
+ RemoteOutcome: remoteOutcome,
+ }
q.associateActivityInternal(&result)
err := q.mutateInternal(ctx, request.EnvironmentID, request.JobID, func(record *st.QueueRecord) error {
if request.Trigger == "scheduled" || request.Trigger == "recovery" {
record.LastEnqueuedAt = now
}
for _, existing := range record.Runs {
- if existing.ID == request.RunID {
- result = existing
- return nil
- }
- if request.Trigger != "manual" && request.Trigger != "remote" && existing.Trigger != "manual" && existing.Trigger != "remote" && !existing.Status.Terminal() && existing.Status != st.Running && existing.Status != st.NeedsAttention {
+ if existing.ID == request.RunID || coalescesPendingInternal(request, existing) {
result = existing
return nil
}
@@ -102,6 +135,24 @@ func (q *Coordinator) signalInternal() {
}
}
+// Manual and remote work never joins another run. Scheduled and recovery work
+// joins an existing run only while it is still waiting to start.
+func coalescesPendingInternal(request st.Request, existing st.Run) bool {
+ if request.ObservedAgentRun != nil || separateTriggerInternal(request.Trigger) || separateTriggerInternal(existing.Trigger) {
+ return false
+ }
+ return existing.Status == st.Queued || existing.Status == st.Waiting || existing.Status == st.Retrying
+}
+
+func separateTriggerInternal(trigger string) bool {
+ switch trigger {
+ case "manual", "remote":
+ return true
+ default:
+ return false
+ }
+}
+
// Checkpoint records the next due occurrence atomically with overdue recovery.
func (q *Coordinator) Checkpoint(ctx context.Context, jobID, schedule string, next time.Time) error {
q.mu.Lock()
@@ -167,10 +218,11 @@ func (q *Coordinator) Start(ctx context.Context) error {
}
for _, record := range records {
for _, run := range record.Runs {
- if q.observer != nil {
- if err := q.UpdateRun(ctx, run, func(*st.Run) error { return nil }); err != nil {
- return err
- }
+ if err := q.UpdateRun(ctx, run, func(current *st.Run) error {
+ normalizeLegacyRunInternal(current)
+ return nil
+ }); err != nil {
+ return err
}
}
}
@@ -270,7 +322,7 @@ func (q *Coordinator) persistOutcomeInternal(ctx context.Context, run st.Run, ou
} else {
current.NextAttempt = nil
}
- if outcome.Status.Terminal() {
+ if outcome.Status.Terminal() && (current.EnvironmentID == "0" || current.FinishedAt == nil) {
current.FinishedAt = &now
}
if len(current.Attempts) > 0 {
@@ -297,7 +349,7 @@ func (q *Coordinator) persistOutcomeInternal(ctx context.Context, run st.Run, ou
func (q *Coordinator) invokeInternal(ctx context.Context, run st.Run, reconcile bool) (outcome st.Outcome, err error) {
defer func() {
if recovered := recover(); recovered != nil {
- outcome = st.Outcome{Status: st.NeedsAttention, Message: "Job panicked; inspect logs before retrying"}
+ outcome = st.Outcome{Status: st.Failed, Message: "Job panicked; inspect logs before retrying"}
err = fmt.Errorf("job panic: %v", recovered)
slog.ErrorContext(ctx, "Job panicked", "runId", run.ID, "error", err)
}
@@ -423,7 +475,8 @@ func mergeTargetProgressInternal(previous, outcome st.Outcome) st.Outcome {
func (q *Coordinator) Active() bool { return q.enabled.Load() }
-// Resolve releases reviewed, inactive work without discarding its execution evidence.
+// Resolve releases legacy reviewed work without discarding execution evidence.
+// TODO(v3): remove this deprecated mixed-version compatibility contract.
func (q *Coordinator) Resolve(ctx context.Context, environmentID, jobID, runID, resolvedBy string) (st.Run, error) {
run, err := q.Get(ctx, environmentID, jobID, runID)
if err != nil {
diff --git a/backend/pkg/scheduler/runs/coordinator_store.go b/backend/pkg/scheduler/runs/coordinator_store.go
index a31670e2c..f8474a0d8 100644
--- a/backend/pkg/scheduler/runs/coordinator_store.go
+++ b/backend/pkg/scheduler/runs/coordinator_store.go
@@ -10,6 +10,7 @@ import (
"time"
"uuid"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/kv"
st "github.com/getarcaneapp/arcane/types/v2/scheduler"
"github.com/italypaleale/francis/actor"
@@ -20,6 +21,7 @@ const queuePrefixInternal = "jobs."
func (q *Coordinator) mutateInternal(ctx context.Context, environmentID, jobID string, change func(*st.QueueRecord) error) error {
id := kit.SHA256Hex(environmentID + "\x00" + jobID)
+ q.runtime.NameActor(id, jobID+"@"+environmentID)
for range 20 {
var state st.CoordinatorState
err := q.service.GetState(ctx, coordinatorTypeInternal, id, &state)
@@ -129,6 +131,7 @@ func (q *Coordinator) Records(ctx context.Context) ([]st.QueueRecord, error) {
return nil, err
}
normalizeRecordTimesInternal(&state.Record)
+ q.runtime.NameActor(entry.ActorID, state.Record.JobID+"@"+state.Record.EnvironmentID)
records = append(records, state.Record)
}
cursor = page.AfterID()
@@ -251,7 +254,7 @@ func pruneRunsInternal(record *st.QueueRecord, now time.Time) {
if terminal <= 100 && now.Sub(run.UpdatedAt) < 7*24*time.Hour {
continue
}
- if run.Trigger == "manual" || run.Trigger == "remote" {
+ if separateTriggerInternal(run.Trigger) {
if record.Receipts == nil {
record.Receipts = make(map[string]st.Run)
}
@@ -358,6 +361,26 @@ func normalizeRunTimesInternal(run *st.Run) {
}
}
+func normalizeLegacyRunInternal(run *st.Run) {
+ automaticResolution := run.Status == st.Canceled && run.Resolution != nil && run.Resolution.ResolvedBy == common.SystemUser.Username
+ if run.Status != st.NeedsAttention && !automaticResolution {
+ return
+ }
+ run.Owner = ""
+ run.NextAttempt = nil
+ if run.EnvironmentID != "0" && (run.RemoteDeliveryAttempted || run.RemoteAccepted) && !run.RemoteSettled {
+ run.Status = st.Waiting
+ run.FinishedAt = nil
+ } else {
+ run.Status = st.Failed
+ run.Outcome.Status = st.Failed
+ if run.FinishedAt == nil {
+ run.FinishedAt = new(run.UpdatedAt)
+ }
+ }
+ run.UpdatedAt = time.Now().UTC()
+}
+
func normalizeTimePointerInternal(value *time.Time) {
if value != nil {
*value = value.UTC()
diff --git a/backend/pkg/scheduler/runs/resolution_test.go b/backend/pkg/scheduler/runs/resolution_test.go
index 27d6fd87c..e4d262c3b 100644
--- a/backend/pkg/scheduler/runs/resolution_test.go
+++ b/backend/pkg/scheduler/runs/resolution_test.go
@@ -78,6 +78,43 @@ func TestResolvePreservesEvidenceWithoutBlockingSchedule(t *testing.T) {
require.Equal(t, resolved.Outcome, duplicate.Outcome)
require.Equal(t, resolved.Resolution.ResolvedBy, duplicate.Resolution.ResolvedBy)
require.True(t, resolved.Resolution.ResolvedAt.Equal(duplicate.Resolution.ResolvedAt))
+
+ // Startup closes legacy local failures and reconciles unsettled remote work.
+ localLegacy, err := q.Submit(ctx, st.Request{JobID: "gitops-sync:project", Trigger: "scheduled"})
+ require.NoError(t, err)
+ remoteLegacy, err := q.Submit(ctx, st.Request{JobID: "gitops-sync:project", EnvironmentID: "remote", Trigger: "scheduled"})
+ require.NoError(t, err)
+ for _, legacy := range []st.Run{localLegacy, remoteLegacy} {
+ require.NoError(t, q.UpdateRun(ctx, legacy, func(current *st.Run) error {
+ current.Status = st.NeedsAttention
+ current.Outcome = evidence
+ current.Attempts = []st.Attempt{{Number: 1, Outcome: evidence}}
+ current.RemoteDeliveryAttempted = current.EnvironmentID != "0"
+ return nil
+ }))
+ }
+ q.SetExecutor(func(context.Context, st.Run) (st.Outcome, error) {
+ t.Error("startup must not repeat legacy work")
+ return st.Outcome{Status: st.Failed}, nil
+ }, nil)
+ require.NoError(t, q.Start(ctx))
+ t.Cleanup(func() { require.NoError(t, q.Stop(context.WithoutCancel(ctx))) })
+ localLegacy, err = q.Get(ctx, "0", localLegacy.JobID, localLegacy.ID)
+ require.NoError(t, err)
+ require.Equal(t, st.Failed, localLegacy.Status)
+ require.Equal(t, evidence.Message, localLegacy.Outcome.Message)
+ require.Equal(t, evidence.Targets, localLegacy.Outcome.Targets)
+ require.Equal(t, evidence, localLegacy.Attempts[0].Outcome)
+ require.NotNil(t, localLegacy.FinishedAt)
+ remoteLegacy, err = q.Get(ctx, "remote", remoteLegacy.JobID, remoteLegacy.ID)
+ require.NoError(t, err)
+ require.Equal(t, st.Waiting, remoteLegacy.Status)
+ require.Equal(t, evidence, remoteLegacy.Outcome)
+ require.False(t, remoteLegacy.RemoteSettled)
+ require.Nil(t, remoteLegacy.FinishedAt)
+ future, err := q.Submit(ctx, st.Request{JobID: localLegacy.JobID, Trigger: "scheduled"})
+ require.NoError(t, err)
+ require.NotEqual(t, localLegacy.ID, future.ID)
}
func TestResolveRejectsUnsafeStates(t *testing.T) {
diff --git a/backend/pkg/scheduler/scheduler.go b/backend/pkg/scheduler/scheduler.go
index ea0e92329..75ef453a6 100644
--- a/backend/pkg/scheduler/scheduler.go
+++ b/backend/pkg/scheduler/scheduler.go
@@ -130,7 +130,10 @@ func (js *jobSchedulerInternal) StartScheduler() error {
return err
}
for _, record := range records {
- if record.EnvironmentID == "0" && record.Schedule != "" && !js.HasJob(record.JobID) {
+ js.mu.RLock()
+ _, watching := js.watchers[record.JobID]
+ js.mu.RUnlock()
+ if record.EnvironmentID == "0" && record.Schedule != "" && !js.HasJob(record.JobID) && !watching {
if err := js.coordinator.Checkpoint(js.context, record.JobID, "", time.Time{}); err != nil {
return err
}
diff --git a/backend/pkg/scheduler/scheduler_test.go b/backend/pkg/scheduler/scheduler_test.go
index e40be8d0c..026c901b2 100644
--- a/backend/pkg/scheduler/scheduler_test.go
+++ b/backend/pkg/scheduler/scheduler_test.go
@@ -270,10 +270,26 @@ func TestJobScheduler_RescheduleJob_AddsEntryWhenEnabled(t *testing.T) {
func TestJobScheduler_StartScheduler_SchedulesNonConditionalJobs(t *testing.T) {
js := newJobSchedulerForTestInternal(t, context.Background(), nil)
+ watcher := &testBusWatcherInternal{
+ name: "image-polling",
+ started: make(chan struct{}),
+ stopped: make(chan struct{}),
+ }
+ require.NoError(t, js.RegisterBusWatcher(watcher, true))
+ nextPolling := time.Now().UTC().Add(time.Hour)
+ require.NoError(t, js.coordinator.Checkpoint(t.Context(), watcher.Name(), "0 0 * * * *", nextPolling))
+ require.NoError(t, js.coordinator.Checkpoint(t.Context(), "removed-job", "0 0 * * * *", nextPolling))
+ executed := make(chan struct{}, 1)
job := &testSchedulerJob{
name: "test-non-conditional-startup",
schedule: "*/1 * * * * *",
+ run: func(context.Context) {
+ select {
+ case executed <- struct{}{}:
+ default:
+ }
+ },
}
require.NoError(t, js.RegisterJob(job))
@@ -283,6 +299,19 @@ func TestJobScheduler_StartScheduler_SchedulesNonConditionalJobs(t *testing.T) {
require.True(t, ok)
require.True(t, state.Scheduled)
requireScheduledJobInternal(t, js, job.Name())
+ polling, err := js.coordinator.ScheduleState(t.Context(), watcher.Name())
+ require.NoError(t, err)
+ require.Equal(t, "0 0 * * * *", polling.Schedule)
+ require.True(t, nextPolling.Equal(polling.NextRun))
+ removed, err := js.coordinator.ScheduleState(t.Context(), "removed-job")
+ require.NoError(t, err)
+ require.Empty(t, removed.Schedule)
+ require.True(t, removed.NextRun.IsZero())
+ select {
+ case <-executed:
+ case <-time.After(5 * time.Second):
+ require.FailNow(t, "scheduled job did not execute through Francis")
+ }
}
func TestJobScheduler_RescheduleJob_UsesProvidedContext(t *testing.T) {
diff --git a/backend/pkg/utils/francis/logging.go b/backend/pkg/utils/francis/logging.go
new file mode 100644
index 000000000..0fb39c0ef
--- /dev/null
+++ b/backend/pkg/utils/francis/logging.go
@@ -0,0 +1,97 @@
+package francis
+
+import (
+ "context"
+ "log/slog"
+ "slices"
+ "strings"
+ "sync"
+)
+
+// NameActor changes only the log display; durable actor IDs remain unchanged.
+func (r *Runtime) NameActor(actorID, name string) {
+ r.actorNames.Store(actorID, name)
+}
+
+type actorLogHandlerInternal struct {
+ slog.Handler
+
+ names *sync.Map
+ steps []actorLogStepInternal
+}
+
+type actorLogStepInternal struct {
+ attrs []slog.Attr
+ group string
+}
+
+func (h *actorLogHandlerInternal) Handle(ctx context.Context, record slog.Record) error {
+ handler := h.Handler
+ // Resolve at emission, including loggers created before restored jobs are loaded.
+ for _, step := range h.steps {
+ if step.group != "" {
+ handler = handler.WithGroup(step.group)
+ continue
+ }
+ attrs := make([]slog.Attr, len(step.attrs))
+ for i, attr := range step.attrs {
+ attrs[i] = h.nameAttrInternal(attr)
+ }
+ handler = handler.WithAttrs(attrs)
+ }
+ named := slog.NewRecord(record.Time, record.Level, record.Message, record.PC)
+ record.Attrs(func(attr slog.Attr) bool {
+ named.AddAttrs(h.nameAttrInternal(attr))
+ return true
+ })
+ return handler.Handle(ctx, named)
+}
+
+func (h *actorLogHandlerInternal) WithAttrs(attrs []slog.Attr) slog.Handler {
+ clone := *h
+ clone.steps = append(slices.Clone(h.steps), actorLogStepInternal{attrs: slices.Clone(attrs)})
+ return &clone
+}
+
+func (h *actorLogHandlerInternal) WithGroup(name string) slog.Handler {
+ if name == "" {
+ return h
+ }
+ clone := *h
+ clone.steps = append(slices.Clone(h.steps), actorLogStepInternal{group: name})
+ return &clone
+}
+
+func (h *actorLogHandlerInternal) nameAttrInternal(attr slog.Attr) slog.Attr {
+ attr.Value = attr.Value.Resolve()
+ if attr.Value.Kind() == slog.KindGroup {
+ attrs := slices.Clone(attr.Value.Group())
+ for i := range attrs {
+ attrs[i] = h.nameAttrInternal(attrs[i])
+ }
+ attr.Value = slog.GroupValue(attrs...)
+ return attr
+ }
+ if (attr.Key != "id" && attr.Key != "actorRef") || attr.Value.Kind() != slog.KindString {
+ return attr
+ }
+ actorType, rest, ok := strings.Cut(attr.Value.String(), "/")
+ if !ok {
+ return attr
+ }
+ actorID, suffix, hasSuffix := strings.Cut(rest, "/")
+ name, found := h.names.Load(actorID)
+ if !found {
+ return attr
+ }
+ displayName, ok := name.(string)
+ if !ok {
+ return attr
+ }
+ value := actorType + "/" + displayName
+ if hasSuffix {
+ value += "/" + suffix
+ }
+ attr.Value = slog.StringValue(value)
+ return attr
+}
diff --git a/backend/pkg/utils/francis/runtime.go b/backend/pkg/utils/francis/runtime.go
index f5ddbf924..a49585f5b 100644
--- a/backend/pkg/utils/francis/runtime.go
+++ b/backend/pkg/utils/francis/runtime.go
@@ -37,6 +37,7 @@ type Runtime struct {
started bool
runError error
address string
+ actorNames sync.Map
}
type registrationInternal struct {
@@ -62,12 +63,15 @@ func New(databaseURL, encryptionKey, instanceID, port string, options ...local.H
service: &actor.Service{}, ready: make(chan struct{}), done: make(chan struct{}), address: address,
options: []local.HostOption{
local.WithAddress(address), providerOption,
- local.WithLogger(slog.Default().With("scope", "actor-host")),
local.WithMaxHosts(1), local.WithHostHealthCheckDeadline(90 * time.Second),
local.WithShutdownGracePeriod(10 * time.Second), local.WithAlarmsPollInterval(time.Second),
local.WithAlarmsFetchAheadInterval(30 * time.Second), local.WithAlarmsLeaseDuration(180 * time.Second),
},
}
+ runtime.options = append(runtime.options, local.WithLogger(slog.New(&actorLogHandlerInternal{
+ Handler: slog.Default().With("scope", "actor-host").Handler(),
+ names: &runtime.actorNames,
+ })))
if encryptionKey != "" && instanceID != "" {
if err := runtime.ConfigureIdentity(encryptionKey, instanceID); err != nil {
return nil, err
diff --git a/backend/pkg/utils/francis/runtime_test.go b/backend/pkg/utils/francis/runtime_test.go
index 41bdea34a..3f1d94ceb 100644
--- a/backend/pkg/utils/francis/runtime_test.go
+++ b/backend/pkg/utils/francis/runtime_test.go
@@ -1,7 +1,9 @@
package francis
import (
+ "bytes"
"context"
+ "log/slog"
"net"
"os"
"path/filepath"
@@ -16,6 +18,23 @@ import (
func TestSQLiteHostRestartPersistence(t *testing.T) {
databaseURL := "file:" + filepath.ToSlash(filepath.Join(t.TempDir(), "actors.db"))
testRestartPersistenceInternal(t, databaseURL)
+
+ var output bytes.Buffer
+ runtime := &Runtime{}
+ logger := slog.New(&actorLogHandlerInternal{
+ Handler: slog.NewTextHandler(&output, nil),
+ names: &runtime.actorNames,
+ })
+ bound := logger.With("id", "job-executor/durable-hash/run-1").WithGroup("alarm")
+ runtime.NameActor("durable-hash", "image-polling@0")
+ bound.WarnContext(t.Context(), "Lease was lost", "actorRef", "job-coordinator/durable-hash")
+ require.Contains(t, output.String(), "id=job-executor/image-polling@0/run-1")
+ require.Contains(t, output.String(), "alarm.actorRef=job-coordinator/image-polling@0")
+ output.Reset()
+ logger.InfoContext(t.Context(), "unknown", "id", "other/unknown/run-1", "hostId", "durable-hash", slog.Group("nested", "actorRef", "job-health-executor/durable-hash"))
+ require.Contains(t, output.String(), "id=other/unknown/run-1")
+ require.Contains(t, output.String(), "hostId=durable-hash")
+ require.Contains(t, output.String(), "nested.actorRef=job-health-executor/image-polling@0")
}
func TestPostgresHostRestartPersistence(t *testing.T) {
@@ -35,6 +54,7 @@ func testRestartPersistenceInternal(t *testing.T, databaseURL string) {
first, err := New(databaseURL, "test-encryption-key", "test-instance", port)
require.NoError(t, err)
service := first.Service()
+ first.NameActor(actorID, "durable@0")
require.NoError(t, first.RegisterActor("test", func(string, *actor.Service) actor.Actor { return struct{}{} }))
require.NoError(t, first.Start(ctx, ctx, func(err error) { t.Errorf("unexpected host failure: %v", err) }))
t.Cleanup(func() { require.NoError(t, first.Stop(context.WithoutCancel(ctx))) })
@@ -48,6 +68,7 @@ func testRestartPersistenceInternal(t *testing.T, databaseURL string) {
second, err := New(databaseURL, "test-encryption-key", "test-instance", port)
require.NoError(t, err)
+ second.NameActor(actorID, "durable@0")
require.NoError(t, second.RegisterActor("test", func(string, *actor.Service) actor.Actor { return struct{}{} }))
require.NoError(t, second.Start(ctx, ctx, nil))
t.Cleanup(func() { require.NoError(t, second.Stop(context.WithoutCancel(ctx))) })
diff --git a/frontend/src/lib/components/job-card/job-card.svelte b/frontend/src/lib/components/job-card/job-card.svelte
index cefe72771..22967324a 100644
--- a/frontend/src/lib/components/job-card/job-card.svelte
+++ b/frontend/src/lib/components/job-card/job-card.svelte
@@ -96,7 +96,7 @@
id: 'worker-health',
visible: job.workerHealth,
label: m.jobs_worker_health(),
- value: jobStatusLabel(job.workerHealth?.status ?? '')
+ value: jobStatusLabel(job.workerHealth?.status ?? '', true)
}
].filter((item) => item.visible)
);
@@ -107,7 +107,7 @@
{
id: 'last',
visible: job.lastError,
- text: `${m.jobs_last_error()}: ${job.lastError}`,
+ text: job.lastError,
class: 'text-xs break-words text-destructive'
},
{
@@ -189,6 +189,7 @@
</div>
{#each errors as error (error.id)}<p class={error.class}>{error.text}</p>{/each}
+ {#if run?.status === 'waiting' && run.outcome.message}<p class="text-xs text-muted-foreground">{run.outcome.message}</p>{/if}
{#if job.workerHealth?.nextRetry}<p class="text-xs text-muted-foreground">
{m.jobs_next_retry()}: {formatDateTimeShort(job.workerHealth.nextRetry)}
</p>{/if}
diff --git a/frontend/src/lib/components/job-card/job-run-history.svelte b/frontend/src/lib/components/job-card/job-run-history.svelte
index f6fd79a73..db6901173 100644
--- a/frontend/src/lib/components/job-card/job-run-history.svelte
+++ b/frontend/src/lib/components/job-card/job-run-history.svelte
@@ -1,5 +1,4 @@
<script lang="ts">
- import { openConfirmDialog } from '#lib/components/confirm-dialog/index.js';
import IfPermitted from '#lib/components/if-permitted.svelte';
import { queryKeys } from '#lib/query/query-keys.js';
import { createQuery, createMutation } from '@tanstack/svelte-query';
@@ -32,7 +31,7 @@
refetchInterval: 5000
}));
const action = createMutation(() => ({
- mutationFn: ({ runId, action }: { runId: string; action: 'retry' | 'cancel' | 'resolve' }) =>
+ mutationFn: ({ runId, action }: { runId: string; action: 'retry' | 'cancel' }) =>
jobScheduleService.updateRun(jobId, runId, action, environmentId),
onSuccess: () => {
void runs.refetch();
@@ -87,16 +86,6 @@
}
].filter((item) => item.visible)
);
- function resolveRun(runId: string) {
- openConfirmDialog({
- title: m.jobs_resolve_run(),
- message: m.jobs_resolve_description(),
- confirm: {
- label: m.jobs_resolve_run(),
- action: () => action.mutateAsync({ runId, action: 'resolve' }).then(() => undefined)
- }
- });
- }
</script>
{#snippet outcomeMessage(message: string | undefined)}
@@ -170,8 +159,7 @@
{#each metadata as item (item.id)}<p class={item.class}>{item.label ? `${item.label}: ` : ''}{item.value}</p>{/each}
{@render outcomeActivity(run.activityId, m.jobs_activity_summary(), run.activityEnvironmentId ?? environmentId)}
{@render outcomeActivity(run.outcome.activityId)}
- {#if run.status === 'needs_attention'}<p>{m.jobs_attention_schedule_blocked()}</p>{/if}
- {#if run.resolution}
+ {#if run.resolution && run.resolution.resolvedBy !== 'System'}
<p>
{m.jobs_resolution_details({ user: run.resolution.resolvedBy, date: formatDateTimeShort(run.resolution.resolvedAt) })}
</p>
@@ -196,11 +184,6 @@
{/if}
{#if action.error}<p class="text-destructive">{action.error.message}</p>{/if}
<IfPermitted perm="jobs:manage" envId={environmentId}>
- {#if run.status === 'needs_attention'}
- <Button variant="outline" disabled={action.isPending} onclick={() => resolveRun(run.id)}
- >{m.jobs_resolve_run()}</Button
- >
- {/if}
{#each availableActions as item (item.id)}
<Button
variant={item.variant}
diff --git a/frontend/src/lib/components/job-card/job-status.ts b/frontend/src/lib/components/job-card/job-status.ts
index bded58d6b..2089ecda4 100644
--- a/frontend/src/lib/components/job-card/job-status.ts
+++ b/frontend/src/lib/components/job-card/job-status.ts
@@ -1,6 +1,6 @@
import type { JobStatus } from '#lib/types/settings.js';
import { m } from '#lib/paraglide/messages.js';
-export function jobStatusLabel(status: string): string {
+export function jobStatusLabel(status: string, isWorker = false): string {
const labels: Record<string, () => string> = {
queued: m.jobs_status_queued,
waiting: m.jobs_status_waiting,
@@ -10,7 +10,7 @@ export function jobStatusLabel(status: string): string {
partial: m.jobs_status_partial,
skipped: m.jobs_status_skipped,
failed: m.jobs_status_failed,
- needs_attention: m.jobs_status_needs_attention,
+ needs_attention: isWorker ? m.jobs_status_needs_attention : m.jobs_status_failed,
canceled: m.jobs_status_canceled,
healthy: m.jobs_status_healthy,
degraded: m.jobs_status_degraded,
diff --git a/frontend/src/lib/services/job-schedule-service.ts b/frontend/src/lib/services/job-schedule-service.ts
index bbdf980f7..5ef3c77f7 100644
--- a/frontend/src/lib/services/job-schedule-service.ts
+++ b/frontend/src/lib/services/job-schedule-service.ts
@@ -28,7 +28,7 @@ class JobScheduleService extends BaseAPIService {
this.api.get(`/environments/${environmentId}/jobs/${encodeURIComponent(jobId)}/runs/${encodeURIComponent(runId)}`)
);
}
- async updateRun(jobId: string, runId: string, action: 'retry' | 'cancel' | 'resolve', environmentId: string): Promise<JobRun> {
+ async updateRun(jobId: string, runId: string, action: 'retry' | 'cancel', environmentId: string): Promise<JobRun> {
return this.handleResponse(
this.api.post(
`/environments/${environmentId}/jobs/${encodeURIComponent(jobId)}/runs/${encodeURIComponent(runId)}/${action}`
diff --git a/types/jobschedule/runs.go b/types/jobschedule/runs.go
index 8e114eb9d..c46bad1f7 100644
--- a/types/jobschedule/runs.go
+++ b/types/jobschedule/runs.go
@@ -23,6 +23,7 @@ type (
)
// ResolveRunInput accepts the original operator identity only from trusted agent transport.
+// TODO(v3): remove this deprecated mixed-version compatibility contract.
type ResolveRunInput struct {
RunInput
diff --git a/types/scheduler/job.go b/types/scheduler/job.go
index f5fa02da5..0401a65f1 100644
--- a/types/scheduler/job.go
+++ b/types/scheduler/job.go
@@ -113,5 +113,5 @@ func (g *GenericJob) Reconcile(ctx context.Context, previous Run) (Outcome, erro
if g.ReconcileFn != nil {
return g.ReconcileFn(ctx, previous)
}
- return Outcome{Status: NeedsAttention, Message: "The interrupted operation has no confirmed result; review its domain records before retrying", Targets: previous.Outcome.Targets}, nil
+ return Outcome{Status: Failed, Message: "The interrupted operation has no confirmed completion", Targets: previous.Outcome.Targets}, nil
}
diff --git a/types/scheduler/run.go b/types/scheduler/run.go
index f4d91e70c..7ec541ed5 100644
--- a/types/scheduler/run.go
+++ b/types/scheduler/run.go
@@ -10,14 +10,15 @@ import (
type RunStatus string
const (
- Queued RunStatus = "queued"
- Waiting RunStatus = "waiting"
- Running RunStatus = "running"
- Retrying RunStatus = "retrying"
- Succeeded RunStatus = "succeeded"
- Partial RunStatus = "partial"
- Skipped RunStatus = "skipped"
- Failed RunStatus = "failed"
+ Queued RunStatus = "queued"
+ Waiting RunStatus = "waiting"
+ Running RunStatus = "running"
+ Retrying RunStatus = "retrying"
+ Succeeded RunStatus = "succeeded"
+ Partial RunStatus = "partial"
+ Skipped RunStatus = "skipped"
+ Failed RunStatus = "failed"
+ // NeedsAttention preserves domain recovery evidence and legacy job outcomes.
NeedsAttention RunStatus = "needs_attention"
Canceled RunStatus = "canceled"
)
@@ -43,6 +44,9 @@ type Outcome struct {
}
type Request struct {
+ // ObservedAgentRun adopts existing agent work without another submission.
+ // TODO(v3): remove legacy-agent adoption.
+ ObservedAgentRun *Run `json:"-"`
RunID string `json:"runId,omitempty"`
JobID string `json:"jobId"`
EnvironmentID string `json:"environmentId"`
@@ -58,6 +62,8 @@ type Attempt struct {
Outcome Outcome `json:"outcome"`
}
+// RunResolution preserves legacy resolution history.
+// TODO(v3): remove this deprecated mixed-version compatibility contract.
type RunResolution struct {
ResolvedBy string `json:"resolvedBy"`
ResolvedAt time.Time `json:"resolvedAt"`