Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
feat: add durable job execution and offline recovery (#3863)
Kyle Mendell58823c2ac4af
diff --git a/backend/internal/apns/service.go b/backend/internal/apns/service.goindex 48233ec0c..861a9a2bf 100644--- a/backend/internal/apns/service.go+++ b/backend/internal/apns/service.go@@ -487,7 +487,12 @@ func (s *ApnsService) Enqueue(ctx context.Context, environmentID, environmentNamif err := s.db.WithContext(ctx).Create(&entry).Error; err != nil {return errors.WrapIf(err, "failed to enqueue push notification")}- go s.DrainOutbox(context.WithoutCancel(ctx))+ go func() {+ drainCtx := context.WithoutCancel(ctx)+ if err := s.DrainOutbox(drainCtx); err != nil {+ slog.WarnContext(drainCtx, "Push outbox drain failed", "error", err)+ }+ }()return nil}@@ -523,44 +528,48 @@ func (s *ApnsService) recipientsInternal(ctx context.Context, environmentID strireturn recipients, nil}-func (s *ApnsService) DrainOutbox(ctx context.Context) {+func (s *ApnsService) DrainOutbox(ctx context.Context) error {s.drainPending.Store(true)if !s.drainMu.TryLock() {- return+ return nil}defer s.drainMu.Unlock()for s.drainPending.Swap(false) {- s.drainOutboxInternal(ctx)+ if err := s.drainOutboxInternal(ctx); err != nil {+ return err+ }}+ return nil}-func (s *ApnsService) drainOutboxInternal(ctx context.Context) {+func (s *ApnsService) drainOutboxInternal(ctx context.Context) error {if !s.Enabled(ctx) {- return+ return nil}var entries []OutboxEntryif err := s.db.WithContext(ctx).Where("next_attempt_at <= ?", time.Now()).Order("next_attempt_at ASC").Limit(outboxBatchSize).Find(&entries).Error; err != nil {slog.ErrorContext(ctx, "Failed to load push outbox", "error", err)- return+ return err}if len(entries) == 0 {- return+ return nil}channelID, err := s.EnsureChannel(ctx)if err != nil {slog.WarnContext(ctx, "Push outbox drain skipped", "error", err)- return+ return err}signer, err := s.signerInternal(ctx)if err != nil {slog.WarnContext(ctx, "Push outbox drain skipped", "error", err)- return+ return err}for _, entry := range entries {if channelRevoked := s.sendOutboxEntryInternal(ctx, entry, channelID, signer); channelRevoked {- return+ return errors.New("push channel revoked")}}+ return nil}func (s *ApnsService) sendOutboxEntryInternal(ctx context.Context, entry OutboxEntry, channelID string, signer *signerInternal) bool {diff --git a/backend/internal/auth/huma_middleware.go b/backend/internal/auth/huma_middleware.goindex dcc7e25bf..d982abe96 100644--- a/backend/internal/auth/huma_middleware.go+++ b/backend/internal/auth/huma_middleware.go@@ -276,6 +276,7 @@ func handleApiKeyAuthInternal(api huma.API, ctx huma.Context, authService *AuthSps = resolveApiKeyPermissionsInternal(ctx.Context(), permResolver, key.ID)}newCtx := setUserInContextInternal(ctx.Context(), user, ps)+ newCtx = context.WithValue(newCtx, middleware.ContextKeyApiKeyID, key.ID)next(huma.WithContext(ctx, newCtx))return}diff --git a/backend/internal/backup/policies.go b/backend/internal/backup/policies.goindex 408976e02..a494e7d4c 100644--- a/backend/internal/backup/policies.go+++ b/backend/internal/backup/policies.go@@ -9,7 +9,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/database"s3domain "github.com/getarcaneapp/arcane/backend/v2/internal/s3"- "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/schedule"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/schedule"backuptypes "github.com/getarcaneapp/arcane/types/v2/backup""gorm.io/gorm")diff --git a/backend/internal/bootstrap/bootstrap_test.go b/backend/internal/bootstrap/bootstrap_test.goindex 154bdd445..c4994c2c4 100644--- a/backend/internal/bootstrap/bootstrap_test.go+++ b/backend/internal/bootstrap/bootstrap_test.go@@ -16,16 +16,20 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/auth""github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/database"+ "github.com/getarcaneapp/arcane/backend/v2/internal/job"+ "github.com/getarcaneapp/arcane/backend/v2/internal/kv""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"tunnelpb "github.com/getarcaneapp/arcane/backend/v2/proto/tunnel/v1""github.com/labstack/echo/v5"+ "github.com/libtnb/sqlite""github.com/stretchr/testify/assert""github.com/stretchr/testify/require"libcrypto "go.getarcane.app/sys/crypto""go.uber.org/fx""go.uber.org/fx/fxtest""golang.org/x/net/http2"+ "gorm.io/gorm")type blockingBusWatcherInternal struct {@@ -381,7 +385,15 @@ func TestJobSchedulerStopCancelsItsPrivateContextInternal(t *testing.T) {lifecycle := fxtest.NewLifecycle(t)runtime, err := actors.NewRuntime(appCtx, lifecycle)require.NoError(t, err)- jobScheduler, err := newJobScheduler(appCtx, lifecycle, &config.Config{}, runtime, nil, nil, nil, nil)+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})+ require.NoError(t, err)+ require.NoError(t, db.AutoMigrate(&kv.KVEntry{}))+ sqlDB, err := db.DB()+ require.NoError(t, err)+ sqlDB.SetMaxOpenConns(1)+ t.Cleanup(func() { _ = sqlDB.Close() })+ jobService := job.NewJobService(&database.DB{DB: db}, nil, &config.Config{})+ jobScheduler, err := newJobScheduler(appCtx, lifecycle, &config.Config{}, runtime, nil, nil, nil, nil, jobService)require.NoError(t, err)watcher := &blockingBusWatcherInternal{started: make(chan struct{}),diff --git a/backend/internal/bootstrap/jobs_bootstrap.go b/backend/internal/bootstrap/jobs_bootstrap.goindex 0283bc4bc..0662ada5b 100644--- a/backend/internal/bootstrap/jobs_bootstrap.go+++ b/backend/internal/bootstrap/jobs_bootstrap.go@@ -7,6 +7,8 @@ import ("net/http""slices"+ "emperror.dev/errors"+"github.com/getarcaneapp/arcane/backend/v2/internal/activity""github.com/getarcaneapp/arcane/backend/v2/internal/actors""github.com/getarcaneapp/arcane/backend/v2/internal/apns"@@ -24,17 +26,19 @@ import ("go.uber.org/fx")-func newJobScheduler(appCtx context.Context, lc fx.Lifecycle, cfg *config.Config, runtime *actors.Runtime, _ *actors.Gate[actors.AdmissionKey], imageUpdateWatcher *scheduler.ImageUpdateWatcher, analytics *scheduler.AnalyticsJob, systemUpgrade *system.SystemUpgradeService) (schedulertypes.JobScheduler, error) {+func newJobScheduler(appCtx context.Context, lc fx.Lifecycle, cfg *config.Config, runtime *actors.Runtime, _ *actors.Gate[actors.AdmissionKey], imageUpdateWatcher *scheduler.ImageUpdateWatcher, analytics *scheduler.AnalyticsJob, systemUpgrade *system.SystemUpgradeService, jobService *job.JobService) (schedulertypes.JobScheduler, error) {schedulerCtx, cancelScheduler := context.WithCancel(appCtx)jobScheduler, err := scheduler.NewJobScheduler(schedulerCtx, runtime, cfg.GetLocation())if err != nil {cancelScheduler()return nil, err}+ jobService.SetScheduler(schedulerCtx, jobScheduler)lc.Append(fx.Hook{OnStart: func(context.Context) error {slog.InfoContext(appCtx, "Starting scheduler")if imageUpdateWatcher != nil {+ imageUpdateWatcher.SetDispatcher(jobService.Queue)if err := jobScheduler.RegisterBusWatcher(imageUpdateWatcher, true); err != nil {return err}@@ -42,8 +46,13 @@ func newJobScheduler(appCtx context.Context, lc fx.Lifecycle, cfg *config.Configif err := jobScheduler.StartScheduler(); err != nil {return err}+ if err := jobService.Queue.Start(schedulerCtx); err != nil {+ return err+ }if analytics != nil {- go analytics.Run(schedulerCtx)+ if _, err := jobService.Queue.Submit(schedulerCtx, schedulertypes.Request{JobID: analytics.Name(), EnvironmentID: "0", Trigger: "startup"}); err != nil {+ return err+ }}if !cfg.AgentMode && systemUpgrade != nil {go systemUpgrade.ResumeUpdateAllOnStartup(schedulerCtx)@@ -51,8 +60,8 @@ func newJobScheduler(appCtx context.Context, lc fx.Lifecycle, cfg *config.Configreturn nil},OnStop: func(ctx context.Context) error {- err := jobScheduler.Stop(ctx)cancelScheduler()+ err := errors.Combine(jobService.Queue.Stop(ctx), jobScheduler.Stop(ctx))if err != nil {slog.ErrorContext(ctx, "Job scheduler exited with error", "error", err)return errdiff --git a/backend/internal/database/database_test.go b/backend/internal/database/database_test.goindex a67d63518..028c940fc 100644--- a/backend/internal/database/database_test.go+++ b/backend/internal/database/database_test.go@@ -777,7 +777,9 @@ func TestIdentityNormalizationMigration(t *testing.T) {}var version int64require.NoError(t, db.QueryRow("SELECT MAX(version_id) FROM goose_db_version WHERE is_applied = 1").Scan(&version))- require.Equal(t, int64(81), version)+ highest, err := getHighestEmbeddedMigrationVersionInternal("sqlite")+ require.NoError(t, err)+ require.Equal(t, highest, version)})}}@@ -804,7 +806,9 @@ func TestIdentityNormalizationGooseUpgrade(t *testing.T) {require.Equal(t, "unchanged", password)var version int64require.NoError(t, db.QueryRow("SELECT MAX(version_id) FROM goose_db_version WHERE is_applied = 1").Scan(&version))- require.Equal(t, int64(81), version)+ highest, err := getHighestEmbeddedMigrationVersionInternal("sqlite")+ require.NoError(t, err)+ require.Equal(t, highest, version)}func TestIdentityNormalizationPostgresUpgrade(t *testing.T) {@@ -856,5 +860,7 @@ func TestIdentityNormalizationPostgresUpgrade(t *testing.T) {}var version int64require.NoError(t, db.QueryRowContext(ctx, "SELECT MAX(version_id) FROM goose_db_version WHERE is_applied").Scan(&version))- require.Equal(t, int64(81), version)+ highest, err := getHighestEmbeddedMigrationVersionInternal("postgres")+ require.NoError(t, err)+ require.Equal(t, highest, version)}diff --git a/backend/internal/di/providers.go b/backend/internal/di/providers.goindex bfcc3f1ec..e9b5e8d51 100644--- a/backend/internal/di/providers.go+++ b/backend/internal/di/providers.go@@ -5,6 +5,7 @@ import ("embed""log/slog""net/http"+ "time""github.com/getarcaneapp/arcane/backend/v2/internal/activity""github.com/getarcaneapp/arcane/backend/v2/internal/apikey"@@ -361,8 +362,8 @@ func provideUserModuleInternal(service *user.UserService, auth *auth.AuthServicereturn user.New(user.Dependencies{Service: service, InvalidateUserTokenCache: auth.InvalidateUserTokenCache, Settings: settingsService})}-func provideJobModuleInternal(db *database.DB, settingsService *settings.SettingsService, cfg *config.Config, environment *environment.EnvironmentService) *job.Module {- return job.New(job.Dependencies{DB: db, Settings: settingsService, Config: cfg, Environment: environment})+func provideJobModuleInternal(db *database.DB, settingsService *settings.SettingsService, cfg *config.Config, environment *environment.EnvironmentService, roles *role.RoleService) *job.Module {+ return job.New(job.Dependencies{DB: db, Settings: settingsService, Config: cfg, Environment: environment, Roles: roles})}func provideJobServiceInternal(module *job.Module) *job.JobService {@@ -411,8 +412,22 @@ func provideFilesystemWatcherJobInternal(ctx context.Context, lc fx.Lifecycle, alc.Append(fx.Hook{OnStart: func(context.Context) error {runner, err = actors.NewRunner(ctx, actorRuntime, "filesystem-watcher", "lifecycle", "filesystem watcher", 3, func(runCtx context.Context) error {- if startErr := job.Start(runCtx); startErr != nil {- return startErr+ delay := 5 * time.Second+ for runCtx.Err() == nil {+ startErr := job.Start(runCtx)+ if startErr == nil {+ break+ }+ slog.WarnContext(runCtx, "Filesystem watcher startup failed; retrying", "error", startErr)++ timer := time.NewTimer(delay)+ select {+ case <-runCtx.Done():+ timer.Stop()+ return nil+ case <-timer.C:+ }+ delay = min(delay*2, 5*time.Minute)}<-runCtx.Done()return nildiff --git a/backend/internal/environment/environment_health.go b/backend/internal/environment/environment_health.goindex 5923475e1..6e6e48432 100644--- a/backend/internal/environment/environment_health.go+++ b/backend/internal/environment/environment_health.go@@ -11,20 +11,19 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/actors""github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext""github.com/getarcaneapp/arcane/types/v2/environment"schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler""github.com/moby/moby/client")const (- defaultEnvironmentHealthInterval = "0 */2 * * * *"- environmentHealthCheckTimeout = 90 * time.Second+ defaultEnvironmentHealthInterval = "0 */2 * * * *"+ environmentHealthCheckTimeout = 90 * time.Second+ environmentHealthJobPrefix = "environment-health:"+ environmentHealthAdmissionScopeInternal = "environment-health")-const environmentHealthJobPrefix = "environment-health:"--const environmentHealthAdmissionScopeInternal = "environment-health"-// SetScheduler injects the job scheduler and app lifecycle context. Called during// bootstrap on the manager only (agent mode leaves scheduler nil, so all health-job// registration becomes a no-op).@@ -44,7 +43,10 @@ func (s *EnvironmentService) registerHealthJobInternal(ctx context.Context, envI}return sched},- func(ctx context.Context) { s.runHealthCheckInternal(ctx, envID) },+ func(ctx context.Context) (schedulertypes.Outcome, error) { return s.runHealthCheckInternal(ctx, envID) },+ func(context.Context, schedulertypes.Run) (schedulertypes.Outcome, error) {+ return schedulertypes.Outcome{Status: schedulertypes.Retrying, Message: "Connection checks can safely resume"}, nil+ },)}@@ -94,63 +96,92 @@ func (s *EnvironmentService) RescheduleHealthJobs(ctx context.Context) {s.registerAllEnabledHealthJobsInternal(ctx)}-// RunHealthChecksNow runs every enabled environment's health check synchronously.-// Backs the "run now" button for the environment-health job in the Jobs UI.+// RunHealthChecksNow queues each enabled environment's health check.+// Child runs retain the parent identity so authorization is checked at execution.func (s *EnvironmentService) RunHealthChecksNow(ctx context.Context) error {+ if !s.jobs.Enabled() {+ return errors.New("environment health scheduler unavailable")+ }ids, err := s.ListEnabledEnvironmentIDs(ctx)if err != nil {return err}+ parent, hasParent := jobcontext.Run(ctx)+ request := schedulertypes.Request{EnvironmentID: "0", Trigger: "internal"}+ if hasParent {+ request.Trigger = parent.Trigger+ request.RequestedBy = parent.RequestedBy+ request.RequestedWithKey = parent.RequestedWithKey+ }for _, id := range ids {- s.runHealthCheckInternal(ctx, id)+ request.JobID = s.jobs.JobName(id)+ if _, err := s.jobs.Scheduler().Submit(ctx, request); err != nil {+ return err+ }}return nil}// runHealthCheckInternal tests one environment's connection (updating its DB status)// and, for online remotes, syncs registries and repositories to it.-func (s *EnvironmentService) runHealthCheckInternal(ctx context.Context, envID string) {+func (s *EnvironmentService) runHealthCheckInternal(ctx context.Context, envID string) (schedulertypes.Outcome, error) {lease, admitted, err := s.jobs.TryAcquire(ctx, envID)if err != nil {slog.ErrorContext(ctx, "environment health check admission failed", "environment_id", envID, "error", err)- return+ return schedulertypes.Outcome{}, err}if !admitted {slog.WarnContext(ctx, "environment health check skipped; previous run still in progress", "environment_id", envID)- return+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil}defer lease.Release()+ environment, err := s.GetEnvironmentByID(ctx, envID)+ if err != nil {+ return schedulertypes.Outcome{}, err+ }+ if !environment.Enabled {+ return schedulertypes.Outcome{Status: schedulertypes.Canceled, Message: "Environment disabled"}, nil+ }status, err := s.TestConnection(ctx, envID, nil)switch {case err != nil:slog.WarnContext(ctx, "environment health check failed", "environment_id", envID, "status", status, "error", err)- return+ return schedulertypes.Outcome{Status: schedulertypes.Retrying}, errcase status != "online":- return+ return schedulertypes.Outcome{Status: schedulertypes.Retrying, Message: "Environment is offline"}, errors.New("environment is offline")}// Local environment (ID "0") has no registries/repositories to push.if envID == "0" {- return+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil}+ var syncErrors []errorsyncCtx, cancel := context.WithTimeout(ctx, environmentHealthCheckTimeout)defer cancel()if err := s.SyncRegistriesToEnvironment(syncCtx, envID); err != nil {slog.WarnContext(syncCtx, "failed to sync registries during health check", "environment_id", envID, "error", err)+ syncErrors = append(syncErrors, err)}if err := s.SyncS3DestinationsToEnvironment(syncCtx, envID); err != nil {slog.WarnContext(syncCtx, "failed to sync S3 destinations during health check", "environment_id", envID, "error", err)+ syncErrors = append(syncErrors, err)}if err := s.SyncRepositoriesToEnvironment(syncCtx, envID); err != nil {slog.WarnContext(syncCtx, "failed to sync git repositories during health check", "environment_id", envID, "error", err)+ syncErrors = append(syncErrors, err)}if s.variableSyncer != nil {if err := s.variableSyncer.SyncEnvironment(syncCtx, envID); err != nil {slog.WarnContext(syncCtx, "failed to sync global variables during health check", "environment_id", envID, "error", err)+ syncErrors = append(syncErrors, err)}}+ if err := errors.Combine(syncErrors...); err != nil {+ return schedulertypes.Outcome{Status: schedulertypes.Partial}, err+ }+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil}func (s *EnvironmentService) TestConnection(ctx context.Context, id string, customApiUrl *string) (string, error) {diff --git a/backend/internal/environment/service_test.go b/backend/internal/environment/service_test.goindex ba9beca20..3b511c34c 100644--- a/backend/internal/environment/service_test.go+++ b/backend/internal/environment/service_test.go@@ -67,8 +67,9 @@ func TestEnvironmentService_OverlappingHealthCheckIsSkippedInternal(t *testing.T}type environmentTestSchedulerInternal struct {- added []string- removed []string+ submitted []schedulertypes.Request+ added []string+ removed []string}func (s *environmentTestSchedulerInternal) AddJob(_ context.Context, job schedulertypes.Job) error {@@ -1429,3 +1430,8 @@ func TestEnvironmentService_ExecuteRemoteRequest_RejectsInvalidEnvironmentURL(trequire.Error(t, err)require.Contains(t, err.Error(), "invalid environment API URL")}++func (s *environmentTestSchedulerInternal) Submit(_ context.Context, request schedulertypes.Request) (schedulertypes.Run, error) {+ s.submitted = append(s.submitted, request)+ return schedulertypes.Run{ID: request.RunID, JobID: request.JobID, EnvironmentID: request.EnvironmentID, Status: schedulertypes.Queued}, nil+}diff --git a/backend/internal/gitops/gitops_sync.go b/backend/internal/gitops/gitops_sync.goindex b6efc0796..b7b30675b 100644--- a/backend/internal/gitops/gitops_sync.go+++ b/backend/internal/gitops/gitops_sync.go@@ -15,6 +15,8 @@ import ("strings""time"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"+"emperror.dev/errors""github.com/getarcaneapp/arcane/backend/v2/internal/actors""github.com/getarcaneapp/arcane/backend/v2/internal/common"@@ -375,23 +377,41 @@ func (s *GitOpsSyncService) SetScheduler(ctx context.Context, scheduler schedule// sync each run so a row toggled to AutoSync=false self-cancels. If the row is gone// (e.g. deleted out-of-band via raw SQL), the job unregisters itself instead of// firing forever; transient load errors are skipped and retried next tick.-func (s *GitOpsSyncService) runScheduledSyncInternal(ctx context.Context, environmentID, syncID string) {+func (s *GitOpsSyncService) runScheduledSyncInternal(ctx context.Context, environmentID, syncID string) (schedulertypes.Outcome, error) {syncRecord, err := s.getSyncRecordByIDInternal(ctx, environmentID, syncID)if err != nil {if errors.Is(err, common.ErrNotFound) {slog.InfoContext(ctx, "gitops auto-sync job unregistering; sync no longer exists", "syncId", syncID)s.unregisterSyncJobInternal(ctx, syncID)- return+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil}slog.DebugContext(ctx, "gitops auto-sync skipped; failed to load sync", "syncId", syncID, "error", err)- return+ return schedulertypes.Outcome{}, err}if !syncRecord.AutoSync {- return+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil+ }+ if previous, ok := jobcontext.Run(ctx); ok {+ outcome := jobcontext.ConfirmedTarget(previous, syncID)+ if outcome.Status == schedulertypes.Succeeded {+ return outcome, nil+ }+ }+ if err := jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ID: syncID, Status: schedulertypes.Running}); err != nil {+ return schedulertypes.Outcome{}, err}- if _, err := s.PerformSync(ctx, environmentID, syncID, common.SystemUser); err != nil {+ result, err := s.PerformSync(ctx, environmentID, syncID, common.SystemUser)+ if err != nil {slog.ErrorContext(ctx, "gitops auto-sync run failed", "syncId", syncID, "error", err)+ return schedulertypes.Outcome{}, err+ }+ if !result.Success {+ return schedulertypes.Outcome{Status: schedulertypes.Skipped, Message: result.Message}, nil}+ if err := jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ID: syncID, Status: schedulertypes.Succeeded}); err != nil {+ return schedulertypes.Outcome{}, err+ }+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded, Message: result.Message}, nil}// registerSyncJobInternal schedules a single sync on a fixed "@every Nm"@@ -402,7 +422,12 @@ func (s *GitOpsSyncService) registerSyncJobInternal(ctx context.Context, syncID,schedule := fmt.Sprintf("@every %dm", max(intervalMinutes, 1))s.jobs.Register(ctx, syncID,func(_ context.Context) string { return schedule },- func(ctx context.Context) { s.runScheduledSyncInternal(ctx, environmentID, syncID) },+ func(ctx context.Context) (schedulertypes.Outcome, error) {+ return s.runScheduledSyncInternal(ctx, environmentID, syncID)+ },+ func(_ context.Context, previous schedulertypes.Run) (schedulertypes.Outcome, error) {+ return jobcontext.ConfirmedTarget(previous, syncID), nil+ },)}@@ -413,13 +438,13 @@ func (s *GitOpsSyncService) unregisterSyncJobInternal(ctx context.Context, syncI// kickSyncInternal runs a sync once in the background on the app lifecycle context.// Used when auto-sync is freshly enabled or when a sync is overdue at startup, so// the first run does not wait a full interval.-func (s *GitOpsSyncService) kickSyncInternal(ctx context.Context, syncID, environmentID string) {- ctx = s.jobs.Context(ctx)- go func() {- if _, err := s.PerformSync(ctx, environmentID, syncID, common.SystemUser); err != nil {- slog.ErrorContext(ctx, "gitops immediate sync kick failed", "syncId", syncID, "error", err)- }- }()+func (s *GitOpsSyncService) kickSyncInternal(ctx context.Context, syncID string) {+ if !s.jobs.Enabled() {+ return+ }+ if _, err := s.jobs.Scheduler().Submit(ctx, schedulertypes.Request{JobID: s.jobs.JobName(syncID), EnvironmentID: "0", Trigger: "startup"}); err != nil {+ slog.ErrorContext(ctx, "gitops immediate sync admission failed", "syncId", syncID, "error", err)+ }}// RegisterAutoSyncJobsOnStartup registers a dynamic job for every auto-sync-enabled@@ -440,7 +465,7 @@ func (s *GitOpsSyncService) RegisterAutoSyncJobsOnStartup(ctx context.Context) {syncRecord := syncs[i]s.registerSyncJobInternal(ctx, syncRecord.ID, syncRecord.EnvironmentID, syncRecord.SyncInterval)if isGitOpsSyncOverdueInternal(&syncRecord) {- s.kickSyncInternal(ctx, syncRecord.ID, syncRecord.EnvironmentID)+ s.kickSyncInternal(ctx, syncRecord.ID)}}slog.InfoContext(ctx, "Registered gitops auto-sync jobs on startup", "count", len(syncs))@@ -814,7 +839,7 @@ func (s *GitOpsSyncService) UpdateSync(ctx context.Context, environmentID, id sts.registerSyncJobInternal(ctx, syncRecord.ID, syncRecord.EnvironmentID, newInterval)if !oldAutoSync {// Freshly enabled — kick a run now so it doesn't wait a full interval.- s.kickSyncInternal(ctx, syncRecord.ID, syncRecord.EnvironmentID)+ s.kickSyncInternal(ctx, syncRecord.ID)}default:s.unregisterSyncJobInternal(ctx, syncRecord.ID)diff --git a/backend/internal/gitops/service_test.go b/backend/internal/gitops/service_test.goindex ac64e4426..a0f80e9d5 100644--- a/backend/internal/gitops/service_test.go+++ b/backend/internal/gitops/service_test.go@@ -123,8 +123,9 @@ func TestGitOpsSyncService_OverlappingSyncPreservesSuccessShapedSkipInternal(t *}type gitOpsSyncTestSchedulerInternal struct {- added []string- removed []string+ submitted []schedulertypes.Request+ added []string+ removed []string}func (s *gitOpsSyncTestSchedulerInternal) AddJob(_ context.Context, job schedulertypes.Job) error {@@ -1877,3 +1878,8 @@ func TestMarkSyncRedeployFailedInternal_PersistsErrorOnSyncRow(t *testing.T) {require.Contains(t, *stored.SyncedFiles, "compose.yml")require.Contains(t, *stored.SyncedFiles, "scripts/pre-deploy.sh")}++func (s *gitOpsSyncTestSchedulerInternal) Submit(_ context.Context, request schedulertypes.Request) (schedulertypes.Run, error) {+ s.submitted = append(s.submitted, request)+ return schedulertypes.Run{ID: request.RunID, JobID: request.JobID, EnvironmentID: request.EnvironmentID, Status: schedulertypes.Queued}, nil+}diff --git a/backend/internal/job/execution.go b/backend/internal/job/execution.gonew file mode 100644index 000000000..7cc734e8f--- /dev/null+++ b/backend/internal/job/execution.go@@ -0,0 +1,287 @@+package job++import (+ "context"+ "log/slog"+ "net"+ "strings"+ "time"++ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/queue"++ "emperror.dev/errors"+ "github.com/getarcaneapp/arcane/backend/v2/internal/activity"+ "github.com/getarcaneapp/arcane/backend/v2/internal/apikey"+ "github.com/getarcaneapp/arcane/backend/v2/internal/common"+ "github.com/getarcaneapp/arcane/backend/v2/internal/role"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/authz"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"+ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"+ "github.com/getarcaneapp/arcane/types/v2/meta"+ st "github.com/getarcaneapp/arcane/types/v2/scheduler"+)++// Submit validates job eligibility and persists acceptance before execution.+// Managers may accept requests for offline environments. Reusing a run ID+// deduplicates admission within the same job and environment.+func (s *JobService) Submit(ctx context.Context, request st.Request) (st.Run, error) {+ if request.EnvironmentID == "" {+ request.EnvironmentID = "0"+ }+ if request.EnvironmentID != "0" {+ if s.cfg.AgentMode {+ return st.Run{}, errors.New("agents cannot queue work for another environment")+ }+ env, err := s.environment.GetEnvironmentByID(ctx, request.EnvironmentID)+ if err != nil {+ return st.Run{}, err+ }+ if !env.Enabled {+ return st.Run{}, errors.New("environment is disabled")+ }+ metadata, ok := meta.GetJobMetadata(request.JobID)+ dynamicRemote := strings.HasPrefix(request.JobID, "gitops-sync:") || strings.HasPrefix(request.JobID, "volume-backup:")+ if !dynamicRemote && (!ok || metadata.ManagerOnly || !metadata.CanRunManually) {+ return st.Run{}, errors.New("job is not remotely runnable")+ }+ } else if err := s.validateLocalJobInternal(ctx, request.JobID); err != nil {+ return st.Run{}, err+ }+ return s.Queue.Submit(ctx, request)+}++func (s *JobService) validateLocalJobInternal(ctx context.Context, jobID string) error {+ if metadata, ok := meta.GetJobMetadata(jobID); ok {+ if !metadata.CanRunManually {+ return errors.New("job cannot be run manually")+ }+ if s.cfg.AgentMode && metadata.ManagerOnly {+ return errors.New("job is manager-only")+ }+ if !s.isJobEnabledInternal(ctx, metadata) {+ return errors.New("job is disabled")+ }+ for _, prereq := range s.evaluatePrerequisitesInternal(ctx, metadata) {+ if !prereq.IsMet {+ return errors.New("job prerequisites are not met")+ }+ }+ if metadata.IsContinuous || jobID == "environment-health" {+ return nil+ }+ }+ if s.scheduler == nil {+ return errors.New("scheduler unavailable")+ }+ job, ok := s.scheduler.GetJob(jobID)+ if !ok {+ return errors.New("job is not registered")+ }+ if conditional, ok := job.(st.ConditionalJob); ok && !conditional.ShouldSchedule(ctx) {+ return errors.New("job is disabled")+ }+ return nil+}++func (s *JobService) authorizeRunInternal(ctx context.Context, run st.Run) error {+ if run.RequestedBy == "" {+ if run.Trigger == "manual" && !s.cfg.AgentMode {+ return errors.New("requesting user unavailable")+ }+ return nil+ }+ if s.roles == nil {+ return errors.New("permission resolver unavailable")+ }+ var user common.User+ if err := s.db.WithContext(ctx).First(&user, "id = ?", run.RequestedBy).Error; err != nil {+ return err+ }+ if run.RequestedWithKey != "" {+ return s.authorizeKeyInternal(ctx, run)+ }+ permissions, err := s.roles.ResolveUserPermissionsInDB(ctx, s.db.WithContext(ctx), user.ID)+ if err != nil {+ return err+ }+ if !permissions.Allows(authz.PermJobsManage, run.EnvironmentID) {+ return errors.New("requesting user no longer has permission to manage jobs")+ }+ return nil+}++func (s *JobService) executeRunInternal(ctx context.Context, run st.Run) (st.Outcome, error) {+ if err := s.authorizeRunInternal(ctx, run); err != nil {+ return st.Outcome{Status: st.NeedsAttention}, err+ }+ if run.EnvironmentID != "0" {+ return s.deliverRemoteInternal(ctx, run)+ }+ if requiresDockerInternal(run.JobID) && s.environment != nil {+ status, err := s.environment.TestConnection(ctx, "0", nil)+ if err != nil || status != "online" {+ return st.Outcome{Status: st.Waiting, Message: "Waiting for Docker"}, err+ }+ }+ ctx = s.runContextInternal(ctx, run)+ if run.JobID == "environment-health" && s.RunEnvironmentHealthNow != nil {+ err := s.RunEnvironmentHealthNow(ctx)+ return classifyOutcomeInternal(run.JobID, st.Outcome{}, err)+ }+ if metadata, ok := meta.GetJobMetadata(run.JobID); ok && metadata.IsContinuous {+ if err := s.validateLocalJobInternal(ctx, run.JobID); err != nil {+ return st.Outcome{Status: st.Canceled}, err+ }+ err := s.scheduler.RunBusWatcherNow(ctx, run.JobID)+ return classifyOutcomeInternal(run.JobID, st.Outcome{}, err)+ }+ job, ok := s.scheduler.GetJob(run.JobID)+ if !ok {+ return st.Outcome{Status: st.Canceled, Message: "Job or target no longer exists"}, nil+ }+ if conditional, ok := job.(st.ConditionalJob); ok && !conditional.ShouldSchedule(ctx) {+ return st.Outcome{Status: st.Canceled, Message: "Job is disabled"}, nil+ }+ outcome, err := job.Run(ctx)+ return classifyOutcomeInternal(run.JobID, outcome, err)+}++func classifyOutcomeInternal(jobID string, outcome st.Outcome, err error) (st.Outcome, error) {+ var resultErr *st.OutcomeError+ if errors.As(err, &resultErr) {+ return resultErr.Outcome, err+ }+ if err == nil {+ if outcome.Status == "" {+ outcome.Status = st.Succeeded+ }+ return outcome, nil+ }+ if outcome.Status == st.Waiting || outcome.Status == st.Retrying {+ return outcome, err+ }+ if outcome.Status == st.NeedsAttention || outcome.Status == st.Partial || outcome.Status == st.Canceled || outcome.Status == st.Skipped {+ return outcome, err+ }+ var networkErr net.Error+ switch {+ case safeJobInternal(jobID) && (errors.As(err, &networkErr) || errors.Is(err, context.DeadlineExceeded)):+ outcome.Status = st.Retrying+ case !safeJobInternal(jobID):+ outcome.Status = st.NeedsAttention+ case outcome.Status == "":+ outcome.Status = st.Failed+ }++ return outcome, err+}++func safeJobInternal(jobID string) bool {+ switch jobID {+ case "image-polling", "environment-health", "docker-client-refresh", "event-cleanup", "expired-sessions-cleanup", "activity-sweep", "upload-sessions-cleanup", "git-clone-scratch-cleanup", "analytics-heartbeat", "apns-outbox", "vulnerability-scan":+ return true+ }+ return strings.HasPrefix(jobID, "environment-health:")+}++func (s *JobService) reconcileRunInternal(ctx context.Context, run st.Run) (st.Outcome, error) {+ if err := s.authorizeRunInternal(ctx, run); err != nil {+ return st.Outcome{Status: st.NeedsAttention}, err+ }+ if run.EnvironmentID != "0" {+ return s.executeRunInternal(ctx, run)+ }+ if safeJobInternal(run.JobID) {+ return s.executeRunInternal(ctx, run)+ }+ // A successful activity proves only that target, never the entire batch.+ for index, target := range run.Outcome.Targets {+ if target.Status == st.Succeeded || target.Status == st.Skipped {+ continue+ }+ if target.ActivityID == "" {+ return st.Outcome{Status: st.NeedsAttention, Message: "Interrupted operation has no confirmed outcome", Targets: run.Outcome.Targets}, nil+ }+ var record activity.Activity+ if err := s.db.WithContext(ctx).First(&record, "id = ?", target.ActivityID).Error; err != nil {+ return st.Outcome{Status: st.NeedsAttention}, err+ }+ if record.Status != activitytypes.StatusSuccess {+ return st.Outcome{Status: st.NeedsAttention, Message: "Interrupted operation requires review", Targets: run.Outcome.Targets}, nil+ }+ run.Outcome.Targets[index].Status = st.Succeeded+ }+ if job, ok := s.scheduler.GetJob(run.JobID); ok {+ if reconciler, ok := job.(st.Reconciler); ok {+ outcome, err := reconciler.Reconcile(s.runContextInternal(ctx, run), run)+ if err != nil {+ slog.WarnContext(ctx, "Job reconciliation requires attention", "runId", run.ID, "error", err)+ }+ return outcome, err+ }+ }+ return st.Outcome{Status: st.NeedsAttention, Message: "Interrupted operation requires review before retrying", Targets: run.Outcome.Targets}, nil+}++func (s *JobService) runContextInternal(ctx context.Context, run st.Run) context.Context {+ ctx = jobcontext.WithExecution(ctx, run, func(target st.TargetOutcome) error {+ return s.Queue.UpdateRun(ctx, run, func(current *st.Run) error {+ if current.Status != st.Running || current.Owner != run.Owner {+ return queue.ErrRunConflict+ }+ for index := range current.Outcome.Targets {+ if current.Outcome.Targets[index].ID == target.ID {+ current.Outcome.Targets[index] = target+ return nil+ }+ }+ current.Outcome.Targets = append(current.Outcome.Targets, target)+ return nil+ })+ })+ return ctx+}++func (s *JobService) authorizeKeyInternal(ctx context.Context, run st.Run) error {+ var key apikey.ApiKey+ if err := s.db.WithContext(ctx).First(&key, "id = ?", run.RequestedWithKey).Error; err != nil {+ return err+ }+ if key.UserID == nil || *key.UserID != run.RequestedBy || (key.ExpiresAt != nil && !key.ExpiresAt.After(time.Now())) {+ return errors.New("requesting API key is no longer valid")+ }+ if key.Kind == apikey.ApiKeyKindPersonal {+ permissions, err := s.roles.ResolveUserPermissionsInDB(ctx, s.db.WithContext(ctx), run.RequestedBy)+ if err != nil {+ return err+ }+ if !permissions.Allows(authz.PermJobsManage, run.EnvironmentID) {+ return errors.New("requesting user no longer has permission to manage jobs")+ }+ return nil+ }+ var grants []role.ApiKeyPermission+ if err := s.db.WithContext(ctx).Where("api_key_id = ?", key.ID).Find(&grants).Error; err != nil {+ return err+ }+ permissions := authz.NewPermissionSet()+ for _, grant := range grants {+ if grant.EnvironmentID == nil {+ permissions.AddGlobal(grant.Permission)+ } else {+ permissions.AddEnv(*grant.EnvironmentID, grant.Permission)+ }+ }+ if !permissions.Allows(authz.PermJobsManage, run.EnvironmentID) {+ return errors.New("requesting API key no longer has permission to manage jobs")+ }+ return nil+}++func requiresDockerInternal(jobID string) bool {+ switch jobID {+ case "auto-update", "auto-patch", "auto-heal", "scheduled-prune", "vulnerability-scan", "image-polling":+ return true+ }+ return strings.HasPrefix(jobID, "gitops-sync:") || strings.HasPrefix(jobID, "volume-backup:") || strings.HasPrefix(jobID, "system-backup:")+}diff --git a/backend/internal/job/handler.go b/backend/internal/job/handler.goindex fffa2baff..434701e8b 100644--- a/backend/internal/job/handler.go+++ b/backend/internal/job/handler.go@@ -11,6 +11,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil""github.com/getarcaneapp/arcane/types/v2/base""github.com/getarcaneapp/arcane/types/v2/jobschedule"+ st "github.com/getarcaneapp/arcane/types/v2/scheduler")type GetJobSchedulesInput struct {@@ -35,8 +36,10 @@ type GetJobsOutput struct {}type RunJobInput struct {- ID string `path:"id" doc:"Environment ID"`- JobID string `path:"jobId" minLength:"1" doc:"Job ID to run"`+ ID string `path:"id" doc:"Environment ID"`+ JobID string `path:"jobId" minLength:"1" doc:"Job ID to run"`+ RequestID string `header:"Idempotency-Key"`+ Body *jobschedule.SubmitRunInput `required:"false"`}type RunJobOutput struct {@@ -80,14 +83,16 @@ func RegisterJobSchedules(api huma.API, jobSvc *JobService, envSvc *environment.}, authz.PermJobsManage, h.ListJobs)middleware.RegisterWithPermission(api, huma.Operation{- OperationID: "run-job",- Method: http.MethodPost,- Path: "/environments/{id}/jobs/{jobId}/run",- Summary: "Run a job now",- Description: "Manually trigger a background job to run immediately",- Tags: []string{"JobSchedules"},- Security: handlerutil.DefaultOperationSecurity(),+ OperationID: "run-job",+ DefaultStatus: http.StatusAccepted,+ Method: http.MethodPost,+ Path: "/environments/{id}/jobs/{jobId}/run",+ Summary: "Run a job now",+ Description: "Manually trigger a background job to run immediately",+ Tags: []string{"JobSchedules"},+ Security: handlerutil.DefaultOperationSecurity(),}, authz.PermJobsManage, h.RunJob)+ h.registerRunRoutesInternal(api)}type JobSchedulesHandler struct {@@ -96,42 +101,39 @@ type JobSchedulesHandler struct {}func (h *JobSchedulesHandler) ListJobs(ctx context.Context, input *ListJobsInput) (*GetJobsOutput, error) {+ var jobs *jobschedule.JobListResponse+ var err errorif input.ID != "0" {- jobs, err := h.proxyRemoteJSON.JSON[jobschedule.JobListResponse](ctx, input.ID, http.MethodGet, "/api/environments/0/jobs", nil)- if err != nil {- return nil, err- }- return &GetJobsOutput{Body: *jobs}, nil+ jobs, err = h.jobService.ListRemoteJobs(ctx, input.ID)+ } else {+ jobs, err = h.jobService.ListJobs(ctx)}-- jobs, err := h.jobService.ListJobs(ctx)if err != nil {- return nil, huma.Error500InternalServerError(err.Error())+ return nil, jobHTTPErrorInternal(err)}-return &GetJobsOutput{Body: *jobs}, nil}func (h *JobSchedulesHandler) RunJob(ctx context.Context, input *RunJobInput) (*RunJobOutput, error) {- if input.ID != "0" {- runResp, err := h.proxyRemoteJSON.JSON[jobschedule.JobRunResponse](ctx, input.ID, http.MethodPost, "/api/environments/0/jobs/"+input.JobID+"/run", nil)- if err != nil {- return nil, err+ runID := input.RequestID+ if input.Body != nil && input.Body.RunID != "" {+ if runID != "" && runID != input.Body.RunID {+ return nil, huma.Error400BadRequest("Conflicting run IDs")}- return &RunJobOutput{Body: *runResp}, nil+ runID = input.Body.RunID}-- err := h.jobService.RunJobNowInline(ctx, input.JobID)+ userID, _ := middleware.GetUserIDFromContext(ctx)+ keyID, _ := ctx.Value(middleware.ContextKeyApiKeyID).(string)+ trigger := "manual"+ if h.jobService.cfg.AgentMode && userID == "agent" {+ userID = ""+ trigger = "remote"+ }+ run, err := h.jobService.Submit(ctx, st.Request{RunID: runID, JobID: input.JobID, EnvironmentID: input.ID, Trigger: trigger, RequestedBy: userID, RequestedWithKey: keyID})if err != nil {- return nil, huma.Error400BadRequest(err.Error())+ return nil, jobHTTPErrorInternal(err)}-- return &RunJobOutput{- Body: jobschedule.JobRunResponse{- Success: true,- Message: "Job completed successfully",- },- }, nil+ return &RunJobOutput{Body: jobschedule.JobRunResponse{Success: true, Message: "Job accepted", RunID: run.ID, Status: run.Status}}, nil}func (h *JobSchedulesHandler) Get(ctx context.Context, input *GetJobSchedulesInput) (*GetJobSchedulesOutput, error) {diff --git a/backend/internal/job/module.go b/backend/internal/job/module.goindex e5120bc8d..b296a7f66 100644--- a/backend/internal/job/module.go+++ b/backend/internal/job/module.go@@ -7,10 +7,12 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/environment"+ "github.com/getarcaneapp/arcane/backend/v2/internal/role""github.com/getarcaneapp/arcane/backend/v2/internal/settings")type Dependencies struct {+ Roles *role.RoleServiceDB *database.DBSettings *settings.SettingsServiceConfig *config.Config@@ -23,8 +25,11 @@ type Module struct {}func New(deps Dependencies) *Module {+ service := NewJobService(deps.DB, deps.Settings, deps.Config)+ service.environment = deps.Environment+ service.roles = deps.Rolesreturn &Module{- service: NewJobService(deps.DB, deps.Settings, deps.Config),+ service: service,environment: deps.Environment,}}diff --git a/backend/internal/job/remote.go b/backend/internal/job/remote.gonew file mode 100644index 000000000..2e55f657b--- /dev/null+++ b/backend/internal/job/remote.go@@ -0,0 +1,284 @@+package job++import (+ "context"+ "crypto/sha256"+ "encoding/hex"+ "encoding/json/v2"+ "net/http"+ "net/url"+ "sort"+ "time"++ "emperror.dev/errors"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/remenv"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/queue"+ "github.com/getarcaneapp/arcane/types/v2/jobschedule"+ "github.com/getarcaneapp/arcane/types/v2/meta"+ st "github.com/getarcaneapp/arcane/types/v2/scheduler"+)++func remoteFailureInternal(err error) (st.Outcome, error) {+ var status *remenv.StatusError+ if errors.As(err, &status) && status.StatusCode >= 400 && status.StatusCode < 500 && status.StatusCode != http.StatusRequestTimeout && status.StatusCode != http.StatusTooManyRequests {+ return st.Outcome{Status: st.NeedsAttention, Message: "Remote agent rejected the request; check configuration and permissions"}, err+ }+ return st.Outcome{Status: st.Waiting, Message: "Waiting for environment"}, err+}++func (s *JobService) deliverRemoteInternal(ctx context.Context, run st.Run) (st.Outcome, error) {+ env, err := s.environment.GetEnvironmentByID(ctx, run.EnvironmentID)+ if err != nil {+ return st.Outcome{Status: st.NeedsAttention}, err+ }+ if !env.Enabled {+ return st.Outcome{Status: st.NeedsAttention, Message: "Environment is disabled"}, nil+ }+ basePath := "/api/environments/0/jobs/" + url.PathEscape(run.JobID)+ runPath := basePath + "/runs/" + url.PathEscape(run.ID)+ // A previously confirmed terminal result needs only acknowledgement, never execution.+ if !run.RemoteRetryRequested && run.RemoteAccepted && run.RemoteOutcome != nil && run.RemoteOutcome.Status.Terminal() {+ return s.acknowledgeRemoteInternal(ctx, run, runPath, *run.RemoteOutcome)+ }+ var catalog jobschedule.JobListResponse+ if err := s.environment.ProxyJSONRequest(ctx, run.EnvironmentID, http.MethodGet, "/api/environments/0/jobs", nil, &catalog); err != nil {+ return remoteFailureInternal(err)+ }+ if !catalog.DurableRuns {+ return st.Outcome{Status: st.NeedsAttention, Message: "Upgrade required: agent does not support durable job runs"}, nil+ }+ var remoteRun st.Run+ err = s.environment.ProxyJSONRequest(ctx, run.EnvironmentID, http.MethodGet, runPath, nil, &remoteRun)+ if err != nil {+ var status *remenv.StatusError+ if !errors.As(err, &status) || status.StatusCode != http.StatusNotFound {+ return remoteFailureInternal(err)+ }+ remoteRun, err = s.admitRemoteInternal(ctx, run, catalog)+ if err != nil {+ if errors.Is(err, errRemoteIneligibleInternal) {+ return st.Outcome{Status: st.Canceled, Message: err.Error()}, nil+ }+ if errors.Is(err, errRemoteReceiptMissingInternal) {+ return st.Outcome{Status: st.NeedsAttention, Message: err.Error()}, nil+ }+ return remoteFailureInternal(err)+ }+ }+ if remoteRun.ID != run.ID || remoteRun.JobID != run.JobID {+ return st.Outcome{Status: st.NeedsAttention, Message: "Agent returned an inconsistent run identity"}, nil+ }+ if run.RemoteRetryRequested && (remoteRun.Status.Terminal() || remoteRun.Status == st.NeedsAttention) && remoteRun.AttemptCount <= run.RemoteAttemptCount {+ remoteRun, err = s.retryDeliveryInternal(ctx, run, runPath)+ if err != nil {+ if errors.Is(err, errRemoteRetryUncertainInternal) {+ return st.Outcome{Status: st.NeedsAttention, Message: err.Error()}, nil+ }+ return remoteFailureInternal(err)+ }+ }+ return s.confirmRemoteRunInternal(ctx, run, remoteRun, runPath)+}++func (s *JobService) confirmRemoteRunInternal(ctx context.Context, run, remoteRun st.Run, runPath string) (st.Outcome, error) {+ now := time.Now().UTC()+ outcome := remoteRun.Outcome+ outcome.Status = remoteRun.Status+ if err := s.Queue.UpdateRun(ctx, run, func(current *st.Run) error {+ current.RemoteAccepted = true+ current.RemoteDeliveryAttempted = true+ current.LastConfirmedAt = &now+ current.RemoteOutcome = &outcome+ current.RemoteAttemptCount = remoteRun.AttemptCount+ current.RemoteRetryRequested = false+ current.Outcome = outcome+ return nil+ }); err != nil {+ return st.Outcome{Status: st.Retrying}, err+ }+ if outcome.Status.Terminal() {+ return s.acknowledgeRemoteInternal(ctx, run, runPath, outcome)+ }+ if outcome.Status == st.NeedsAttention {+ return outcome, nil+ }+ return st.Outcome{Status: st.Waiting, Message: "Waiting for the agent to complete this run", Targets: outcome.Targets, ActivityID: outcome.ActivityID}, nil+}++func (s *JobService) acknowledgeRemoteInternal(ctx context.Context, run st.Run, path string, outcome st.Outcome) (st.Outcome, error) {+ var response st.Run+ if err := s.environment.ProxyJSONRequest(ctx, run.EnvironmentID, http.MethodPost, path+"/ack", nil, &response); err != nil {+ return st.Outcome{Status: st.Waiting, Message: "Remote operation completed; waiting for delivery acknowledgement", Targets: outcome.Targets, ActivityID: outcome.ActivityID}, err+ }+ if response.ID != run.ID || !response.RemoteSettled {+ return st.Outcome{Status: st.Waiting, Message: "Remote operation completed; acknowledgement not confirmed"}, nil+ }+ if err := s.Queue.UpdateRun(ctx, run, func(current *st.Run) error { current.RemoteSettled = true; return nil }); err != nil {+ return st.Outcome{Status: st.Retrying}, err+ }+ return outcome, nil+}++// ListRemoteJobs retains the last observed agent catalog while its environment is offline.+func (s *JobService) ListRemoteJobs(ctx context.Context, environmentID string) (*jobschedule.JobListResponse, error) {+ if _, err := s.environment.GetEnvironmentByID(ctx, environmentID); err != nil {+ return nil, err+ }+ digest := sha256.Sum256([]byte(environmentID))+ key := "jobs-catalog/" + hex.EncodeToString(digest[:])+ var catalog jobschedule.JobListResponse+ remoteErr := s.environment.ProxyJSONRequest(ctx, environmentID, http.MethodGet, "/api/environments/0/jobs", nil, &catalog)+ if remoteErr == nil {+ catalog.ObservedAt = time.Now().UTC()+ catalog.Offline = false+ raw, err := json.Marshal(catalog)+ if err != nil {+ return nil, err+ }+ if err := s.store.Set(ctx, key, string(raw)); err != nil {+ return nil, err+ }+ } else {+ var status *remenv.StatusError+ if errors.As(remoteErr, &status) && (status.StatusCode == http.StatusUnauthorized || status.StatusCode == http.StatusForbidden) {+ return nil, remoteErr+ }+ raw, found, err := s.store.Get(ctx, key)+ if err != nil {+ return nil, err+ }+ if found {+ if err := json.Unmarshal([]byte(raw), &catalog); err != nil {+ return nil, err+ }+ } else {+ catalog.IsAgent = true+ for _, metadata := range meta.GetAllJobMetadata() {+ if !metadata.ManagerOnly {+ catalog.Jobs = append(catalog.Jobs, metadata.ToJobStatus("", nil, metadata.CanRunManually, []jobschedule.JobPrerequisite{}))+ }+ }+ }+ catalog.Offline = true+ }+ if err := s.addRuntimeStatusInternal(ctx, environmentID, &catalog.Jobs); err != nil {+ return nil, err+ }+ sort.Slice(catalog.Jobs, func(i, j int) bool { return catalog.Jobs[i].ID < catalog.Jobs[j].ID })+ return &catalog, nil+}++// RetryRemoteRun queues one explicit retry while retaining the agent's target progress.+func (s *JobService) RetryRemoteRun(ctx context.Context, environmentID, jobID, runID string) (st.Run, error) {+ run, err := s.Queue.Get(ctx, environmentID, jobID, runID)+ if errors.Is(err, queue.ErrRunNotFound) {+ return s.mutateAgentRunInternal(ctx, environmentID, jobID, runID, "retry")+ }+ if err != nil {+ return run, err+ }+ if err := s.authorizeRunInternal(ctx, run); err != nil {+ return run, err+ }+ err = s.Queue.UpdateRun(ctx, run, func(current *st.Run) error {+ if current.Status != st.Failed && current.Status != st.Partial && current.Status != st.NeedsAttention {+ return errors.New("run is not eligible for retry")+ }+ if !current.RemoteAccepted {+ if current.RemoteDeliveryAttempted {+ return errors.New("remote delivery is uncertain; inspect the agent before submitting new work")+ }+ } else {+ if current.RemoteOutcome == nil || (!current.RemoteOutcome.Status.Terminal() && current.RemoteOutcome.Status != st.NeedsAttention) {+ return errors.New("remote outcome must be confirmed before retry")+ }+ current.RemoteRetryRequested = true+ current.RemoteRetryAttempted = false+ current.RemoteSettled = false+ }+ current.Status = st.Queued+ current.Owner = ""+ current.NextAttempt = nil+ current.FinishedAt = nil+ current.UpdatedAt = time.Now().UTC()+ run = *current+ return nil+ })+ return run, err+}++var (+ errRemoteIneligibleInternal = errors.Sentinel("Remote job is disabled or no longer eligible")+ errRemoteReceiptMissingInternal = errors.Sentinel("Agent no longer has the delivery receipt; verify the operation before retrying")+ errRemoteRetryUncertainInternal = errors.Sentinel("Remote retry acknowledgement was lost; inspect the agent before retrying again")+)++func remoteJobEligibleInternal(jobs []jobschedule.JobStatus, id string) bool {+ for _, job := range jobs {+ if job.ID != id {+ if remoteJobEligibleInternal(job.Children, id) {+ return true+ }+ continue+ }+ if !job.Enabled || !job.CanRunManually || job.ManagerOnly {+ return false+ }+ for _, prerequisite := range job.Prerequisites {+ if !prerequisite.IsMet {+ return false+ }+ }+ return true+ }+ return false+}++func (s *JobService) admitRemoteInternal(ctx context.Context, run st.Run, catalog jobschedule.JobListResponse) (st.Run, error) {+ if run.RemoteAccepted {+ return st.Run{}, errRemoteReceiptMissingInternal+ }+ if !remoteJobEligibleInternal(catalog.Jobs, run.JobID) {+ return st.Run{}, errRemoteIneligibleInternal+ }+ // A confirmed 404 permits another delivery of the same deduplicated ID.+ if err := s.Queue.UpdateRun(ctx, run, func(current *st.Run) error {+ current.RemoteDeliveryAttempted = true+ return nil+ }); err != nil {+ return st.Run{}, err+ }+ body, err := json.Marshal(map[string]string{"runId": run.ID})+ if err != nil {+ return st.Run{}, err+ }+ path := "/api/environments/0/jobs/" + url.PathEscape(run.JobID) + "/run"+ var accepted jobschedule.JobRunResponse+ if err := s.environment.ProxyJSONRequest(ctx, run.EnvironmentID, http.MethodPost, path, body, &accepted); err != nil {+ return st.Run{}, err+ }+ if accepted.RunID != run.ID {+ return st.Run{}, errors.New("Agent returned an inconsistent delivery receipt")+ }+ return st.Run{ID: run.ID, JobID: run.JobID, Status: accepted.Status, Outcome: st.Outcome{Status: accepted.Status}}, nil+}++func (s *JobService) retryDeliveryInternal(ctx context.Context, run st.Run, path string) (st.Run, error) {+ if run.RemoteRetryAttempted {+ return st.Run{}, errRemoteRetryUncertainInternal+ }+ if err := s.Queue.UpdateRun(ctx, run, func(current *st.Run) error {+ current.RemoteRetryAttempted = true+ return nil+ }); err != nil {+ return st.Run{}, err+ }+ var retried st.Run+ if err := s.environment.ProxyJSONRequest(ctx, run.EnvironmentID, http.MethodPost, path+"/retry", nil, &retried); err != nil {+ return st.Run{}, err+ }+ if retried.ID != run.ID || retried.JobID != run.JobID {+ return st.Run{}, errors.New("Agent returned an inconsistent retry receipt")+ }+ return retried, nil+}diff --git a/backend/internal/job/remote_history.go b/backend/internal/job/remote_history.gonew file mode 100644index 000000000..f6338e12b--- /dev/null+++ b/backend/internal/job/remote_history.go@@ -0,0 +1,130 @@+package job++import (+ "context"+ "net/http"+ "net/url"+ "sort"+ "strconv"++ "emperror.dev/errors"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/remenv"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/queue"+ st "github.com/getarcaneapp/arcane/types/v2/scheduler"+)++// ListRuns merges agent history with requests accepted by this manager.+func (s *JobService) ListRuns(ctx context.Context, environmentID, jobID string, page, limit int) (st.RunList, error) {+ if environmentID == "0" {+ return s.Queue.List(ctx, environmentID, jobID, page, limit)+ }+ if _, err := s.environment.GetEnvironmentByID(ctx, environmentID); err != nil {+ return st.RunList{}, err+ }+ records, err := s.Queue.Records(ctx)+ if err != nil {+ return st.RunList{}, err+ }+ merged, err := s.remoteHistoryInternal(ctx, environmentID, jobID)+ if err != nil {+ var status *remenv.StatusError+ if errors.As(err, &status) && (status.StatusCode == http.StatusForbidden || status.StatusCode == http.StatusUnauthorized) {+ return st.RunList{}, err+ }+ merged = make(map[string]st.Run)+ }+ for _, record := range records {+ if record.EnvironmentID != environmentID || record.JobID != jobID {+ continue+ }+ for _, run := range record.Runs {+ merged[run.ID] = run+ }+ }+ runs := make([]st.Run, 0, len(merged))+ for _, run := range merged {+ runs = append(runs, run)+ }+ sort.Slice(runs, func(i, j int) bool {+ if runs[i].CreatedAt.Equal(runs[j].CreatedAt) {+ return runs[i].ID < runs[j].ID+ }+ return runs[i].CreatedAt.After(runs[j].CreatedAt)+ })+ page = max(page, 1)+ limit = min(max(limit, 1), 100)+ start := len(runs)+ if page-1 <= len(runs)/limit {+ start = min((page-1)*limit, len(runs))+ }+ return st.RunList{Runs: runs[start:min(start+limit, len(runs))], Total: len(runs), Page: page, Limit: limit}, nil+}++func (s *JobService) remoteHistoryInternal(ctx context.Context, environmentID, jobID string) (map[string]st.Run, error) {+ runs := make(map[string]st.Run)+ for page := 1; ; page++ {+ path := "/api/environments/0/jobs/" + url.PathEscape(jobID) + "/runs?page=" + strconv.Itoa(page) + "&limit=100"+ var response st.RunList+ if err := s.environment.ProxyJSONRequest(ctx, environmentID, http.MethodGet, path, nil, &response); err != nil {+ return nil, err+ }+ previous := len(runs)+ for _, run := range response.Runs {+ run.EnvironmentID = environmentID+ runs[run.ID] = run+ }+ if len(runs) >= response.Total || len(runs) == previous {+ return runs, nil+ }+ }+}++// GetRun prefers the manager's durable delivery record, then queries agent-owned history.+func (s *JobService) GetRun(ctx context.Context, environmentID, jobID, runID string) (st.Run, error) {+ run, err := s.Queue.Get(ctx, environmentID, jobID, runID)+ if environmentID == "0" || !errors.Is(err, queue.ErrRunNotFound) {+ return run, err+ }+ path := "/api/environments/0/jobs/" + url.PathEscape(jobID) + "/runs/" + url.PathEscape(runID)+ if err := s.environment.ProxyJSONRequest(ctx, environmentID, http.MethodGet, path, nil, &run); err != nil {+ return st.Run{}, err+ }+ if run.ID != runID || run.JobID != jobID {+ return st.Run{}, errors.New("agent returned an inconsistent run identity")+ }+ run.EnvironmentID = environmentID+ return run, nil+}++// CancelRun cancels manager admission or explicitly forwards an agent-owned cancellation.+func (s *JobService) CancelRun(ctx context.Context, environmentID, jobID, runID string) (st.Run, error) {+ _, err := s.Queue.Get(ctx, environmentID, jobID, runID)+ if environmentID != "0" && errors.Is(err, queue.ErrRunNotFound) {+ return s.mutateAgentRunInternal(ctx, environmentID, jobID, runID, "cancel")+ }+ if err != nil {+ return st.Run{}, err+ }+ return s.Queue.Cancel(ctx, environmentID, jobID, runID)+}++func (s *JobService) mutateAgentRunInternal(ctx context.Context, environmentID, jobID, runID, action string) (st.Run, error) {+ environment, err := s.environment.GetEnvironmentByID(ctx, environmentID)+ if err != nil {+ return st.Run{}, err+ }+ if !environment.Enabled {+ return st.Run{}, errors.New("environment is disabled")+ }+ path := "/api/environments/0/jobs/" + url.PathEscape(jobID) + "/runs/" + url.PathEscape(runID) + "/" + action+ var run st.Run+ // This is an explicit operator action. Never replay a lost mutation response.+ if err := s.environment.ProxyJSONRequest(ctx, environmentID, http.MethodPost, path, nil, &run); err != nil {+ return st.Run{}, err+ }+ if run.ID != runID || run.JobID != jobID {+ return st.Run{}, errors.New("agent returned an inconsistent run identity")+ }+ run.EnvironmentID = environmentID+ return run, nil+}diff --git a/backend/internal/job/run_routes.go b/backend/internal/job/run_routes.gonew file mode 100644index 000000000..cc7053f2b--- /dev/null+++ b/backend/internal/job/run_routes.go@@ -0,0 +1,133 @@+package job++import (+ "context"+ "net/http"+ "time"++ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/queue"++ "emperror.dev/errors"+ "github.com/danielgtaylor/huma/v2"+ "github.com/getarcaneapp/arcane/backend/v2/internal/middleware"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/authz"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"+ "github.com/getarcaneapp/arcane/types/v2/jobschedule"+ st "github.com/getarcaneapp/arcane/types/v2/scheduler"+)++func (h *JobSchedulesHandler) registerRunRoutesInternal(api huma.API) {+ base := "/environments/{id}/jobs/{jobId}"+ middleware.RegisterWithPermission(api, huma.Operation{OperationID: "list-job-runs", Method: http.MethodGet, Path: base + "/runs", Summary: "List job runs", Security: handlerutil.DefaultOperationSecurity()}, authz.PermJobsManage, h.ListRuns)+ middleware.RegisterWithPermission(api, huma.Operation{OperationID: "get-job-run", Method: http.MethodGet, Path: base + "/runs/{runId}", Summary: "Get job run", Security: handlerutil.DefaultOperationSecurity()}, authz.PermJobsManage, h.GetRun)+ middleware.RegisterWithPermission(api, huma.Operation{OperationID: "retry-job-run", Method: http.MethodPost, Path: base + "/runs/{runId}/retry", Summary: "Retry job run", Security: handlerutil.DefaultOperationSecurity()}, authz.PermJobsManage, h.RetryRun)+ middleware.RegisterWithPermission(api, huma.Operation{OperationID: "cancel-job-run", Method: http.MethodPost, Path: base + "/runs/{runId}/cancel", Summary: "Cancel pending job run", Security: handlerutil.DefaultOperationSecurity()}, authz.PermJobsManage, h.CancelRun)+ middleware.RegisterWithPermission(api, huma.Operation{OperationID: "ack-job-run", Method: http.MethodPost, Path: base + "/runs/{runId}/ack", Summary: "Acknowledge remote run completion", Security: handlerutil.DefaultOperationSecurity()}, authz.PermJobsManage, h.AcknowledgeRun)+ middleware.RegisterWithPermission(api, huma.Operation{OperationID: "restart-job-worker", Method: http.MethodPost, Path: base + "/restart", Summary: "Restart continuous job worker", Security: handlerutil.DefaultOperationSecurity()}, authz.PermJobsManage, h.RestartWorker)+}++// ListRuns returns paginated history for the requested job and environment.+// Remote history combines agent runs with the manager's delivery records.+func (h *JobSchedulesHandler) ListRuns(ctx context.Context, input *jobschedule.ListRunsInput) (*jobschedule.ListRunsOutput, error) {+ result, err := h.jobService.ListRuns(ctx, input.ID, input.JobID, input.Page, input.Limit)+ if err != nil {+ return nil, jobHTTPErrorInternal(err)+ }+ return &jobschedule.ListRunsOutput{Body: result}, nil+}++// GetRun returns a persisted run or receipt. For remote work, it queries the+// agent when the manager has no delivery record for the requested run.+func (h *JobSchedulesHandler) GetRun(ctx context.Context, input *jobschedule.RunInput) (*jobschedule.RunOutput, error) {+ result, err := h.jobService.GetRun(ctx, input.ID, input.JobID, input.RunID)+ if err != nil {+ return nil, jobHTTPErrorInternal(err)+ }+ return &jobschedule.RunOutput{Body: result}, nil+}++// RetryRun requests an explicit retry while preserving the run ID and target+// progress. Eligibility and remote-outcome checks are enforced by the service.+func (h *JobSchedulesHandler) RetryRun(ctx context.Context, input *jobschedule.RunInput) (*jobschedule.RunOutput, error) {+ result, err := h.jobService.RetryRun(ctx, input.ID, input.JobID, input.RunID)+ if err != nil {+ return nil, jobHTTPErrorInternal(err)+ }+ return &jobschedule.RunOutput{Body: result}, nil+}++// CancelRun cancels queued work only when it has never been attempted or sent+// remotely. Agent-owned cancellations are forwarded to the owning environment.+func (h *JobSchedulesHandler) CancelRun(ctx context.Context, input *jobschedule.RunInput) (*jobschedule.RunOutput, error) {+ result, err := h.jobService.CancelRun(ctx, input.ID, input.JobID, input.RunID)+ if err != nil {+ return nil, jobHTTPErrorInternal(err)+ }+ return &jobschedule.RunOutput{Body: result}, nil+}++// AcknowledgeRun marks a terminal run's remote delivery settled so retention can+// compact its history. Nonterminal runs cannot be acknowledged.+func (h *JobSchedulesHandler) AcknowledgeRun(ctx context.Context, input *jobschedule.RunInput) (*jobschedule.RunOutput, error) {+ result, err := h.jobService.GetRun(ctx, input.ID, input.JobID, input.RunID)+ if err != nil {+ return nil, jobHTTPErrorInternal(err)+ }+ err = h.jobService.Queue.UpdateRun(ctx, result, func(current *st.Run) error {+ if !current.Status.Terminal() {+ return errors.New("run is not terminal")+ }+ current.RemoteSettled = true+ current.UpdatedAt = time.Now().UTC()+ result = *current+ return nil+ })+ if err != nil {+ return nil, jobHTTPErrorInternal(err)+ }+ return &jobschedule.RunOutput{Body: result}, nil+}++// RestartWorker requests a continuous watcher's restart on the owning runtime,+// forwarding remote requests to the agent. It does not enqueue a scheduled run.+func (h *JobSchedulesHandler) RestartWorker(ctx context.Context, input *jobschedule.JobInput) (*RunJobOutput, error) {+ if input.ID != "0" {+ result, err := h.proxyRemoteJSON.JSON[jobschedule.JobRunResponse](ctx, input.ID, http.MethodPost, "/api/environments/0/jobs/"+input.JobID+"/restart", nil)+ if err != nil {+ return nil, err+ }+ return &RunJobOutput{Body: *result}, nil+ }+ runtime, ok := h.jobService.scheduler.(st.WorkerController)+ if !ok {+ return nil, huma.Error503ServiceUnavailable("Scheduler unavailable")+ }+ if err := runtime.RestartWatcher(ctx, input.JobID); err != nil {+ return nil, jobHTTPErrorInternal(err)+ }+ return &RunJobOutput{Body: jobschedule.JobRunResponse{Success: true, Message: "Worker restart requested"}}, nil+}++func jobHTTPErrorInternal(err error) error {+ if errors.Is(err, queue.ErrRunNotFound) {+ return huma.Error404NotFound("Job run not found")+ }+ if errors.Is(err, queue.ErrRunConflict) {+ return huma.Error409Conflict("Job run state changed")+ }+ return huma.Error400BadRequest(err.Error())+}++// RetryRun revalidates local job eligibility before requeuing the existing run.+// Remote retries follow the delivery protocol and retain confirmed target progress.+func (s *JobService) RetryRun(ctx context.Context, environmentID, jobID, runID string) (st.Run, error) {+ if environmentID != "0" {+ return s.RetryRemoteRun(ctx, environmentID, jobID, runID)+ }+ if environmentID == "0" {+ if err := s.validateLocalJobInternal(ctx, jobID); err != nil {+ return st.Run{}, err+ }+ }+ return s.Queue.Retry(ctx, environmentID, jobID, runID)+}diff --git a/backend/internal/job/runtime_status.go b/backend/internal/job/runtime_status.gonew file mode 100644index 000000000..45ae4cea4--- /dev/null+++ b/backend/internal/job/runtime_status.go@@ -0,0 +1,235 @@+package job++import (+ "context"+ "slices"+ "strings"+ "time"++ "emperror.dev/errors"+ "github.com/getarcaneapp/arcane/types/v2/jobschedule"+ st "github.com/getarcaneapp/arcane/types/v2/scheduler"+)++func (s *JobService) addRuntimeStatusInternal(ctx context.Context, environmentID string, jobs *[]jobschedule.JobStatus) error {+ runsByJob := make(map[string][]st.Run)+ if s.Queue != nil {+ records, err := s.Queue.Records(ctx)+ if err != nil {+ return errors.WrapIf(err, "load durable job status")+ }+ for _, record := range records {+ if record.EnvironmentID == environmentID {+ runsByJob[record.JobID] = append(runsByJob[record.JobID], record.Runs...)+ }+ }+ }+ runtime, hasRuntime := s.scheduler.(st.JobScheduler)+ if environmentID == "0" && hasRuntime {+ appendDynamicJobStatusesInternal(ctx, runtime, jobs)+ }++ for index := range *jobs {+ status := &(*jobs)[index]+ applyRunStatusInternal(status, runsByJob[status.ID])+ if environmentID == "0" && hasRuntime {+ setRuntimeHealthInternal(ctx, runtime, status)+ }+ for childIndex := range status.Children {+ child := &status.Children[childIndex]+ applyRunStatusInternal(child, runsByJob[child.ID])+ }+ }+ return nil+}++func appendDynamicJobStatusesInternal(ctx context.Context, runtime st.JobScheduler, jobs *[]jobschedule.JobStatus) {+ parentIndices := make(map[string]int, len(*jobs))+ for index, status := range *jobs {+ parentIndices[status.ID] = index+ }+ registered := runtime.ListRegisteredJobs()+ slices.SortFunc(registered, func(left, right st.Job) int {+ return strings.Compare(left.Name(), right.Name())+ })+ for _, registeredJob := range registered {+ parentID, _, dynamic := strings.Cut(registeredJob.Name(), ":")+ if !dynamic {+ continue+ }+ if parentID == "environment-health" {+ if registeredJob.Name() == "environment-health:0" {+ if index, exists := parentIndices[parentID]; exists {+ health := dynamicJobStatusInternal(ctx, runtime, registeredJob, "monitoring", true)+ health.Name = (*jobs)[index].Name+ (*jobs)[index] = health+ }+ }+ continue+ }+ category, managerOnly, supported := dynamicJobCategoryInternal(parentID)+ if !supported {+ continue+ }+ parentIndex, exists := parentIndices[parentID]+ if !exists {+ parentIndex = len(*jobs)+ parentIndices[parentID] = parentIndex+ *jobs = append(*jobs, jobschedule.JobStatus{+ ID: parentID,+ Name: parentID,+ Category: category,+ Enabled: true,+ ManagerOnly: managerOnly,+ Prerequisites: []jobschedule.JobPrerequisite{},+ })+ }+ parent := &(*jobs)[parentIndex]+ duplicate := slices.ContainsFunc(parent.Children, func(child jobschedule.JobStatus) bool {+ return child.ID == registeredJob.Name()+ })+ if duplicate {+ continue+ }+ parent.Children = append(parent.Children, dynamicJobStatusInternal(ctx, runtime, registeredJob, category, managerOnly))+ }+}++func dynamicJobCategoryInternal(parentID string) (category string, managerOnly, supported bool) {+ switch parentID {+ case "environment-health":+ return "monitoring", true, true+ case "gitops-sync":+ return "sync", false, true+ case "system-backup":+ return "maintenance", true, true+ case "volume-backup":+ return "maintenance", false, true+ default:+ return "", false, false+ }+}++func dynamicJobStatusInternal(ctx context.Context, runtime st.JobScheduler, job st.Job, category string, managerOnly bool) jobschedule.JobStatus {+ child := jobschedule.JobStatus{+ ID: job.Name(),+ Name: job.Name(),+ Category: category,+ Schedule: job.Schedule(ctx),+ Enabled: true,+ ManagerOnly: managerOnly,+ CanRunManually: true,+ Prerequisites: []jobschedule.JobPrerequisite{},+ }+ if conditional, ok := job.(st.ConditionalJob); ok {+ child.Enabled = conditional.ShouldSchedule(ctx)+ }+ if state, ok := runtime.GetJobRuntimeState(child.ID); ok {+ child.Schedule = state.Schedule+ child.NextRun = state.NextRun+ }+ return child+}++func applyRunStatusInternal(status *jobschedule.JobStatus, runs []st.Run) {+ var errorAt time.Time+ if status.LastError != "" {+ for _, run := range []*st.Run{status.CurrentRun, status.LastRun} {+ if run != nil && run.UpdatedAt.After(errorAt) {+ errorAt = run.UpdatedAt+ }+ }+ }+ for _, run := range runs {+ updateLatestRunInternal(status, run)+ updateLastSuccessInternal(status, run)+ if run.UpdatedAt.After(errorAt) && run.Outcome.Message != "" && runHasErrorInternal(run.Status) {+ status.LastError = run.Outcome.Message+ errorAt = run.UpdatedAt+ }+ }+}++func updateLatestRunInternal(status *jobschedule.JobStatus, run st.Run) {+ if !run.Status.Terminal() {+ if status.CurrentRun == nil || preferCurrentRunInternal(run, *status.CurrentRun) {+ status.CurrentRun = new(run)+ }+ return+ }+ current := status.CurrentRun+ if current != nil && current.ID == run.ID && run.UpdatedAt.After(current.UpdatedAt) && (!run.RemoteAccepted || run.RemoteSettled) {+ status.CurrentRun = nil+ }+ if status.LastRun == nil || run.UpdatedAt.After(status.LastRun.UpdatedAt) {+ status.LastRun = new(run)+ }+}++func updateLastSuccessInternal(status *jobschedule.JobStatus, run st.Run) {+ if run.Status != st.Succeeded {+ return+ }+ succeededAt := run.UpdatedAt+ if run.FinishedAt != nil {+ succeededAt = *run.FinishedAt+ }+ if status.LastSuccess == nil || succeededAt.After(*status.LastSuccess) {+ status.LastSuccess = new(succeededAt)+ }+}++func runHasErrorInternal(status st.RunStatus) bool {+ switch status {+ case st.Failed, st.Partial, st.NeedsAttention, st.Retrying, st.Waiting:+ return true+ case st.Queued, st.Running, st.Succeeded, st.Skipped, st.Canceled:+ return false+ default:+ return false+ }+}++func preferCurrentRunInternal(candidate, current st.Run) bool {+ if candidate.ID == current.ID {+ return candidate.UpdatedAt.After(current.UpdatedAt)+ }+ candidatePriority := currentRunPriorityInternal(candidate.Status)+ currentPriority := currentRunPriorityInternal(current.Status)+ if candidatePriority != currentPriority {+ return candidatePriority < currentPriority+ }+ return candidate.CreatedAt.Before(current.CreatedAt)+}++func currentRunPriorityInternal(status st.RunStatus) int {+ switch status {+ case st.Running:+ return 0+ case st.NeedsAttention:+ return 1+ case st.Queued, st.Waiting, st.Retrying, st.Succeeded, st.Partial, st.Skipped, st.Failed, st.Canceled:+ return 2+ default:+ return 2+ }+}++func setRuntimeHealthInternal(ctx context.Context, runtime st.JobScheduler, status *jobschedule.JobStatus) {+ if health, found := runtime.WatcherHealth(status.ID); found {+ status.WorkerHealth = &health+ return+ }+ registered, found := runtime.GetJob(status.ID)+ if !found {+ return+ }+ if conditional, ok := registered.(st.ConditionalJob); ok && !conditional.ShouldSchedule(ctx) {+ return+ }+ state, found := runtime.GetJobRuntimeState(status.ID)+ if !found || !state.Scheduled {+ status.NextRun = nil+ status.WorkerHealth = &st.WorkerHealth{Status: "needs_attention", LastError: "Job has no installed schedule", UpdatedAt: time.Now().UTC()}+ }+}diff --git a/backend/internal/job/service.go b/backend/internal/job/service.goindex 30d7d4308..3689ef7ad 100644--- a/backend/internal/job/service.go+++ b/backend/internal/job/service.go@@ -9,8 +9,13 @@ import ("emperror.dev/errors""github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/database"+ "github.com/getarcaneapp/arcane/backend/v2/internal/environment"+ "github.com/getarcaneapp/arcane/backend/v2/internal/kv"+ "github.com/getarcaneapp/arcane/backend/v2/internal/role""github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/queue"+ scheduleutil "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/schedule""github.com/getarcaneapp/arcane/types/v2/jobschedule""github.com/getarcaneapp/arcane/types/v2/meta"schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"@@ -26,6 +31,10 @@ import (// NOTE: This is intentionally separate from settings.SettingsService to keep the API// surface job-focused and to centralize schedule validation/rescheduling.type JobService struct {+ Queue *queue.Queue+ store *kv.KVService+ environment *environment.EnvironmentService+ roles *role.RoleServicedb *database.DBsettings *settings.SettingsServicecfg *config.Config@@ -42,12 +51,15 @@ type JobService struct {}func NewJobService(db *database.DB, settings *settings.SettingsService, cfg *config.Config) *JobService {- return &JobService{+ service := &JobService{+ store: kv.NewKVService(db),db: db,settings: settings,cfg: cfg,location: cfg.GetLocation(),}+ service.Queue = queue.New(service.store, service.executeRunInternal, service.reconcileRunInternal)+ return service}func (s *JobService) SetScheduler(ctx context.Context, scheduler schedulertypes.JobController) { //nolint:contextcheck // scheduler jobs must capture the app lifecycle context, not request contexts@@ -56,6 +68,9 @@ func (s *JobService) SetScheduler(ctx context.Context, scheduler schedulertypes.}s.lifecycleCtx = ctxs.scheduler = scheduler+ if runtime, ok := scheduler.(schedulertypes.JobScheduler); ok {+ runtime.SetDispatcher(s.Queue)+ }}func (s *JobService) GetJobSchedules(ctx context.Context) jobschedule.Config {@@ -104,7 +119,7 @@ func (s *JobService) UpdateJobSchedules(ctx context.Context, updates jobschedule}// Validate inputs (cron expressions)- parser := cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow)+ parser := scheduleutil.Parser()for _, field := range fields {if field.update == nil || *field.update == "" {continue@@ -280,8 +295,10 @@ func (s *JobService) ListJobs(ctx context.Context) (*jobschedule.JobListResponseprerequisites := s.evaluatePrerequisitesInternal(ctx, jobMeta)if s.scheduler != nil && jobMeta.SettingsKey != "" && jobMeta.ID != "environment-health" {- if runtimeState, ok := s.scheduler.GetJobRuntimeState(jobMeta.ID); ok && runtimeState.Schedule != "" {- schedule = runtimeState.Schedule+ if runtimeState, ok := s.scheduler.GetJobRuntimeState(jobMeta.ID); ok {+ if runtimeState.Schedule != "" {+ schedule = runtimeState.Schedule+ }nextRun = runtimeState.NextRun}}@@ -290,6 +307,10 @@ func (s *JobService) ListJobs(ctx context.Context) (*jobschedule.JobListResponsejobs = append(jobs, jobStatus)}+ if err := s.addRuntimeStatusInternal(ctx, "0", &jobs); err != nil {+ return nil, err+ }+// Sort jobs by ID to ensure stable UI ordersort.Slice(jobs, func(i, j int) bool {return jobs[i].ID < jobs[j].ID@@ -298,62 +319,12 @@ func (s *JobService) ListJobs(ctx context.Context) (*jobschedule.JobListResponseisAgent := s.cfg != nil && s.cfg.AgentModereturn &jobschedule.JobListResponse{- Jobs: jobs,- IsAgent: isAgent,+ Jobs: jobs,+ IsAgent: isAgent,+ DurableRuns: true, ObservedAt: time.Now().UTC(),}, nil}-func (s *JobService) RunJobNowInline(ctx context.Context, jobID string) error {- // environment-health runs as per-environment dynamic jobs; "run now" fans out- // through environment.EnvironmentService rather than a single registered job.- if jobID == "environment-health" && s.RunEnvironmentHealthNow != nil {- return s.RunEnvironmentHealthNow(context.WithoutCancel(ctx))- }- jobMeta, ok := meta.GetJobMetadata(jobID)- if ok && jobMeta.IsContinuous && jobMeta.CanRunManually {- if s == nil || s.scheduler == nil {- return errors.New("job service or scheduler not initialized")- }- return s.scheduler.RunBusWatcherNow(context.WithoutCancel(ctx), jobID)- }-- job, err := s.getRunnableJobInternal(jobID)- if err != nil {- return err- }-- runCtx := context.WithoutCancel(ctx)- job.Run(runCtx)-- return nil-}--func (s *JobService) getRunnableJobInternal(jobID string) (schedulertypes.Job, error) {- if s == nil || s.scheduler == nil {- return nil, errors.New("job service or scheduler not initialized")- }-- jobMeta, ok := meta.GetJobMetadata(jobID)- if !ok {- return nil, errors.Errorf("unknown job: %s", jobID)- }-- if !jobMeta.CanRunManually {- return nil, errors.Errorf("job %s cannot be run manually", jobID)- }-- if s.cfg != nil && s.cfg.AgentMode && jobMeta.ManagerOnly {- return nil, errors.Errorf("job %s is manager-only and cannot run in agent mode", jobID)- }-- job, ok := s.scheduler.GetJob(jobID)- if !ok {- return nil, errors.Errorf("job %s not found in scheduler", jobID)- }-- return job, nil-}-func (s *JobService) getJobScheduleInternal(ctx context.Context, meta meta.JobMetadata) string {// Continuous jobs with a settings key (image-polling) are event-driven but// also poll on that cron schedule, so surface the real expression.@@ -411,7 +382,7 @@ func (s *JobService) calculateNextRunInternal(schedule string) *time.Time {}// Parse schedule and force it to use the same timezone as the scheduler.- parser := cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow)+ parser := scheduleutil.Parser()sched, err := parser.Parse(schedule)if err != nil {return nildiff --git a/backend/internal/job/service_test.go b/backend/internal/job/service_test.goindex fca9032c0..c769da482 100644--- a/backend/internal/job/service_test.go+++ b/backend/internal/job/service_test.go@@ -2,13 +2,14 @@ package jobimport ("context"- "errors"+ "emperror.dev/errors""testing""time""github.com/getarcaneapp/arcane/backend/v2/internal/actors""github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/database"+ "github.com/getarcaneapp/arcane/backend/v2/internal/kv""github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/types/v2/jobschedule"schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"@@ -22,7 +23,7 @@ func setupSettingsTestDBInternal(t *testing.T) *database.DB {t.Helper()db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})require.NoError(t, err)- require.NoError(t, db.AutoMigrate(&settings.SettingVariable{}))+ require.NoError(t, db.AutoMigrate(&settings.SettingVariable{}, &kv.KVEntry{}))return &database.DB{DB: db}}@@ -330,17 +331,21 @@ func TestJobService_UpdateJobSchedules_DelegatesEnvironmentHealthReschedule(t *trequire.Empty(t, scheduler.rescheduled)}-func TestJobService_RunJobNowInline_DelegatesImageUpdateWatcherWithDetachedContext(t *testing.T) {- ctx, cancel := context.WithCancel(context.Background())- cancel()-+func TestJobService_Submit_PersistsImageUpdateWatcherRun(t *testing.T) {+ ctx := t.Context()+ db := setupSettingsTestDBInternal(t)+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)+ require.NoError(t, err)scheduler := newFakeJobSchedulerInternal()- jobSvc := &JobService{scheduler: scheduler}-- require.NoError(t, jobSvc.RunJobNowInline(ctx, "image-polling"))- require.Equal(t, []string{"image-polling"}, scheduler.busWatcherRuns)- require.Len(t, scheduler.busWatcherContexts, 1)- require.NoError(t, scheduler.busWatcherContexts[0].Err())+ jobSvc := NewJobService(db, settingsSvc, &config.Config{})+ jobSvc.SetScheduler(ctx, scheduler)+ run, err := jobSvc.Submit(ctx, schedulertypes.Request{JobID: "image-polling", EnvironmentID: "0", Trigger: "manual"})+ require.NoError(t, err)+ require.Equal(t, schedulertypes.Queued, run.Status)+ persisted, err := jobSvc.Queue.Get(ctx, "0", "image-polling", run.ID)+ require.NoError(t, err)+ require.Equal(t, run.ID, persisted.ID)+ require.Empty(t, scheduler.busWatcherRuns)}func findJobStatusByIDInternal(t *testing.T, jobs []jobschedule.JobStatus, id string) jobschedule.JobStatus {@@ -419,4 +424,6 @@ func (j fakeJobInternal) Schedule(context.Context) string {return "0 0 0 * * *"}-func (j fakeJobInternal) Run(context.Context) {}+func (j fakeJobInternal) Run(context.Context) (schedulertypes.Outcome, error) {+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil+}diff --git a/backend/internal/kv/service.go b/backend/internal/kv/service.goindex e5dece753..0b49d3c7e 100644--- a/backend/internal/kv/service.go+++ b/backend/internal/kv/service.go@@ -4,6 +4,7 @@ import ("context""strconv""strings"+ "time""emperror.dev/errors"@@ -126,3 +127,15 @@ func (s *KVService) IncrementInt64(ctx context.Context, key string, delta int64)return nextValue, nil}++// CreateIfAbsent atomically creates an entry without replacing an existing value.+func (s *KVService) CreateIfAbsent(ctx context.Context, key, value string) (bool, error) {+ result := s.db.WithContext(ctx).Clauses(clause.OnConflict{DoNothing: true}).Create(&KVEntry{Key: key, Value: value})+ return result.RowsAffected == 1, errors.WrapIf(result.Error, "create kv entry")+}++// CompareAndSwap updates only the exact value read by the caller.+func (s *KVService) CompareAndSwap(ctx context.Context, key, previous, next string) (bool, error) {+ result := s.db.WithContext(ctx).Model(&KVEntry{}).Where("key = ? AND value = ?", key, previous).Updates(map[string]any{"value": next, "updated_at": time.Now()})+ return result.RowsAffected == 1, errors.WrapIf(result.Error, "compare and swap kv entry")+}diff --git a/backend/internal/middleware/context.go b/backend/internal/middleware/context.goindex 528b2fb5e..30d3131e2 100644--- a/backend/internal/middleware/context.go+++ b/backend/internal/middleware/context.go@@ -10,6 +10,7 @@ import (type ContextKey stringconst (+ ContextKeyApiKeyID ContextKey = "apiKeyID"// ContextKeyUserID is the context key for the authenticated user's ID.ContextKeyUserID ContextKey = "userID"// ContextKeyCurrentSessionID is the context key for the authenticated session ID.diff --git a/backend/internal/middleware/environment_middleware.go b/backend/internal/middleware/environment_middleware.goindex bb2b14810..a05374a3d 100644--- a/backend/internal/middleware/environment_middleware.go+++ b/backend/internal/middleware/environment_middleware.go@@ -427,6 +427,9 @@ func (m *EnvironmentMiddleware) hasResourcePath(c *echo.Context, envID string) b}func isManagementPathInternal(method, suffix string) bool {+ if suffix == "/jobs" || strings.HasPrefix(suffix, "/jobs/") {+ return true+ }if isCentralSwarmManagementPathInternal(method, suffix) {return true}diff --git a/backend/internal/system/system.go b/backend/internal/system/system.goindex 24beb2bc4..573147f4a 100644--- a/backend/internal/system/system.go+++ b/backend/internal/system/system.go@@ -2,6 +2,8 @@ package systemimport ("github.com/getarcaneapp/arcane/backend/v2/internal/imageupdate"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler""github.com/getarcaneapp/arcane/backend/v2/internal/common"@@ -97,6 +99,12 @@ func (s *SystemService) PruneAll(ctx context.Context, environmentID string, reqdefer s.finishSystemPruneInternal(environmentID)ctx = s.activityService.Track(ctx, activityID)+ if err := jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ID: "scheduled-prune", Status: schedulertypes.Running, ActivityID: activityID}); err != nil {+ result.Success = false+ result.Errors = append(result.Errors, err.Error())+ s.completeSystemPruneActivityInternal(ctx, activityID, result)+ return result, true, err+ }s.runSystemPruneInternal(ctx, req, activityID, result)return result, true, nildiff --git a/backend/internal/systembackup/service.go b/backend/internal/systembackup/service.goindex 35b73cc19..55cbf0bbc 100644--- a/backend/internal/systembackup/service.go+++ b/backend/internal/systembackup/service.go@@ -18,6 +18,8 @@ import ("strings""time"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"+"emperror.dev/errors""github.com/getarcaneapp/arcane/backend/v2/internal/activity"@@ -1565,13 +1567,28 @@ func (s *SystemBackupService) SetScheduler(ctx context.Context, scheduler schedureturn s.jobs.SetScheduler(ctx, scheduler, admissionGate)}-func (s *SystemBackupService) runScheduledBackupInternal(ctx context.Context, policyID string) {+func (s *SystemBackupService) runScheduledBackupInternal(ctx context.Context, policyID string) (schedulertypes.Outcome, error) {policy, loadErr := s.loadPolicyInternal(ctx, policyID)- if loadErr != nil || policy == nil || !policy.Enabled {- return+ if loadErr != nil {+ return schedulertypes.Outcome{}, loadErr+ }+ if policy == nil || !policy.Enabled {+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil+ }+ if previous, ok := jobcontext.Run(ctx); ok {+ outcome := jobcontext.ConfirmedTarget(previous, policy.ID)+ if outcome.Status == schedulertypes.Succeeded {+ if policy.RetentionCount > 0 {+ if err := s.applyRetentionInternal(ctx, policy.ID, policy.RetentionCount); err != nil {+ outcome.Status = schedulertypes.Partial+ return outcome, err+ }+ }+ return outcome, nil+ }}var run *SystemBackupRun- _, runErr := activitylib.RunHandlerActivity(ctx, s.activityService, activitylib.HandlerOptions{+ activityID, runErr := activitylib.RunHandlerActivity(ctx, s.activityService, activitylib.HandlerOptions{EnvironmentID: "0", Type: activitytypes.TypeResourceAction, ResourceType: "system_backup",ResourceID: policy.ID, ResourceName: "Arcane", User: &common.SystemUser,Step: "Creating scheduled system backup", Message: "Creating scheduled Arcane system backup",@@ -1587,18 +1604,20 @@ func (s *SystemBackupService) runScheduledBackupInternal(ctx context.Context, po})if errors.Is(runErr, ErrSystemBackupAlreadyRunning) {slog.InfoContext(ctx, "Scheduled Arcane system backup skipped; another backup is running", "policyId", policy.ID)- return+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil}if runErr != nil {slog.ErrorContext(ctx, "Scheduled Arcane system backup failed", "policyId", policy.ID, "error", runErr)- return+ return schedulertypes.Outcome{ActivityID: activityID}, runErr}if policy.RetentionCount > 0 {if retentionErr := s.applyRetentionInternal(ctx, policy.ID, policy.RetentionCount); retentionErr != nil {slog.ErrorContext(ctx, "System backup retention failed", "policyId", policy.ID, "error", retentionErr)+ return schedulertypes.Outcome{Status: schedulertypes.Partial, ActivityID: activityID, Message: "Backup completed but retention failed"}, retentionErr}}slog.InfoContext(ctx, "Scheduled Arcane system backup completed", "backupId", run.ID, "policyId", policy.ID)+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded, ActivityID: activityID}, nil}func (s *SystemBackupService) rescheduleSystemBackupPolicyInternal(ctx context.Context, policy *SystemBackupPolicy) {@@ -1618,8 +1637,25 @@ func (s *SystemBackupService) rescheduleSystemBackupPolicyInternal(ctx context.C}return current.Schedule},- func(ctx context.Context) {- s.runScheduledBackupInternal(ctx, policyID)+ func(ctx context.Context) (schedulertypes.Outcome, error) {+ return s.runScheduledBackupInternal(ctx, policyID)+ },+ func(ctx context.Context, previous schedulertypes.Run) (schedulertypes.Outcome, error) {+ outcome := jobcontext.ConfirmedTarget(previous, policy.ID)+ if outcome.Status == schedulertypes.Succeeded {+ current, err := s.loadPolicyInternal(ctx, policyID)+ if err != nil {+ return outcome, err+ }+ if current != nil && current.RetentionCount > 0 {+ if err := s.applyRetentionInternal(ctx, policyID, current.RetentionCount); err != nil {+ outcome.Status = schedulertypes.Partial+ outcome.Message = "Backup completed but retention failed"+ return outcome, err+ }+ }+ }+ return outcome, nil},)}diff --git a/backend/internal/systembackup/service_test.go b/backend/internal/systembackup/service_test.goindex 72c1c4178..cf08d592b 100644--- a/backend/internal/systembackup/service_test.go+++ b/backend/internal/systembackup/service_test.go@@ -40,7 +40,8 @@ func newSystemBackupAdmissionGateForTestInternal(t testing.TB) *actors.Gate[acto}type systemBackupPolicySchedulerInternal struct {- jobs map[string]schedulertypes.Job+ submitted []schedulertypes.Request+ jobs map[string]schedulertypes.Job}func (s *systemBackupPolicySchedulerInternal) AddJob(_ context.Context, job schedulertypes.Job) error {@@ -293,3 +294,8 @@ func TestNormalizeSystemBackupSelectionInternal(t *testing.T) {require.NoError(t, err)require.Equal(t, []backuptypes.BackupFileEntry{{Path: "demo", Name: "demo", IsDirectory: true}}, selected)}++func (s *systemBackupPolicySchedulerInternal) Submit(_ context.Context, request schedulertypes.Request) (schedulertypes.Run, error) {+ s.submitted = append(s.submitted, request)+ return schedulertypes.Run{ID: request.RunID, JobID: request.JobID, EnvironmentID: request.EnvironmentID, Status: schedulertypes.Queued}, nil+}diff --git a/backend/internal/systembackup/system_volume.go b/backend/internal/systembackup/system_volume.goindex 821504e73..c29db4397 100644--- a/backend/internal/systembackup/system_volume.go+++ b/backend/internal/systembackup/system_volume.go@@ -10,6 +10,9 @@ import ("slices""strings"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"+"emperror.dev/errors""github.com/getarcaneapp/arcane/backend/v2/internal/actors"@@ -322,6 +325,19 @@ func (s *SystemBackupService) executeSystemVolumeBackupsInternal(ctx context.ConS3DestinationID: policyConfig.S3DestinationID,}for _, candidate := range candidates {+ if previous, ok := jobcontext.Run(ctx); ok {+ completed := false+ for _, target := range previous.Outcome.Targets {+ if target.ID == candidate.Name && target.Status == schedulertypes.Succeeded {+ completed = true+ break+ }+ }+ if completed {+ result.Succeeded+++ continue+ }+ }if err := ctx.Err(); err != nil {return result, err}@@ -340,6 +356,9 @@ func (s *SystemBackupService) executeSystemVolumeBackupsInternal(ctx context.Conif manualPolicy {seriesID = systemVolumeManualPolicyIDInternal(candidate.Name)}+ if progressErr := jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ID: candidate.Name, Status: schedulertypes.Running}); progressErr != nil {+ return result, progressErr+ }_, backupErr := s.volumeService.CreateSystemManagedBackup(ctx, candidate.Name, common.SystemUser, trigger, seriesID, policy)if errors.Is(backupErr, volume.ErrVolumeBackupAlreadyRunning) {result.Skipped++@@ -350,26 +369,40 @@ func (s *SystemBackupService) executeSystemVolumeBackupsInternal(ctx context.Conresult.Failures = append(result.Failures, backuptypes.SystemVolumeBackupFailure{VolumeName: candidate.Name, Error: backupErr.Error()})continue}+ if progressErr := jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ID: candidate.Name, Status: schedulertypes.Succeeded}); progressErr != nil {+ return result, progressErr+ }result.Succeeded++}return result, nil}-func (s *SystemBackupService) runScheduledSystemVolumeBackupInternal(ctx context.Context, policyID string) {+func (s *SystemBackupService) runScheduledSystemVolumeBackupInternal(ctx context.Context, policyID string) (schedulertypes.Outcome, error) {policy, err := s.systemVolumeBackupPolicyInternal(policyID)- if err != nil || policy == nil || !policy.Enabled {- return+ if err != nil {+ return schedulertypes.Outcome{}, err+ }+ if policy == nil || !policy.Enabled {+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil}result, err := s.runSystemVolumeBackupsInternal(ctx, backuptypes.RunSystemVolumeBackupsRequest{PolicyID: policyID}, volume.VolumeBackupTriggerScheduled)if errors.Is(err, ErrSystemBackupAlreadyRunning) {slog.InfoContext(ctx, "Scheduled system-managed volume backups skipped; a system backup is running", "policyId", policyID)- return+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil}if err != nil {slog.ErrorContext(ctx, "Scheduled system-managed volume backups failed", "policyId", policyID, "error", err)- return+ return schedulertypes.Outcome{}, err}slog.InfoContext(ctx, "Scheduled system-managed volume backups completed", "policyId", policyID, "matched", result.Matched, "succeeded", result.Succeeded, "failed", result.Failed, "skipped", result.Skipped)+ outcome := schedulertypes.Outcome{Status: schedulertypes.Succeeded}+ if result.Failed > 0 {+ outcome.Status = schedulertypes.Partial+ }+ for _, failure := range result.Failures {+ outcome.Targets = append(outcome.Targets, schedulertypes.TargetOutcome{ID: failure.VolumeName, Status: schedulertypes.Failed, Message: failure.Error})+ }+ return outcome, nil}func (s *SystemBackupService) rescheduleSystemVolumeBackupInternal(ctx context.Context, policy *backuptypes.SystemVolumeBackupPolicy) {@@ -388,8 +421,15 @@ func (s *SystemBackupService) rescheduleSystemVolumeBackupInternal(ctx context.Creturn defaultSystemVolumeSchedule}return current.Schedule- }, func(ctx context.Context) {- s.runScheduledSystemVolumeBackupInternal(ctx, policyID)+ }, func(ctx context.Context) (schedulertypes.Outcome, error) {+ return s.runScheduledSystemVolumeBackupInternal(ctx, policyID)+ }, func(ctx context.Context, previous schedulertypes.Run) (schedulertypes.Outcome, error) {+ for _, target := range previous.Outcome.Targets {+ if target.Status != schedulertypes.Succeeded && target.Status != schedulertypes.Skipped {+ return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Targets: previous.Outcome.Targets, Message: "A volume backup has an unconfirmed outcome"}, nil+ }+ }+ return s.runScheduledSystemVolumeBackupInternal(jobcontext.WithExecution(ctx, previous, nil), policyID)})}diff --git a/backend/internal/updater/service.go b/backend/internal/updater/service.goindex f8f58fa6f..34d0b249a 100644--- a/backend/internal/updater/service.go+++ b/backend/internal/updater/service.go@@ -2,7 +2,6 @@ package updaterimport ("context"- stderrors "errors""fmt""io""log/slog"@@ -12,6 +11,9 @@ import ("sync""time"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"+activitytypes "github.com/getarcaneapp/arcane/types/v2/activity""emperror.dev/errors"@@ -190,6 +192,10 @@ func (s *UpdaterService) ApplyPending(ctx context.Context, options arcaneupdaters.completeAutoUpdateActivityInternal(ctx, activityID, out, err)}()+ if err = jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ID: "auto-update", Status: schedulertypes.Running, ActivityID: activityID}); err != nil {+ return out, err+ }+if activityID != "" && s.deps.Activity != nil {// Bounded slot wait: an unbounded wait behind other long-running runs// would strand the queued activity row (the completion defer above@@ -267,7 +273,12 @@ func (s *UpdaterService) applyScopedUpdatesInternal(ctx context.Context, optionsengineOpts := updater.Options{Force: options.ForceUpdate, DryRun: options.DryRun}var engineErrs []errorfor _, containerID := range containerIDs {+ target := schedulertypes.TargetOutcome{ID: containerID, ResourceType: "container", Status: schedulertypes.Running}+ if err := jobcontext.Progress(ctx, target); err != nil {+ return err+ }moduleResult, engineErr := s.engineInternal().UpdateContainer(ctx, containerID, engineOpts)+ target.Status = schedulertypes.NeedsAttentionif moduleResult != nil {partial := resultFromModuleInternal(moduleResult)out.Checked += partial.Checked@@ -276,6 +287,12 @@ func (s *UpdaterService) applyScopedUpdatesInternal(ctx context.Context, optionsout.Skipped += partial.Skippedout.Failed += partial.Failedout.Items = append(out.Items, partial.Items...)+ target.Status = schedulertypes.Succeeded+ if partial.Failed > 0 {+ target.Status = schedulertypes.Failed+ } else if partial.Skipped > 0 {+ target.Status = schedulertypes.Skipped+ }}if engineErr != nil {out.Failed++@@ -286,13 +303,18 @@ func (s *UpdaterService) applyScopedUpdatesInternal(ctx context.Context, optionsError: engineErr.Error(),})engineErrs = append(engineErrs, errors.WrapIff(engineErr, "%s", containerID))+ target.Status = schedulertypes.Failed+ target.Message = engineErr.Error()+ }+ if err := jobcontext.Progress(ctx, target); err != nil {+ return errors.Combine(err, errors.Combine(engineErrs...))}}s.logResultItemsInternal(ctx, out)out.Success = out.Failed == 0// Engine errors propagate like the unscoped path's engine error does —// the remaining containers were still attempted and recorded above.- return stderrors.Join(engineErrs...)+ return errors.Combine(engineErrs...)}// resolveScopedContainerIDsInternal maps a scoped options payload to the@@ -1163,7 +1185,7 @@ func (s *UpdaterService) CollectUsedImages(ctx context.Context) (map[string]stru}if successfulSources == 0 {- return nil, stderrors.Join(errs...)+ return nil, errors.Combine(errs...)}s.loggerInternal().DebugContext(ctx, "collectUsedImages: collected used images", "count", len(out))diff --git a/backend/internal/volume/backup.go b/backend/internal/volume/backup.goindex f413ac767..b12487fd5 100644--- a/backend/internal/volume/backup.go+++ b/backend/internal/volume/backup.go@@ -15,6 +15,9 @@ import ("time""uuid"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"+"emperror.dev/errors""gorm.io/gorm"@@ -1775,13 +1778,22 @@ func (s *VolumeService) applyVolumeBackupRetentionInternal(ctx context.Context,return s.deleteRusticBackupsInternal(ctx, entries, nil)}-func (s *VolumeService) runScheduledBackupInternal(ctx context.Context, policyID string) {+func (s *VolumeService) runScheduledBackupInternal(ctx context.Context, policyID string) (schedulertypes.Outcome, error) {var policy VolumeBackupPolicyif err := s.db.WithContext(ctx).Where("id = ? AND enabled = ?", policyID, true).First(&policy).Error; err != nil {- return+ if errors.Is(err, gorm.ErrRecordNotFound) {+ return schedulertypes.Outcome{Status: schedulertypes.Canceled, Message: "Backup policy disabled or deleted"}, nil+ }+ return schedulertypes.Outcome{}, err+ }+ if previous, ok := jobcontext.Run(ctx); ok {+ outcome := jobcontext.ConfirmedTarget(previous, policy.VolumeName)+ if outcome.Status == schedulertypes.Succeeded {+ return outcome, nil+ }}var entry *VolumeBackup- _, err := activitylib.RunHandlerActivity(ctx, s.activityService, activitylib.HandlerOptions{+ activityID, err := activitylib.RunHandlerActivity(ctx, s.activityService, activitylib.HandlerOptions{EnvironmentID: "0",Type: activitytypes.TypeResourceAction,ResourceType: "volume_backup",@@ -1809,13 +1821,14 @@ func (s *VolumeService) runScheduledBackupInternal(ctx context.Context, policyID})if errors.Is(err, ErrVolumeBackupAlreadyRunning) {slog.InfoContext(ctx, "Scheduled volume backup skipped; another backup is running", "volume", policy.VolumeName)- return+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil}if err != nil {slog.ErrorContext(ctx, "Scheduled volume backup failed", "volume", policy.VolumeName, "error", err)- return+ return schedulertypes.Outcome{}, err}slog.InfoContext(ctx, "Scheduled volume backup completed", "volume", policy.VolumeName, "backup_id", entry.ID, "remote_snapshot_id", entry.RemoteSnapshotID)+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded, ActivityID: activityID}, nil}func (s *VolumeService) rescheduleVolumeBackupPolicyInternal(ctx context.Context, policy *VolumeBackupPolicy) {@@ -1835,8 +1848,11 @@ func (s *VolumeService) rescheduleVolumeBackupPolicyInternal(ctx context.Context}return current.Schedule},- func(ctx context.Context) {- s.runScheduledBackupInternal(ctx, policyID)+ func(ctx context.Context) (schedulertypes.Outcome, error) {+ return s.runScheduledBackupInternal(ctx, policyID)+ },+ func(_ context.Context, previous schedulertypes.Run) (schedulertypes.Outcome, error) {+ return jobcontext.ConfirmedTarget(previous, policy.VolumeName), nil},)}diff --git a/backend/internal/volume/service_test.go b/backend/internal/volume/service_test.goindex 5f4d92224..8dc58399e 100644--- a/backend/internal/volume/service_test.go+++ b/backend/internal/volume/service_test.go@@ -624,7 +624,8 @@ func TestTouchHelperInternal(t *testing.T) {}type volumeBackupPolicySchedulerInternal struct {- jobs map[string]schedulertypes.Job+ submitted []schedulertypes.Request+ jobs map[string]schedulertypes.Job}func (s *volumeBackupPolicySchedulerInternal) AddJob(_ context.Context, job schedulertypes.Job) error {@@ -1042,3 +1043,8 @@ func TestBackupPolicyDestinationLookupFailureInternal(t *testing.T) {require.Empty(t, collection.Policies[0].S3Bucket)}}++func (s *volumeBackupPolicySchedulerInternal) Submit(_ context.Context, request schedulertypes.Request) (schedulertypes.Run, error) {+ s.submitted = append(s.submitted, request)+ return schedulertypes.Run{ID: request.RunID, JobID: request.JobID, EnvironmentID: request.EnvironmentID, Status: schedulertypes.Queued}, nil+}diff --git a/backend/pkg/fswatch/watcher.go b/backend/pkg/fswatch/watcher.goindex 980d22697..087a806aa 100644--- a/backend/pkg/fswatch/watcher.go+++ b/backend/pkg/fswatch/watcher.go@@ -143,6 +143,9 @@ func (fw *Watcher) watchLoop(ctx context.Context) {debounceTimer := time.NewTimer(fw.debounce)debounceTimer.Stop()+ defer debounceTimer.Stop()+ healthTicker := time.NewTicker(5 * time.Second)+ defer healthTicker.Stop()debouncePending := falselastGoroutineLog := time.Time{}@@ -154,15 +157,24 @@ func (fw *Watcher) watchLoop(ctx context.Context) {returncase event, ok := <-fw.watcher.Events:if fw.processEventInternal(ctx, event, ok, debounceTimer, &debouncePending) {- return+ if !fw.reconnectInternal(ctx) {+ return+ }}case <-debounceTimer.C:if fw.fireDebounceInternal(ctx, &debouncePending, &lastGoroutineLog) {continue}+ case <-healthTicker.C:+ if len(fw.watcher.WatchList()) == 0 && !fw.reconnectInternal(ctx) {+ return+ }case err, ok := <-fw.watcher.Errors:if !ok {- return+ if !fw.reconnectInternal(ctx) {+ return+ }+ continue}slog.ErrorContext(ctx, "Filesystem watcher error", "error", err)}@@ -430,3 +442,47 @@ func (fw *Watcher) isWatchableDirectory(path string) bool {return resolvedInfo.IsDir()}++// reconnectInternal replaces a lost filesystem subscription without abandoning its owner.+func (fw *Watcher) reconnectInternal(ctx context.Context) bool {+ if err := fw.watcher.Close(); err != nil {+ slog.DebugContext(ctx, "Closing lost filesystem subscription", "error", err)+ }+ delay := 5 * time.Second+ for {+ timer := time.NewTimer(delay)+ select {+ case <-ctx.Done():+ timer.Stop()+ return false+ case <-fw.stopCh:+ timer.Stop()+ return false+ case <-timer.C:+ }+ watcher, err := fsnotify.NewWatcher()+ if err == nil {+ err = watcher.Add(fw.watchedPath)+ if err == nil {+ fw.watcher = watcher+ fw.watchAliases = make(map[string]string)+ if err := fw.addExistingDirectories(fw.watchedPath); err != nil {+ slog.WarnContext(ctx, "Some filesystem subscriptions could not be restored", "error", err)+ }+ // Changes while disconnected need a full reconciliation.+ if fw.onChange != nil {+ fw.onChange(ctx)+ }+ if fw.onChangePaths != nil {+ fw.onChangePaths(ctx, []string{fw.watchedPath})+ }+ return true+ }+ if closeErr := watcher.Close(); closeErr != nil {+ slog.DebugContext(ctx, "Closing failed filesystem subscription", "error", closeErr)+ }+ }+ slog.WarnContext(ctx, "Filesystem subscription unavailable; retrying", "path", fw.watchedPath, "error", err)+ delay = min(delay*2, 5*time.Minute)+ }+}diff --git a/backend/pkg/libarcane/activity/handler.go b/backend/pkg/libarcane/activity/handler.goindex e2827f5a5..18ccc77b7 100644--- a/backend/pkg/libarcane/activity/handler.go+++ b/backend/pkg/libarcane/activity/handler.go@@ -2,6 +2,8 @@ package activityimport ("github.com/getarcaneapp/arcane/backend/v2/internal/common"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler""github.com/getarcaneapp/arcane/backend/v2/internal/database"@@ -190,8 +192,19 @@ func RunHandlerActivity(ctx context.Context, activityService Service, opts HandlAwaitHandlerActivitySlot(workCtx, activityService, activityID, opts.EnvironmentID)}- err := action(workCtx)+ err := jobcontext.Progress(workCtx, schedulertypes.TargetOutcome{ID: opts.ResourceID, Status: schedulertypes.Running, ActivityID: activityID})+ if err == nil {+ err = action(workCtx)+ }CompleteHandlerActivity(workCtx, activityService, activityID, opts.SuccessMessage, err)+ target := schedulertypes.TargetOutcome{ID: opts.ResourceID, Status: schedulertypes.Succeeded, ActivityID: activityID}+ if err != nil {+ target.Status = schedulertypes.Failed+ target.Message = err.Error()+ }+ if progressErr := jobcontext.Progress(workCtx, target); progressErr != nil && err == nil {+ err = progressErr+ }return activityID, err}diff --git a/backend/pkg/scheduler/activity_sweep_job.go b/backend/pkg/scheduler/activity_sweep_job.goindex b73231ec5..f8c0792f0 100644--- a/backend/pkg/scheduler/activity_sweep_job.go+++ b/backend/pkg/scheduler/activity_sweep_job.go@@ -4,6 +4,8 @@ import ("context""log/slog"+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"+"github.com/getarcaneapp/arcane/backend/v2/internal/activity")@@ -30,13 +32,14 @@ func (j *ActivitySweepJob) Schedule(_ context.Context) string {return "0 */5 * * * *"}-func (j *ActivitySweepJob) Run(ctx context.Context) {+func (j *ActivitySweepJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {swept, err := j.activityService.FailAbandonedActivities(ctx)if err != nil {slog.WarnContext(ctx, "activity sweep failed", "jobName", ActivitySweepJobName, "swept", swept, "error", err)- return+ return schedulertypes.Outcome{}, err}if swept > 0 {slog.InfoContext(ctx, "marked abandoned activities as failed", "jobName", ActivitySweepJobName, "count", swept)}+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil}diff --git a/backend/pkg/scheduler/actor_test.go b/backend/pkg/scheduler/actor_test.goindex 8c30e74f7..99b080a1c 100644--- a/backend/pkg/scheduler/actor_test.go+++ b/backend/pkg/scheduler/actor_test.go@@ -2,6 +2,8 @@ package schedulerimport ("context"+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"+ "sync""testing""time"@@ -36,6 +38,14 @@ func newJobSchedulerForTestInternal(t testing.TB, ctx context.Context, locationcreated, err := NewJobScheduler(ctx, runtime, location)require.NoError(t, err)scheduler := created.(*jobSchedulerInternal)+ scheduler.SetDispatcher(&testDispatcherInternal{run: func(ctx context.Context, request schedulertypes.Request) error {+ job, ok := scheduler.GetJob(request.JobID)+ if !ok {+ return nil+ }+ _, err := job.Run(ctx)+ return err+ }})t.Cleanup(func() {stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)defer cancel()@@ -63,3 +73,21 @@ func newSettingsServiceForTestInternal(t testing.TB, ctx context.Context, db *da})return settings.NewSettingsService(ctx, db, executor, effects)}++type testDispatcherInternal struct {+ run func(context.Context, schedulertypes.Request) error+ locks sync.Map+}++func (d *testDispatcherInternal) Submit(ctx context.Context, request schedulertypes.Request) (schedulertypes.Run, error) {+ value, _ := d.locks.LoadOrStore(request.JobID, &sync.Mutex{})+ lock := value.(*sync.Mutex)+ if !lock.TryLock() {+ return schedulertypes.Run{}, nil+ }+ defer lock.Unlock()+ return schedulertypes.Run{JobID: request.JobID}, d.run(ctx, request)+}+func (*testDispatcherInternal) Checkpoint(context.Context, string, string, time.Time) error {+ return nil+}diff --git a/backend/pkg/scheduler/analytics_job.go b/backend/pkg/scheduler/analytics_job.goindex e37bb3ef9..821ce6b8c 100644--- a/backend/pkg/scheduler/analytics_job.go+++ b/backend/pkg/scheduler/analytics_job.go@@ -11,6 +11,8 @@ import ("sync""time"+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"+"emperror.dev/errors""github.com/cenkalti/backoff/v5"@@ -69,23 +71,23 @@ func (j *AnalyticsJob) Schedule(_ context.Context) string {return analyticsHeartbeatCheckSchedule}-func (j *AnalyticsJob) Run(ctx context.Context) {+func (j *AnalyticsJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {if j.cfg.AnalyticsDisabled {slog.DebugContext(ctx, "analytics disabled; skipping heartbeat", "analyticsDisabled", j.cfg.AnalyticsDisabled)- return+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil}if j.cfg.Environment.IsTestEnvironment() {slog.DebugContext(ctx, "test environment; skipping heartbeat", "env", j.cfg.Environment)- return+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil}allowed, err := j.claimHeartbeatAttemptWindowInternal(ctx)if err != nil {slog.ErrorContext(ctx, "failed to acquire analytics heartbeat send window", "error", err)- return+ return schedulertypes.Outcome{}, err}if !allowed {- return+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil}instanceID := j.settingsService.GetStringSetting(ctx, "instanceId", "")@@ -103,7 +105,7 @@ func (j *AnalyticsJob) Run(ctx context.Context) {body, err := json.Marshal(payload)if err != nil {slog.ErrorContext(ctx, "failed to marshal analytics heartbeat body", "error", err)- return+ return schedulertypes.Outcome{}, err}slog.InfoContext(@@ -167,7 +169,7 @@ func (j *AnalyticsJob) Run(ctx context.Context) {)if err != nil {slog.ErrorContext(ctx, "analytics heartbeat failed", "error", err)- return+ return schedulertypes.Outcome{}, err}slog.InfoContext(@@ -186,6 +188,7 @@ func (j *AnalyticsJob) Run(ctx context.Context) {"env",j.cfg.Environment,)+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil}func (j *AnalyticsJob) Reschedule(ctx context.Context) error {diff --git a/backend/pkg/scheduler/analytics_job_test.go b/backend/pkg/scheduler/analytics_job_test.goindex dc255dbc2..9b570d957 100644--- a/backend/pkg/scheduler/analytics_job_test.go+++ b/backend/pkg/scheduler/analytics_job_test.go@@ -67,7 +67,11 @@ func TestAnalyticsJob_Run_ManagerPayload(t *testing.T) {job := NewAnalyticsJob(settingsService, kvService, server.Client(), cfg)job.heartbeatURL = server.URL- job.Run(ctx)+ if _, err := job.Run(ctx); !assert.NoError(t, err) {++ return++ }var body []byteselect {@@ -93,7 +97,11 @@ func TestAnalyticsJob_Run_AgentPayload(t *testing.T) {job := NewAnalyticsJob(settingsService, kvService, server.Client(), cfg)job.heartbeatURL = server.URL- job.Run(ctx)+ if _, err := job.Run(ctx); !assert.NoError(t, err) {++ return++ }var body []byteselect {@@ -117,7 +125,11 @@ func TestAnalyticsJob_Run_SkipsWhenDisabled(t *testing.T) {job := NewAnalyticsJob(settingsService, kvService, server.Client(), cfg)job.heartbeatURL = server.URL- job.Run(ctx)+ if _, err := job.Run(ctx); !assert.NoError(t, err) {++ return++ }select {case <-bodyCh:@@ -136,7 +148,11 @@ func TestAnalyticsJob_Run_SkipsWhenTestEnv(t *testing.T) {job := NewAnalyticsJob(settingsService, kvService, server.Client(), cfg)job.heartbeatURL = server.URL- job.Run(ctx)+ if _, err := job.Run(ctx); !assert.NoError(t, err) {++ return++ }select {case <-bodyCh:@@ -157,14 +173,18 @@ func TestAnalyticsJob_Run_SkipsWithinHeartbeatWindowAfterRestart(t *testing.T) {job := NewAnalyticsJob(settingsService, kvService, server.Client(), cfg)job.heartbeatURL = server.URLjob.now = func() time.Time { return firstAttemptAt }- job.Run(ctx)+ if _, err := job.Run(ctx); !assert.NoError(t, err) {+ return+ }reloadedSettingsService, err := newSettingsServiceForTestInternal(t, ctx, wrappedDB)require.NoError(t, err)restartedJob := NewAnalyticsJob(reloadedSettingsService, kv.NewKVService(wrappedDB), server.Client(), cfg)restartedJob.heartbeatURL = server.URLrestartedJob.now = func() time.Time { return firstAttemptAt.Add(19 * time.Minute) }- restartedJob.Run(ctx)+ if _, err := restartedJob.Run(ctx); !assert.NoError(t, err) {+ return+ }require.Equal(t, int32(1), requestCount.Load())}@@ -181,12 +201,16 @@ func TestAnalyticsJob_Run_AllowsSendAfterHeartbeatWindow(t *testing.T) {firstJob := NewAnalyticsJob(settingsService, kvService, server.Client(), cfg)firstJob.heartbeatURL = server.URLfirstJob.now = func() time.Time { return firstAttemptAt }- firstJob.Run(ctx)+ if _, err := firstJob.Run(ctx); !assert.NoError(t, err) {+ return+ }secondJob := NewAnalyticsJob(settingsService, kvService, server.Client(), cfg)secondJob.heartbeatURL = server.URLsecondJob.now = func() time.Time { return firstAttemptAt.Add(25 * time.Hour) }- secondJob.Run(ctx)+ if _, err := secondJob.Run(ctx); !assert.NoError(t, err) {+ return+ }require.Equal(t, int32(2), requestCount.Load())}@@ -209,7 +233,9 @@ func TestAnalyticsJob_Run_ConcurrentRunsSendOnce(t *testing.T) {for range 2 {wg.Go(func() {<-start- job.Run(ctx)+ if _, err := job.Run(ctx); !assert.NoError(t, err) {+ return+ }})}diff --git a/backend/pkg/scheduler/apns_outbox_job.go b/backend/pkg/scheduler/apns_outbox_job.goindex 0c8a663af..adad24a17 100644--- a/backend/pkg/scheduler/apns_outbox_job.go+++ b/backend/pkg/scheduler/apns_outbox_job.go@@ -3,6 +3,8 @@ package schedulerimport ("context"+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"+"github.com/getarcaneapp/arcane/backend/v2/internal/apns""github.com/getarcaneapp/arcane/backend/v2/internal/config")@@ -30,6 +32,9 @@ func (j *ApnsOutboxJob) ShouldSchedule(ctx context.Context) bool {return (j.config == nil || !j.config.AgentMode) && j.service.Enabled(ctx)}-func (j *ApnsOutboxJob) Run(ctx context.Context) {- j.service.DrainOutbox(ctx)+func (j *ApnsOutboxJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {+ if err := j.service.DrainOutbox(ctx); err != nil {+ return schedulertypes.Outcome{}, err+ }+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil}diff --git a/backend/pkg/scheduler/auto_heal_job.go b/backend/pkg/scheduler/auto_heal_job.goindex dce30c091..ed86cc6da 100644--- a/backend/pkg/scheduler/auto_heal_job.go+++ b/backend/pkg/scheduler/auto_heal_job.go@@ -2,6 +2,8 @@ package schedulerimport ("github.com/getarcaneapp/arcane/backend/v2/internal/database"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler""context""log/slog"@@ -14,9 +16,9 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/notification""emperror.dev/errors"+ scheduleutil "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/schedule""github.com/moby/moby/api/types/container""github.com/moby/moby/client"- "github.com/robfig/cron/v3""golang.org/x/sync/errgroup""github.com/getarcaneapp/arcane/backend/v2/internal/actors"@@ -91,7 +93,7 @@ func (j *AutoHealJob) Schedule(ctx context.Context) string {schedule = "*/30 * * * * *"}- parser := cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow)+ parser := scheduleutil.Parser()if _, err := parser.Parse(schedule); err != nil {slog.WarnContext(ctx, "Invalid cron expression for auto-heal, using default", "invalid_schedule", schedule, "error", err)return "*/30 * * * * *"@@ -100,34 +102,34 @@ func (j *AutoHealJob) Schedule(ctx context.Context) string {return schedule}-func (j *AutoHealJob) Run(ctx context.Context) {+func (j *AutoHealJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {enabled := j.settingsService.GetBoolSetting(ctx, "autoHealEnabled", false)if !enabled {slog.DebugContext(ctx, "auto-heal disabled; skipping run")- return+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil}lease, admitted, err := j.admissionGate.TryAcquire(ctx, actors.AdmissionKey{Scope: autoHealAdmissionScopeInternal})if err != nil {slog.ErrorContext(ctx, "auto-heal admission failed", "error", err)- return+ return schedulertypes.Outcome{}, err}if !admitted {slog.WarnContext(ctx, "auto-heal run still in progress; skipping overlapping run")- return+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil}defer lease.Release()dockerClient, err := j.getDockerClientInternal(ctx)if err != nil {slog.ErrorContext(ctx, "auto-heal failed to get Docker client", "error", err)- return+ return schedulertypes.Outcome{}, err}containerList, err := j.ListContainers(ctx, dockerClient)if err != nil {slog.ErrorContext(ctx, "auto-heal failed to list containers", "error", err)- return+ return schedulertypes.Outcome{}, err}containers := containerList@@ -140,18 +142,30 @@ func (j *AutoHealJob) Run(ctx context.Context) {candidates := j.filterCandidatesInternal(containers, excludedContainers, selfID)g, groupCtx := errgroup.WithContext(ctx)+ var resultMu sync.Mutex+ outcome := schedulertypes.Outcome{Status: schedulertypes.Succeeded}g.SetLimit(autoHealInspectConcurrency)for _, candidate := range candidates {g.Go(func() (workerErr error) {defer utils.RecoverToError(&workerErr, "auto-heal worker")- j.processCandidateInternal(groupCtx, dockerClient, candidate, maxRestarts, restartWindow, restartWindowMinutes)+ target, err := j.processCandidateInternal(groupCtx, dockerClient, candidate, maxRestarts, restartWindow, restartWindowMinutes)+ resultMu.Lock()+ outcome.Targets = append(outcome.Targets, target)+ if err != nil {+ outcome.Status = schedulertypes.Partial+ }+ resultMu.Unlock()+ if progressErr := jobcontext.Progress(groupCtx, target); progressErr != nil {+ return progressErr+ }return nil})}- _ = g.Wait()+ err = g.Wait()+ return outcome, err}// selfContainerIDInternal resolves and caches the ID (full 64-char or short@@ -207,8 +221,16 @@ func (j *AutoHealJob) processCandidateInternal(maxRestarts int,restartWindow time.Duration,restartWindowMinutes int,-) {+) (schedulertypes.TargetOutcome, error) {containerID := candidate.ID+ target := schedulertypes.TargetOutcome{ID: containerID, Status: schedulertypes.Skipped}+ if previous, ok := jobcontext.Run(ctx); ok {+ for _, completed := range previous.Outcome.Targets {+ if completed.ID == containerID && completed.Status == schedulertypes.Succeeded {+ return completed, nil+ }+ }+ }containerName := dockerutil.ContainerNameFromNames(candidate.Names)// The daemon-side health filter already selected unhealthy containers; the@@ -217,14 +239,16 @@ func (j *AutoHealJob) processCandidateInternal(inspect, err := j.inspectContainerInternal(ctx, dockerClient, containerID)if err != nil {slog.WarnContext(ctx, "auto-heal failed to inspect container", "container", containerName, "error", err)- return+ target.Status = schedulertypes.Failed+ target.Message = err.Error()+ return target, err}if inspect.State == nil || inspect.State.Health == nil {- return+ return target, nil}if inspect.State.Health.Status != container.Unhealthy {- return+ return target, nil}releaseSlot, reserved := j.reserveRestartSlotInternal(containerID, maxRestarts, restartWindow)@@ -234,18 +258,27 @@ func (j *AutoHealJob) processCandidateInternal("max_restarts", maxRestarts,"window_minutes", restartWindowMinutes,)- return+ return target, nil}+ target.Status = schedulertypes.Running+ if err := jobcontext.Progress(ctx, target); err != nil {+ releaseSlot()+ return target, err+ }if err := j.restartContainerInternal(ctx, dockerClient, containerID); err != nil {releaseSlot()slog.ErrorContext(ctx, "auto-heal failed to restart container", "container", containerName, "error", err)- return+ target.Status = schedulertypes.Failed+ target.Message = err.Error()+ return target, err}j.postRestartActionsInternal(ctx, containerID, containerName)slog.InfoContext(ctx, "auto-heal restarted unhealthy container", "container", containerName, "container_id", containerID)+ target.Status = schedulertypes.Succeeded+ return target, nil}func (j *AutoHealJob) postRestartActionsInternal(ctx context.Context, containerID, containerName string) {@@ -478,3 +511,12 @@ func (j *AutoHealJob) RecordRestartAtExported(containerID string, t time.Time) {record.timestamps = append(record.timestamps, t)}++func (j *AutoHealJob) Reconcile(ctx context.Context, previous schedulertypes.Run) (schedulertypes.Outcome, error) {+ for _, target := range previous.Outcome.Targets {+ if target.Status != schedulertypes.Succeeded && target.Status != schedulertypes.Skipped {+ return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: "A container restart has an unconfirmed outcome", Targets: previous.Outcome.Targets}, nil+ }+ }+ return j.Run(jobcontext.WithExecution(ctx, previous, nil))+}diff --git a/backend/pkg/scheduler/auto_patch_job.go b/backend/pkg/scheduler/auto_patch_job.goindex 255515ce5..fbf20d2ad 100644--- a/backend/pkg/scheduler/auto_patch_job.go+++ b/backend/pkg/scheduler/auto_patch_job.go@@ -4,11 +4,13 @@ import ("context""log/slog"+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"+"github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/internal/imagepatch""github.com/getarcaneapp/arcane/backend/v2/internal/settings"+ scheduleutil "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/schedule""github.com/getarcaneapp/arcane/types/v2"- "github.com/robfig/cron/v3")const AutoPatchJobName = "auto-patch"@@ -48,7 +50,7 @@ func (j *AutoPatchJob) Schedule(ctx context.Context) string {schedule = "0 0 3 * * *"}- parser := cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow)+ parser := scheduleutil.Parser()if _, err := parser.Parse(schedule); err != nil {slog.WarnContext(ctx, "Invalid cron expression for auto-patch, using default", "invalid_schedule", schedule, "error", err)return "0 0 3 * * *"@@ -57,10 +59,10 @@ func (j *AutoPatchJob) Schedule(ctx context.Context) string {return schedule}-func (j *AutoPatchJob) Run(ctx context.Context) {+func (j *AutoPatchJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {if !j.settingsService.GetBoolSetting(ctx, "imageAutoPatchEnabled", false) {slog.DebugContext(ctx, "scheduled image patching disabled; skipping run")- return+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil}slog.InfoContext(ctx, "scheduled image patching started")@@ -68,11 +70,15 @@ func (j *AutoPatchJob) Run(ctx context.Context) {patched, skipped, err := j.imagePatchService.PatchFlaggedImages(ctx, types.LocalDockerEnvironmentID, autoPatchSystemUser)if err != nil {slog.ErrorContext(ctx, "scheduled image patching failed", "error", err)- return+ if patched > 0 {+ return schedulertypes.Outcome{Status: schedulertypes.Partial}, err+ }+ return schedulertypes.Outcome{}, err}slog.InfoContext(ctx, "scheduled image patching completed","patched", patched,"skipped", skipped,)+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil}diff --git a/backend/pkg/scheduler/auto_update_job.go b/backend/pkg/scheduler/auto_update_job.goindex 2bd847e5d..65bf96989 100644--- a/backend/pkg/scheduler/auto_update_job.go+++ b/backend/pkg/scheduler/auto_update_job.go@@ -4,6 +4,9 @@ import ("context""log/slog"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"+"github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/backend/v2/internal/updater"@@ -55,32 +58,47 @@ func (j *AutoUpdateJob) Schedule(ctx context.Context) string {return s}-func (j *AutoUpdateJob) Run(ctx context.Context) {+func (j *AutoUpdateJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {+ options := updatertypes.Options{}+ unresolvedTargets := false+ if previous, ok := jobcontext.Run(ctx); ok {+ var priorOutcome schedulertypes.Outcome+ options, priorOutcome, unresolvedTargets = autoUpdateRetryOptionsInternal(previous)+ if priorOutcome.Status != "" {+ return priorOutcome, nil+ }+ }+enabled := j.settingsService.GetBoolSetting(ctx, "autoUpdate", false)pollingEnabled := j.settingsService.GetBoolSetting(ctx, "pollingEnabled", true)if !enabled || !pollingEnabled {slog.DebugContext(ctx, "auto-update disabled or polling disabled; skipping run","autoUpdate", enabled, "pollingEnabled", pollingEnabled)- return+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil}lease, admitted, err := j.admissionGate.TryAcquire(ctx, actors.AdmissionKey{Scope: autoUpdateAdmissionScopeInternal})if err != nil {slog.ErrorContext(ctx, "auto-update admission failed", "error", err)- return+ return schedulertypes.Outcome{}, err}if !admitted {slog.WarnContext(ctx, "auto-update run still in progress; skipping overlapping run")- return+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil}defer lease.Release()slog.InfoContext(ctx, "auto-update run started")- result, err := j.updaterService.ApplyPending(ctx, updatertypes.Options{})+ result, err := j.updaterService.ApplyPending(ctx, options)if err != nil {slog.ErrorContext(ctx, "auto-update run failed", "err", err)- return+ if result == nil {+ return schedulertypes.Outcome{}, err+ }+ }+ if result == nil {+ return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: "Updater returned no operation result"}, nil}slog.InfoContext(ctx, "auto-update run completed",@@ -90,9 +108,66 @@ func (j *AutoUpdateJob) Run(ctx context.Context) {"skipped", result.Skipped,"failed", result.Failed,)+ outcome := schedulertypes.Outcome{Status: schedulertypes.Succeeded}+ if result.ActivityID != nil {+ outcome.ActivityID = *result.ActivityID+ }+ for _, item := range result.Items {+ status := schedulertypes.Succeeded+ if item.Status == "failed" {+ status = schedulertypes.Failed+ }+ if item.Status == "skipped" {+ status = schedulertypes.Skipped+ }+ outcome.Targets = append(outcome.Targets, schedulertypes.TargetOutcome{ID: item.ResourceID, ResourceType: item.ResourceType, Status: status, Message: item.Error})+ }+ if result.Failed > 0 || err != nil || unresolvedTargets {+ outcome.Status = schedulertypes.Partial+ outcome.Message = "Some updates failed; review target outcomes before retrying"+ }+ return outcome, err}func (j *AutoUpdateJob) Reschedule(ctx context.Context) error {slog.InfoContext(ctx, "rescheduling auto-update job in new scheduler; currently requires restart")return nil}++func (j *AutoUpdateJob) Reconcile(_ context.Context, previous schedulertypes.Run) (schedulertypes.Outcome, error) {+ return jobcontext.ConfirmedTarget(previous, "auto-update"), nil+}++func autoUpdateRetryOptionsInternal(previous schedulertypes.Run) (updatertypes.Options, schedulertypes.Outcome, bool) {+ options := updatertypes.Options{}+ unresolvedTargets := false+ confirmed := jobcontext.ConfirmedTarget(previous, "auto-update")+ if confirmed.Status == schedulertypes.Succeeded {+ return options, confirmed, false+ }+ hasResourceTargets := false+ for _, target := range previous.Outcome.Targets {+ if target.ID == "auto-update" {+ continue+ }+ hasResourceTargets = true+ if target.Status == schedulertypes.Succeeded || target.Status == schedulertypes.Skipped {+ continue+ }+ if target.ResourceType != "container" || target.Status != schedulertypes.Failed {+ unresolvedTargets = true+ continue+ }+ options.ResourceIds = append(options.ResourceIds, target.ID)+ }+ if hasResourceTargets && len(options.ResourceIds) == 0 {+ if unresolvedTargets {+ return options, schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: "Remaining image or unconfirmed targets need review before retrying", Targets: previous.Outcome.Targets}, true+ }+ return options, schedulertypes.Outcome{Status: schedulertypes.Succeeded, Targets: previous.Outcome.Targets}, false+ }+ if len(options.ResourceIds) > 0 {+ options.Type = "container"+ }+ return options, schedulertypes.Outcome{}, unresolvedTargets+}diff --git a/backend/pkg/scheduler/docker_client_refresh_job.go b/backend/pkg/scheduler/docker_client_refresh_job.goindex b3ab4cc64..5dc5d924d 100644--- a/backend/pkg/scheduler/docker_client_refresh_job.go+++ b/backend/pkg/scheduler/docker_client_refresh_job.go@@ -4,10 +4,12 @@ import ("context""log/slog"+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"+"github.com/getarcaneapp/arcane/backend/v2/internal/docker""github.com/getarcaneapp/arcane/backend/v2/internal/settings"- "github.com/robfig/cron/v3"+ scheduleutil "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/schedule")const DockerClientRefreshJobName = "docker-client-refresh"@@ -38,7 +40,7 @@ func (j *DockerClientRefreshJob) Schedule(ctx context.Context) string {schedule = dockerClientRefreshDefaultSchedule}- parser := cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow)+ parser := scheduleutil.Parser()if _, err := parser.Parse(schedule); err != nil {slog.WarnContext(ctx, "Invalid cron expression for Docker client refresh, using default", "invalid_schedule", schedule, "error", err)return dockerClientRefreshDefaultSchedule@@ -47,11 +49,12 @@ func (j *DockerClientRefreshJob) Schedule(ctx context.Context) string {return schedule}-func (j *DockerClientRefreshJob) Run(ctx context.Context) {+func (j *DockerClientRefreshJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {if err := j.dockerClientService.RefreshClient(ctx); err != nil {slog.WarnContext(ctx, "Docker client refresh failed", "error", err)- return+ return schedulertypes.Outcome{}, err}slog.DebugContext(ctx, "Docker client refresh completed")+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil}diff --git a/backend/pkg/scheduler/entityjobs/entityjobs.go b/backend/pkg/scheduler/entityjobs/entityjobs.goindex 3c854880f..855df80a6 100644--- a/backend/pkg/scheduler/entityjobs/entityjobs.go+++ b/backend/pkg/scheduler/entityjobs/entityjobs.go@@ -77,7 +77,7 @@ func (r *Registry) Context(ctx context.Context) context.Context {func (r *Registry) JobName(entityID string) string { return r.jobPrefix + entityID }// Register adds (or replaces) the dynamic job for entityID.-func (r *Registry) Register(ctx context.Context, entityID string, schedule func(context.Context) string, run func(context.Context)) {+func (r *Registry) Register(ctx context.Context, entityID string, schedule func(context.Context) string, run func(context.Context) (schedulertypes.Outcome, error), reconcile ...func(context.Context, schedulertypes.Run) (schedulertypes.Outcome, error)) {if r.scheduler == nil {return}@@ -87,6 +87,9 @@ func (r *Registry) Register(ctx context.Context, entityID string, schedule func(ScheduleFn: schedule,RunFn: run,}+ if len(reconcile) > 0 {+ job.ReconcileFn = reconcile[0]+ }if err := r.scheduler.AddJob(schedulerCtx, job); err != nil {slog.ErrorContext(schedulerCtx, "failed to register scheduled job", "job", job.JobName, "error", err)}diff --git a/backend/pkg/scheduler/event_cleanup_job.go b/backend/pkg/scheduler/event_cleanup_job.goindex 2444d0fa1..949cdb7d4 100644--- a/backend/pkg/scheduler/event_cleanup_job.go+++ b/backend/pkg/scheduler/event_cleanup_job.go@@ -5,6 +5,8 @@ import ("log/slog""time"+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"+"github.com/getarcaneapp/arcane/backend/v2/internal/activity""github.com/getarcaneapp/arcane/backend/v2/internal/event""github.com/getarcaneapp/arcane/backend/v2/internal/settings"@@ -38,14 +40,14 @@ func (j *EventCleanupJob) Schedule(ctx context.Context) string {return s}-func (j *EventCleanupJob) Run(ctx context.Context) {+func (j *EventCleanupJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {slog.InfoContext(ctx, "Running event cleanup job", "jobName", EventCleanupJobName)// Delete events older than 36 hoursolderThan := 36 * time.Hourif err := j.eventService.DeleteOldEvents(ctx, olderThan); err != nil {slog.ErrorContext(ctx, "Failed to delete old events", "jobName", EventCleanupJobName, "olderThan", olderThan.String(), "error", err)- return+ return schedulertypes.Outcome{}, err}slog.InfoContext(ctx, "Event cleanup job completed successfully",@@ -58,7 +60,7 @@ func (j *EventCleanupJob) Run(ctx context.Context) {deleted, err := j.activityService.PruneHistory(ctx, retentionDays, maxEntries)if err != nil {slog.ErrorContext(ctx, "Failed to prune activity history", "jobName", EventCleanupJobName, "error", err)- return+ return schedulertypes.Outcome{}, err}slog.InfoContext(ctx, "Activity history cleanup completed successfully",@@ -67,6 +69,7 @@ func (j *EventCleanupJob) Run(ctx context.Context) {"maxEntries", maxEntries,"deleted", deleted)}+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil}func (j *EventCleanupJob) Reschedule(ctx context.Context) error {diff --git a/backend/pkg/scheduler/expired_sessions_cleanup_job.go b/backend/pkg/scheduler/expired_sessions_cleanup_job.goindex feec763bc..078b92c72 100644--- a/backend/pkg/scheduler/expired_sessions_cleanup_job.go+++ b/backend/pkg/scheduler/expired_sessions_cleanup_job.go@@ -5,6 +5,8 @@ import ("log/slog""time"+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"+"github.com/getarcaneapp/arcane/backend/v2/internal/session""github.com/getarcaneapp/arcane/backend/v2/internal/settings")@@ -35,20 +37,21 @@ func (j *ExpiredSessionsCleanupJob) Schedule(ctx context.Context) string {return s}-func (j *ExpiredSessionsCleanupJob) Run(ctx context.Context) {+func (j *ExpiredSessionsCleanupJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {slog.InfoContext(ctx, "Running expired sessions cleanup job", "jobName", ExpiredSessionsCleanupJobName)revokedRetention := 7 * 24 * time.Hourdeleted, err := j.sessionService.DeleteExpiredSessions(ctx, revokedRetention)if err != nil {slog.ErrorContext(ctx, "Failed to delete expired user sessions", "jobName", ExpiredSessionsCleanupJobName, "revokedRetention", revokedRetention.String(), "error", err)- return+ return schedulertypes.Outcome{}, err}slog.InfoContext(ctx, "Expired sessions cleanup job completed successfully","jobName", ExpiredSessionsCleanupJobName,"revokedRetention", revokedRetention.String(),"deleted", deleted)+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil}func (j *ExpiredSessionsCleanupJob) Reschedule(ctx context.Context) error {diff --git a/backend/pkg/scheduler/git_clone_cleanup_job.go b/backend/pkg/scheduler/git_clone_cleanup_job.goindex 1133e1d80..2150462d5 100644--- a/backend/pkg/scheduler/git_clone_cleanup_job.go+++ b/backend/pkg/scheduler/git_clone_cleanup_job.go@@ -5,6 +5,8 @@ import ("log/slog""time"+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"+"github.com/getarcaneapp/arcane/backend/v2/internal/gitrepo""github.com/getarcaneapp/arcane/backend/v2/internal/settings")@@ -40,9 +42,9 @@ func (j *GitCloneCleanupJob) Schedule(_ context.Context) string {return "0 30 * * * *"}-func (j *GitCloneCleanupJob) Run(ctx context.Context) {+func (j *GitCloneCleanupJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {if j.repoService == nil || j.repoService.Client == nil {- return+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil}maxAge := gitCloneScratchMinAge@@ -54,11 +56,12 @@ func (j *GitCloneCleanupJob) Run(ctx context.Context) {removed, err := j.repoService.PurgeScratchDirs(ctx, maxAge)if err != nil {slog.ErrorContext(ctx, "Failed to purge leaked git clone scratch dirs", "jobName", GitCloneCleanupJobName, "removed", removed, "error", err)- return+ return schedulertypes.Outcome{}, err}if removed > 0 {slog.InfoContext(ctx, "Purged leaked git clone scratch dirs", "jobName", GitCloneCleanupJobName, "removed", removed)}+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil}func (j *GitCloneCleanupJob) Reschedule(_ context.Context) error {diff --git a/backend/pkg/scheduler/image_update_watcher.go b/backend/pkg/scheduler/image_update_watcher.goindex 0cf1ee514..94e486f58 100644--- a/backend/pkg/scheduler/image_update_watcher.go+++ b/backend/pkg/scheduler/image_update_watcher.go@@ -20,6 +20,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/types/v2/containerregistry"imageupdatetypes "github.com/getarcaneapp/arcane/types/v2/imageupdate"+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler""github.com/moby/moby/api/types/events""go.getarcane.app/streams/bus")@@ -79,6 +80,9 @@ type ImageUpdateWatcher struct {environmentService registryCredentialLoaderInternaldockerService dockerEventBusProviderInternalprojectService projectImageRefsBackfillerInternal+ dispatchMu sync.RWMutex+ dispatcher schedulertypes.Dispatcher+ eventDegraded atomic.BoolactorRuntime *actors.RuntimeactorProcess atomic.Pointer[actors.Actor]stopping atomic.Bool@@ -147,7 +151,9 @@ func (w *ImageUpdateWatcher) Start(ctx context.Context) error {return errors.New("docker event bus unavailable")}eventCh, unsubscribe := eventBus.Subscribe(events.ImageEventType, bus.WithSubscriberBuffer(16))- defer unsubscribe()+ defer func() { unsubscribe() }()+ reconnect := time.NewTicker(5 * time.Second)+ defer reconnect.Stop()process, err := actors.NewActor(ctx,@@ -174,11 +180,12 @@ func (w *ImageUpdateWatcher) Start(ctx context.Context) error {slog.InfoContext(ctx, "image update watcher started")defer func() {- w.actorProcess.CompareAndSwap(process, nil)stopCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), imageUpdateWatcherStopTimeout)defer cancel()if err := process.Stop(stopCtx); err != nil {slog.ErrorContext(context.WithoutCancel(ctx), "failed to stop image update watcher actor", "error", err)+ } else {+ w.actorProcess.CompareAndSwap(process, nil)}slog.InfoContext(context.WithoutCancel(ctx), "image update watcher stopped")}()@@ -192,10 +199,19 @@ func (w *ImageUpdateWatcher) Start(ctx context.Context) error {return nil}return errors.New("image update watcher actor stopped unexpectedly")+ case <-reconnect.C:+ if eventCh == nil {+ if currentBus := w.dockerService.EventBus(); currentBus != nil {+ unsubscribe()+ eventCh, unsubscribe = currentBus.Subscribe(events.ImageEventType, bus.WithSubscriberBuffer(16))+ w.eventDegraded.Store(false)+ }+ }case _, ok := <-eventCh:if !ok {slog.WarnContext(ctx, "docker image event subscription closed; scheduled image polling remains active")eventCh = nil+ w.eventDegraded.Store(true)continue}if !w.settingsService.GetBoolSetting(ctx, "imageEventWatcherEnabled", false) {@@ -267,12 +283,8 @@ func (w *ImageUpdateWatcher) RunNow(ctx context.Context) error {return admission.Err}- select {- case <-ctx.Done():- return ctx.Err()- case err := <-admission.Value.Done():- return err- }+ // Cancellation reaches the scan context; retain the execution guard until its worker exits.+ return <-admission.Value.Done()}func (w *ImageUpdateWatcher) executeScanInternal(ctx context.Context) error {@@ -298,12 +310,14 @@ func (w *ImageUpdateWatcher) executeScanInternal(ctx context.Context) error {updates := 0scanErrors := 0- for _, result := range results {+ targets := make([]schedulertypes.TargetOutcome, 0, len(results))+ for imageID, result := range results {if result == nil {continue}if result.Error != "" {scanErrors+++ targets = append(targets, schedulertypes.TargetOutcome{ID: imageID, Status: schedulertypes.Failed, Message: result.Error})continue}if result.HasUpdate {@@ -311,6 +325,9 @@ func (w *ImageUpdateWatcher) executeScanInternal(ctx context.Context) error {}}slog.InfoContext(ctx, "image scan run completed", "checked", len(results), "updates", updates, "errors", scanErrors)+ if scanErrors > 0 {+ return &schedulertypes.OutcomeError{Outcome: schedulertypes.Outcome{Status: schedulertypes.Partial, Message: "Some image update checks failed", Targets: targets}}+ }return nil}@@ -331,6 +348,7 @@ func (w *ImageUpdateWatcher) backfillProjectImageRefsInternal(ctx context.Contextype imageUpdateWatcherActorInternal struct {watcher *ImageUpdateWatcherscanRunning bool+ scanSubmitting boolautomaticPending booltriggerGeneration uint64metadataReady bool@@ -378,10 +396,14 @@ func (a *imageUpdateWatcherActorInternal) receiveInternal(ctx *actors.Context, rif message.Kind != imageUpdateScanCompletedMessageInternal {return}- a.scanRunning = falseif message.Value.automatic {- a.watcher.triggerIngress.Acknowledge(message.Value.triggerGeneration)+ a.scanSubmitting = false+ if message.Err == nil {+ a.watcher.triggerIngress.Acknowledge(message.Value.triggerGeneration)+ }a.automaticPending = a.watcher.triggerIngress.Pending()+ } else {+ a.scanRunning = false}if message.Value.automatic && message.Err != nil && !errors.Is(message.Err, context.Canceled) {slog.ErrorContext(ctx.Context(), "image update watcher scan failed", "error", message.Err)@@ -400,7 +422,7 @@ func (a *imageUpdateWatcherActorInternal) receiveSignalInternal(ctx *actors.ConttriggeredAt, generation := a.watcher.triggerIngress.Begin()a.automaticPending = truea.triggerGeneration = generation- if a.metadataReady && !a.scanRunning {+ if a.metadataReady && !a.scanRunning && !a.scanSubmitting {a.debounceTimer.Reset(ctx, imageUpdateDebounceElapsedMessageInternal, max(time.Until(triggeredAt.Add(a.watcher.debounce)), 0))}return@@ -425,7 +447,7 @@ func (a *imageUpdateWatcherActorInternal) receiveSignalInternal(ctx *actors.Contfunc (a *imageUpdateWatcherActorInternal) receiveTimerInternal(ctx *actors.Context, message actors.Message[imageUpdateMessageKindInternal, uint64]) {if message.Kind == imageUpdateDebounceElapsedMessageInternal {- if !a.debounceTimer.Current(message.Value) || !a.automaticPending || a.scanRunning {+ if !a.debounceTimer.Current(message.Value) || !a.automaticPending || a.scanRunning || a.scanSubmitting {return}if remaining := time.Until(a.watcher.triggerIngress.Latest().Add(a.watcher.debounce)); remaining > 0 {@@ -461,7 +483,11 @@ func (a *imageUpdateWatcherActorInternal) startBackfillInternal(ctx *actors.Cont}func (a *imageUpdateWatcherActorInternal) startScanInternal(scanCtx context.Context, ctx *actors.Context, automatic bool, triggerGeneration uint64, done *actors.Promise[error]) {- a.scanRunning = true+ if automatic {+ a.scanSubmitting = true+ } else {+ a.scanRunning = true+ }worker := actors.Worker[imageUpdateMessageKindInternal, imageUpdateScanCompletionInternal]{Actor: ctx,WorkContext: scanCtx,@@ -469,11 +495,24 @@ func (a *imageUpdateWatcherActorInternal) startScanInternal(scanCtx context.ContCompletionKind: imageUpdateScanCompletedMessageInternal,PanicValue: imageUpdateScanCompletionInternal{automatic: automatic, triggerGeneration: triggerGeneration, done: done},ActorStopped: func(_ imageUpdateScanCompletionInternal, err error) {- done.Resolve(err)+ if done != nil {+ done.Resolve(err)+ }},}worker.Start(func(workerCtx context.Context) (imageUpdateScanCompletionInternal, error) {- return imageUpdateScanCompletionInternal{automatic: automatic, triggerGeneration: triggerGeneration, done: done}, a.watcher.executeScanInternal(workerCtx)+ completion := imageUpdateScanCompletionInternal{automatic: automatic, triggerGeneration: triggerGeneration, done: done}+ if automatic {+ a.watcher.dispatchMu.RLock()+ dispatcher := a.watcher.dispatcher+ a.watcher.dispatchMu.RUnlock()+ if dispatcher == nil {+ return completion, errors.New("durable job dispatcher unavailable")+ }+ _, err := dispatcher.Submit(workerCtx, schedulertypes.Request{JobID: a.watcher.Name(), EnvironmentID: "0", Trigger: "watcher"})+ return completion, err+ }+ return completion, a.watcher.executeScanInternal(workerCtx)})}@@ -489,6 +528,21 @@ func (a *imageUpdateWatcherActorInternal) resetScheduleInternal(ctx *actors.Cont}}- delay := time.Until(schedule.Next(time.Now().In(a.watcher.location)))- a.scheduleTimer.Reset(ctx, imageUpdateScheduledPollMessageInternal, delay)+ next := schedule.Next(time.Now().In(a.watcher.location))+ a.watcher.dispatchMu.RLock()+ dispatcher := a.watcher.dispatcher+ a.watcher.dispatchMu.RUnlock()+ if dispatcher != nil {+ if err := dispatcher.Checkpoint(ctx.Context(), a.watcher.Name(), spec, next); err != nil {+ slog.ErrorContext(ctx.Context(), "Failed to checkpoint image polling schedule", "error", err)+ }+ }+ a.scheduleTimer.Reset(ctx, imageUpdateScheduledPollMessageInternal, time.Until(next))+}++// SetDispatcher connects automatic image scans to durable admission.+func (w *ImageUpdateWatcher) SetDispatcher(dispatcher schedulertypes.Dispatcher) {+ w.dispatchMu.Lock()+ defer w.dispatchMu.Unlock()+ w.dispatcher = dispatcher}diff --git a/backend/pkg/scheduler/image_update_watcher_test.go b/backend/pkg/scheduler/image_update_watcher_test.goindex 9dcaa7a9e..a1f5777c0 100644--- a/backend/pkg/scheduler/image_update_watcher_test.go+++ b/backend/pkg/scheduler/image_update_watcher_test.go@@ -4,6 +4,7 @@ import ("bytes""context""fmt"+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler""log/slog""sync""testing"@@ -196,7 +197,7 @@ func newImageUpdateWatcherForTestInternal(t *testing.T, scanner imageUpdateScanndefer cancel()require.NoError(t, lifecycle.Stop(stopCtx))})- return &ImageUpdateWatcher{+ watcher := &ImageUpdateWatcher{imageUpdateService: scanner,settingsService: settings,environmentService: registryCredentialLoaderFakeInternal{},@@ -212,6 +213,8 @@ func newImageUpdateWatcherForTestInternal(t *testing.T, scanner imageUpdateScannstarted: make(chan struct{}),stopped: make(chan struct{}),}+ watcher.SetDispatcher(&testDispatcherInternal{run: func(ctx context.Context, _ schedulertypes.Request) error { return watcher.RunNow(ctx) }})+ return watcher}func startImageUpdateWatcherForTestInternal(t *testing.T, watcher *ImageUpdateWatcher) (context.CancelFunc, <-chan error) {diff --git a/backend/pkg/scheduler/job_schedule_integration_test.go b/backend/pkg/scheduler/job_schedule_integration_test.goindex f1d9a4394..3534c5032 100644--- a/backend/pkg/scheduler/job_schedule_integration_test.go+++ b/backend/pkg/scheduler/job_schedule_integration_test.go@@ -8,6 +8,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/job"+ "github.com/getarcaneapp/arcane/backend/v2/internal/kv""github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/types/v2/jobschedule""github.com/libtnb/sqlite"@@ -103,7 +104,7 @@ func openJobScheduleTestDatabaseInternal(t *testing.T, ctx context.Context, datagormDB, err := gorm.Open(sqlite.Open(databasePath), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)})require.NoError(t, err)- require.NoError(t, gormDB.AutoMigrate(&settings.SettingVariable{}))+ require.NoError(t, gormDB.AutoMigrate(&settings.SettingVariable{}, &kv.KVEntry{}))db := &database.DB{DB: gormDB}settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)diff --git a/backend/pkg/scheduler/jobcontext/context.go b/backend/pkg/scheduler/jobcontext/context.gonew file mode 100644index 000000000..a75662b0c--- /dev/null+++ b/backend/pkg/scheduler/jobcontext/context.go@@ -0,0 +1,42 @@+// Package jobcontext carries durable execution progress through domain operations.+package jobcontext++import (+ "context"++ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"+)++type progressKeyInternal struct{}+type runKeyInternal struct{}++func WithExecution(ctx context.Context, run schedulertypes.Run, persist func(schedulertypes.TargetOutcome) error) context.Context {+ ctx = context.WithValue(ctx, runKeyInternal{}, run)+ if persist != nil {+ ctx = context.WithValue(ctx, progressKeyInternal{}, persist)+ }+ return ctx+}++func Progress(ctx context.Context, target schedulertypes.TargetOutcome) error {+ if persist, ok := ctx.Value(progressKeyInternal{}).(func(schedulertypes.TargetOutcome) error); ok {+ return persist(target)+ }+ return nil+}++func Run(ctx context.Context) (schedulertypes.Run, bool) {+ run, ok := ctx.Value(runKeyInternal{}).(schedulertypes.Run)+ return run, ok+}++// ConfirmedTarget returns a terminal outcome only for a target whose completion+// was persisted or confirmed against its activity by the execution coordinator.+func ConfirmedTarget(run schedulertypes.Run, targetID string) schedulertypes.Outcome {+ for _, target := range run.Outcome.Targets {+ if target.ID == targetID && target.Status == schedulertypes.Succeeded {+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded, ActivityID: target.ActivityID, Targets: run.Outcome.Targets}+ }+ }+ return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: "The interrupted operation has no confirmed completion", Targets: run.Outcome.Targets}+}diff --git a/backend/pkg/scheduler/queue/queue.go b/backend/pkg/scheduler/queue/queue.gonew file mode 100644index 000000000..c6a8da88a--- /dev/null+++ b/backend/pkg/scheduler/queue/queue.go@@ -0,0 +1,464 @@+// Package queue persists scheduler runs and coordinates their execution and recovery.+package queue++import (+ "context"+ "crypto/rand"+ "io"+ "log/slog"+ "strings"+ "sync"+ "time"++ "emperror.dev/errors"+ "github.com/getarcaneapp/arcane/backend/v2/internal/kv"+ st "github.com/getarcaneapp/arcane/types/v2/scheduler"+ "github.com/google/uuid"+)++// Queue persists work before dispatching it to the owning domain.+type Queue struct {+ store *kv.KVService+ owner string+ mu sync.Mutex+ active map[string]bool+ checkpointed map[string]bool+ ordinary int+ health int+ stopping bool+ started bool+ cancel context.CancelFunc+ done chan struct{}+ wake chan struct{}+ workers sync.WaitGroup+ execute func(context.Context, st.Run) (st.Outcome, error)+ reconcile func(context.Context, st.Run) (st.Outcome, error)+}++// Submit persists acceptance before waking the dispatcher. Run IDs deduplicate+// requests within a job and environment. Manual and remote requests stay separate;+// other triggers coalesce pending work.+// Cancellation of ctx affects admission only, not an accepted run's lifetime.+func (q *Queue) Submit(ctx context.Context, request st.Request) (st.Run, error) {+ q.mu.Lock()+ defer q.mu.Unlock()+ if q.stopping {+ return st.Run{}, errors.New("job queue is stopping")+ }+ if request.EnvironmentID == "" {+ request.EnvironmentID = "0"+ }+ if request.JobID == "" {+ return st.Run{}, errors.New("job ID required")+ }+ if request.Trigger == "" {+ request.Trigger = "manual"+ }+ if request.RunID == "" {+ request.RunID = uuid.NewString()+ }+ if _, err := uuid.Parse(request.RunID); err != nil {+ return st.Run{}, errors.New("invalid run ID")+ }+ now := time.Now().UTC()+ result := st.Run{ID: request.RunID, JobID: request.JobID, EnvironmentID: request.EnvironmentID, Trigger: request.Trigger, RequestedBy: request.RequestedBy, RequestedWithKey: request.RequestedWithKey, RemoteAccepted: request.Trigger == "remote", Status: st.Queued, CreatedAt: now, UpdatedAt: now}+ err := q.mutateInternal(ctx, request.EnvironmentID, request.JobID, func(record *st.QueueRecord) error {+ if request.Trigger == "scheduled" || request.Trigger == "recovery" {+ record.LastEnqueuedAt = now+ }+ for _, existing := range record.Runs {+ if existing.ID == request.RunID {+ result = existing+ return nil+ }+ if request.Trigger != "manual" && request.Trigger != "remote" && existing.Trigger != "manual" && existing.Trigger != "remote" && !existing.Status.Terminal() && existing.Status != st.Running {+ result = existing+ return nil+ }+ }+ if existing, ok := record.Receipts[request.RunID]; ok {+ result = existing+ return nil+ }+ pruneRunsInternal(record, now)+ record.Runs = append(record.Runs, result)+ return nil+ })+ if err == nil {+ q.signalInternal()+ }+ return result, err+}++func (q *Queue) signalInternal() {+ select {+ case q.wake <- struct{}{}:+ default:+ }+}++// Checkpoint records the next due occurrence atomically with overdue recovery.+func (q *Queue) Checkpoint(ctx context.Context, jobID, schedule string, next time.Time) error {+ q.mu.Lock()+ defer q.mu.Unlock()+ first := !q.checkpointed[jobID]+ recovered := false+ err := q.mutateInternal(ctx, "0", jobID, func(record *st.QueueRecord) error {+ now := time.Now().UTC()+ if first && record.Schedule == schedule && !record.NextRun.IsZero() && !record.NextRun.After(now) && record.LastEnqueuedAt.Before(record.NextRun) {+ pending := false+ for _, run := range record.Runs {+ if !run.Status.Terminal() {+ pending = true+ break+ }+ }+ if !pending {+ recovered = true+ record.LastEnqueuedAt = now+ record.Runs = append(record.Runs, st.Run{ID: uuid.NewString(), JobID: jobID, EnvironmentID: "0", Trigger: "recovery", Status: st.Queued, CreatedAt: now, UpdatedAt: now})+ }+ }+ record.Schedule = schedule+ record.NextRun = next+ return nil+ })+ if err == nil {+ q.checkpointed[jobID] = true+ if recovered {+ q.signalInternal()+ }+ }+ return err+}++// Start validates persisted state and starts dispatching on ctx. Repeated calls+// are harmless, but a stopped queue cannot be restarted; create a new instance.+func (q *Queue) Start(ctx context.Context) error {+ q.mu.Lock()+ defer q.mu.Unlock()+ if q.started {+ return nil+ }+ if q.execute == nil {+ return errors.New("job executor unavailable")+ }+ // Fail startup if persisted state cannot be read. Never replace it with empty state.+ if _, err := q.Records(ctx); err != nil {+ return err+ }+ runCtx, cancel := context.WithCancel(ctx)+ q.cancel = cancel+ q.started = true+ go q.dispatchInternal(runCtx)+ return nil+}++// Stop rejects new submissions, cancels execution, and waits for workers until+// ctx expires. Unfinished claims remain persisted for startup reconciliation.+func (q *Queue) Stop(ctx context.Context) error {+ q.mu.Lock()+ q.stopping = true+ started := q.started+ if q.cancel != nil {+ q.cancel()+ }+ q.mu.Unlock()+ if !started {+ return nil+ }+ select {+ case <-q.done:+ return nil+ case <-ctx.Done():+ return ctx.Err()+ }+}++func (q *Queue) dispatchInternal(ctx context.Context) {+ defer close(q.done)+ defer q.workers.Wait()+ nextRetention := time.Now().Add(time.Hour)+ for ctx.Err() == nil {+ if !time.Now().Before(nextRetention) {+ if err := q.pruneInternal(ctx); err != nil && ctx.Err() == nil {+ slog.ErrorContext(ctx, "Job retention failed", "error", err)+ }+ nextRetention = time.Now().Add(time.Hour)+ }+ nextRetry, err := q.dispatchReadyInternal(ctx)+ if err != nil && ctx.Err() == nil {+ slog.ErrorContext(ctx, "Job queue dispatch failed", "error", err)+ nextRetry = time.Now().Add(5 * time.Second)+ }+ nextWake := nextRetention+ if !nextRetry.IsZero() && nextRetry.Before(nextWake) {+ nextWake = nextRetry+ }+ timer := time.NewTimer(max(time.Until(nextWake), 0))+ select {+ case <-ctx.Done():+ timer.Stop()+ return+ case <-q.wake:+ case <-timer.C:+ }+ timer.Stop()+ }+}++func (q *Queue) dispatchReadyInternal(ctx context.Context) (time.Time, error) {+ records, err := q.Records(ctx)+ if err != nil {+ return time.Time{}, err+ }+ var nextRetry time.Time+ for _, record := range records {+ key := queueKeyInternal(record.EnvironmentID, record.JobID)+ reserved := strings.HasPrefix(record.JobID, "environment-health") || record.JobID == "docker-client-refresh" || record.JobID == "activity-sweep"+ q.mu.Lock()+ if q.active[key] || q.stopping || (!reserved && q.ordinary >= 4) || (reserved && q.health >= 2) {+ q.mu.Unlock()+ continue+ }+ selected, retryAt := q.nextRunInternal(record.Runs)+ if !retryAt.IsZero() && (nextRetry.IsZero() || retryAt.Before(nextRetry)) {+ nextRetry = retryAt+ }+ if selected == nil {+ q.mu.Unlock()+ continue+ }+ run := *selected+ reconcile := run.Status == st.Running+ err = q.UpdateRun(ctx, run, func(current *st.Run) error {+ if current.Status != run.Status || current.Owner != run.Owner {+ return ErrRunConflict+ }+ now := time.Now().UTC()+ current.Status = st.Running+ current.Owner = q.owner+ current.UpdatedAt = now+ current.StartedAt = &now+ current.NextAttempt = nil+ current.AttemptCount+++ current.Attempts = append(current.Attempts, st.Attempt{Number: current.AttemptCount, StartedAt: now})+ if len(current.Attempts) > 100 {+ current.Attempts = current.Attempts[len(current.Attempts)-100:]+ }+ // Preserve the previous outcome for reconciliation and completed target filtering.+ run = *current+ return nil+ })+ if err != nil {+ q.mu.Unlock()+ return time.Time{}, err+ }+ q.active[key] = true+ if reserved {+ q.health+++ } else {+ q.ordinary+++ }+ q.workers.Add(1)+ q.mu.Unlock()+ go q.executeInternal(ctx, key, run, reserved, reconcile)+ }+ return nextRetry, nil+}++func (q *Queue) executeInternal(ctx context.Context, key string, run st.Run, reserved, reconcile bool) {+ defer q.workers.Done()+ defer func() {+ q.mu.Lock()+ delete(q.active, key)+ if reserved {+ q.health--+ } else {+ q.ordinary--+ }+ q.mu.Unlock()+ q.signalInternal()+ }()+ outcome, err := q.invokeInternal(ctx, run, reconcile)+ if outcome.Status == "" {+ if err != nil {+ outcome.Status = st.Failed+ } else {+ outcome.Status = st.Succeeded+ }+ }+ if err != nil && outcome.Message == "" {+ outcome.Message = err.Error()+ }+ // Shutdown leaves a running claim for startup reconciliation, never a false success.+ if ctx.Err() != nil {+ return+ }+ q.persistOutcomeInternal(ctx, run, outcome)+}++func (q *Queue) persistOutcomeInternal(ctx context.Context, run st.Run, outcome st.Outcome) {+ // Keep the execution slot until the terminal result has been durably recorded.+ for {+ err := q.UpdateRun(ctx, run, func(current *st.Run) error {+ if current.Owner != q.owner || current.Status != st.Running {+ return ErrRunConflict+ }+ now := time.Now().UTC()+ current.Status = outcome.Status+ current.UpdatedAt = now+ outcome = mergeTargetProgressInternal(current.Outcome, outcome)+ current.Outcome = outcome+ if outcome.Status == st.Retrying || outcome.Status == st.Waiting {+ delay := 5 * time.Second * time.Duration(1<<min(run.AttemptCount-1, 6))+ delay = min(delay, 5*time.Minute)+ var jitter [1]byte+ if _, jitterErr := io.ReadFull(rand.Reader, jitter[:]); jitterErr != nil {+ // Keep the unjittered backoff when randomness is unavailable.+ slog.WarnContext(ctx, "Retry jitter unavailable; using exponential backoff", "runId", run.ID, "error", jitterErr)+ } else {+ delay = time.Duration(float64(delay) * (0.8 + float64(jitter[0])/255*0.2))+ }+ next := now.Add(delay)+ current.NextAttempt = &next+ } else {+ current.NextAttempt = nil+ }+ if outcome.Status.Terminal() {+ current.FinishedAt = &now+ }+ if len(current.Attempts) > 0 {+ last := ¤t.Attempts[len(current.Attempts)-1]+ last.FinishedAt = &now+ last.Outcome = outcome+ }+ return nil+ })+ if err == nil || errors.Is(err, ErrRunConflict) {+ return+ }+ slog.ErrorContext(ctx, "Unable to persist job result; retaining execution guard", "runId", run.ID, "error", err)+ timer := time.NewTimer(time.Second)+ select {+ case <-ctx.Done():+ timer.Stop()+ return+ case <-timer.C:+ }+ }+}++func (q *Queue) invokeInternal(ctx context.Context, run st.Run, reconcile bool) (outcome st.Outcome, err error) {+ defer func() {+ if recovered := recover(); recovered != nil {+ outcome = st.Outcome{Status: st.NeedsAttention, Message: "Job panicked; inspect logs before retrying"}+ err = errors.Errorf("job panic: %v", recovered)+ slog.ErrorContext(ctx, "Job panicked", "runId", run.ID, "error", err)+ }+ }()+ if reconcile && q.reconcile != nil {+ return q.reconcile(ctx, run)+ }+ return q.execute(ctx, run)+}++// Cancel marks an unstarted queued run canceled. Runs already attempted locally+// or dispatched remotely must be reconciled rather than canceled as pending work.+func (q *Queue) Cancel(ctx context.Context, environmentID, jobID, runID string) (st.Run, error) {+ run, err := q.Get(ctx, environmentID, jobID, runID)+ if err != nil {+ return run, err+ }+ err = q.UpdateRun(ctx, run, func(current *st.Run) error {+ if current.Status != st.Queued || current.AttemptCount != 0 || current.RemoteDeliveryAttempted {+ return errors.New("only unstarted pending runs can be canceled")+ }+ now := time.Now().UTC()+ current.Status = st.Canceled+ current.UpdatedAt = now+ current.FinishedAt = &now+ run = *current+ return nil+ })+ return run, err+}++// Retry requeues a failed, partial, or attention-required run without changing its+// ID or dropping target progress. Unconfirmed remote work cannot be retried here.+func (q *Queue) Retry(ctx context.Context, environmentID, jobID, runID string) (st.Run, error) {+ run, err := q.Get(ctx, environmentID, jobID, runID)+ if err != nil {+ return run, err+ }+ err = q.UpdateRun(ctx, run, func(current *st.Run) error {+ if current.Status != st.Failed && current.Status != st.NeedsAttention && current.Status != st.Partial {+ return errors.New("run is not eligible for retry")+ }+ if current.RemoteAccepted && !current.RemoteSettled {+ return errors.New("remote outcome must be confirmed before retry")+ }+ current.Status = st.Queued+ current.Owner = ""+ current.NextAttempt = nil+ current.FinishedAt = nil+ current.UpdatedAt = time.Now().UTC()+ run = *current+ return nil+ })+ if err == nil {+ q.signalInternal()+ }+ return run, err+}++func (q *Queue) nextRunInternal(runs []st.Run) (*st.Run, time.Time) {+ var interrupted *st.Run+ blocked := false+ for index := range runs {+ run := &runs[index]+ if run.Status == st.NeedsAttention || (run.Status == st.Running && run.Owner == q.owner) {+ blocked = true+ }+ if run.Status == st.Running && run.Owner != q.owner && interrupted == nil {+ interrupted = run+ }+ }+ if interrupted != nil || blocked {+ return interrupted, time.Time{}+ }+ var nextRetry time.Time+ now := time.Now()+ for index := range runs {+ run := &runs[index]+ if run.Status.Terminal() {+ continue+ }+ if run.NextAttempt == nil || !run.NextAttempt.After(now) {+ return run, time.Time{}+ }+ if nextRetry.IsZero() || run.NextAttempt.Before(nextRetry) {+ nextRetry = *run.NextAttempt+ }+ }+ return nil, nextRetry+}++func mergeTargetProgressInternal(previous, outcome st.Outcome) st.Outcome {+ for _, target := range previous.Targets {+ found := false+ for _, updated := range outcome.Targets {+ if updated.ID == target.ID {+ found = true+ break+ }+ }+ if !found {+ outcome.Targets = append(outcome.Targets, target)+ }+ }+ if outcome.ActivityID == "" {+ outcome.ActivityID = previous.ActivityID+ }+ return outcome+}diff --git a/backend/pkg/scheduler/queue/queue_store.go b/backend/pkg/scheduler/queue/queue_store.gonew file mode 100644index 000000000..92df79e87--- /dev/null+++ b/backend/pkg/scheduler/queue/queue_store.go@@ -0,0 +1,212 @@+package queue++import (+ "context"+ "crypto/sha256"+ "encoding/hex"+ "encoding/json/v2"+ "slices"+ "sort"+ "time"++ "emperror.dev/errors"+ "github.com/getarcaneapp/arcane/backend/v2/internal/kv"+ st "github.com/getarcaneapp/arcane/types/v2/scheduler"+ "github.com/google/uuid"+)++const queuePrefixInternal = "jobs."++func queueKeyInternal(environmentID, jobID string) string {+ sum := sha256.Sum256([]byte(environmentID + "\x00" + jobID))+ return queuePrefixInternal + hex.EncodeToString(sum[:])+}++func (q *Queue) mutateInternal(ctx context.Context, environmentID, jobID string, change func(*st.QueueRecord) error) error {+ key := queueKeyInternal(environmentID, jobID)+ for range 20 {+ previous, found, err := q.store.Get(ctx, key)+ if err != nil {+ return err+ }+ record := st.QueueRecord{JobID: jobID, EnvironmentID: environmentID}+ if found {+ if err := json.Unmarshal([]byte(previous), &record); err != nil {+ return errors.WrapIf(err, "decode job queue")+ }+ }+ if err := change(&record); err != nil {+ return err+ }+ next, err := json.Marshal(record)+ if err != nil {+ return err+ }+ if found && string(next) == previous {+ return nil+ }+ var swapped bool+ if found {+ swapped, err = q.store.CompareAndSwap(ctx, key, previous, string(next))+ } else {+ swapped, err = q.store.CreateIfAbsent(ctx, key, string(next))+ }+ if err != nil {+ return err+ }+ if swapped {+ return nil+ }+ }+ return errors.New("job queue contention; retry request with the same run ID")+}++// Records reads all persisted queue records, including run history and receipts.+// Storage or decoding errors fail the read instead of returning an incomplete snapshot.+func (q *Queue) Records(ctx context.Context) ([]st.QueueRecord, error) {+ entries, err := q.store.ListByPrefix(ctx, queuePrefixInternal)+ if err != nil {+ return nil, err+ }+ records := make([]st.QueueRecord, 0, len(entries))+ for _, entry := range entries {+ var record st.QueueRecord+ if err := json.Unmarshal([]byte(entry.Value), &record); err != nil {+ return nil, errors.WrapIff(err, "decode job queue %s", entry.Key)+ }+ records = append(records, record)+ }+ return records, nil+}++// Get retrieves a run by environment, job, and run ID, including compacted+// idempotency receipts. It returns ErrRunNotFound when no matching record exists.+func (q *Queue) Get(ctx context.Context, environmentID, jobID, runID string) (st.Run, error) {+ raw, found, err := q.store.Get(ctx, queueKeyInternal(environmentID, jobID))+ if err != nil {+ return st.Run{}, err+ }+ if !found {+ return st.Run{}, ErrRunNotFound+ }+ var record st.QueueRecord+ if err := json.Unmarshal([]byte(raw), &record); err != nil {+ return st.Run{}, err+ }+ for _, run := range record.Runs {+ if run.ID == runID {+ return run, nil+ }+ }+ if run, ok := record.Receipts[runID]; ok {+ return run, nil+ }+ return st.Run{}, ErrRunNotFound+}++// List returns retained run history newest first, excluding compacted receipts.+// An empty job ID includes every job in the environment; pages are one-based.+func (q *Queue) List(ctx context.Context, environmentID, jobID string, page, limit int) (st.RunList, error) {+ records, err := q.Records(ctx)+ if err != nil {+ return st.RunList{}, err+ }+ runs := []st.Run{}+ for _, record := range records {+ if record.EnvironmentID == environmentID && (jobID == "" || record.JobID == jobID) {+ runs = append(runs, record.Runs...)+ }+ }+ sort.Slice(runs, func(i, j int) bool { return runs[i].CreatedAt.After(runs[j].CreatedAt) })+ page = max(1, page)+ limit = min(100, max(1, limit))+ start := len(runs)+ if page-1 <= len(runs)/limit {+ start = min(len(runs), (page-1)*limit)+ }+ end := min(len(runs), start+limit)+ return st.RunList{Runs: runs[start:end], Total: len(runs), Page: page, Limit: limit}, nil+}++// UpdateRun atomically changes a persisted run or terminal receipt using compare+// and swap. The callback may run more than once and must not perform external side effects.+func (q *Queue) UpdateRun(ctx context.Context, run st.Run, change func(*st.Run) error) error {+ wake := false+ err := q.mutateInternal(ctx, run.EnvironmentID, run.JobID, func(record *st.QueueRecord) error {+ for index := range record.Runs {+ if record.Runs[index].ID == run.ID {+ previous := record.Runs[index].Status+ if err := change(&record.Runs[index]); err != nil {+ return err+ }+ wake = previous != st.Queued && record.Runs[index].Status == st.Queued+ return nil+ }+ }+ if receipt, found := record.Receipts[run.ID]; found {+ if err := change(&receipt); err != nil {+ return err+ }+ if !receipt.Status.Terminal() {+ return errors.New("archived runs cannot be restarted")+ }+ record.Receipts[run.ID] = receipt+ return nil+ }+ return ErrRunNotFound+ })+ if err == nil && wake {+ q.signalInternal()+ }+ return err+}++func (q *Queue) pruneInternal(ctx context.Context) error {+ records, err := q.Records(ctx)+ if err != nil {+ return err+ }+ now := time.Now().UTC()+ for _, record := range records {+ if err := q.mutateInternal(ctx, record.EnvironmentID, record.JobID, func(current *st.QueueRecord) error {+ pruneRunsInternal(current, now)+ return nil+ }); err != nil {+ return err+ }+ }+ return nil+}++func pruneRunsInternal(record *st.QueueRecord, now time.Time) {+ terminal := 0+ for index, run := range slices.Backward(record.Runs) {+ if !run.Status.Terminal() || (run.RemoteAccepted && !run.RemoteSettled) {+ continue+ }+ terminal+++ if terminal <= 100 && now.Sub(run.UpdatedAt) < 7*24*time.Hour {+ continue+ }+ if run.Trigger == "manual" || run.Trigger == "remote" {+ if record.Receipts == nil {+ record.Receipts = make(map[string]st.Run)+ }+ run.Attempts = nil+ run.Outcome.Targets = nil+ record.Receipts[run.ID] = run+ }+ record.Runs = append(record.Runs[:index], record.Runs[index+1:]...)+ }+}++// ErrRunNotFound indicates that neither a run nor its receipt exists.+var ErrRunNotFound = errors.Sentinel("job run not found")++// ErrRunConflict indicates that execution ownership or state changed.+var ErrRunConflict = errors.Sentinel("job run state changed")++// New uses the application's existing KV store; no in-memory admission exists.+func New(store *kv.KVService, execute, reconcile func(context.Context, st.Run) (st.Outcome, error)) *Queue {+ return &Queue{execute: execute, reconcile: reconcile, store: store, owner: uuid.NewString(), active: make(map[string]bool), checkpointed: make(map[string]bool), wake: make(chan struct{}, 1), done: make(chan struct{})}+}diff --git a/backend/pkg/scheduler/schedule/cron.go b/backend/pkg/scheduler/schedule/cron.gonew file mode 100644index 000000000..cf91062a3--- /dev/null+++ b/backend/pkg/scheduler/schedule/cron.go@@ -0,0 +1,32 @@+// Package schedule provides reusable schedule validation.+package schedule++import (+ "strings"++ "emperror.dev/errors"++ "github.com/robfig/cron/v3"+)++// NormalizeSixField validates and normalizes a six-field cron schedule.+func NormalizeSixField(value, subject string) (string, error) {+ fields := strings.Fields(value)+ if len(fields) == 0 {+ return "", errors.Errorf("%s schedule is required", subject)+ }+ if len(fields) != 6 {+ return "", errors.Errorf("invalid %s schedule %q: expected six fields", subject, strings.TrimSpace(value))+ }+ normalized := strings.Join(fields, " ")+ parser := Parser()+ if _, err := parser.Parse(normalized); err != nil {+ return "", errors.WrapIff(err, "invalid %s schedule %q", subject, normalized)+ }+ return normalized, nil+}++// Parser is shared by schedule validation, execution, and next-run display.+func Parser() cron.Parser {+ return cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor)+}diff --git a/backend/pkg/scheduler/scheduled_prune_job.go b/backend/pkg/scheduler/scheduled_prune_job.goindex bb701b80a..7c058b407 100644--- a/backend/pkg/scheduler/scheduled_prune_job.go+++ b/backend/pkg/scheduler/scheduled_prune_job.go@@ -4,12 +4,15 @@ import ("context""log/slog"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"+"github.com/getarcaneapp/arcane/backend/v2/internal/notification""github.com/getarcaneapp/arcane/backend/v2/internal/settings"systemtypes "github.com/getarcaneapp/arcane/types/v2/system""github.com/getarcaneapp/arcane/backend/v2/internal/system"- "github.com/robfig/cron/v3"+ scheduleutil "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/schedule")const ScheduledPruneJobName = "scheduled-prune"@@ -42,7 +45,7 @@ func (j *ScheduledPruneJob) Schedule(ctx context.Context) string {schedule = "0 0 0 * * *"}- parser := cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow)+ parser := scheduleutil.Parser()if _, err := parser.Parse(schedule); err != nil {slog.WarnContext(ctx, "Invalid cron expression for scheduled-prune, using default", "invalid_schedule", schedule, "error", err)return "0 0 0 * * *"@@ -51,18 +54,18 @@ func (j *ScheduledPruneJob) Schedule(ctx context.Context) string {return schedule}-func (j *ScheduledPruneJob) Run(ctx context.Context) {+func (j *ScheduledPruneJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {enabled := j.settingsService.GetBoolSetting(ctx, "scheduledPruneEnabled", false)if !enabled {slog.DebugContext(ctx, "scheduled prune disabled; skipping run")- return+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil}req := buildScheduledPruneRequestInternal(ctx, j.settingsService)if !hasScheduledPruneTargetsInternal(req) {slog.InfoContext(ctx, "scheduled prune run skipped; no resource types selected")- return+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil}slog.InfoContext(ctx, "scheduled prune run started",@@ -76,11 +79,11 @@ func (j *ScheduledPruneJob) Run(ctx context.Context) {result, started, err := j.systemService.PruneAll(ctx, "0", req)if err != nil {slog.ErrorContext(ctx, "scheduled prune run failed", "error", err)- return+ return schedulertypes.Outcome{}, err}if !started {slog.InfoContext(ctx, "scheduled prune run skipped; prune already in progress", "activityId", result.ActivityID)- return+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil}slog.InfoContext(ctx, "scheduled prune run completed",@@ -100,6 +103,15 @@ func (j *ScheduledPruneJob) Run(ctx context.Context) {if err := j.notificationService.SendPruneReportNotification(ctx, result); err != nil {slog.WarnContext(ctx, "failed to send prune report notification", "error", err)}+ outcome := schedulertypes.Outcome{Status: schedulertypes.Succeeded}+ if result.ActivityID != nil {+ outcome.ActivityID = *result.ActivityID+ }+ if len(result.Errors) > 0 {+ outcome.Status = schedulertypes.Partial+ outcome.Message = "Some resources could not be pruned; see activity details"+ }+ return outcome, nil}func (j *ScheduledPruneJob) Reschedule(ctx context.Context) error {@@ -177,3 +189,7 @@ func buildScheduledBuildCachePruneOptionsInternal(ctx context.Context, settingsSUntil: settingsService.GetStringSetting(ctx, "pruneBuildCacheUntil", ""),}}++func (j *ScheduledPruneJob) Reconcile(_ context.Context, previous schedulertypes.Run) (schedulertypes.Outcome, error) {+ return jobcontext.ConfirmedTarget(previous, "scheduled-prune"), nil+}diff --git a/backend/pkg/scheduler/scheduler.go b/backend/pkg/scheduler/scheduler.goindex de206d1ba..5c4638772 100644--- a/backend/pkg/scheduler/scheduler.go+++ b/backend/pkg/scheduler/scheduler.go@@ -10,6 +10,7 @@ import ("emperror.dev/errors""github.com/getarcaneapp/arcane/backend/v2/internal/actors"+ scheduleutil "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/schedule""github.com/getarcaneapp/arcane/backend/v2/pkg/utils"schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler""github.com/robfig/cron/v3"@@ -18,7 +19,7 @@ import (// cronScheduleParser is the shared parser for all cron settings: six fields// with seconds, plus @-descriptors. The image update watcher parses its poll// schedule with the same spec so Jobs-UI cron values behave identically.-var cronScheduleParser = cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor)+var cronScheduleParser = scheduleutil.Parser()var errJobSchedulerStoppedInternal = errors.Sentinel("job scheduler stopped")@@ -30,7 +31,7 @@ type schedulerStateInternal struct {runners map[string]*actors.RunnerentryIDs map[string]cron.EntryIDschedules map[string]string- runLocks map[string]*sync.Mutex+ supervisors map[string]*watcherSupervisorInternalstopping bool}@@ -47,12 +48,15 @@ type schedulerShutdownInternal struct {// jobSchedulerInternal serializes its control plane through one shared actor executor.// robfig/cron remains the timing engine and invokes job bodies outside the actor.type jobSchedulerInternal struct {- cron *cron.Cron- parser cron.Parser- context context.Context- location *time.Location- runtime *actors.Runtime- state *actors.State[schedulerStateInternal]+ cron *cron.Cron+ parser cron.Parser+ context context.Context+ location *time.Location+ runtime *actors.Runtime+ state *actors.State[schedulerStateInternal]+ cancel context.CancelFunc+ dispatchMu sync.RWMutex+ dispatcher schedulertypes.Dispatcher}// NewJobScheduler creates an actor-owned scheduler control plane.@@ -74,7 +78,7 @@ func NewJobScheduler(ctx context.Context, runtime *actors.Runtime, location *timrunners: make(map[string]*actors.Runner),entryIDs: make(map[string]cron.EntryID),schedules: make(map[string]string),- runLocks: make(map[string]*sync.Mutex),+ supervisors: make(map[string]*watcherSupervisorInternal),}state, err := actors.NewState(ctx, runtime, "scheduler", "control-plane", 3, initial, func(value schedulerStateInternal) schedulerStateInternal {value.jobs = slices.Clone(value.jobs)@@ -84,17 +88,19 @@ func NewJobScheduler(ctx context.Context, runtime *actors.Runtime, location *timvalue.runners = maps.Clone(value.runners)value.entryIDs = maps.Clone(value.entryIDs)value.schedules = maps.Clone(value.schedules)- value.runLocks = maps.Clone(value.runLocks)+ value.supervisors = maps.Clone(value.supervisors)return value})if err != nil {return nil, err}+ lifetime, cancel := context.WithCancel(ctx)parser := cronScheduleParserscheduler := &jobSchedulerInternal{cron: cron.New(cron.WithParser(parser), cron.WithLocation(location)),parser: parser,- context: ctx,+ context: lifetime,+ cancel: cancel,location: location,runtime: runtime,state: state,@@ -127,11 +133,16 @@ func (js *jobSchedulerInternal) RegisterBusWatcher(watcher schedulertypes.BusWatif state.stopping {return errJobSchedulerStoppedInternal}- runner, err := actors.NewRunner(js.context, js.runtime, "scheduler-watchers", watcher.Name(), "bus watcher "+watcher.Name(), 3, watcher.Start) //nolint:contextcheck // watcher lifetime belongs to the scheduler, not one state-actor receiver incarnation.+ if _, exists := state.allWatchers[watcher.Name()]; exists {+ return errors.New("watcher already registered")+ }+ supervisor := newWatcherSupervisorInternal(watcher)+ runner, err := actors.NewRunner(js.context, js.runtime, "scheduler-watchers", watcher.Name(), "bus watcher "+watcher.Name(), 3, supervisor.runInternal) //nolint:contextcheck // watcher lifetime belongs to the scheduler, not one state-actor receiver incarnation.if err != nil {return err}state.runners[watcher.Name()] = runner+ state.supervisors[watcher.Name()] = supervisorstate.allWatchers[watcher.Name()] = watcherif canRunManually {state.watchers[watcher.Name()] = watcher@@ -301,6 +312,7 @@ func (js *jobSchedulerInternal) Stop(ctx context.Context) error {}}+ js.cancel()for _, registration := range shutdown.watchers {stopErr = errors.Combine(stopErr, registration.runner.Stop(ctx))if stopper, ok := registration.watcher.(schedulertypes.StoppableBusWatcher); ok {@@ -344,17 +356,17 @@ func (js *jobSchedulerInternal) upsertJobInternal(ctx context.Context, state *scslog.DebugContext(ctx, "Job disabled; not scheduling", "name", jobName)}+ if shouldSchedule {+ if err := js.checkpointInternal(ctx, jobName, schedule, parsedSchedule.Next(time.Now().In(js.location))); err != nil {+ return err+ }+ }if hadPreviousEntry {js.cron.Remove(previousEntryID)delete(state.entryIDs, jobName)}if shouldSchedule {- runLock, ok := state.runLocks[jobName]- if !ok {- runLock = &sync.Mutex{}- state.runLocks[jobName] = runLock- }- entryID, nextRun = js.addCronEntryInternal(job, schedule, parsedSchedule, runLock)+ entryID, nextRun = js.addCronEntryInternal(job, schedule, parsedSchedule)state.entryIDs[jobName] = entryID}state.jobsByID[jobName] = job@@ -369,17 +381,16 @@ func (js *jobSchedulerInternal) upsertJobInternal(ctx context.Context, state *screturn nil}-func (js *jobSchedulerInternal) addCronEntryInternal(job schedulertypes.Job, schedule string, parsedSchedule cron.Schedule, runLock *sync.Mutex) (cron.EntryID, *time.Time) {+func (js *jobSchedulerInternal) addCronEntryInternal(job schedulertypes.Job, schedule string, parsedSchedule cron.Schedule) (cron.EntryID, *time.Time) {entryID := js.cron.Schedule(parsedSchedule, cron.FuncJob(func() {- if !runLock.TryLock() {- slog.WarnContext(js.context, "Job skipped; previous run still in progress", "name", job.Name(), "schedule", schedule)+ defer utils.RecoverToError(nil, "scheduled job admission", "name", job.Name())+ if _, err := js.Submit(js.context, schedulertypes.Request{JobID: job.Name(), EnvironmentID: "0", Trigger: "scheduled"}); err != nil {+ slog.ErrorContext(js.context, "Failed to queue scheduled job", "name", job.Name(), "error", err)return}- defer runLock.Unlock()- defer utils.RecoverToError(nil, "scheduled job", "name", job.Name(), "schedule", schedule)- slog.InfoContext(js.context, "Job starting", "name", job.Name(), "schedule", schedule)- job.Run(js.context)- slog.InfoContext(js.context, "Job finished", "name", job.Name())+ if err := js.checkpointInternal(js.context, job.Name(), schedule, parsedSchedule.Next(time.Now().In(js.location))); err != nil {+ slog.ErrorContext(js.context, "Failed to checkpoint scheduled job", "name", job.Name(), "error", err)+ }}))entry := js.cron.Entry(entryID)diff --git a/backend/pkg/scheduler/scheduler_test.go b/backend/pkg/scheduler/scheduler_test.goindex 13fd2884f..e73c67240 100644--- a/backend/pkg/scheduler/scheduler_test.go+++ b/backend/pkg/scheduler/scheduler_test.go@@ -21,10 +21,11 @@ func (j *testSchedulerJob) Name() string { return j.name }func (j *testSchedulerJob) Schedule(context.Context) string { return j.schedule }-func (j *testSchedulerJob) Run(ctx context.Context) {+func (j *testSchedulerJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {if j.run != nil {j.run(ctx)}+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil}type conditionalTestSchedulerJob struct {@@ -415,7 +416,7 @@ func TestJobScheduler_AddJob_GenericJobWithoutShouldRunIsScheduled(t *testing.T)job := &schedulertypes.GenericJob{JobName: "generic-dyn",ScheduleFn: func(context.Context) string { return "@every 1m" },- RunFn: func(context.Context) {},+ RunFn: func(context.Context) (schedulertypes.Outcome, error) { return schedulertypes.Outcome{}, nil },}require.NoError(t, js.AddJob(context.Background(), job))require.True(t, js.HasJob(job.Name()))diff --git a/backend/pkg/scheduler/supervision.go b/backend/pkg/scheduler/supervision.gonew file mode 100644index 000000000..f642279b1--- /dev/null+++ b/backend/pkg/scheduler/supervision.go@@ -0,0 +1,192 @@+package scheduler++import (+ "context"+ "crypto/rand"+ "math/big"+ "sync"+ "time"++ "emperror.dev/errors"+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"+)++type watcherSupervisorInternal struct {+ watcher schedulertypes.BusWatcher+ mu sync.Mutex+ health schedulertypes.WorkerHealth+ restart chan struct{}+ cancel context.CancelFunc+}++func newWatcherSupervisorInternal(watcher schedulertypes.BusWatcher) *watcherSupervisorInternal {+ return &watcherSupervisorInternal{watcher: watcher, restart: make(chan struct{}, 1), health: schedulertypes.WorkerHealth{Status: "starting", UpdatedAt: time.Now()}}+}++func (s *watcherSupervisorInternal) setHealthInternal(status string, err error, next *time.Time) {+ s.mu.Lock()+ defer s.mu.Unlock()+ s.health = schedulertypes.WorkerHealth{Status: status, NextRetry: next, UpdatedAt: time.Now()}+ if err != nil {+ s.health.LastError = err.Error()+ }+}++func (s *watcherSupervisorInternal) runInternal(ctx context.Context) error {+ delay := 5 * time.Second+ unexpected := 0+ for ctx.Err() == nil {+ runCtx, cancel := context.WithCancel(ctx)+ s.mu.Lock()+ s.cancel = cancel+ s.mu.Unlock()+ s.setHealthInternal("running", nil, nil)+ started := time.Now()+ panicked, err := s.startInternal(runCtx)+ cancel()+ if ctx.Err() != nil {+ break+ }+ select {+ case <-s.restart:+ unexpected = 0+ delay = 5 * time.Second+ continue+ default:+ }+ if panicked || err == nil {+ unexpected+++ }+ if err == nil {+ err = errors.New("worker exited unexpectedly")+ }+ if unexpected >= 3 {+ s.setHealthInternal("needs_attention", err, nil)+ select {+ case <-ctx.Done():+ case <-s.restart:+ unexpected = 0+ delay = 5 * time.Second+ }+ continue+ }+ if time.Since(started) > 5*time.Minute {+ delay = 5 * time.Second+ unexpected = 0+ }+ jitter, jitterErr := rand.Int(rand.Reader, big.NewInt(int64(delay/2)))+ if jitterErr != nil {+ return errors.WrapIf(jitterErr, "generate watcher retry delay")+ }+ next := time.Now().Add(delay/2 + time.Duration(jitter.Int64()))+ s.setHealthInternal("retrying", err, &next)+ timer := time.NewTimer(time.Until(next))+ select {+ case <-ctx.Done():+ case <-s.restart:+ unexpected = 0+ delay = 5 * time.Second+ case <-timer.C:+ delay = min(2*delay, 5*time.Minute)+ }+ timer.Stop()+ }+ s.setHealthInternal("stopped", nil, nil)+ return ctx.Err()+}++func (s *watcherSupervisorInternal) startInternal(ctx context.Context) (panicked bool, err error) {+ defer func() {+ if recovered := recover(); recovered != nil {+ err = errors.Errorf("worker panic: %v", recovered)+ panicked = true+ }+ }()+ return false, s.watcher.Start(ctx)+}++func (js *jobSchedulerInternal) SetDispatcher(dispatcher schedulertypes.Dispatcher) {+ js.dispatchMu.Lock()+ defer js.dispatchMu.Unlock()+ js.dispatcher = dispatcher+}++func (js *jobSchedulerInternal) dispatcherInternal() schedulertypes.Dispatcher {+ js.dispatchMu.RLock()+ defer js.dispatchMu.RUnlock()+ return js.dispatcher+}++func (js *jobSchedulerInternal) Submit(ctx context.Context, request schedulertypes.Request) (schedulertypes.Run, error) {+ if js.context.Err() != nil {+ return schedulertypes.Run{}, errJobSchedulerStoppedInternal+ }+ dispatcher := js.dispatcherInternal()+ if dispatcher == nil {+ return schedulertypes.Run{}, errors.New("durable job dispatcher unavailable")+ }+ return dispatcher.Submit(ctx, request)+}++func (js *jobSchedulerInternal) checkpointInternal(ctx context.Context, jobID, schedule string, next time.Time) error {+ dispatcher := js.dispatcherInternal()+ if dispatcher == nil {+ return errors.New("durable job dispatcher unavailable")+ }+ return dispatcher.Checkpoint(ctx, jobID, schedule, next)+}++func (js *jobSchedulerInternal) ListRegisteredJobs() []schedulertypes.Job {+ snapshot, ok := js.state.Load()+ if !ok {+ return nil+ }+ jobs := make([]schedulertypes.Job, 0, len(snapshot.jobsByID))+ for _, job := range snapshot.jobsByID {+ jobs = append(jobs, job)+ }+ return jobs+}++func (js *jobSchedulerInternal) RestartWatcher(ctx context.Context, id string) error {+ if err := ctx.Err(); err != nil {+ return err+ }+ snapshot, ok := js.state.Load()+ if !ok || snapshot.stopping {+ return errJobSchedulerStoppedInternal+ }+ supervisor, ok := snapshot.supervisors[id]+ if !ok {+ return errors.New("watcher not found")+ }+ supervisor.mu.Lock()+ defer supervisor.mu.Unlock()+ select {+ case supervisor.restart <- struct{}{}:+ default:+ }+ if supervisor.cancel != nil {+ supervisor.cancel()+ }+ return nil+}++func (js *jobSchedulerInternal) WatcherHealth(id string) (schedulertypes.WorkerHealth, bool) {+ snapshot, ok := js.state.Load()+ if !ok {+ return schedulertypes.WorkerHealth{}, false+ }+ supervisor, ok := snapshot.supervisors[id]+ if !ok {+ return schedulertypes.WorkerHealth{}, false+ }+ supervisor.mu.Lock()+ defer supervisor.mu.Unlock()+ health := supervisor.health+ if watcher, ok := supervisor.watcher.(*ImageUpdateWatcher); ok && watcher.eventDegraded.Load() && health.Status == "running" {+ health.Status = "degraded"+ health.LastError = "Docker image event subscription unavailable; polling remains active"+ }+ return health, true+}diff --git a/backend/pkg/scheduler/upload_sessions_cleanup_job.go b/backend/pkg/scheduler/upload_sessions_cleanup_job.goindex f1ecf4bff..6d4600aad 100644--- a/backend/pkg/scheduler/upload_sessions_cleanup_job.go+++ b/backend/pkg/scheduler/upload_sessions_cleanup_job.go@@ -5,6 +5,8 @@ import ("log/slog""time"+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"+"github.com/getarcaneapp/arcane/backend/v2/internal/upload")@@ -34,15 +36,16 @@ func (j *UploadSessionsCleanupJob) Schedule(ctx context.Context) string {return "0 0 * * * *"}-func (j *UploadSessionsCleanupJob) Run(ctx context.Context) {+func (j *UploadSessionsCleanupJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {removed, err := j.uploadService.PurgeExpiredSessions(ctx, uploadSessionMaxAge)if err != nil {slog.ErrorContext(ctx, "Failed to purge expired upload sessions", "jobName", UploadSessionsCleanupJobName, "removed", removed, "error", err)- return+ return schedulertypes.Outcome{}, err}if removed > 0 {slog.InfoContext(ctx, "Purged expired upload sessions", "jobName", UploadSessionsCleanupJobName, "removed", removed)}+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil}func (j *UploadSessionsCleanupJob) Reschedule(ctx context.Context) error {diff --git a/backend/pkg/scheduler/volume_helper_reaper_job.go b/backend/pkg/scheduler/volume_helper_reaper_job.goindex be664f109..eea29cd8e 100644--- a/backend/pkg/scheduler/volume_helper_reaper_job.go+++ b/backend/pkg/scheduler/volume_helper_reaper_job.go@@ -5,6 +5,8 @@ import ("log/slog""time"+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"+"github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/backend/v2/internal/volume")@@ -44,9 +46,9 @@ func (j *PruningVolumeHelperJob) Schedule(ctx context.Context) string {return volumeHelperPruningSchedule}-func (j *PruningVolumeHelperJob) Run(ctx context.Context) {+func (j *PruningVolumeHelperJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {if j.volumeService == nil {- return+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil}minutes := defaultVolumeHelperIdleTimeoutMinutes@@ -55,13 +57,13 @@ func (j *PruningVolumeHelperJob) Run(ctx context.Context) {}if minutes <= 0 {// 0 (or negative) disables idle pruning.- return+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil}removed, err := j.volumeService.ReapIdleHelpers(ctx, time.Duration(minutes)*time.Minute)if err != nil {slog.ErrorContext(ctx, "volume helper pruning failed", "jobName", PruningVolumeHelperJobName, "error", err)- return+ return schedulertypes.Outcome{}, err}if removed > 0 {slog.InfoContext(ctx, "volume helper pruning completed",@@ -69,6 +71,7 @@ func (j *PruningVolumeHelperJob) Run(ctx context.Context) {"removed", removed,"idleTimeoutMinutes", minutes)}+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil}func (j *PruningVolumeHelperJob) Reschedule(ctx context.Context) error {diff --git a/backend/pkg/scheduler/vulnerability_scan_job.go b/backend/pkg/scheduler/vulnerability_scan_job.goindex 9894ddf68..4b1c008f8 100644--- a/backend/pkg/scheduler/vulnerability_scan_job.go+++ b/backend/pkg/scheduler/vulnerability_scan_job.go@@ -2,14 +2,15 @@ package schedulerimport ("github.com/getarcaneapp/arcane/backend/v2/internal/common"+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler""context""log/slog""github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability"+ scheduleutil "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/schedule""github.com/getarcaneapp/arcane/types/v2"- "github.com/robfig/cron/v3")const VulnerabilityScanJobName = "vulnerability-scan"@@ -49,7 +50,7 @@ func (j *VulnerabilityScanJob) Schedule(ctx context.Context) string {schedule = "0 0 0 * * *"}- parser := cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow)+ parser := scheduleutil.Parser()if _, err := parser.Parse(schedule); err != nil {slog.WarnContext(ctx, "Invalid cron expression for vulnerability-scan, using default", "invalid_schedule", schedule, "error", err)return "0 0 0 * * *"@@ -58,19 +59,19 @@ func (j *VulnerabilityScanJob) Schedule(ctx context.Context) string {return schedule}-func (j *VulnerabilityScanJob) Run(ctx context.Context) {+func (j *VulnerabilityScanJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {enabled := j.settingsService.GetBoolSetting(ctx, "vulnerabilityScanEnabled", false)if !enabled {slog.DebugContext(ctx, "scheduled vulnerability scan disabled; skipping run")- return+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil}slog.InfoContext(ctx, "scheduled vulnerability scan started")// Remove scan records for images that no longer exist (fixes e.g. "5/3 images scanned").- deleted, err := j.vulnerabilityService.CleanupOrphanedScanRecords(ctx)- if err != nil {- slog.WarnContext(ctx, "orphaned vulnerability scan cleanup failed", "error", err)+ deleted, cleanupErr := j.vulnerabilityService.CleanupOrphanedScanRecords(ctx)+ if cleanupErr != nil {+ slog.WarnContext(ctx, "orphaned vulnerability scan cleanup failed", "error", cleanupErr)}if deleted > 0 {slog.InfoContext(ctx, "cleaned up orphaned vulnerability scan records", "deleted", deleted)@@ -79,11 +80,15 @@ func (j *VulnerabilityScanJob) Run(ctx context.Context) {scanned, failed, err := j.vulnerabilityService.ScanAllImages(ctx, types.LocalDockerEnvironmentID, vulnerabilityScanSystemUser, false)if err != nil {slog.ErrorContext(ctx, "scheduled vulnerability scan failed", "error", err)- return+ return schedulertypes.Outcome{}, err}slog.InfoContext(ctx, "scheduled vulnerability scan completed","scanned", scanned,"failed", failed,)+ if failed > 0 || cleanupErr != nil {+ return schedulertypes.Outcome{Status: schedulertypes.Partial, Message: "Some scans or orphaned record cleanup failed"}, cleanupErr+ }+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil}diff --git a/backend/pkg/utils/schedule/cron.go b/backend/pkg/utils/schedule/cron.godeleted file mode 100644index 30ac6f37e..000000000--- a/backend/pkg/utils/schedule/cron.go+++ /dev/null@@ -1,26 +0,0 @@-// Package schedule provides reusable schedule validation.-package schedule--import (- "fmt"- "strings"-- "github.com/robfig/cron/v3"-)--// NormalizeSixField validates and normalizes a six-field cron schedule.-func NormalizeSixField(value, subject string) (string, error) {- fields := strings.Fields(value)- if len(fields) == 0 {- return "", fmt.Errorf("%s schedule is required", subject)- }- if len(fields) != 6 {- return "", fmt.Errorf("invalid %s schedule %q: expected six fields", subject, strings.TrimSpace(value))- }- normalized := strings.Join(fields, " ")- parser := cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor)- if _, err := parser.Parse(normalized); err != nil {- return "", fmt.Errorf("invalid %s schedule %q: %w", subject, normalized, err)- }- return normalized, nil-}diff --git a/backend/resources/migrations/postgres/082_remove_job_key_versions.sql b/backend/resources/migrations/postgres/082_remove_job_key_versions.sqlnew file mode 100644index 000000000..c48d44bf3--- /dev/null+++ b/backend/resources/migrations/postgres/082_remove_job_key_versions.sql@@ -0,0 +1,7 @@+-- +goose Up+UPDATE kv SET "key" = REPLACE("key", 'jobs.v1.', 'jobs.') WHERE "key" LIKE 'jobs.v1.%';+UPDATE kv SET "key" = REPLACE("key", 'jobs-catalog.v1/', 'jobs-catalog/') WHERE "key" LIKE 'jobs-catalog.v1/%';++-- +goose Down+UPDATE kv SET "key" = REPLACE("key", 'jobs.', 'jobs.v1.') WHERE "key" LIKE 'jobs.%';+UPDATE kv SET "key" = REPLACE("key", 'jobs-catalog/', 'jobs-catalog.v1/') WHERE "key" LIKE 'jobs-catalog/%';diff --git a/backend/resources/migrations/sqlite/082_remove_job_key_versions.sql b/backend/resources/migrations/sqlite/082_remove_job_key_versions.sqlnew file mode 100644index 000000000..c48d44bf3--- /dev/null+++ b/backend/resources/migrations/sqlite/082_remove_job_key_versions.sql@@ -0,0 +1,7 @@+-- +goose Up+UPDATE kv SET "key" = REPLACE("key", 'jobs.v1.', 'jobs.') WHERE "key" LIKE 'jobs.v1.%';+UPDATE kv SET "key" = REPLACE("key", 'jobs-catalog.v1/', 'jobs-catalog/') WHERE "key" LIKE 'jobs-catalog.v1/%';++-- +goose Down+UPDATE kv SET "key" = REPLACE("key", 'jobs.', 'jobs.v1.') WHERE "key" LIKE 'jobs.%';+UPDATE kv SET "key" = REPLACE("key", 'jobs-catalog/', 'jobs-catalog.v1/') WHERE "key" LIKE 'jobs-catalog/%';diff --git a/cli/internal/client/client.go b/cli/internal/client/client.goindex 720549835..6d76f0ae5 100644--- a/cli/internal/client/client.go+++ b/cli/internal/client/client.go@@ -29,6 +29,7 @@ import ("encoding/json/v2"stderrors "errors""io"+ "maps""net""net/http""net/url"@@ -244,7 +245,7 @@ type APIResponse[T any] struct {// It constructs the full URL from the base URL and path, serializes the body// as JSON (if provided), and includes authentication headers. The caller is// responsible for closing the response body.-func (c *Client) Request(ctx context.Context, method, path string, body any) (*http.Response, error) {+func (c *Client) Request(ctx context.Context, method, path string, body any, headers ...http.Header) (*http.Response, error) {fullURL, err := c.resolveURL(path)if err != nil {return nil, err@@ -266,6 +267,10 @@ func (c *Client) Request(ctx context.Context, method, path string, body any) (*hc.applyAuth(req)req.Header.Set("Content-Type", "application/json")req.Header.Set("Accept", "application/json")+ for _, values := range headers {+ maps.Copy(req.Header, values.Clone())+ }+start := time.Now()logger.GetLogger().Debug("Sending request", "method", method, "url", fullURL, "env_id", c.envID, "streaming_body", true)resp, err := c.httpClient.Do(req)@@ -284,7 +289,7 @@ func (c *Client) Request(ctx context.Context, method, path string, body any) (*h}}- return c.doRequest(ctx, method, fullURL, bodyBytes, true)+ return c.doRequestInternal(ctx, method, fullURL, bodyBytes, true, headers...)}// RequestRaw makes an HTTP request with a raw body and custom headers.@@ -341,7 +346,7 @@ func (c *Client) resolveURL(path string) (string, error) {return c.baseURLParsed.ResolveReference(rel).String(), nil}-func (c *Client) doRequest(ctx context.Context, method, fullURL string, bodyBytes []byte, allowRefresh bool) (*http.Response, error) {+func (c *Client) doRequestInternal(ctx context.Context, method, fullURL string, bodyBytes []byte, allowRefresh bool, headers ...http.Header) (*http.Response, error) {attempts := max(c.maxAttempts, 1)for attempt := 1; attempt <= attempts; attempt++ {@@ -358,6 +363,9 @@ func (c *Client) doRequest(ctx context.Context, method, fullURL string, bodyBytec.applyAuth(req)req.Header.Set("Content-Type", "application/json")req.Header.Set("Accept", "application/json")+ for _, values := range headers {+ maps.Copy(req.Header, values.Clone())+ }start := time.Now()logger.GetLogger().Debug("Sending request", "method", method, "url", fullURL, "env_id", c.envID, "attempt", attempt, "max_attempts", attempts, "body_bytes", len(bodyBytes))@@ -570,9 +578,9 @@ func (c *Client) DeleteWithBody(ctx context.Context, path string, body any) (*ht// DoJSON performs a request and decodes the response JSON, enforcing a// successful HTTP status. Unlike GetJSON/PostJSON it does not require the// standard API envelope or its Success flag.-func (c *Client) DoJSON[T any](ctx context.Context, method, path string, body any) (T, error) {+func (c *Client) DoJSON[T any](ctx context.Context, method, path string, body any, headers ...http.Header) (T, error) {var out T- resp, err := c.Request(ctx, method, path, body)+ resp, err := c.Request(ctx, method, path, body, headers...)if err != nil {return out, err}diff --git a/cli/pkg/jobs/cmd.go b/cli/pkg/jobs/cmd.goindex 9a2266928..9c628957e 100644--- a/cli/pkg/jobs/cmd.go+++ b/cli/pkg/jobs/cmd.go@@ -1,6 +1,7 @@package jobsimport (+ "crypto/rand""fmt""net/http""strconv"@@ -16,7 +17,10 @@ import ("github.com/spf13/cobra")-var jsonOutput bool+var (+ jsonOutput bool+ requestID string+)// JobsCmd is the parent command for job schedule operations.var JobsCmd = &cobra.Command{@@ -145,7 +149,7 @@ var listCmd = &cobra.Command{var runCmd = &cobra.Command{Use: "run <job-id>",- Short: "Run a background job now",+ Short: "Queue a background job now",Args: cobra.ExactArgs(1),SilenceUsage: true,RunE: func(cmd *cobra.Command, args []string) error {@@ -154,9 +158,17 @@ var runCmd = &cobra.Command{return err}- result, err := c.DoJSON[jobschedule.JobRunResponse](cmd.Context(), http.MethodPost, types.JobRun(c.EnvID(), args[0]), nil)+ id := requestID+ if id == "" {+ var value [16]byte+ _, _ = rand.Read(value[:])+ value[6] = (value[6] & 0x0f) | 0x40+ value[8] = (value[8] & 0x3f) | 0x80+ id = fmt.Sprintf("%x-%x-%x-%x-%x", value[0:4], value[4:6], value[6:8], value[8:10], value[10:16])+ }+ result, err := c.DoJSON[jobschedule.JobRunResponse](cmd.Context(), http.MethodPost, types.JobRun(c.EnvID(), args[0]), nil, http.Header{"Idempotency-Key": []string{id}})if err != nil {- return errors.WrapIf(err, "failed to run job")+ return errors.WrapIff(err, "failed to queue job (reuse --request-id %s if delivery is uncertain)", id)}if jsonOutput {@@ -167,6 +179,8 @@ var runCmd = &cobra.Command{return errors.Errorf("job %s failed: %s", args[0], result.Message)}output.Success("%s", result.Message)+ output.KeyValue("Run ID", result.RunID)+ output.KeyValue("Status", string(result.Status))return nil},}@@ -180,6 +194,7 @@ func init() {getCmd.Flags().BoolVar(&jsonOutput, "json", false, "Output in JSON format")updateCmd.Flags().BoolVar(&jsonOutput, "json", false, "Output in JSON format")listCmd.Flags().BoolVar(&jsonOutput, "json", false, "Output in JSON format")+ runCmd.Flags().StringVar(&requestID, "request-id", "", "Stable request ID to reuse when retrying an uncertain submission")runCmd.Flags().BoolVar(&jsonOutput, "json", false, "Output in JSON format")updateCmd.Flags().StringVar(&environmentHealthInterval, "environment-health-interval", "", "Environment health job interval (cron expression)")diff --git a/frontend/messages/en.json b/frontend/messages/en.jsonindex 41785eea4..5cd2608bd 100644--- a/frontend/messages/en.json+++ b/frontend/messages/en.json@@ -3452,5 +3452,54 @@"system_volume_backups_unavailable": "Unavailable","system_volume_backups_no_matching_volumes": "No matching volumes.","system_volume_backups_ignore_anonymous": "Ignore anonymous volumes",- "system_volume_backups_ignore_anonymous_description": "Anonymous Docker volumes remain visible above but are skipped during backup runs."+ "system_volume_backups_ignore_anonymous_description": "Anonymous Docker volumes remain visible above but are skipped during backup runs.",+ "jobs_run_queued": "Run queued",+ "jobs_submitting": "Queuing…",+ "jobs_run_history": "Run history",+ "jobs_run_attempts": "Attempts",+ "jobs_last_success": "Last success",+ "jobs_last_error": "Last error",+ "jobs_next_retry": "Next retry",+ "jobs_restart_worker": "Restart worker",+ "jobs_retry_run": "Retry run",+ "jobs_cancel_run": "Cancel pending run",+ "jobs_last_confirmed": "Last confirmed",+ "jobs_run_empty": "No runs recorded yet.",+ "jobs_upgrade_required": "Upgrade this agent to use durable jobs.",+ "jobs_offline_status": "This environment is offline. Showing its last confirmed job status.",+ "jobs_run_id": "Run ID",+ "jobs_run_targets": "Target outcomes",+ "jobs_worker_health": "Worker health",+ "jobs_run_previous": "Previous page",+ "jobs_run_next": "Next page",+ "jobs_run_error": "Unable to load job runs.",+ "jobs_worker_restarted": "Worker restart requested",+ "jobs_status_unknown": "Unknown",+ "jobs_status_healthy": "Healthy",+ "jobs_status_degraded": "Degraded",+ "jobs_status_stopped": "Stopped",+ "jobs_status_starting": "Starting",+ "jobs_status_queued": "Queued",+ "jobs_status_waiting": "Waiting for environment",+ "jobs_status_running": "Running",+ "jobs_status_retrying": "Retrying",+ "jobs_status_succeeded": "Succeeded",+ "jobs_status_partial": "Partial",+ "jobs_status_skipped": "Skipped",+ "jobs_status_failed": "Failed",+ "jobs_status_needs_attention": "Needs attention",+ "jobs_status_canceled": "Canceled",+ "jobs_containers_load_error": "Failed to load containers",+ "jobs_container_label_excluded": "Excluded by label",+ "jobs_volume_backup_group": "Volume backups",+ "jobs_system_backup_group": "System backups",+ "jobs_dynamic_job_name": "{name} · {target}",+ "jobs_capability_unknown": "This agent has not been reached yet. Job capabilities will be checked when it reconnects.",+ "jobs_remote_status": "Agent status",+ "jobs_target_runs": "Targets ({count})",+ "jobs_run_all": "Run all",+ "jobs_local_health_check": "Local environment health",+ "jobs_health_check": "Environment health",+ "jobs_settings_details": "Settings and details",+ "jobs_health_scope_description": "Checks the health and connectivity of this environment."}diff --git a/frontend/src/lib/components/job-card/job-card.svelte b/frontend/src/lib/components/job-card/job-card.svelteindex 34e987872..c136c3332 100644--- a/frontend/src/lib/components/job-card/job-card.svelte+++ b/frontend/src/lib/components/job-card/job-card.svelte@@ -3,37 +3,60 @@import { m } from '#lib/paraglide/messages';import { Badge } from '#lib/components/ui/badge';import { Button } from '#lib/components/ui/button';- import * as Card from '#lib/components/ui/card';import { Spinner } from '#lib/components/ui/spinner';import { jobScheduleService } from '#lib/services/job-schedule-service';import { formatDateTimeShort, formatRelativeTime, parseInstant } from '#lib/utils/formatting';import type { Snippet } from 'svelte';import type { JobStatus } from '#lib/types/settings';import JobScheduleDialog from './job-schedule-dialog.svelte';+ import JobRunHistory from './job-run-history.svelte';+ import { jobStatusLabel, jobNameLabel } from './job-status';+ import { toast } from 'svelte-sonner';import { createMutation } from '@tanstack/svelte-query';let {job,environmentId = '0',isAgent = false,+ durableRuns = false,onScheduleUpdate,children,enabledOverride,- headerAccessory+ headerAccessory,+ collapsibleSettings = false}: {job: JobStatus;environmentId?: string;isAgent?: boolean;+ durableRuns?: boolean;onScheduleUpdate?: () => void;children?: Snippet;enabledOverride?: boolean;headerAccessory?: Snippet;+ collapsibleSettings?: boolean;} = $props();+ let requestId: string | undefined;let showScheduleDialog = $state(false);const runJobMutation = createMutation(() => ({- mutationFn: () => jobScheduleService.runJob(job.id, environmentId)+ mutationFn: () => {+ requestId ??= crypto.randomUUID();+ return jobScheduleService.runJob(job.id, environmentId, requestId);+ },+ onSuccess: () => {+ requestId = undefined;+ toast.success(m.jobs_run_queued());+ onScheduleUpdate?.();+ }}));+ const restartMutation = createMutation(() => ({+ mutationFn: () => jobScheduleService.restartWorker(job.id, environmentId),+ onSuccess: () => {+ toast.success(m.jobs_worker_restarted());+ onScheduleUpdate?.();+ }+ }));+ const run = $derived(job.currentRun ?? job.lastRun);const isRunning = $derived(runJobMutation.isPending);const nextRunText = $derived.by(() => {@@ -47,7 +70,7 @@const isEnabled = $derived.by(() => enabledOverride ?? job.enabled);- const canRun = $derived(isEnabled && job.canRunManually && !isRunning && !(isAgent && job.managerOnly));+ const canRun = $derived(durableRuns && isEnabled && job.canRunManually && !isRunning && !(isAgent && job.managerOnly));function runJobNow() {if (!canRun) return;@@ -64,76 +87,100 @@}</script>-<Card.Root class="h-full rounded-xl border border-white/10 bg-background/60 shadow-none backdrop-blur-md">- <Card.Header class="flex flex-row items-center justify-between space-y-0 px-4 py-4">- <div class="flex flex-col gap-1.5">- <div class="flex items-center gap-2">- <Card.Title class="text-sm leading-none font-medium">{job.name}</Card.Title>- {#if job.isContinuous}+<article class="flex min-w-0 flex-col gap-3 py-5">+ <div class="flex min-w-0 flex-wrap items-start justify-between gap-x-6 gap-y-3">+ <div class="flex min-w-0 flex-[1_1_20rem] flex-col gap-1.5">+ <div class="flex flex-wrap items-center gap-2">+ {#if headerAccessory}<div class="mr-1 flex items-center">{@render headerAccessory()}</div>{/if}+ <h4 class="text-base font-semibold">{jobNameLabel(job)}</h4>+ {#if !isEnabled}+ <Badge variant="secondary" size="sm">{m.common_disabled()}</Badge>+ {:else if run}+ <Badge variant={run.status === 'failed' || run.status === 'needs_attention' ? 'destructive' : 'outline'} size="sm"+ >{jobStatusLabel(run.status)}</Badge+ >+ {:else if job.isContinuous}<Badge variant="outline" size="sm">{m.jobs_continuous()}</Badge>{/if}</div>- <Card.Description class="line-clamp-1 text-xs text-muted-foreground/80">{job.description}</Card.Description>+ {#if job.id.startsWith('environment-health:')}+ <p class="text-sm leading-relaxed text-muted-foreground">{m.jobs_health_scope_description()}</p>+ {:else if job.description}<p class="text-sm leading-relaxed text-muted-foreground">{job.description}</p>{/if}+ {#if job.lastSuccess}<p class="text-xs text-muted-foreground">+ {m.jobs_last_success()}: {formatDateTimeShort(job.lastSuccess)}+ </p>{/if}++ <div class="mt-1 flex min-w-0 flex-wrap items-center gap-x-4 gap-y-1 text-xs text-muted-foreground">+ {#if isEnabled}+ {#if job.schedule && (!job.isContinuous || job.settingsKey)}+ <div class="flex items-center gap-2">+ <ClockIcon class="size-3.5 shrink-0" />+ <span class="font-medium">{m.jobs_schedule()}</span>+ <code class="break-all text-foreground">{job.schedule}</code>+ </div>+ {#if nextRunText}<p>{m.jobs_next_run()}: {nextRunText}</p>{/if}+ {:else if job.isContinuous}<p>{m.jobs_continuous()}</p>{/if}+ {#if run?.nextAttempt}<p>{m.jobs_next_retry()}: {formatDateTimeShort(run.nextAttempt)}</p>{/if}+ {/if}+ {#if job.workerHealth}<p>{m.jobs_worker_health()}: {jobStatusLabel(job.workerHealth.status)}</p>{/if}+ </div></div>- <div class="flex items-center gap-2">- {#if headerAccessory}- {@render headerAccessory()}- {/if}+ <div class="flex max-w-full flex-wrap items-center gap-1">{#if isEnabled && job.settingsKey}<Buttonvariant="ghost"size="icon"onclick={openScheduleDialog}disabled={isAgent && job.managerOnly}- class="size-7 opacity-75 hover:bg-white/5 hover:opacity-100"+ aria-label={m.jobs_edit_schedule()}+ title={m.jobs_edit_schedule()}>- <EditIcon class="size-3.5" />+ <EditIcon /></Button>{/if}+ {#if durableRuns && !job.children?.length}<JobRunHistory jobId={job.id} {environmentId} onUpdate={onScheduleUpdate} />{/if}{#if isEnabled && job.canRunManually}- <Button- variant="outline"- size="sm"- onclick={runJobNow}- disabled={!canRun}- class="h-7 border-white/10 bg-transparent px-2.5 text-xs font-medium shadow-none hover:bg-white/5"- >- {#if isRunning}- <Spinner class="mr-1.5 size-3" />- {m.common_running()}- {:else}- <StartIcon class="mr-1.5 size-3.5" />- {m.jobs_run_now()}- {/if}+ <Button variant="outline" size="sm" onclick={runJobNow} disabled={!canRun}>+ {#if isRunning}<Spinner data-icon="inline-start" />{m.jobs_submitting()}+ {:else}<StartIcon data-icon="inline-start" />{job.children?.length ? m.jobs_run_all() : m.jobs_run_now()}{/if}</Button>{/if}+ {#if durableRuns && job.isContinuous && isEnabled}+ <Button variant="ghost" size="sm" disabled={restartMutation.isPending} onclick={() => restartMutation.mutate()}+ >{m.jobs_restart_worker()}</Button+ >+ {/if}</div>- </Card.Header>- <Card.Content class="space-y-2 px-4 pt-0 pb-4">- {#if isEnabled}- <div class="grid gap-2 text-xs text-muted-foreground">- {#if !job.isContinuous || job.settingsKey}- <div class="flex items-center gap-2">- <ClockIcon class="h-3.5 w-3.5" />- <span>{m.jobs_schedule()}:</span>- <code class="rounded bg-muted px-2 py-0.5 text-[0.65rem]">{job.schedule}</code>- </div>- {#if nextRunText}- <div>- {m.jobs_next_run()}: {nextRunText}- </div>- {/if}- {:else}- <div>{m.jobs_continuous()}</div>- {/if}- </div>- {/if}+ </div>- {#if children}+ {#if runJobMutation.error}<p class="text-xs text-destructive">{runJobMutation.error.message}</p>{/if}+ {#if restartMutation.error}<p class="text-xs text-destructive">{restartMutation.error.message}</p>{/if}+ {#if job.lastError}<p class="text-xs break-words text-destructive">{m.jobs_last_error()}: {job.lastError}</p>{/if}+ {#if job.workerHealth?.lastError}<p class="text-xs break-words text-destructive">{job.workerHealth.lastError}</p>{/if}+ {#if job.workerHealth?.nextRetry}<p class="text-xs text-muted-foreground">+ {m.jobs_next_retry()}: {formatDateTimeShort(job.workerHealth.nextRetry)}+ </p>{/if}+ {#if run?.status === 'waiting' && run.remoteOutcome}+ <p class="text-xs text-muted-foreground">+ {m.jobs_remote_status()}: {jobStatusLabel(run.remoteOutcome.status)}+ {#if run.lastConfirmedAt}<span class="ml-2">{m.jobs_last_confirmed()}: {formatDateTimeShort(run.lastConfirmedAt)}</span+ >{/if}+ </p>+ {/if}+ {#if children}+ {#if collapsibleSettings}+ <details class="group">+ <summary+ class="w-fit cursor-pointer rounded-sm text-sm text-muted-foreground outline-none hover:text-foreground focus-visible:ring-2 focus-visible:ring-ring"+ >{m.jobs_settings_details()}</summary+ >+ <div class="mt-4 max-w-3xl">{@render children()}</div>+ </details>+ {:else}{@render children()}{/if}- </Card.Content>-</Card.Root>+ {/if}+</article>{#if showScheduleDialog}<JobScheduleDialog {job} {environmentId} bind:open={showScheduleDialog} onUpdate={handleScheduleUpdated} />diff --git a/frontend/src/lib/components/job-card/job-run-history.svelte b/frontend/src/lib/components/job-card/job-run-history.sveltenew file mode 100644index 000000000..d5445ec9c--- /dev/null+++ b/frontend/src/lib/components/job-card/job-run-history.svelte@@ -0,0 +1,134 @@+<script lang="ts">+ import { queryKeys } from '#lib/query/query-keys';+ import { createQuery, createMutation } from '@tanstack/svelte-query';+ import { m } from '#lib/paraglide/messages';+ import { Button } from '#lib/components/ui/button';+ import * as Dialog from '#lib/components/ui/dialog';+ import { jobScheduleService } from '#lib/services/job-schedule-service';+ import { activityStore } from '#lib/stores/activity.store.svelte';+ import { formatDateTimeShort } from '#lib/utils/formatting';+ import { jobStatusLabel } from './job-status';+ let { jobId, environmentId, onUpdate }: { jobId: string; environmentId: string; onUpdate?: () => void } = $props();+ let open = $state(false);+ let page = $state(1);+ let selected = $state<string | undefined>();+ const runs = createQuery(() => ({+ queryKey: queryKeys.jobs.runs(environmentId, jobId, page),+ queryFn: () => jobScheduleService.listRuns(jobId, environmentId, page),+ enabled: open,+ refetchInterval: 5000+ }));+ const detail = createQuery(() => ({+ queryKey: queryKeys.jobs.run(environmentId, jobId, selected),+ queryFn: () => jobScheduleService.getRun(jobId, selected!, environmentId),+ enabled: open && !!selected,+ refetchInterval: 5000+ }));+ const action = createMutation(() => ({+ mutationFn: ({ runId, action }: { runId: string; action: 'retry' | 'cancel' }) =>+ jobScheduleService.updateRun(jobId, runId, action, environmentId),+ onSuccess: () => {+ void runs.refetch();+ void detail.refetch();+ onUpdate?.();+ }+ }));+</script>++<Button+ variant="ghost"+ size="sm"+ onclick={() => {+ open = true;+ page = 1;+ selected = undefined;+ }}>{m.jobs_run_history()}</Button+>+<Dialog.Root bind:open>+ <Dialog.Content class="max-h-[85vh] overflow-y-auto sm:max-w-2xl">+ <Dialog.Header+ ><Dialog.Title>{m.jobs_run_history()}</Dialog.Title><Dialog.Description>{jobId}</Dialog.Description></Dialog.Header+ >+ {#if runs.error}<p class="text-sm text-destructive">{runs.error.message}</p>{/if}+ {#each runs.data?.runs ?? [] as run (run.id)}+ <Button+ variant="outline"+ class="h-auto w-full justify-between gap-2 text-left"+ onclick={() => {+ selected = run.id;+ }}+ >+ <span class="truncate">{formatDateTimeShort(run.createdAt)}</span><span>{jobStatusLabel(run.status)}</span>+ </Button>+ {:else}+ <p class="text-sm text-muted-foreground">{runs.isPending ? m.common_loading() : m.jobs_run_empty()}</p>+ {/each}+ <div class="flex justify-between gap-2">+ <Button+ variant="outline"+ size="sm"+ disabled={page <= 1 || runs.isFetching}+ onclick={() => {+ page--;+ selected = undefined;+ }}>{m.jobs_run_previous()}</Button+ >+ <Button+ variant="outline"+ size="sm"+ disabled={!runs.data || page * runs.data.limit >= runs.data.total || runs.isFetching}+ onclick={() => {+ page++;+ selected = undefined;+ }}>{m.jobs_run_next()}</Button+ >+ </div>+ {#if detail.error}<p class="text-sm text-destructive">{detail.error.message}</p>{/if}+ {#if selected && detail.data}+ {@const run = detail.data}+ <div class="space-y-3 border-t pt-4 text-sm">+ <p class="break-all"><strong>{m.jobs_run_id()}:</strong> {run.id}</p>+ <p>{jobStatusLabel(run.status)}</p>+ {#if run.status === 'waiting' && run.remoteOutcome}<p>+ {m.jobs_remote_status()}: {jobStatusLabel(run.remoteOutcome.status)}+ </p>{/if}+ {#if run.outcome.message}<p class="break-words whitespace-pre-wrap">{run.outcome.message}</p>{/if}+ {#if run.lastConfirmedAt}<p>{m.jobs_last_confirmed()}: {formatDateTimeShort(run.lastConfirmedAt)}</p>{/if}+ {#if run.nextAttempt}<p>{m.jobs_next_retry()}: {formatDateTimeShort(run.nextAttempt)}</p>{/if}+ {#if run.outcome.activityId}<Button variant="link" onclick={() => activityStore.openCenter(run.outcome.activityId)}+ >{m.activity()}</Button+ >{/if}+ <h4 class="font-medium">{m.jobs_run_attempts()}: {run.attemptCount}</h4>+ {#each run.attempts ?? [] as attempt (attempt.number)}+ <div class="space-y-1 rounded border p-2">+ <p>{attempt.number} · {formatDateTimeShort(attempt.startedAt)} · {jobStatusLabel(attempt.outcome.status)}</p>+ {#if attempt.outcome.message}<p class="break-words whitespace-pre-wrap">{attempt.outcome.message}</p>{/if}+ </div>+ {/each}+ {#if run.outcome.targets?.length}+ <h4 class="font-medium">{m.jobs_run_targets()}</h4>+ {#each run.outcome.targets as target (target.id)}+ <div class="space-y-1 rounded border p-2">+ <p class="break-all">{target.id} · {jobStatusLabel(target.status)}</p>+ {#if target.message}<p class="break-words whitespace-pre-wrap">{target.message}</p>{/if}+ {#if target.activityId}<Button variant="link" onclick={() => activityStore.openCenter(target.activityId)}+ >{m.activity()}</Button+ >{/if}+ </div>+ {/each}+ {/if}+ {#if action.error}<p class="text-destructive">{action.error.message}</p>{/if}+ {#if ['failed', 'partial', 'needs_attention'].includes(run.status)}+ <Button disabled={action.isPending} onclick={() => action.mutate({ runId: run.id, action: 'retry' })}+ >{m.jobs_retry_run()}</Button+ >+ {/if}+ {#if run.status === 'queued' && run.attemptCount === 0 && !run.remoteDeliveryAttempted}+ <Button variant="outline" disabled={action.isPending} onclick={() => action.mutate({ runId: run.id, action: 'cancel' })}+ >{m.jobs_cancel_run()}</Button+ >+ {/if}+ </div>+ {/if}+ </Dialog.Content>+</Dialog.Root>diff --git a/frontend/src/lib/components/job-card/job-status.ts b/frontend/src/lib/components/job-card/job-status.tsnew file mode 100644index 000000000..e47653938--- /dev/null+++ b/frontend/src/lib/components/job-card/job-status.ts@@ -0,0 +1,34 @@+import type { JobStatus } from '#lib/types/settings';+import { m } from '#lib/paraglide/messages';+export function jobStatusLabel(status: string): string {+ const labels: Record<string, () => string> = {+ queued: m.jobs_status_queued,+ waiting: m.jobs_status_waiting,+ running: m.jobs_status_running,+ retrying: m.jobs_status_retrying,+ succeeded: m.jobs_status_succeeded,+ partial: m.jobs_status_partial,+ skipped: m.jobs_status_skipped,+ failed: m.jobs_status_failed,+ needs_attention: m.jobs_status_needs_attention,+ canceled: m.jobs_status_canceled,+ healthy: m.jobs_status_healthy,+ degraded: m.jobs_status_degraded,+ stopped: m.jobs_status_stopped,+ starting: m.jobs_status_starting+ };+ return labels[status]?.() ?? m.jobs_status_unknown();+}++export function jobNameLabel(job: JobStatus): string {+ const [group = '', ...target] = job.id.split(':');+ const labels: Record<string, () => string> = {+ 'gitops-sync': m.gitops,+ 'volume-backup': m.jobs_volume_backup_group,+ 'system-backup': m.jobs_system_backup_group+ };+ if (group === 'environment-health' && target.length) return m.jobs_health_check();+ const label = labels[group]?.();+ if (!label) return job.name;+ return target.length ? m.jobs_dynamic_job_name({ name: label, target: target.join(':') }) : label;+}diff --git a/frontend/src/lib/components/section-card.svelte b/frontend/src/lib/components/section-card.svelteindex d5a204b67..95edec352 100644--- a/frontend/src/lib/components/section-card.svelte+++ b/frontend/src/lib/components/section-card.svelte@@ -2,10 +2,11 @@import * as Card from '#lib/components/ui/card';import type { IconType } from '#lib/icons';import { cn } from '#lib/utils';- import type { Snippet } from 'svelte';+ import type { ComponentProps, Snippet } from 'svelte';interface Props {title: string;+ variant?: ComponentProps<typeof Card.Root>['variant'];description?: string;icon?: IconType;iconVariant?: 'primary' | 'emerald' | 'red' | 'amber' | 'blue' | 'purple' | 'cyan' | 'orange' | 'indigo' | 'pink';@@ -18,6 +19,7 @@let {title,+ variant = 'default',description,icon,iconVariant = 'primary',@@ -29,7 +31,7 @@}: Props = $props();</script>-<Card.Root class={className}>+<Card.Root {variant} class={className}><Card.Header {icon} {iconVariant} class={headerClass}><div class="flex min-w-0 flex-1 flex-col space-y-1.5"><Card.Title>diff --git a/frontend/src/lib/components/settings/image-patch-settings.svelte b/frontend/src/lib/components/settings/image-patch-settings.svelteindex d69f3bc4d..09a3265c0 100644--- a/frontend/src/lib/components/settings/image-patch-settings.svelte+++ b/frontend/src/lib/components/settings/image-patch-settings.svelte@@ -29,6 +29,7 @@</script><SectionCard+ variant="transparent"title={m.security_image_patching_heading()}icon={SecurityIcon}class="flex flex-col"diff --git a/frontend/src/lib/components/settings/lifecycle-security-settings.svelte b/frontend/src/lib/components/settings/lifecycle-security-settings.svelteindex 06812a143..732d023c6 100644--- a/frontend/src/lib/components/settings/lifecycle-security-settings.svelte+++ b/frontend/src/lib/components/settings/lifecycle-security-settings.svelte@@ -29,6 +29,7 @@</script><SectionCard+ variant="transparent"title={m.security_lifecycle_hooks_heading()}icon={CodeIcon}class="flex flex-col"diff --git a/frontend/src/lib/components/settings/trivy-security-settings.svelte b/frontend/src/lib/components/settings/trivy-security-settings.svelteindex aa391e9fe..c0ede60ed 100644--- a/frontend/src/lib/components/settings/trivy-security-settings.svelte+++ b/frontend/src/lib/components/settings/trivy-security-settings.svelte@@ -152,6 +152,7 @@</script><SectionCard+ variant="transparent"title={m.security_vulnerability_scanning_heading()}icon={SecurityIcon}class="flex flex-col"diff --git a/frontend/src/lib/components/ui/card/card.svelte b/frontend/src/lib/components/ui/card/card.svelteindex 458af2341..4efa202d0 100644--- a/frontend/src/lib/components/ui/card/card.svelte+++ b/frontend/src/lib/components/ui/card/card.svelte@@ -10,7 +10,10 @@children,...restProps}: WithElementRef<- HTMLAttributes<HTMLDivElement> & { variant?: 'default' | 'subtle' | 'outlined'; onclick?: (e: MouseEvent) => void }+ HTMLAttributes<HTMLDivElement> & {+ variant?: 'default' | 'subtle' | 'outlined' | 'transparent';+ onclick?: (e: MouseEvent) => void;+ }> = $props();function handleClick(e: MouseEvent) {@@ -25,12 +28,14 @@}}- function getVariantClasses(variant: 'default' | 'subtle' | 'outlined') {+ function getVariantClasses(variant: 'default' | 'subtle' | 'outlined' | 'transparent') {switch (variant) {case 'default':return 'backdrop-blur-md bg-card/60 shadow-xs dark:bg-surface/40';case 'subtle':return 'bg-muted/40 dark:bg-surface/30';+ case 'transparent':+ return 'border-border bg-transparent shadow-none';case 'outlined':return 'bg-card/50 backdrop-blur-md border-border/70';default:diff --git a/frontend/src/lib/query/query-keys.ts b/frontend/src/lib/query/query-keys.tsindex 19fbfd82f..c67bdf16a 100644--- a/frontend/src/lib/query/query-keys.ts+++ b/frontend/src/lib/query/query-keys.ts@@ -13,6 +13,11 @@ function stableSerialize(value: unknown): string {}export const queryKeys = {+ jobs: {+ list: (environmentId: string) => ['environment-jobs', environmentId] as const,+ runs: (environmentId: string, jobId: string, page: number) => ['job-runs', environmentId, jobId, page] as const,+ run: (environmentId: string, jobId: string, runId: string | undefined) => ['job-run', environmentId, jobId, runId] as const+ },backupActivities: {active: (environmentId: string, resourceKey: string, discoverFromHistory: boolean) =>['backup-activities', environmentId, resourceKey, discoverFromHistory] as constdiff --git a/frontend/src/lib/services/job-schedule-service.ts b/frontend/src/lib/services/job-schedule-service.tsindex e4b785b99..0c156f380 100644--- a/frontend/src/lib/services/job-schedule-service.ts+++ b/frontend/src/lib/services/job-schedule-service.ts@@ -1,3 +1,4 @@+import type { JobRun, JobRunList } from '#lib/types/job';import BaseAPIService from './api-service';import type { JobSchedules, JobSchedulesUpdate, JobListResponse, JobRunResponse } from '#lib/types/settings';@@ -14,8 +15,32 @@ class JobScheduleService extends BaseAPIService {return this.handleResponse(this.api.get(`/environments/${environmentId}/jobs`));}- async runJob(jobId: string, environmentId: string = '0'): Promise<JobRunResponse> {- return this.handleResponse(this.api.post(`/environments/${environmentId}/jobs/${jobId}/run`));+ async runJob(jobId: string, environmentId: string = '0', requestId: string = crypto.randomUUID()): Promise<JobRunResponse> {+ return this.handleResponse(+ this.api.post(`/environments/${environmentId}/jobs/${encodeURIComponent(jobId)}/run`, undefined, {+ headers: { 'Idempotency-Key': requestId }+ })+ );+ }+ async listRuns(jobId: string, environmentId: string, page = 1): Promise<JobRunList> {+ return this.handleResponse(+ this.api.get(`/environments/${environmentId}/jobs/${encodeURIComponent(jobId)}/runs`, { params: { page, limit: 20 } })+ );+ }+ async getRun(jobId: string, runId: string, environmentId: string): Promise<JobRun> {+ return this.handleResponse(+ this.api.get(`/environments/${environmentId}/jobs/${encodeURIComponent(jobId)}/runs/${encodeURIComponent(runId)}`)+ );+ }+ async updateRun(jobId: string, runId: string, action: 'retry' | 'cancel', environmentId: string): Promise<JobRun> {+ return this.handleResponse(+ this.api.post(+ `/environments/${environmentId}/jobs/${encodeURIComponent(jobId)}/runs/${encodeURIComponent(runId)}/${action}`+ )+ );+ }+ async restartWorker(jobId: string, environmentId: string): Promise<{ success: boolean; message: string }> {+ return this.handleResponse(this.api.post(`/environments/${environmentId}/jobs/${encodeURIComponent(jobId)}/restart`));}}diff --git a/frontend/src/lib/types/job.ts b/frontend/src/lib/types/job.tsnew file mode 100644index 000000000..486757bf1--- /dev/null+++ b/frontend/src/lib/types/job.ts@@ -0,0 +1,37 @@+export type JobRunStatus =+ | 'queued'+ | 'waiting'+ | 'running'+ | 'retrying'+ | 'succeeded'+ | 'partial'+ | 'skipped'+ | 'failed'+ | 'needs_attention'+ | 'canceled';+export type JobTargetOutcome = { id: string; status: JobRunStatus; message?: string; activityId?: string };+export type JobOutcome = { status: JobRunStatus; message?: string; activityId?: string; targets?: JobTargetOutcome[] };+export type JobAttempt = { number: number; startedAt: string; finishedAt?: string; outcome: JobOutcome };+export type JobRun = {+ id: string;+ jobId: string;+ environmentId: string;+ trigger: string;+ requestedBy?: string;+ status: JobRunStatus;+ createdAt: string;+ updatedAt: string;+ startedAt?: string;+ finishedAt?: string;+ nextAttempt?: string;+ attemptCount: number;+ outcome: JobOutcome;+ attempts?: JobAttempt[];+ remoteAccepted: boolean;+ remoteDeliveryAttempted: boolean;+ remoteOutcome?: JobOutcome;+ remoteSettled: boolean;+ lastConfirmedAt?: string;+};+export type JobRunList = { runs: JobRun[]; total: number; page: number; limit: number };+export type WorkerHealth = { status: string; lastError?: string; nextRetry?: string; updatedAt: string };diff --git a/frontend/src/lib/types/settings.ts b/frontend/src/lib/types/settings.tsindex 120f54430..78d5a1159 100644--- a/frontend/src/lib/types/settings.ts+++ b/frontend/src/lib/types/settings.ts@@ -1,3 +1,4 @@+import type { JobRun, JobRunStatus, WorkerHealth } from './job';import type { TemplateRegistryConfig } from './swarm';// --- Application settings ---@@ -182,6 +183,12 @@ export type JobPrerequisite = {};export type JobStatus = {+ currentRun?: JobRun;+ lastRun?: JobRun;+ lastSuccess?: string;+ lastError?: string;+ workerHealth?: WorkerHealth;+ children?: JobStatus[];id: string;name: string;description: string;@@ -197,11 +204,16 @@ export type JobStatus = {};export type JobListResponse = {+ durableRuns: boolean;+ observedAt: string;+ offline?: boolean;jobs: JobStatus[];isAgent: boolean;};export type JobRunResponse = {+ runId: string;+ status: JobRunStatus;success: boolean;message: string;};diff --git a/frontend/src/routes/(app)/environments/[id]/components/ConnectionEdgeTab.svelte b/frontend/src/routes/(app)/environments/[id]/components/ConnectionEdgeTab.svelteindex 7158b0e95..91d31b7a8 100644--- a/frontend/src/routes/(app)/environments/[id]/components/ConnectionEdgeTab.svelte+++ b/frontend/src/routes/(app)/environments/[id]/components/ConnectionEdgeTab.svelte@@ -48,7 +48,7 @@{/snippet}<div class="space-y-6">- <Card.Root class="flex flex-col">+ <Card.Root variant="transparent" class="flex flex-col"><Card.Header icon={ConnectionIcon}><div class="flex flex-col space-y-1.5"><Card.Title>@@ -63,7 +63,7 @@</Card.Root>{#if showAgentSecurity}- <Card.Root class="flex flex-col">+ <Card.Root variant="transparent" class="flex flex-col"><Card.Header icon={SecurityIcon}><div class="flex flex-col space-y-1.5"><Card.Title>diff --git a/frontend/src/routes/(app)/environments/[id]/components/DockerTab.svelte b/frontend/src/routes/(app)/environments/[id]/components/DockerTab.svelteindex 22033b948..f5402e717 100644--- a/frontend/src/routes/(app)/environments/[id]/components/DockerTab.svelte+++ b/frontend/src/routes/(app)/environments/[id]/components/DockerTab.svelte@@ -50,7 +50,7 @@];</script>-<Card.Root class="flex flex-col">+<Card.Root variant="transparent" class="flex flex-col"><Card.Header icon={DockerBrandIcon}><div class="flex flex-col space-y-1.5"><Card.Title>diff --git a/frontend/src/routes/(app)/environments/[id]/components/JobsTab.svelte b/frontend/src/routes/(app)/environments/[id]/components/JobsTab.svelteindex e880d69fb..87bbc6c18 100644--- a/frontend/src/routes/(app)/environments/[id]/components/JobsTab.svelte+++ b/frontend/src/routes/(app)/environments/[id]/components/JobsTab.svelte@@ -1,4 +1,6 @@<script lang="ts">+ import { queryKeys } from '#lib/query/query-keys';+ import { createQuery } from '@tanstack/svelte-query';import { SvelteSet } from 'svelte/reactivity';import { jobScheduleService } from '#lib/services/job-schedule-service';import { containerService } from '#lib/services/container-service';@@ -7,7 +9,6 @@import { Spinner } from '#lib/components/ui/spinner';import { m } from '#lib/paraglide/messages';import * as Alert from '#lib/components/ui/alert';- import * as Card from '#lib/components/ui/card';import { Label } from '#lib/components/ui/label';import { Switch } from '#lib/components/ui/switch';import { Input } from '#lib/components/ui/input';@@ -21,29 +22,25 @@let { formInputs, environmentId }: JobsTabProps = $props();- let refreshSignal = $state(0);-- const jobsPromise = $derived.by(async () => {- refreshSignal; // trigger dependency- if (!environmentId) return null;-- const result = await tryCatch(jobScheduleService.listJobs(environmentId));-- if (result.error) {- throw result.error;- }-- return {- ...result.data,- jobs: result.data.jobs.map((job) => ({- ...job,- prerequisites: job.prerequisites.map((prereq) => ({- ...prereq,- settingsUrl: resolveSettingsUrl(job, prereq)+ const jobsQuery = createQuery(() => ({+ queryKey: queryKeys.jobs.list(environmentId),+ queryFn: async () => {+ const response = await jobScheduleService.listJobs(environmentId);+ return {+ ...response,+ jobs: response.jobs.map((job) => ({+ ...job,+ prerequisites: (job.prerequisites ?? []).map((prereq) => ({ ...prereq, settingsUrl: resolveSettingsUrl(job, prereq) }))}))- }))- };- });+ };+ },+ enabled: !!environmentId,+ refetchInterval: 5000+ }));+ const jobsResponse = $derived(jobsQuery.data);+ const capabilityUnknown = $derived(+ !!jobsResponse?.offline && (!jobsResponse.observedAt || jobsResponse.observedAt.startsWith('0001-'))+ );const containersPromise = $derived.by(async () => {if (!environmentId) return [];@@ -95,7 +92,7 @@}function loadJobs() {- refreshSignal++;+ void jobsQuery.refetch();}function toggleContainerExclusion(containerName: string) {@@ -132,11 +129,8 @@{ id: 'telemetry', label: m.jobs_telemetry_heading() }];- const hiddenJobIds = new Set(['analytics-heartbeat', 'filesystem-watcher']);-function getJobsByCategory(categoryId: string, jobs: JobStatus[]): JobStatus[] {return jobs.filter((j) => {- if (hiddenJobIds.has(j.id)) return false;if (j.category !== categoryId) return false;// Only show manager-only jobs on the local environment (ID "0")if (j.managerOnly && environmentId !== '0') return false;@@ -173,7 +167,7 @@value: name,label: name,disabled: labelExcluded,- hint: labelExcluded ? '(Label)' : undefined,+ hint: labelExcluded ? m.jobs_container_label_excluded() : undefined,selected: excludedContainers.has(name)};}@@ -230,178 +224,206 @@{/if}{:catch error}<div class="p-2 text-sm text-destructive">- {(error instanceof Error ? error.message : '') || 'Failed to load containers'}+ {(error instanceof Error ? error.message : '') || m.jobs_containers_load_error()}</div>{/await}</div></ScrollArea.Root>{/snippet}-<div class="space-y-6">- <Card.Root>- <Card.Header icon={JobsIcon}>- <div class="flex flex-col space-y-1.5">- <Card.Title>- <h2>{m.automations()}</h2>- </Card.Title>- <Card.Description>{m.jobs_environment_scope_description()}</Card.Description>+<section class="flex w-full min-w-0 flex-col gap-6">+ <header class="flex items-start gap-3">+ <JobsIcon class="mt-0.5 size-5 text-muted-foreground" />+ <div class="flex flex-col gap-1.5">+ <h2 class="text-lg font-semibold">{m.automations()}</h2>+ <p class="text-sm text-muted-foreground">{m.jobs_environment_scope_description()}</p>+ </div>+ </header>+ <div>+ {#if jobsQuery.isPending}+ <div class="flex h-32 items-center justify-center">+ <Spinner class="size-8" /></div>- </Card.Header>- <Card.Content class="p-4 sm:p-6">- {#await jobsPromise}- <div class="flex h-32 items-center justify-center">- <Spinner class="size-8" />- </div>- {:then jobsResponse}- {#if jobsResponse}- <div class="space-y-8">- {#each categories as category (category.id)}- {@const categoryJobs = getJobsByCategory(category.id, jobsResponse.jobs)}- {#if categoryJobs.length > 0}- <div class="space-y-4">- <h3 class="text-sm font-semibold tracking-tight text-muted-foreground uppercase">- {category.label}- </h3>- <div class="grid gap-4 sm:grid-cols-2 lg:grid-cols-2">- {#each categoryJobs as job (job.id)}- <JobCard- {job}- {environmentId}- isAgent={jobsResponse.isAgent}- onScheduleUpdate={loadJobs}- enabledOverride={getEnabledOverride(job)}- >- {#snippet headerAccessory()}- {#if job.id === 'image-polling'}- <Switch bind:checked={$formInputs.pollingEnabled.value} />- {:else if job.id === 'auto-update'}- <Switch bind:checked={$formInputs.autoUpdate.value} disabled={!$formInputs.pollingEnabled.value} />- {:else if job.id === 'scheduled-prune'}- <Switch bind:checked={$formInputs.scheduledPruneEnabled.value} />- {:else if job.id === 'vulnerability-scan'}- <Switch bind:checked={$formInputs.vulnerabilityScanEnabled.value} />- {:else if job.id === 'auto-heal'}- <Switch bind:checked={$formInputs.autoHealEnabled.value} />- {/if}- {/snippet}-- {#if job.id === 'auto-update' && $formInputs.autoUpdate.value}- <div class="space-y-3 border-t border-border/20 pt-3">- <div class="space-y-1">- <Label class="text-sm font-medium">- {m.excluded_containers()}- {#await containersPromise then containers}- <span class="ml-1 font-normal text-muted-foreground">- ({containers.filter((c) => excludedContainers.has(getContainerName(c))).length})- </span>- {/await}- </Label>- <p class="text-xs text-muted-foreground">{m.auto_update_exclude_description()}</p>- </div>-- <div class="space-y-2">- <Input type="search" placeholder={m.jobs_search_containers()} class="h-8" bind:value={searchTerm} />- {@render ContainerExclusionList({- term: searchTerm,- mapItem: mapContainerToItem,- idPrefix: 'container-',- onToggle: toggleContainerExclusion- })}- </div>- </div>+ {:else}+ {#if jobsResponse}+ {#if capabilityUnknown}<p class="mb-4 text-sm text-muted-foreground">{m.jobs_capability_unknown()}</p>+ {:else if !jobsResponse.durableRuns}<p class="mb-4 text-sm text-muted-foreground">{m.jobs_upgrade_required()}</p>{/if}+ {#if jobsResponse.offline}<p class="mb-4 text-sm text-muted-foreground">+ {m.jobs_offline_status()}+ {#if !capabilityUnknown}{m.jobs_last_confirmed()}: {jobsResponse.observedAt}{/if}+ </p>{/if}+ <div class="grid items-start gap-x-12 gap-y-10 2xl:grid-cols-2">+ {#each categories as category (category.id)}+ {@const categoryJobs = getJobsByCategory(category.id, jobsResponse.jobs)}+ {#if categoryJobs.length > 0}+ <div+ class="grid min-w-0 gap-x-6 gap-y-2 rounded-xl border border-border bg-transparent px-4 py-2 sm:px-5 lg:grid-cols-[7rem_minmax(0,1fr)]"+ >+ <h3 class="text-sm font-semibold text-foreground lg:pt-5">+ {category.label}+ </h3>+ <div class="min-w-0 divide-y divide-border/50">+ {#each categoryJobs as job (job.id)}+ <JobCard+ {job}+ {environmentId}+ isAgent={jobsResponse.isAgent}+ durableRuns={jobsResponse.durableRuns || capabilityUnknown}+ onScheduleUpdate={loadJobs}+ enabledOverride={capabilityUnknown ? undefined : getEnabledOverride(job)}+ collapsibleSettings={job.id === 'image-polling' ||+ (job.id === 'auto-update' && $formInputs.autoUpdate.value) ||+ (job.id === 'auto-heal' && $formInputs.autoHealEnabled.value)}+ >+ {#snippet headerAccessory()}+ {#if job.id === 'image-polling'}+ <Switch aria-label={job.name} bind:checked={$formInputs.pollingEnabled.value} />+ {:else if job.id === 'auto-update'}+ <Switch+ aria-label={job.name}+ bind:checked={$formInputs.autoUpdate.value}+ disabled={!$formInputs.pollingEnabled.value}+ />+ {:else if job.id === 'scheduled-prune'}+ <Switch aria-label={job.name} bind:checked={$formInputs.scheduledPruneEnabled.value} />+ {:else if job.id === 'vulnerability-scan'}+ <Switch aria-label={job.name} bind:checked={$formInputs.vulnerabilityScanEnabled.value} />+ {:else if job.id === 'auto-heal'}+ <Switch aria-label={job.name} bind:checked={$formInputs.autoHealEnabled.value} />{/if}+ {/snippet}++ {#if job.id === 'auto-update' && $formInputs.autoUpdate.value}+ <div class="space-y-3 border-t border-border/20 pt-3">+ <div class="space-y-1">+ <Label class="text-sm font-medium">+ {m.excluded_containers()}+ {#await containersPromise then containers}+ <span class="ml-1 font-normal text-muted-foreground">+ ({containers.filter((c) => excludedContainers.has(getContainerName(c))).length})+ </span>+ {/await}+ </Label>+ <p class="text-xs text-muted-foreground">{m.auto_update_exclude_description()}</p>+ </div>- {#if job.id === 'image-polling'}- <div class="space-y-3 border-t border-border/20 pt-3">- <div class="flex items-center justify-between gap-3">- <div class="space-y-1">- <Label class="text-sm font-medium">{m.jobs_image_event_watcher_label()}</Label>- <p class="text-xs text-muted-foreground">{m.jobs_image_event_watcher_description()}</p>- </div>- <Switch- id="image-event-watcher-enabled"- bind:checked={$formInputs.imageEventWatcherEnabled.value}- />- </div>- <Alert.Root variant="warning" class="py-2 [&>svg]:top-2">- <AlertTriangleIcon class="size-4" />- <Alert.Description class="text-xs">{m.jobs_image_event_watcher_warning()}</Alert.Description>- </Alert.Root>+ <div class="space-y-2">+ <Input type="search" placeholder={m.jobs_search_containers()} class="h-8" bind:value={searchTerm} />+ {@render ContainerExclusionList({+ term: searchTerm,+ mapItem: mapContainerToItem,+ idPrefix: 'container-',+ onToggle: toggleContainerExclusion+ })}</div>- {/if}+ </div>+ {/if}- {#if job.id === 'auto-heal' && $formInputs.autoHealEnabled.value}- <div class="space-y-3 border-t border-border/20 pt-3">- <div class="grid gap-3 sm:grid-cols-2">- <div class="space-y-1">- <Label for="auto-heal-max-restarts" class="text-sm font-medium"- >{m.auto_heal_max_restarts_label()}</Label- >- <p class="text-xs text-muted-foreground">{m.auto_heal_max_restarts_description()}</p>- <Input- id="auto-heal-max-restarts"- type="number"- min="1"- class="h-8 w-full"- bind:value={$formInputs.autoHealMaxRestarts.value}- />- </div>- <div class="space-y-1">- <Label for="auto-heal-restart-window" class="text-sm font-medium"- >{m.auto_heal_restart_window_label()}</Label- >- <p class="text-xs text-muted-foreground">{m.auto_heal_restart_window_description()}</p>- <Input- id="auto-heal-restart-window"- type="number"- min="1"- class="h-8 w-full"- bind:value={$formInputs.autoHealRestartWindow.value}- />- </div>+ {#if job.id === 'image-polling'}+ <div class="space-y-3 border-t border-border/20 pt-3">+ <div class="flex items-center justify-between gap-3">+ <div class="space-y-1">+ <Label class="text-sm font-medium">{m.jobs_image_event_watcher_label()}</Label>+ <p class="text-xs text-muted-foreground">{m.jobs_image_event_watcher_description()}</p></div>-+ <Switch id="image-event-watcher-enabled" bind:checked={$formInputs.imageEventWatcherEnabled.value} />+ </div>+ <Alert.Root variant="warning" class="py-2 [&>svg]:top-2">+ <AlertTriangleIcon class="size-4" />+ <Alert.Description class="text-xs">{m.jobs_image_event_watcher_warning()}</Alert.Description>+ </Alert.Root>+ </div>+ {/if}++ {#if job.id === 'auto-heal' && $formInputs.autoHealEnabled.value}+ <div class="space-y-3 border-t border-border/20 pt-3">+ <div class="grid gap-3 sm:grid-cols-2"><div class="space-y-1">- <Label class="text-sm font-medium">- {m.excluded_containers()}- {#await containersPromise then containers}- <span class="ml-1 font-normal text-muted-foreground">- ({containers.filter((c) => autoHealExcludedContainers.has(getContainerName(c))).length})- </span>- {/await}- </Label>- <p class="text-xs text-muted-foreground">{m.auto_heal_exclude_description()}</p>+ <Label for="auto-heal-max-restarts" class="text-sm font-medium"+ >{m.auto_heal_max_restarts_label()}</Label+ >+ <p class="text-xs text-muted-foreground">{m.auto_heal_max_restarts_description()}</p>+ <Input+ id="auto-heal-max-restarts"+ type="number"+ min="1"+ class="h-8 w-full"+ bind:value={$formInputs.autoHealMaxRestarts.value}+ /></div>-- <div class="space-y-2">+ <div class="space-y-1">+ <Label for="auto-heal-restart-window" class="text-sm font-medium"+ >{m.auto_heal_restart_window_label()}</Label+ >+ <p class="text-xs text-muted-foreground">{m.auto_heal_restart_window_description()}</p><Input- type="search"- placeholder={m.jobs_search_containers()}- class="h-8"- bind:value={autoHealSearchTerm}+ id="auto-heal-restart-window"+ type="number"+ min="1"+ class="h-8 w-full"+ bind:value={$formInputs.autoHealRestartWindow.value}/>- {@render ContainerExclusionList({- term: autoHealSearchTerm,- mapItem: mapContainerToAutoHealItem,- idPrefix: 'auto-heal-container-',- onToggle: toggleAutoHealContainerExclusion- })}</div></div>- {/if}- </JobCard>- {/each}- </div>++ <div class="space-y-1">+ <Label class="text-sm font-medium">+ {m.excluded_containers()}+ {#await containersPromise then containers}+ <span class="ml-1 font-normal text-muted-foreground">+ ({containers.filter((c) => autoHealExcludedContainers.has(getContainerName(c))).length})+ </span>+ {/await}+ </Label>+ <p class="text-xs text-muted-foreground">{m.auto_heal_exclude_description()}</p>+ </div>++ <div class="space-y-2">+ <Input+ type="search"+ placeholder={m.jobs_search_containers()}+ class="h-8"+ bind:value={autoHealSearchTerm}+ />+ {@render ContainerExclusionList({+ term: autoHealSearchTerm,+ mapItem: mapContainerToAutoHealItem,+ idPrefix: 'auto-heal-container-',+ onToggle: toggleAutoHealContainerExclusion+ })}+ </div>+ </div>+ {/if}+ {#if job.children?.length}+ <details class="mt-1">+ <summary+ class="w-fit cursor-pointer rounded-sm text-sm text-muted-foreground outline-none hover:text-foreground focus-visible:ring-2 focus-visible:ring-ring"+ >{m.jobs_target_runs({ count: job.children.length })}</summary+ >+ <div class="mt-3 divide-y divide-border border-l border-border pl-4 sm:pl-6">+ {#each job.children ?? [] as child (child.id)}+ <JobCard+ job={child}+ {environmentId}+ isAgent={jobsResponse.isAgent}+ durableRuns={jobsResponse.durableRuns || capabilityUnknown}+ onScheduleUpdate={loadJobs}+ />+ {/each}+ </div>+ </details>+ {/if}+ </JobCard>+ {/each}</div>- {/if}- {/each}- </div>- {/if}- {:catch error}- <div class="rounded-lg border border-destructive/50 bg-destructive/10 p-4 text-destructive">- {(error instanceof Error ? error.message : '') || String(error)}+ </div>+ {/if}+ {/each}</div>- {/await}- </Card.Content>- </Card.Root>-</div>+ {/if}+ {/if}+ {#if jobsQuery.error}<div class="rounded-lg border border-destructive/50 bg-destructive/10 p-4 text-destructive">+ {jobsQuery.error.message}+ </div>{/if}+ </div>+</section>diff --git a/frontend/src/routes/(app)/environments/[id]/components/StorageTab.svelte b/frontend/src/routes/(app)/environments/[id]/components/StorageTab.svelteindex 0bc35e864..ed2b8f4c9 100644--- a/frontend/src/routes/(app)/environments/[id]/components/StorageTab.svelte+++ b/frontend/src/routes/(app)/environments/[id]/components/StorageTab.svelte@@ -11,7 +11,7 @@</script><div class="space-y-6">- <Card.Root class="flex flex-col">+ <Card.Root variant="transparent" class="flex flex-col"><Card.Header icon={FolderOpenIcon}><div class="flex flex-col space-y-1.5"><Card.Title>@@ -64,7 +64,7 @@</Card.Content></Card.Root>- <Card.Root class="flex flex-col">+ <Card.Root variant="transparent" class="flex flex-col"><Card.Header icon={UploadIcon}><div class="flex flex-col space-y-1.5"><Card.Title>diff --git a/types/jobschedule/jobschedule.go b/types/jobschedule/jobschedule.goindex 9cb551f92..6e16362ca 100644--- a/types/jobschedule/jobschedule.go+++ b/types/jobschedule/jobschedule.go@@ -1,11 +1,12 @@package jobschedule-import "time"+import (+ "time"-// Config represents the configured intervals (in minutes) for Arcane background jobs.-//-// All fields are in minutes.-// This makes conversion to time.Duration straightforward in the backend.+ "github.com/getarcaneapp/arcane/types/v2/scheduler"+)++// Config contains cron schedules for Arcane background jobs.type Config struct {EnvironmentHealthInterval string `json:"environmentHealthInterval"`EventCleanupInterval string `json:"eventCleanupInterval"`@@ -19,7 +20,7 @@ type Config struct {AutoHealInterval string `json:"autoHealInterval"`}-// Update is used to update job schedule intervals (in minutes).+// Update changes job cron schedules.//// Any nil field is ignored.type Update struct {@@ -37,18 +38,24 @@ type Update struct {// JobStatus represents the current status and metadata for a background job.type JobStatus struct {- ID string `json:"id"`- Name string `json:"name"`- Description string `json:"description"`- Category string `json:"category"`- Schedule string `json:"schedule"`- NextRun *time.Time `json:"nextRun,omitempty"`- Enabled bool `json:"enabled"`- ManagerOnly bool `json:"managerOnly"`- IsContinuous bool `json:"isContinuous"`- CanRunManually bool `json:"canRunManually"`- Prerequisites []JobPrerequisite `json:"prerequisites"`- SettingsKey string `json:"settingsKey,omitempty"`+ CurrentRun *scheduler.Run `json:"currentRun,omitempty"`+ LastRun *scheduler.Run `json:"lastRun,omitempty"`+ LastSuccess *time.Time `json:"lastSuccess,omitempty"`+ LastError string `json:"lastError,omitempty"`+ WorkerHealth *scheduler.WorkerHealth `json:"workerHealth,omitempty"`+ Children []JobStatus `json:"children,omitempty"`+ ID string `json:"id"`+ Name string `json:"name"`+ Description string `json:"description"`+ Category string `json:"category"`+ Schedule string `json:"schedule"`+ NextRun *time.Time `json:"nextRun,omitempty"`+ Enabled bool `json:"enabled"`+ ManagerOnly bool `json:"managerOnly"`+ IsContinuous bool `json:"isContinuous"`+ CanRunManually bool `json:"canRunManually"`+ Prerequisites []JobPrerequisite `json:"prerequisites"`+ SettingsKey string `json:"settingsKey,omitempty"`}// JobPrerequisite represents a requirement that must be met for a job to run.@@ -61,8 +68,11 @@ type JobPrerequisite struct {// JobListResponse contains all jobs and system mode information.type JobListResponse struct {- Jobs []JobStatus `json:"jobs"`- IsAgent bool `json:"isAgent"`+ DurableRuns bool `json:"durableRuns"`+ ObservedAt time.Time `json:"observedAt"`+ Offline bool `json:"offline,omitempty"`+ Jobs []JobStatus `json:"jobs"`+ IsAgent bool `json:"isAgent"`}// JobRunRequest is the request to manually run a job.@@ -72,6 +82,13 @@ type JobRunRequest struct {// JobRunResponse is the response after manually triggering a job.type JobRunResponse struct {- Success bool `json:"success"`- Message string `json:"message"`+ RunID string `json:"runId,omitempty"`+ Status scheduler.RunStatus `json:"status,omitempty"`+ Success bool `json:"success"`+ Message string `json:"message"`+}++// SubmitRunInput preserves identity across manager-to-agent delivery retries.+type SubmitRunInput struct {+ RunID string `json:"runId"`}diff --git a/types/jobschedule/runs.go b/types/jobschedule/runs.gonew file mode 100644index 000000000..72d6c4c74--- /dev/null+++ b/types/jobschedule/runs.go@@ -0,0 +1,21 @@+package jobschedule++import st "github.com/getarcaneapp/arcane/types/v2/scheduler"++type JobInput struct {+ ID string `path:"id"`+ JobID string `path:"jobId" minLength:"1"`+}+type ListRunsInput struct {+ ID string `path:"id"`+ JobID string `path:"jobId" minLength:"1"`+ Page int `query:"page" default:"1" minimum:"1"`+ Limit int `query:"limit" default:"20" minimum:"1" maximum:"100"`+}+type RunInput struct {+ ID string `path:"id"`+ JobID string `path:"jobId" minLength:"1"`+ RunID string `path:"runId" format:"uuid"`+}+type RunOutput struct{ Body st.Run }+type ListRunsOutput struct{ Body st.RunList }diff --git a/types/scheduler/job.go b/types/scheduler/job.goindex 80872402c..049793c67 100644--- a/types/scheduler/job.go+++ b/types/scheduler/job.go@@ -16,7 +16,7 @@ type JobRuntimeState struct {type Job interface {Name() stringSchedule(ctx context.Context) string- Run(ctx context.Context)+ Run(ctx context.Context) (Outcome, error)}// ConditionalJob allows a job to opt out of cron registration when it is disabled.@@ -41,6 +41,7 @@ type StoppableBusWatcher interface {// DynamicScheduler owns jobs registered and removed while the application is running.type DynamicScheduler interface {+ Submit(ctx context.Context, request Request) (Run, error)AddJob(ctx context.Context, job Job) errorRemoveJob(ctx context.Context, name string)HasJob(name string) bool@@ -54,9 +55,18 @@ type JobController interface {RunBusWatcherNow(ctx context.Context, watcherID string) error}+// WorkerController exposes continuous-worker health and recovery operations.+type WorkerController interface {+ RestartWatcher(ctx context.Context, watcherID string) error+ WatcherHealth(watcherID string) (WorkerHealth, bool)+}+// JobScheduler is the shared public contract for the actor-owned scheduler.// Concrete cron and actor state remains private to the backend implementation.type JobScheduler interface {+ SetDispatcher(dispatcher Dispatcher)+ ListRegisteredJobs() []Job+ WorkerControllerDynamicSchedulerJobController@@ -79,7 +89,8 @@ type JobScheduler interface {type GenericJob struct {JobName stringScheduleFn func(ctx context.Context) string- RunFn func(ctx context.Context)+ RunFn func(ctx context.Context) (Outcome, error)+ ReconcileFn func(ctx context.Context, previous Run) (Outcome, error)ShouldRunFn func(ctx context.Context) bool}@@ -87,7 +98,7 @@ func (g *GenericJob) Name() string { return g.JobName }func (g *GenericJob) Schedule(ctx context.Context) string { return g.ScheduleFn(ctx) }-func (g *GenericJob) Run(ctx context.Context) { g.RunFn(ctx) }+func (g *GenericJob) Run(ctx context.Context) (Outcome, error) { return g.RunFn(ctx) }// ShouldSchedule satisfies ConditionalJob. A GenericJob without a ShouldRunFn is// always scheduled; the scheduler treats a ConditionalJob returning false as "do@@ -98,3 +109,10 @@ func (g *GenericJob) ShouldSchedule(ctx context.Context) bool {}return g.ShouldRunFn(ctx)}++func (g *GenericJob) Reconcile(ctx context.Context, previous Run) (Outcome, error) {+ if g.ReconcileFn != nil {+ return g.ReconcileFn(ctx, previous)+ }+ return Outcome{Status: NeedsAttention, Message: "The interrupted operation has no confirmed result; review its domain records before retrying", Targets: previous.Outcome.Targets}, nil+}diff --git a/types/scheduler/queue.go b/types/scheduler/queue.gonew file mode 100644index 000000000..fdd27e015--- /dev/null+++ b/types/scheduler/queue.go@@ -0,0 +1,14 @@+package scheduler++import "time"++// QueueRecord holds atomic admission, claims, and checkpoints for one job and target.+type QueueRecord struct {+ JobID string `json:"jobId"`+ EnvironmentID string `json:"environmentId"`+ Schedule string `json:"schedule"`+ LastEnqueuedAt time.Time `json:"lastEnqueuedAt"`+ NextRun time.Time `json:"nextRun"`+ Runs []Run `json:"runs"`+ Receipts map[string]Run `json:"receipts,omitempty"`+}diff --git a/types/scheduler/run.go b/types/scheduler/run.gonew file mode 100644index 000000000..99de46ec4--- /dev/null+++ b/types/scheduler/run.go@@ -0,0 +1,127 @@+package scheduler++import (+ "context"+ "time"+)++// RunStatus describes durable execution, independently of the configured schedule.+type RunStatus string++const (+ Queued RunStatus = "queued"+ Waiting RunStatus = "waiting"+ Running RunStatus = "running"+ Retrying RunStatus = "retrying"+ Succeeded RunStatus = "succeeded"+ Partial RunStatus = "partial"+ Skipped RunStatus = "skipped"+ Failed RunStatus = "failed"+ NeedsAttention RunStatus = "needs_attention"+ Canceled RunStatus = "canceled"+)++func (s RunStatus) Terminal() bool {+ return s == Succeeded || s == Partial || s == Skipped || s == Failed || s == Canceled+}++type TargetOutcome struct {+ ResourceType string `json:"resourceType,omitempty"`+ ID string `json:"id"`+ Status RunStatus `json:"status"`+ Message string `json:"message,omitempty"`+ ActivityID string `json:"activityId,omitempty"`+}++type Outcome struct {+ Status RunStatus `json:"status"`+ Message string `json:"message,omitempty"`+ ActivityID string `json:"activityId,omitempty"`+ Targets []TargetOutcome `json:"targets,omitempty"`+}++type Request struct {+ RunID string `json:"runId,omitempty"`+ JobID string `json:"jobId"`+ EnvironmentID string `json:"environmentId"`+ Trigger string `json:"trigger"`+ RequestedWithKey string `json:"requestedWithKey,omitempty"`+ RequestedBy string `json:"requestedBy,omitempty"`+}++type Attempt struct {+ Number int `json:"number"`+ StartedAt time.Time `json:"startedAt"`+ FinishedAt *time.Time `json:"finishedAt,omitempty"`+ Outcome Outcome `json:"outcome"`+}++type Run struct {+ RequestedWithKey string `json:"requestedWithKey,omitempty"`+ ID string `json:"id"`+ JobID string `json:"jobId"`+ EnvironmentID string `json:"environmentId"`+ Trigger string `json:"trigger"`+ RequestedBy string `json:"requestedBy,omitempty"`+ Status RunStatus `json:"status"`+ CreatedAt time.Time `json:"createdAt"`+ UpdatedAt time.Time `json:"updatedAt"`+ StartedAt *time.Time `json:"startedAt,omitempty"`+ FinishedAt *time.Time `json:"finishedAt,omitempty"`+ NextAttempt *time.Time `json:"nextAttempt,omitempty"`+ AttemptCount int `json:"attemptCount"`+ Owner string `json:"owner,omitempty"`+ Outcome Outcome `json:"outcome"`+ Attempts []Attempt `json:"attempts,omitempty"`+ RemoteDeliveryAttempted bool `json:"remoteDeliveryAttempted"`+ RemoteOutcome *Outcome `json:"remoteOutcome,omitempty"`+ RemoteRetryRequested bool `json:"remoteRetryRequested,omitempty"`+ RemoteRetryAttempted bool `json:"remoteRetryAttempted,omitempty"`+ RemoteAttemptCount int `json:"remoteAttemptCount,omitempty"`+ RemoteAccepted bool `json:"remoteAccepted"`+ RemoteSettled bool `json:"remoteSettled"`+ LastConfirmedAt *time.Time `json:"lastConfirmedAt,omitempty"`+}++type RunList struct {+ Runs []Run `json:"runs"`+ Total int `json:"total"`+ Page int `json:"page"`+ Limit int `json:"limit"`+}++type WorkerHealth struct {+ Status string `json:"status"`+ LastError string `json:"lastError,omitempty"`+ NextRetry *time.Time `json:"nextRetry,omitempty"`+ UpdatedAt time.Time `json:"updatedAt"`+}++// Dispatcher owns durable admission and execution. Cron only submits work.+type Dispatcher interface {+ Submit(ctx context.Context, request Request) (Run, error)+ Checkpoint(ctx context.Context, jobID, schedule string, nextRun time.Time) error+}++// Reconciler consults domain state before repeating interrupted work.+type Reconciler interface {+ Reconcile(ctx context.Context, previous Run) (Outcome, error)+}++// OutcomeError carries partial results through error-only watcher contracts.+type OutcomeError struct {+ Outcome Outcome+ Cause error+}++func (e *OutcomeError) Error() string {+ if e.Outcome.Message != "" {+ return e.Outcome.Message+ }+ if e.Cause != nil {+ return e.Cause.Error()+ }+ return string(e.Outcome.Status)+}++func (e *OutcomeError) Unwrap() error { return e.Cause }