Skip to content

Modern Docker Management, Designed for Everyone

feat: add durable job execution and offline recovery (#3863)

Kyle Mendell58823c2ac4af
diff --git a/backend/internal/apns/service.go b/backend/internal/apns/service.go
index 48233ec0c..861a9a2bf 100644
--- a/backend/internal/apns/service.go
+++ b/backend/internal/apns/service.go
@@ -487,7 +487,12 @@ func (s *ApnsService) Enqueue(ctx context.Context, environmentID, environmentNam
if err := s.db.WithContext(ctx).Create(&entry).Error; err != nil {
return errors.WrapIf(err, "failed to enqueue push notification")
}
- go s.DrainOutbox(context.WithoutCancel(ctx))
+ go func() {
+ drainCtx := context.WithoutCancel(ctx)
+ if err := s.DrainOutbox(drainCtx); err != nil {
+ slog.WarnContext(drainCtx, "Push outbox drain failed", "error", err)
+ }
+ }()
return nil
}
@@ -523,44 +528,48 @@ func (s *ApnsService) recipientsInternal(ctx context.Context, environmentID stri
return recipients, nil
}
-func (s *ApnsService) DrainOutbox(ctx context.Context) {
+func (s *ApnsService) DrainOutbox(ctx context.Context) error {
s.drainPending.Store(true)
if !s.drainMu.TryLock() {
- return
+ return nil
}
defer s.drainMu.Unlock()
for s.drainPending.Swap(false) {
- s.drainOutboxInternal(ctx)
+ if err := s.drainOutboxInternal(ctx); err != nil {
+ return err
+ }
}
+ return nil
}
-func (s *ApnsService) drainOutboxInternal(ctx context.Context) {
+func (s *ApnsService) drainOutboxInternal(ctx context.Context) error {
if !s.Enabled(ctx) {
- return
+ return nil
}
var entries []OutboxEntry
if err := s.db.WithContext(ctx).Where("next_attempt_at <= ?", time.Now()).Order("next_attempt_at ASC").Limit(outboxBatchSize).Find(&entries).Error; err != nil {
slog.ErrorContext(ctx, "Failed to load push outbox", "error", err)
- return
+ return err
}
if len(entries) == 0 {
- return
+ return nil
}
channelID, err := s.EnsureChannel(ctx)
if err != nil {
slog.WarnContext(ctx, "Push outbox drain skipped", "error", err)
- return
+ return err
}
signer, err := s.signerInternal(ctx)
if err != nil {
slog.WarnContext(ctx, "Push outbox drain skipped", "error", err)
- return
+ return err
}
for _, entry := range entries {
if channelRevoked := s.sendOutboxEntryInternal(ctx, entry, channelID, signer); channelRevoked {
- return
+ return errors.New("push channel revoked")
}
}
+ return nil
}
func (s *ApnsService) sendOutboxEntryInternal(ctx context.Context, entry OutboxEntry, channelID string, signer *signerInternal) bool {
diff --git a/backend/internal/auth/huma_middleware.go b/backend/internal/auth/huma_middleware.go
index dcc7e25bf..d982abe96 100644
--- a/backend/internal/auth/huma_middleware.go
+++ b/backend/internal/auth/huma_middleware.go
@@ -276,6 +276,7 @@ func handleApiKeyAuthInternal(api huma.API, ctx huma.Context, authService *AuthS
ps = resolveApiKeyPermissionsInternal(ctx.Context(), permResolver, key.ID)
}
newCtx := setUserInContextInternal(ctx.Context(), user, ps)
+ newCtx = context.WithValue(newCtx, middleware.ContextKeyApiKeyID, key.ID)
next(huma.WithContext(ctx, newCtx))
return
}
diff --git a/backend/internal/backup/policies.go b/backend/internal/backup/policies.go
index 408976e02..a494e7d4c 100644
--- a/backend/internal/backup/policies.go
+++ b/backend/internal/backup/policies.go
@@ -9,7 +9,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
s3domain "github.com/getarcaneapp/arcane/backend/v2/internal/s3"
- "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/schedule"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/schedule"
backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"
"gorm.io/gorm"
)
diff --git a/backend/internal/bootstrap/bootstrap_test.go b/backend/internal/bootstrap/bootstrap_test.go
index 154bdd445..c4994c2c4 100644
--- a/backend/internal/bootstrap/bootstrap_test.go
+++ b/backend/internal/bootstrap/bootstrap_test.go
@@ -16,16 +16,20 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/auth"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/job"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/kv"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
tunnelpb "github.com/getarcaneapp/arcane/backend/v2/proto/tunnel/v1"
"github.com/labstack/echo/v5"
+ "github.com/libtnb/sqlite"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
libcrypto "go.getarcane.app/sys/crypto"
"go.uber.org/fx"
"go.uber.org/fx/fxtest"
"golang.org/x/net/http2"
+ "gorm.io/gorm"
)
type blockingBusWatcherInternal struct {
@@ -381,7 +385,15 @@ func TestJobSchedulerStopCancelsItsPrivateContextInternal(t *testing.T) {
lifecycle := fxtest.NewLifecycle(t)
runtime, err := actors.NewRuntime(appCtx, lifecycle)
require.NoError(t, err)
- jobScheduler, err := newJobScheduler(appCtx, lifecycle, &config.Config{}, runtime, nil, nil, nil, nil)
+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
+ require.NoError(t, err)
+ require.NoError(t, db.AutoMigrate(&kv.KVEntry{}))
+ sqlDB, err := db.DB()
+ require.NoError(t, err)
+ sqlDB.SetMaxOpenConns(1)
+ t.Cleanup(func() { _ = sqlDB.Close() })
+ jobService := job.NewJobService(&database.DB{DB: db}, nil, &config.Config{})
+ jobScheduler, err := newJobScheduler(appCtx, lifecycle, &config.Config{}, runtime, nil, nil, nil, nil, jobService)
require.NoError(t, err)
watcher := &blockingBusWatcherInternal{
started: make(chan struct{}),
diff --git a/backend/internal/bootstrap/jobs_bootstrap.go b/backend/internal/bootstrap/jobs_bootstrap.go
index 0283bc4bc..0662ada5b 100644
--- a/backend/internal/bootstrap/jobs_bootstrap.go
+++ b/backend/internal/bootstrap/jobs_bootstrap.go
@@ -7,6 +7,8 @@ import (
"net/http"
"slices"
+ "emperror.dev/errors"
+
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
"github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/apns"
@@ -24,17 +26,19 @@ import (
"go.uber.org/fx"
)
-func newJobScheduler(appCtx context.Context, lc fx.Lifecycle, cfg *config.Config, runtime *actors.Runtime, _ *actors.Gate[actors.AdmissionKey], imageUpdateWatcher *scheduler.ImageUpdateWatcher, analytics *scheduler.AnalyticsJob, systemUpgrade *system.SystemUpgradeService) (schedulertypes.JobScheduler, error) {
+func newJobScheduler(appCtx context.Context, lc fx.Lifecycle, cfg *config.Config, runtime *actors.Runtime, _ *actors.Gate[actors.AdmissionKey], imageUpdateWatcher *scheduler.ImageUpdateWatcher, analytics *scheduler.AnalyticsJob, systemUpgrade *system.SystemUpgradeService, jobService *job.JobService) (schedulertypes.JobScheduler, error) {
schedulerCtx, cancelScheduler := context.WithCancel(appCtx)
jobScheduler, err := scheduler.NewJobScheduler(schedulerCtx, runtime, cfg.GetLocation())
if err != nil {
cancelScheduler()
return nil, err
}
+ jobService.SetScheduler(schedulerCtx, jobScheduler)
lc.Append(fx.Hook{
OnStart: func(context.Context) error {
slog.InfoContext(appCtx, "Starting scheduler")
if imageUpdateWatcher != nil {
+ imageUpdateWatcher.SetDispatcher(jobService.Queue)
if err := jobScheduler.RegisterBusWatcher(imageUpdateWatcher, true); err != nil {
return err
}
@@ -42,8 +46,13 @@ func newJobScheduler(appCtx context.Context, lc fx.Lifecycle, cfg *config.Config
if err := jobScheduler.StartScheduler(); err != nil {
return err
}
+ if err := jobService.Queue.Start(schedulerCtx); err != nil {
+ return err
+ }
if analytics != nil {
- go analytics.Run(schedulerCtx)
+ if _, err := jobService.Queue.Submit(schedulerCtx, schedulertypes.Request{JobID: analytics.Name(), EnvironmentID: "0", Trigger: "startup"}); err != nil {
+ return err
+ }
}
if !cfg.AgentMode && systemUpgrade != nil {
go systemUpgrade.ResumeUpdateAllOnStartup(schedulerCtx)
@@ -51,8 +60,8 @@ func newJobScheduler(appCtx context.Context, lc fx.Lifecycle, cfg *config.Config
return nil
},
OnStop: func(ctx context.Context) error {
- err := jobScheduler.Stop(ctx)
cancelScheduler()
+ err := errors.Combine(jobService.Queue.Stop(ctx), jobScheduler.Stop(ctx))
if err != nil {
slog.ErrorContext(ctx, "Job scheduler exited with error", "error", err)
return err
diff --git a/backend/internal/database/database_test.go b/backend/internal/database/database_test.go
index a67d63518..028c940fc 100644
--- a/backend/internal/database/database_test.go
+++ b/backend/internal/database/database_test.go
@@ -777,7 +777,9 @@ func TestIdentityNormalizationMigration(t *testing.T) {
}
var version int64
require.NoError(t, db.QueryRow("SELECT MAX(version_id) FROM goose_db_version WHERE is_applied = 1").Scan(&version))
- require.Equal(t, int64(81), version)
+ highest, err := getHighestEmbeddedMigrationVersionInternal("sqlite")
+ require.NoError(t, err)
+ require.Equal(t, highest, version)
})
}
}
@@ -804,7 +806,9 @@ func TestIdentityNormalizationGooseUpgrade(t *testing.T) {
require.Equal(t, "unchanged", password)
var version int64
require.NoError(t, db.QueryRow("SELECT MAX(version_id) FROM goose_db_version WHERE is_applied = 1").Scan(&version))
- require.Equal(t, int64(81), version)
+ highest, err := getHighestEmbeddedMigrationVersionInternal("sqlite")
+ require.NoError(t, err)
+ require.Equal(t, highest, version)
}
func TestIdentityNormalizationPostgresUpgrade(t *testing.T) {
@@ -856,5 +860,7 @@ func TestIdentityNormalizationPostgresUpgrade(t *testing.T) {
}
var version int64
require.NoError(t, db.QueryRowContext(ctx, "SELECT MAX(version_id) FROM goose_db_version WHERE is_applied").Scan(&version))
- require.Equal(t, int64(81), version)
+ highest, err := getHighestEmbeddedMigrationVersionInternal("postgres")
+ require.NoError(t, err)
+ require.Equal(t, highest, version)
}
diff --git a/backend/internal/di/providers.go b/backend/internal/di/providers.go
index bfcc3f1ec..e9b5e8d51 100644
--- a/backend/internal/di/providers.go
+++ b/backend/internal/di/providers.go
@@ -5,6 +5,7 @@ import (
"embed"
"log/slog"
"net/http"
+ "time"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
"github.com/getarcaneapp/arcane/backend/v2/internal/apikey"
@@ -361,8 +362,8 @@ func provideUserModuleInternal(service *user.UserService, auth *auth.AuthService
return user.New(user.Dependencies{Service: service, InvalidateUserTokenCache: auth.InvalidateUserTokenCache, Settings: settingsService})
}
-func provideJobModuleInternal(db *database.DB, settingsService *settings.SettingsService, cfg *config.Config, environment *environment.EnvironmentService) *job.Module {
- return job.New(job.Dependencies{DB: db, Settings: settingsService, Config: cfg, Environment: environment})
+func provideJobModuleInternal(db *database.DB, settingsService *settings.SettingsService, cfg *config.Config, environment *environment.EnvironmentService, roles *role.RoleService) *job.Module {
+ return job.New(job.Dependencies{DB: db, Settings: settingsService, Config: cfg, Environment: environment, Roles: roles})
}
func provideJobServiceInternal(module *job.Module) *job.JobService {
@@ -411,8 +412,22 @@ func provideFilesystemWatcherJobInternal(ctx context.Context, lc fx.Lifecycle, a
lc.Append(fx.Hook{
OnStart: func(context.Context) error {
runner, err = actors.NewRunner(ctx, actorRuntime, "filesystem-watcher", "lifecycle", "filesystem watcher", 3, func(runCtx context.Context) error {
- if startErr := job.Start(runCtx); startErr != nil {
- return startErr
+ delay := 5 * time.Second
+ for runCtx.Err() == nil {
+ startErr := job.Start(runCtx)
+ if startErr == nil {
+ break
+ }
+ slog.WarnContext(runCtx, "Filesystem watcher startup failed; retrying", "error", startErr)
+
+ timer := time.NewTimer(delay)
+ select {
+ case <-runCtx.Done():
+ timer.Stop()
+ return nil
+ case <-timer.C:
+ }
+ delay = min(delay*2, 5*time.Minute)
}
<-runCtx.Done()
return nil
diff --git a/backend/internal/environment/environment_health.go b/backend/internal/environment/environment_health.go
index 5923475e1..6e6e48432 100644
--- a/backend/internal/environment/environment_health.go
+++ b/backend/internal/environment/environment_health.go
@@ -11,20 +11,19 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
"github.com/getarcaneapp/arcane/types/v2/environment"
schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
"github.com/moby/moby/client"
)
const (
- defaultEnvironmentHealthInterval = "0 */2 * * * *"
- environmentHealthCheckTimeout = 90 * time.Second
+ defaultEnvironmentHealthInterval = "0 */2 * * * *"
+ environmentHealthCheckTimeout = 90 * time.Second
+ environmentHealthJobPrefix = "environment-health:"
+ environmentHealthAdmissionScopeInternal = "environment-health"
)
-const environmentHealthJobPrefix = "environment-health:"
-
-const environmentHealthAdmissionScopeInternal = "environment-health"
-
// SetScheduler injects the job scheduler and app lifecycle context. Called during
// bootstrap on the manager only (agent mode leaves scheduler nil, so all health-job
// registration becomes a no-op).
@@ -44,7 +43,10 @@ func (s *EnvironmentService) registerHealthJobInternal(ctx context.Context, envI
}
return sched
},
- func(ctx context.Context) { s.runHealthCheckInternal(ctx, envID) },
+ func(ctx context.Context) (schedulertypes.Outcome, error) { return s.runHealthCheckInternal(ctx, envID) },
+ func(context.Context, schedulertypes.Run) (schedulertypes.Outcome, error) {
+ return schedulertypes.Outcome{Status: schedulertypes.Retrying, Message: "Connection checks can safely resume"}, nil
+ },
)
}
@@ -94,63 +96,92 @@ func (s *EnvironmentService) RescheduleHealthJobs(ctx context.Context) {
s.registerAllEnabledHealthJobsInternal(ctx)
}
-// RunHealthChecksNow runs every enabled environment's health check synchronously.
-// Backs the "run now" button for the environment-health job in the Jobs UI.
+// RunHealthChecksNow queues each enabled environment's health check.
+// Child runs retain the parent identity so authorization is checked at execution.
func (s *EnvironmentService) RunHealthChecksNow(ctx context.Context) error {
+ if !s.jobs.Enabled() {
+ return errors.New("environment health scheduler unavailable")
+ }
ids, err := s.ListEnabledEnvironmentIDs(ctx)
if err != nil {
return err
}
+ parent, hasParent := jobcontext.Run(ctx)
+ request := schedulertypes.Request{EnvironmentID: "0", Trigger: "internal"}
+ if hasParent {
+ request.Trigger = parent.Trigger
+ request.RequestedBy = parent.RequestedBy
+ request.RequestedWithKey = parent.RequestedWithKey
+ }
for _, id := range ids {
- s.runHealthCheckInternal(ctx, id)
+ request.JobID = s.jobs.JobName(id)
+ if _, err := s.jobs.Scheduler().Submit(ctx, request); err != nil {
+ return err
+ }
}
return nil
}
// runHealthCheckInternal tests one environment's connection (updating its DB status)
// and, for online remotes, syncs registries and repositories to it.
-func (s *EnvironmentService) runHealthCheckInternal(ctx context.Context, envID string) {
+func (s *EnvironmentService) runHealthCheckInternal(ctx context.Context, envID string) (schedulertypes.Outcome, error) {
lease, admitted, err := s.jobs.TryAcquire(ctx, envID)
if err != nil {
slog.ErrorContext(ctx, "environment health check admission failed", "environment_id", envID, "error", err)
- return
+ return schedulertypes.Outcome{}, err
}
if !admitted {
slog.WarnContext(ctx, "environment health check skipped; previous run still in progress", "environment_id", envID)
- return
+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil
}
defer lease.Release()
+ environment, err := s.GetEnvironmentByID(ctx, envID)
+ if err != nil {
+ return schedulertypes.Outcome{}, err
+ }
+ if !environment.Enabled {
+ return schedulertypes.Outcome{Status: schedulertypes.Canceled, Message: "Environment disabled"}, nil
+ }
status, err := s.TestConnection(ctx, envID, nil)
switch {
case err != nil:
slog.WarnContext(ctx, "environment health check failed", "environment_id", envID, "status", status, "error", err)
- return
+ return schedulertypes.Outcome{Status: schedulertypes.Retrying}, err
case status != "online":
- return
+ return schedulertypes.Outcome{Status: schedulertypes.Retrying, Message: "Environment is offline"}, errors.New("environment is offline")
}
// Local environment (ID "0") has no registries/repositories to push.
if envID == "0" {
- return
+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil
}
+ var syncErrors []error
syncCtx, cancel := context.WithTimeout(ctx, environmentHealthCheckTimeout)
defer cancel()
if err := s.SyncRegistriesToEnvironment(syncCtx, envID); err != nil {
slog.WarnContext(syncCtx, "failed to sync registries during health check", "environment_id", envID, "error", err)
+ syncErrors = append(syncErrors, err)
}
if err := s.SyncS3DestinationsToEnvironment(syncCtx, envID); err != nil {
slog.WarnContext(syncCtx, "failed to sync S3 destinations during health check", "environment_id", envID, "error", err)
+ syncErrors = append(syncErrors, err)
}
if err := s.SyncRepositoriesToEnvironment(syncCtx, envID); err != nil {
slog.WarnContext(syncCtx, "failed to sync git repositories during health check", "environment_id", envID, "error", err)
+ syncErrors = append(syncErrors, err)
}
if s.variableSyncer != nil {
if err := s.variableSyncer.SyncEnvironment(syncCtx, envID); err != nil {
slog.WarnContext(syncCtx, "failed to sync global variables during health check", "environment_id", envID, "error", err)
+ syncErrors = append(syncErrors, err)
}
}
+ if err := errors.Combine(syncErrors...); err != nil {
+ return schedulertypes.Outcome{Status: schedulertypes.Partial}, err
+ }
+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil
}
func (s *EnvironmentService) TestConnection(ctx context.Context, id string, customApiUrl *string) (string, error) {
diff --git a/backend/internal/environment/service_test.go b/backend/internal/environment/service_test.go
index ba9beca20..3b511c34c 100644
--- a/backend/internal/environment/service_test.go
+++ b/backend/internal/environment/service_test.go
@@ -67,8 +67,9 @@ func TestEnvironmentService_OverlappingHealthCheckIsSkippedInternal(t *testing.T
}
type environmentTestSchedulerInternal struct {
- added []string
- removed []string
+ submitted []schedulertypes.Request
+ added []string
+ removed []string
}
func (s *environmentTestSchedulerInternal) AddJob(_ context.Context, job schedulertypes.Job) error {
@@ -1429,3 +1430,8 @@ func TestEnvironmentService_ExecuteRemoteRequest_RejectsInvalidEnvironmentURL(t
require.Error(t, err)
require.Contains(t, err.Error(), "invalid environment API URL")
}
+
+func (s *environmentTestSchedulerInternal) Submit(_ context.Context, request schedulertypes.Request) (schedulertypes.Run, error) {
+ s.submitted = append(s.submitted, request)
+ return schedulertypes.Run{ID: request.RunID, JobID: request.JobID, EnvironmentID: request.EnvironmentID, Status: schedulertypes.Queued}, nil
+}
diff --git a/backend/internal/gitops/gitops_sync.go b/backend/internal/gitops/gitops_sync.go
index b6efc0796..b7b30675b 100644
--- a/backend/internal/gitops/gitops_sync.go
+++ b/backend/internal/gitops/gitops_sync.go
@@ -15,6 +15,8 @@ import (
"strings"
"time"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
+
"emperror.dev/errors"
"github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
@@ -375,23 +377,41 @@ func (s *GitOpsSyncService) SetScheduler(ctx context.Context, scheduler schedule
// sync each run so a row toggled to AutoSync=false self-cancels. If the row is gone
// (e.g. deleted out-of-band via raw SQL), the job unregisters itself instead of
// firing forever; transient load errors are skipped and retried next tick.
-func (s *GitOpsSyncService) runScheduledSyncInternal(ctx context.Context, environmentID, syncID string) {
+func (s *GitOpsSyncService) runScheduledSyncInternal(ctx context.Context, environmentID, syncID string) (schedulertypes.Outcome, error) {
syncRecord, err := s.getSyncRecordByIDInternal(ctx, environmentID, syncID)
if err != nil {
if errors.Is(err, common.ErrNotFound) {
slog.InfoContext(ctx, "gitops auto-sync job unregistering; sync no longer exists", "syncId", syncID)
s.unregisterSyncJobInternal(ctx, syncID)
- return
+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil
}
slog.DebugContext(ctx, "gitops auto-sync skipped; failed to load sync", "syncId", syncID, "error", err)
- return
+ return schedulertypes.Outcome{}, err
}
if !syncRecord.AutoSync {
- return
+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil
+ }
+ if previous, ok := jobcontext.Run(ctx); ok {
+ outcome := jobcontext.ConfirmedTarget(previous, syncID)
+ if outcome.Status == schedulertypes.Succeeded {
+ return outcome, nil
+ }
+ }
+ if err := jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ID: syncID, Status: schedulertypes.Running}); err != nil {
+ return schedulertypes.Outcome{}, err
}
- if _, err := s.PerformSync(ctx, environmentID, syncID, common.SystemUser); err != nil {
+ result, err := s.PerformSync(ctx, environmentID, syncID, common.SystemUser)
+ if err != nil {
slog.ErrorContext(ctx, "gitops auto-sync run failed", "syncId", syncID, "error", err)
+ return schedulertypes.Outcome{}, err
+ }
+ if !result.Success {
+ return schedulertypes.Outcome{Status: schedulertypes.Skipped, Message: result.Message}, nil
}
+ if err := jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ID: syncID, Status: schedulertypes.Succeeded}); err != nil {
+ return schedulertypes.Outcome{}, err
+ }
+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded, Message: result.Message}, nil
}
// registerSyncJobInternal schedules a single sync on a fixed "@every Nm"
@@ -402,7 +422,12 @@ func (s *GitOpsSyncService) registerSyncJobInternal(ctx context.Context, syncID,
schedule := fmt.Sprintf("@every %dm", max(intervalMinutes, 1))
s.jobs.Register(ctx, syncID,
func(_ context.Context) string { return schedule },
- func(ctx context.Context) { s.runScheduledSyncInternal(ctx, environmentID, syncID) },
+ func(ctx context.Context) (schedulertypes.Outcome, error) {
+ return s.runScheduledSyncInternal(ctx, environmentID, syncID)
+ },
+ func(_ context.Context, previous schedulertypes.Run) (schedulertypes.Outcome, error) {
+ return jobcontext.ConfirmedTarget(previous, syncID), nil
+ },
)
}
@@ -413,13 +438,13 @@ func (s *GitOpsSyncService) unregisterSyncJobInternal(ctx context.Context, syncI
// kickSyncInternal runs a sync once in the background on the app lifecycle context.
// Used when auto-sync is freshly enabled or when a sync is overdue at startup, so
// the first run does not wait a full interval.
-func (s *GitOpsSyncService) kickSyncInternal(ctx context.Context, syncID, environmentID string) {
- ctx = s.jobs.Context(ctx)
- go func() {
- if _, err := s.PerformSync(ctx, environmentID, syncID, common.SystemUser); err != nil {
- slog.ErrorContext(ctx, "gitops immediate sync kick failed", "syncId", syncID, "error", err)
- }
- }()
+func (s *GitOpsSyncService) kickSyncInternal(ctx context.Context, syncID string) {
+ if !s.jobs.Enabled() {
+ return
+ }
+ if _, err := s.jobs.Scheduler().Submit(ctx, schedulertypes.Request{JobID: s.jobs.JobName(syncID), EnvironmentID: "0", Trigger: "startup"}); err != nil {
+ slog.ErrorContext(ctx, "gitops immediate sync admission failed", "syncId", syncID, "error", err)
+ }
}
// RegisterAutoSyncJobsOnStartup registers a dynamic job for every auto-sync-enabled
@@ -440,7 +465,7 @@ func (s *GitOpsSyncService) RegisterAutoSyncJobsOnStartup(ctx context.Context) {
syncRecord := syncs[i]
s.registerSyncJobInternal(ctx, syncRecord.ID, syncRecord.EnvironmentID, syncRecord.SyncInterval)
if isGitOpsSyncOverdueInternal(&syncRecord) {
- s.kickSyncInternal(ctx, syncRecord.ID, syncRecord.EnvironmentID)
+ s.kickSyncInternal(ctx, syncRecord.ID)
}
}
slog.InfoContext(ctx, "Registered gitops auto-sync jobs on startup", "count", len(syncs))
@@ -814,7 +839,7 @@ func (s *GitOpsSyncService) UpdateSync(ctx context.Context, environmentID, id st
s.registerSyncJobInternal(ctx, syncRecord.ID, syncRecord.EnvironmentID, newInterval)
if !oldAutoSync {
// Freshly enabled — kick a run now so it doesn't wait a full interval.
- s.kickSyncInternal(ctx, syncRecord.ID, syncRecord.EnvironmentID)
+ s.kickSyncInternal(ctx, syncRecord.ID)
}
default:
s.unregisterSyncJobInternal(ctx, syncRecord.ID)
diff --git a/backend/internal/gitops/service_test.go b/backend/internal/gitops/service_test.go
index ac64e4426..a0f80e9d5 100644
--- a/backend/internal/gitops/service_test.go
+++ b/backend/internal/gitops/service_test.go
@@ -123,8 +123,9 @@ func TestGitOpsSyncService_OverlappingSyncPreservesSuccessShapedSkipInternal(t *
}
type gitOpsSyncTestSchedulerInternal struct {
- added []string
- removed []string
+ submitted []schedulertypes.Request
+ added []string
+ removed []string
}
func (s *gitOpsSyncTestSchedulerInternal) AddJob(_ context.Context, job schedulertypes.Job) error {
@@ -1877,3 +1878,8 @@ func TestMarkSyncRedeployFailedInternal_PersistsErrorOnSyncRow(t *testing.T) {
require.Contains(t, *stored.SyncedFiles, "compose.yml")
require.Contains(t, *stored.SyncedFiles, "scripts/pre-deploy.sh")
}
+
+func (s *gitOpsSyncTestSchedulerInternal) Submit(_ context.Context, request schedulertypes.Request) (schedulertypes.Run, error) {
+ s.submitted = append(s.submitted, request)
+ return schedulertypes.Run{ID: request.RunID, JobID: request.JobID, EnvironmentID: request.EnvironmentID, Status: schedulertypes.Queued}, nil
+}
diff --git a/backend/internal/job/execution.go b/backend/internal/job/execution.go
new file mode 100644
index 000000000..7cc734e8f
--- /dev/null
+++ b/backend/internal/job/execution.go
@@ -0,0 +1,287 @@
+package job
+
+import (
+ "context"
+ "log/slog"
+ "net"
+ "strings"
+ "time"
+
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/queue"
+
+ "emperror.dev/errors"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/activity"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/apikey"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/common"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/role"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/authz"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
+ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
+ "github.com/getarcaneapp/arcane/types/v2/meta"
+ st "github.com/getarcaneapp/arcane/types/v2/scheduler"
+)
+
+// Submit validates job eligibility and persists acceptance before execution.
+// Managers may accept requests for offline environments. Reusing a run ID
+// deduplicates admission within the same job and environment.
+func (s *JobService) Submit(ctx context.Context, request st.Request) (st.Run, error) {
+ if request.EnvironmentID == "" {
+ request.EnvironmentID = "0"
+ }
+ if request.EnvironmentID != "0" {
+ if s.cfg.AgentMode {
+ return st.Run{}, errors.New("agents cannot queue work for another environment")
+ }
+ env, err := s.environment.GetEnvironmentByID(ctx, request.EnvironmentID)
+ if err != nil {
+ return st.Run{}, err
+ }
+ if !env.Enabled {
+ return st.Run{}, errors.New("environment is disabled")
+ }
+ metadata, ok := meta.GetJobMetadata(request.JobID)
+ dynamicRemote := strings.HasPrefix(request.JobID, "gitops-sync:") || strings.HasPrefix(request.JobID, "volume-backup:")
+ if !dynamicRemote && (!ok || metadata.ManagerOnly || !metadata.CanRunManually) {
+ return st.Run{}, errors.New("job is not remotely runnable")
+ }
+ } else if err := s.validateLocalJobInternal(ctx, request.JobID); err != nil {
+ return st.Run{}, err
+ }
+ return s.Queue.Submit(ctx, request)
+}
+
+func (s *JobService) validateLocalJobInternal(ctx context.Context, jobID string) error {
+ if metadata, ok := meta.GetJobMetadata(jobID); ok {
+ if !metadata.CanRunManually {
+ return errors.New("job cannot be run manually")
+ }
+ if s.cfg.AgentMode && metadata.ManagerOnly {
+ return errors.New("job is manager-only")
+ }
+ if !s.isJobEnabledInternal(ctx, metadata) {
+ return errors.New("job is disabled")
+ }
+ for _, prereq := range s.evaluatePrerequisitesInternal(ctx, metadata) {
+ if !prereq.IsMet {
+ return errors.New("job prerequisites are not met")
+ }
+ }
+ if metadata.IsContinuous || jobID == "environment-health" {
+ return nil
+ }
+ }
+ if s.scheduler == nil {
+ return errors.New("scheduler unavailable")
+ }
+ job, ok := s.scheduler.GetJob(jobID)
+ if !ok {
+ return errors.New("job is not registered")
+ }
+ if conditional, ok := job.(st.ConditionalJob); ok && !conditional.ShouldSchedule(ctx) {
+ return errors.New("job is disabled")
+ }
+ return nil
+}
+
+func (s *JobService) authorizeRunInternal(ctx context.Context, run st.Run) error {
+ if run.RequestedBy == "" {
+ if run.Trigger == "manual" && !s.cfg.AgentMode {
+ return errors.New("requesting user unavailable")
+ }
+ return nil
+ }
+ if s.roles == nil {
+ return errors.New("permission resolver unavailable")
+ }
+ var user common.User
+ if err := s.db.WithContext(ctx).First(&user, "id = ?", run.RequestedBy).Error; err != nil {
+ return err
+ }
+ if run.RequestedWithKey != "" {
+ return s.authorizeKeyInternal(ctx, run)
+ }
+ permissions, err := s.roles.ResolveUserPermissionsInDB(ctx, s.db.WithContext(ctx), user.ID)
+ if err != nil {
+ return err
+ }
+ if !permissions.Allows(authz.PermJobsManage, run.EnvironmentID) {
+ return errors.New("requesting user no longer has permission to manage jobs")
+ }
+ return nil
+}
+
+func (s *JobService) executeRunInternal(ctx context.Context, run st.Run) (st.Outcome, error) {
+ if err := s.authorizeRunInternal(ctx, run); err != nil {
+ return st.Outcome{Status: st.NeedsAttention}, err
+ }
+ if run.EnvironmentID != "0" {
+ return s.deliverRemoteInternal(ctx, run)
+ }
+ if requiresDockerInternal(run.JobID) && s.environment != nil {
+ status, err := s.environment.TestConnection(ctx, "0", nil)
+ if err != nil || status != "online" {
+ return st.Outcome{Status: st.Waiting, Message: "Waiting for Docker"}, err
+ }
+ }
+ ctx = s.runContextInternal(ctx, run)
+ if run.JobID == "environment-health" && s.RunEnvironmentHealthNow != nil {
+ err := s.RunEnvironmentHealthNow(ctx)
+ return classifyOutcomeInternal(run.JobID, st.Outcome{}, err)
+ }
+ if metadata, ok := meta.GetJobMetadata(run.JobID); ok && metadata.IsContinuous {
+ if err := s.validateLocalJobInternal(ctx, run.JobID); err != nil {
+ return st.Outcome{Status: st.Canceled}, err
+ }
+ err := s.scheduler.RunBusWatcherNow(ctx, run.JobID)
+ return classifyOutcomeInternal(run.JobID, st.Outcome{}, err)
+ }
+ job, ok := s.scheduler.GetJob(run.JobID)
+ if !ok {
+ return st.Outcome{Status: st.Canceled, Message: "Job or target no longer exists"}, nil
+ }
+ if conditional, ok := job.(st.ConditionalJob); ok && !conditional.ShouldSchedule(ctx) {
+ return st.Outcome{Status: st.Canceled, Message: "Job is disabled"}, nil
+ }
+ outcome, err := job.Run(ctx)
+ return classifyOutcomeInternal(run.JobID, outcome, err)
+}
+
+func classifyOutcomeInternal(jobID string, outcome st.Outcome, err error) (st.Outcome, error) {
+ var resultErr *st.OutcomeError
+ if errors.As(err, &resultErr) {
+ return resultErr.Outcome, err
+ }
+ if err == nil {
+ if outcome.Status == "" {
+ outcome.Status = st.Succeeded
+ }
+ return outcome, nil
+ }
+ if outcome.Status == st.Waiting || outcome.Status == st.Retrying {
+ return outcome, err
+ }
+ if outcome.Status == st.NeedsAttention || outcome.Status == st.Partial || outcome.Status == st.Canceled || outcome.Status == st.Skipped {
+ return outcome, err
+ }
+ var networkErr net.Error
+ switch {
+ case safeJobInternal(jobID) && (errors.As(err, &networkErr) || errors.Is(err, context.DeadlineExceeded)):
+ outcome.Status = st.Retrying
+ case !safeJobInternal(jobID):
+ outcome.Status = st.NeedsAttention
+ case outcome.Status == "":
+ outcome.Status = st.Failed
+ }
+
+ return outcome, err
+}
+
+func safeJobInternal(jobID string) bool {
+ switch jobID {
+ case "image-polling", "environment-health", "docker-client-refresh", "event-cleanup", "expired-sessions-cleanup", "activity-sweep", "upload-sessions-cleanup", "git-clone-scratch-cleanup", "analytics-heartbeat", "apns-outbox", "vulnerability-scan":
+ return true
+ }
+ return strings.HasPrefix(jobID, "environment-health:")
+}
+
+func (s *JobService) reconcileRunInternal(ctx context.Context, run st.Run) (st.Outcome, error) {
+ if err := s.authorizeRunInternal(ctx, run); err != nil {
+ return st.Outcome{Status: st.NeedsAttention}, err
+ }
+ if run.EnvironmentID != "0" {
+ return s.executeRunInternal(ctx, run)
+ }
+ if safeJobInternal(run.JobID) {
+ return s.executeRunInternal(ctx, run)
+ }
+ // A successful activity proves only that target, never the entire batch.
+ for index, target := range run.Outcome.Targets {
+ if target.Status == st.Succeeded || target.Status == st.Skipped {
+ continue
+ }
+ if target.ActivityID == "" {
+ return st.Outcome{Status: st.NeedsAttention, Message: "Interrupted operation has no confirmed outcome", Targets: run.Outcome.Targets}, nil
+ }
+ var record activity.Activity
+ if err := s.db.WithContext(ctx).First(&record, "id = ?", target.ActivityID).Error; err != nil {
+ return st.Outcome{Status: st.NeedsAttention}, err
+ }
+ if record.Status != activitytypes.StatusSuccess {
+ return st.Outcome{Status: st.NeedsAttention, Message: "Interrupted operation requires review", Targets: run.Outcome.Targets}, nil
+ }
+ run.Outcome.Targets[index].Status = st.Succeeded
+ }
+ if job, ok := s.scheduler.GetJob(run.JobID); ok {
+ if reconciler, ok := job.(st.Reconciler); ok {
+ outcome, err := reconciler.Reconcile(s.runContextInternal(ctx, run), run)
+ if err != nil {
+ slog.WarnContext(ctx, "Job reconciliation requires attention", "runId", run.ID, "error", err)
+ }
+ return outcome, err
+ }
+ }
+ return st.Outcome{Status: st.NeedsAttention, Message: "Interrupted operation requires review before retrying", Targets: run.Outcome.Targets}, nil
+}
+
+func (s *JobService) runContextInternal(ctx context.Context, run st.Run) context.Context {
+ ctx = jobcontext.WithExecution(ctx, run, func(target st.TargetOutcome) error {
+ return s.Queue.UpdateRun(ctx, run, func(current *st.Run) error {
+ if current.Status != st.Running || current.Owner != run.Owner {
+ return queue.ErrRunConflict
+ }
+ for index := range current.Outcome.Targets {
+ if current.Outcome.Targets[index].ID == target.ID {
+ current.Outcome.Targets[index] = target
+ return nil
+ }
+ }
+ current.Outcome.Targets = append(current.Outcome.Targets, target)
+ return nil
+ })
+ })
+ return ctx
+}
+
+func (s *JobService) authorizeKeyInternal(ctx context.Context, run st.Run) error {
+ var key apikey.ApiKey
+ if err := s.db.WithContext(ctx).First(&key, "id = ?", run.RequestedWithKey).Error; err != nil {
+ return err
+ }
+ if key.UserID == nil || *key.UserID != run.RequestedBy || (key.ExpiresAt != nil && !key.ExpiresAt.After(time.Now())) {
+ return errors.New("requesting API key is no longer valid")
+ }
+ if key.Kind == apikey.ApiKeyKindPersonal {
+ permissions, err := s.roles.ResolveUserPermissionsInDB(ctx, s.db.WithContext(ctx), run.RequestedBy)
+ if err != nil {
+ return err
+ }
+ if !permissions.Allows(authz.PermJobsManage, run.EnvironmentID) {
+ return errors.New("requesting user no longer has permission to manage jobs")
+ }
+ return nil
+ }
+ var grants []role.ApiKeyPermission
+ if err := s.db.WithContext(ctx).Where("api_key_id = ?", key.ID).Find(&grants).Error; err != nil {
+ return err
+ }
+ permissions := authz.NewPermissionSet()
+ for _, grant := range grants {
+ if grant.EnvironmentID == nil {
+ permissions.AddGlobal(grant.Permission)
+ } else {
+ permissions.AddEnv(*grant.EnvironmentID, grant.Permission)
+ }
+ }
+ if !permissions.Allows(authz.PermJobsManage, run.EnvironmentID) {
+ return errors.New("requesting API key no longer has permission to manage jobs")
+ }
+ return nil
+}
+
+func requiresDockerInternal(jobID string) bool {
+ switch jobID {
+ case "auto-update", "auto-patch", "auto-heal", "scheduled-prune", "vulnerability-scan", "image-polling":
+ return true
+ }
+ return strings.HasPrefix(jobID, "gitops-sync:") || strings.HasPrefix(jobID, "volume-backup:") || strings.HasPrefix(jobID, "system-backup:")
+}
diff --git a/backend/internal/job/handler.go b/backend/internal/job/handler.go
index fffa2baff..434701e8b 100644
--- a/backend/internal/job/handler.go
+++ b/backend/internal/job/handler.go
@@ -11,6 +11,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"
"github.com/getarcaneapp/arcane/types/v2/base"
"github.com/getarcaneapp/arcane/types/v2/jobschedule"
+ st "github.com/getarcaneapp/arcane/types/v2/scheduler"
)
type GetJobSchedulesInput struct {
@@ -35,8 +36,10 @@ type GetJobsOutput struct {
}
type RunJobInput struct {
- ID string `path:"id" doc:"Environment ID"`
- JobID string `path:"jobId" minLength:"1" doc:"Job ID to run"`
+ ID string `path:"id" doc:"Environment ID"`
+ JobID string `path:"jobId" minLength:"1" doc:"Job ID to run"`
+ RequestID string `header:"Idempotency-Key"`
+ Body *jobschedule.SubmitRunInput `required:"false"`
}
type RunJobOutput struct {
@@ -80,14 +83,16 @@ func RegisterJobSchedules(api huma.API, jobSvc *JobService, envSvc *environment.
}, authz.PermJobsManage, h.ListJobs)
middleware.RegisterWithPermission(api, huma.Operation{
- OperationID: "run-job",
- Method: http.MethodPost,
- Path: "/environments/{id}/jobs/{jobId}/run",
- Summary: "Run a job now",
- Description: "Manually trigger a background job to run immediately",
- Tags: []string{"JobSchedules"},
- Security: handlerutil.DefaultOperationSecurity(),
+ OperationID: "run-job",
+ DefaultStatus: http.StatusAccepted,
+ Method: http.MethodPost,
+ Path: "/environments/{id}/jobs/{jobId}/run",
+ Summary: "Run a job now",
+ Description: "Manually trigger a background job to run immediately",
+ Tags: []string{"JobSchedules"},
+ Security: handlerutil.DefaultOperationSecurity(),
}, authz.PermJobsManage, h.RunJob)
+ h.registerRunRoutesInternal(api)
}
type JobSchedulesHandler struct {
@@ -96,42 +101,39 @@ type JobSchedulesHandler struct {
}
func (h *JobSchedulesHandler) ListJobs(ctx context.Context, input *ListJobsInput) (*GetJobsOutput, error) {
+ var jobs *jobschedule.JobListResponse
+ var err error
if input.ID != "0" {
- jobs, err := h.proxyRemoteJSON.JSON[jobschedule.JobListResponse](ctx, input.ID, http.MethodGet, "/api/environments/0/jobs", nil)
- if err != nil {
- return nil, err
- }
- return &GetJobsOutput{Body: *jobs}, nil
+ jobs, err = h.jobService.ListRemoteJobs(ctx, input.ID)
+ } else {
+ jobs, err = h.jobService.ListJobs(ctx)
}
-
- jobs, err := h.jobService.ListJobs(ctx)
if err != nil {
- return nil, huma.Error500InternalServerError(err.Error())
+ return nil, jobHTTPErrorInternal(err)
}
-
return &GetJobsOutput{Body: *jobs}, nil
}
func (h *JobSchedulesHandler) RunJob(ctx context.Context, input *RunJobInput) (*RunJobOutput, error) {
- if input.ID != "0" {
- runResp, err := h.proxyRemoteJSON.JSON[jobschedule.JobRunResponse](ctx, input.ID, http.MethodPost, "/api/environments/0/jobs/"+input.JobID+"/run", nil)
- if err != nil {
- return nil, err
+ runID := input.RequestID
+ if input.Body != nil && input.Body.RunID != "" {
+ if runID != "" && runID != input.Body.RunID {
+ return nil, huma.Error400BadRequest("Conflicting run IDs")
}
- return &RunJobOutput{Body: *runResp}, nil
+ runID = input.Body.RunID
}
-
- err := h.jobService.RunJobNowInline(ctx, input.JobID)
+ userID, _ := middleware.GetUserIDFromContext(ctx)
+ keyID, _ := ctx.Value(middleware.ContextKeyApiKeyID).(string)
+ trigger := "manual"
+ if h.jobService.cfg.AgentMode && userID == "agent" {
+ userID = ""
+ trigger = "remote"
+ }
+ run, err := h.jobService.Submit(ctx, st.Request{RunID: runID, JobID: input.JobID, EnvironmentID: input.ID, Trigger: trigger, RequestedBy: userID, RequestedWithKey: keyID})
if err != nil {
- return nil, huma.Error400BadRequest(err.Error())
+ return nil, jobHTTPErrorInternal(err)
}
-
- return &RunJobOutput{
- Body: jobschedule.JobRunResponse{
- Success: true,
- Message: "Job completed successfully",
- },
- }, nil
+ return &RunJobOutput{Body: jobschedule.JobRunResponse{Success: true, Message: "Job accepted", RunID: run.ID, Status: run.Status}}, nil
}
func (h *JobSchedulesHandler) Get(ctx context.Context, input *GetJobSchedulesInput) (*GetJobSchedulesOutput, error) {
diff --git a/backend/internal/job/module.go b/backend/internal/job/module.go
index e5120bc8d..b296a7f66 100644
--- a/backend/internal/job/module.go
+++ b/backend/internal/job/module.go
@@ -7,10 +7,12 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/environment"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/role"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
)
type Dependencies struct {
+ Roles *role.RoleService
DB *database.DB
Settings *settings.SettingsService
Config *config.Config
@@ -23,8 +25,11 @@ type Module struct {
}
func New(deps Dependencies) *Module {
+ service := NewJobService(deps.DB, deps.Settings, deps.Config)
+ service.environment = deps.Environment
+ service.roles = deps.Roles
return &Module{
- service: NewJobService(deps.DB, deps.Settings, deps.Config),
+ service: service,
environment: deps.Environment,
}
}
diff --git a/backend/internal/job/remote.go b/backend/internal/job/remote.go
new file mode 100644
index 000000000..2e55f657b
--- /dev/null
+++ b/backend/internal/job/remote.go
@@ -0,0 +1,284 @@
+package job
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json/v2"
+ "net/http"
+ "net/url"
+ "sort"
+ "time"
+
+ "emperror.dev/errors"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/remenv"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/queue"
+ "github.com/getarcaneapp/arcane/types/v2/jobschedule"
+ "github.com/getarcaneapp/arcane/types/v2/meta"
+ st "github.com/getarcaneapp/arcane/types/v2/scheduler"
+)
+
+func remoteFailureInternal(err error) (st.Outcome, error) {
+ var status *remenv.StatusError
+ if errors.As(err, &status) && status.StatusCode >= 400 && status.StatusCode < 500 && status.StatusCode != http.StatusRequestTimeout && status.StatusCode != http.StatusTooManyRequests {
+ return st.Outcome{Status: st.NeedsAttention, Message: "Remote agent rejected the request; check configuration and permissions"}, err
+ }
+ return st.Outcome{Status: st.Waiting, Message: "Waiting for environment"}, err
+}
+
+func (s *JobService) deliverRemoteInternal(ctx context.Context, run st.Run) (st.Outcome, error) {
+ env, err := s.environment.GetEnvironmentByID(ctx, run.EnvironmentID)
+ if err != nil {
+ return st.Outcome{Status: st.NeedsAttention}, err
+ }
+ if !env.Enabled {
+ return st.Outcome{Status: st.NeedsAttention, Message: "Environment is disabled"}, nil
+ }
+ basePath := "/api/environments/0/jobs/" + url.PathEscape(run.JobID)
+ runPath := basePath + "/runs/" + url.PathEscape(run.ID)
+ // A previously confirmed terminal result needs only acknowledgement, never execution.
+ if !run.RemoteRetryRequested && run.RemoteAccepted && run.RemoteOutcome != nil && run.RemoteOutcome.Status.Terminal() {
+ return s.acknowledgeRemoteInternal(ctx, run, runPath, *run.RemoteOutcome)
+ }
+ var catalog jobschedule.JobListResponse
+ if err := s.environment.ProxyJSONRequest(ctx, run.EnvironmentID, http.MethodGet, "/api/environments/0/jobs", nil, &catalog); err != nil {
+ return remoteFailureInternal(err)
+ }
+ if !catalog.DurableRuns {
+ return st.Outcome{Status: st.NeedsAttention, Message: "Upgrade required: agent does not support durable job runs"}, nil
+ }
+ var remoteRun st.Run
+ err = s.environment.ProxyJSONRequest(ctx, run.EnvironmentID, http.MethodGet, runPath, nil, &remoteRun)
+ if err != nil {
+ var status *remenv.StatusError
+ if !errors.As(err, &status) || status.StatusCode != http.StatusNotFound {
+ return remoteFailureInternal(err)
+ }
+ remoteRun, err = s.admitRemoteInternal(ctx, run, catalog)
+ if err != nil {
+ if errors.Is(err, errRemoteIneligibleInternal) {
+ return st.Outcome{Status: st.Canceled, Message: err.Error()}, nil
+ }
+ if errors.Is(err, errRemoteReceiptMissingInternal) {
+ return st.Outcome{Status: st.NeedsAttention, Message: err.Error()}, nil
+ }
+ return remoteFailureInternal(err)
+ }
+ }
+ if remoteRun.ID != run.ID || remoteRun.JobID != run.JobID {
+ return st.Outcome{Status: st.NeedsAttention, Message: "Agent returned an inconsistent run identity"}, nil
+ }
+ if run.RemoteRetryRequested && (remoteRun.Status.Terminal() || remoteRun.Status == st.NeedsAttention) && remoteRun.AttemptCount <= run.RemoteAttemptCount {
+ remoteRun, err = s.retryDeliveryInternal(ctx, run, runPath)
+ if err != nil {
+ if errors.Is(err, errRemoteRetryUncertainInternal) {
+ return st.Outcome{Status: st.NeedsAttention, Message: err.Error()}, nil
+ }
+ return remoteFailureInternal(err)
+ }
+ }
+ return s.confirmRemoteRunInternal(ctx, run, remoteRun, runPath)
+}
+
+func (s *JobService) confirmRemoteRunInternal(ctx context.Context, run, remoteRun st.Run, runPath string) (st.Outcome, error) {
+ now := time.Now().UTC()
+ outcome := remoteRun.Outcome
+ outcome.Status = remoteRun.Status
+ if err := s.Queue.UpdateRun(ctx, run, func(current *st.Run) error {
+ current.RemoteAccepted = true
+ current.RemoteDeliveryAttempted = true
+ current.LastConfirmedAt = &now
+ current.RemoteOutcome = &outcome
+ current.RemoteAttemptCount = remoteRun.AttemptCount
+ current.RemoteRetryRequested = false
+ current.Outcome = outcome
+ return nil
+ }); err != nil {
+ return st.Outcome{Status: st.Retrying}, err
+ }
+ if outcome.Status.Terminal() {
+ return s.acknowledgeRemoteInternal(ctx, run, runPath, outcome)
+ }
+ if outcome.Status == st.NeedsAttention {
+ return outcome, nil
+ }
+ return st.Outcome{Status: st.Waiting, Message: "Waiting for the agent to complete this run", Targets: outcome.Targets, ActivityID: outcome.ActivityID}, nil
+}
+
+func (s *JobService) acknowledgeRemoteInternal(ctx context.Context, run st.Run, path string, outcome st.Outcome) (st.Outcome, error) {
+ var response st.Run
+ if err := s.environment.ProxyJSONRequest(ctx, run.EnvironmentID, http.MethodPost, path+"/ack", nil, &response); err != nil {
+ return st.Outcome{Status: st.Waiting, Message: "Remote operation completed; waiting for delivery acknowledgement", Targets: outcome.Targets, ActivityID: outcome.ActivityID}, err
+ }
+ if response.ID != run.ID || !response.RemoteSettled {
+ return st.Outcome{Status: st.Waiting, Message: "Remote operation completed; acknowledgement not confirmed"}, nil
+ }
+ if err := s.Queue.UpdateRun(ctx, run, func(current *st.Run) error { current.RemoteSettled = true; return nil }); err != nil {
+ return st.Outcome{Status: st.Retrying}, err
+ }
+ return outcome, nil
+}
+
+// ListRemoteJobs retains the last observed agent catalog while its environment is offline.
+func (s *JobService) ListRemoteJobs(ctx context.Context, environmentID string) (*jobschedule.JobListResponse, error) {
+ if _, err := s.environment.GetEnvironmentByID(ctx, environmentID); err != nil {
+ return nil, err
+ }
+ digest := sha256.Sum256([]byte(environmentID))
+ key := "jobs-catalog/" + hex.EncodeToString(digest[:])
+ var catalog jobschedule.JobListResponse
+ remoteErr := s.environment.ProxyJSONRequest(ctx, environmentID, http.MethodGet, "/api/environments/0/jobs", nil, &catalog)
+ if remoteErr == nil {
+ catalog.ObservedAt = time.Now().UTC()
+ catalog.Offline = false
+ raw, err := json.Marshal(catalog)
+ if err != nil {
+ return nil, err
+ }
+ if err := s.store.Set(ctx, key, string(raw)); err != nil {
+ return nil, err
+ }
+ } else {
+ var status *remenv.StatusError
+ if errors.As(remoteErr, &status) && (status.StatusCode == http.StatusUnauthorized || status.StatusCode == http.StatusForbidden) {
+ return nil, remoteErr
+ }
+ raw, found, err := s.store.Get(ctx, key)
+ if err != nil {
+ return nil, err
+ }
+ if found {
+ if err := json.Unmarshal([]byte(raw), &catalog); err != nil {
+ return nil, err
+ }
+ } else {
+ catalog.IsAgent = true
+ for _, metadata := range meta.GetAllJobMetadata() {
+ if !metadata.ManagerOnly {
+ catalog.Jobs = append(catalog.Jobs, metadata.ToJobStatus("", nil, metadata.CanRunManually, []jobschedule.JobPrerequisite{}))
+ }
+ }
+ }
+ catalog.Offline = true
+ }
+ if err := s.addRuntimeStatusInternal(ctx, environmentID, &catalog.Jobs); err != nil {
+ return nil, err
+ }
+ sort.Slice(catalog.Jobs, func(i, j int) bool { return catalog.Jobs[i].ID < catalog.Jobs[j].ID })
+ return &catalog, nil
+}
+
+// RetryRemoteRun queues one explicit retry while retaining the agent's target progress.
+func (s *JobService) RetryRemoteRun(ctx context.Context, environmentID, jobID, runID string) (st.Run, error) {
+ run, err := s.Queue.Get(ctx, environmentID, jobID, runID)
+ if errors.Is(err, queue.ErrRunNotFound) {
+ return s.mutateAgentRunInternal(ctx, environmentID, jobID, runID, "retry")
+ }
+ if err != nil {
+ return run, err
+ }
+ if err := s.authorizeRunInternal(ctx, run); err != nil {
+ return run, err
+ }
+ err = s.Queue.UpdateRun(ctx, run, func(current *st.Run) error {
+ if current.Status != st.Failed && current.Status != st.Partial && current.Status != st.NeedsAttention {
+ return errors.New("run is not eligible for retry")
+ }
+ if !current.RemoteAccepted {
+ if current.RemoteDeliveryAttempted {
+ return errors.New("remote delivery is uncertain; inspect the agent before submitting new work")
+ }
+ } else {
+ if current.RemoteOutcome == nil || (!current.RemoteOutcome.Status.Terminal() && current.RemoteOutcome.Status != st.NeedsAttention) {
+ return errors.New("remote outcome must be confirmed before retry")
+ }
+ current.RemoteRetryRequested = true
+ current.RemoteRetryAttempted = false
+ current.RemoteSettled = false
+ }
+ current.Status = st.Queued
+ current.Owner = ""
+ current.NextAttempt = nil
+ current.FinishedAt = nil
+ current.UpdatedAt = time.Now().UTC()
+ run = *current
+ return nil
+ })
+ return run, err
+}
+
+var (
+ errRemoteIneligibleInternal = errors.Sentinel("Remote job is disabled or no longer eligible")
+ errRemoteReceiptMissingInternal = errors.Sentinel("Agent no longer has the delivery receipt; verify the operation before retrying")
+ errRemoteRetryUncertainInternal = errors.Sentinel("Remote retry acknowledgement was lost; inspect the agent before retrying again")
+)
+
+func remoteJobEligibleInternal(jobs []jobschedule.JobStatus, id string) bool {
+ for _, job := range jobs {
+ if job.ID != id {
+ if remoteJobEligibleInternal(job.Children, id) {
+ return true
+ }
+ continue
+ }
+ if !job.Enabled || !job.CanRunManually || job.ManagerOnly {
+ return false
+ }
+ for _, prerequisite := range job.Prerequisites {
+ if !prerequisite.IsMet {
+ return false
+ }
+ }
+ return true
+ }
+ return false
+}
+
+func (s *JobService) admitRemoteInternal(ctx context.Context, run st.Run, catalog jobschedule.JobListResponse) (st.Run, error) {
+ if run.RemoteAccepted {
+ return st.Run{}, errRemoteReceiptMissingInternal
+ }
+ if !remoteJobEligibleInternal(catalog.Jobs, run.JobID) {
+ return st.Run{}, errRemoteIneligibleInternal
+ }
+ // A confirmed 404 permits another delivery of the same deduplicated ID.
+ if err := s.Queue.UpdateRun(ctx, run, func(current *st.Run) error {
+ current.RemoteDeliveryAttempted = true
+ return nil
+ }); err != nil {
+ return st.Run{}, err
+ }
+ body, err := json.Marshal(map[string]string{"runId": run.ID})
+ if err != nil {
+ return st.Run{}, err
+ }
+ path := "/api/environments/0/jobs/" + url.PathEscape(run.JobID) + "/run"
+ var accepted jobschedule.JobRunResponse
+ if err := s.environment.ProxyJSONRequest(ctx, run.EnvironmentID, http.MethodPost, path, body, &accepted); err != nil {
+ return st.Run{}, err
+ }
+ if accepted.RunID != run.ID {
+ return st.Run{}, errors.New("Agent returned an inconsistent delivery receipt")
+ }
+ return st.Run{ID: run.ID, JobID: run.JobID, Status: accepted.Status, Outcome: st.Outcome{Status: accepted.Status}}, nil
+}
+
+func (s *JobService) retryDeliveryInternal(ctx context.Context, run st.Run, path string) (st.Run, error) {
+ if run.RemoteRetryAttempted {
+ return st.Run{}, errRemoteRetryUncertainInternal
+ }
+ if err := s.Queue.UpdateRun(ctx, run, func(current *st.Run) error {
+ current.RemoteRetryAttempted = true
+ return nil
+ }); err != nil {
+ return st.Run{}, err
+ }
+ var retried st.Run
+ if err := s.environment.ProxyJSONRequest(ctx, run.EnvironmentID, http.MethodPost, path+"/retry", nil, &retried); err != nil {
+ return st.Run{}, err
+ }
+ if retried.ID != run.ID || retried.JobID != run.JobID {
+ return st.Run{}, errors.New("Agent returned an inconsistent retry receipt")
+ }
+ return retried, nil
+}
diff --git a/backend/internal/job/remote_history.go b/backend/internal/job/remote_history.go
new file mode 100644
index 000000000..f6338e12b
--- /dev/null
+++ b/backend/internal/job/remote_history.go
@@ -0,0 +1,130 @@
+package job
+
+import (
+ "context"
+ "net/http"
+ "net/url"
+ "sort"
+ "strconv"
+
+ "emperror.dev/errors"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/remenv"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/queue"
+ st "github.com/getarcaneapp/arcane/types/v2/scheduler"
+)
+
+// ListRuns merges agent history with requests accepted by this manager.
+func (s *JobService) ListRuns(ctx context.Context, environmentID, jobID string, page, limit int) (st.RunList, error) {
+ if environmentID == "0" {
+ return s.Queue.List(ctx, environmentID, jobID, page, limit)
+ }
+ if _, err := s.environment.GetEnvironmentByID(ctx, environmentID); err != nil {
+ return st.RunList{}, err
+ }
+ records, err := s.Queue.Records(ctx)
+ if err != nil {
+ return st.RunList{}, err
+ }
+ merged, err := s.remoteHistoryInternal(ctx, environmentID, jobID)
+ if err != nil {
+ var status *remenv.StatusError
+ if errors.As(err, &status) && (status.StatusCode == http.StatusForbidden || status.StatusCode == http.StatusUnauthorized) {
+ return st.RunList{}, err
+ }
+ merged = make(map[string]st.Run)
+ }
+ for _, record := range records {
+ if record.EnvironmentID != environmentID || record.JobID != jobID {
+ continue
+ }
+ for _, run := range record.Runs {
+ merged[run.ID] = run
+ }
+ }
+ runs := make([]st.Run, 0, len(merged))
+ for _, run := range merged {
+ runs = append(runs, run)
+ }
+ sort.Slice(runs, func(i, j int) bool {
+ if runs[i].CreatedAt.Equal(runs[j].CreatedAt) {
+ return runs[i].ID < runs[j].ID
+ }
+ return runs[i].CreatedAt.After(runs[j].CreatedAt)
+ })
+ page = max(page, 1)
+ limit = min(max(limit, 1), 100)
+ start := len(runs)
+ if page-1 <= len(runs)/limit {
+ start = min((page-1)*limit, len(runs))
+ }
+ return st.RunList{Runs: runs[start:min(start+limit, len(runs))], Total: len(runs), Page: page, Limit: limit}, nil
+}
+
+func (s *JobService) remoteHistoryInternal(ctx context.Context, environmentID, jobID string) (map[string]st.Run, error) {
+ runs := make(map[string]st.Run)
+ for page := 1; ; page++ {
+ path := "/api/environments/0/jobs/" + url.PathEscape(jobID) + "/runs?page=" + strconv.Itoa(page) + "&limit=100"
+ var response st.RunList
+ if err := s.environment.ProxyJSONRequest(ctx, environmentID, http.MethodGet, path, nil, &response); err != nil {
+ return nil, err
+ }
+ previous := len(runs)
+ for _, run := range response.Runs {
+ run.EnvironmentID = environmentID
+ runs[run.ID] = run
+ }
+ if len(runs) >= response.Total || len(runs) == previous {
+ return runs, nil
+ }
+ }
+}
+
+// GetRun prefers the manager's durable delivery record, then queries agent-owned history.
+func (s *JobService) GetRun(ctx context.Context, environmentID, jobID, runID string) (st.Run, error) {
+ run, err := s.Queue.Get(ctx, environmentID, jobID, runID)
+ if environmentID == "0" || !errors.Is(err, queue.ErrRunNotFound) {
+ return run, err
+ }
+ path := "/api/environments/0/jobs/" + url.PathEscape(jobID) + "/runs/" + url.PathEscape(runID)
+ if err := s.environment.ProxyJSONRequest(ctx, environmentID, http.MethodGet, path, nil, &run); err != nil {
+ return st.Run{}, err
+ }
+ if run.ID != runID || run.JobID != jobID {
+ return st.Run{}, errors.New("agent returned an inconsistent run identity")
+ }
+ run.EnvironmentID = environmentID
+ return run, nil
+}
+
+// CancelRun cancels manager admission or explicitly forwards an agent-owned cancellation.
+func (s *JobService) CancelRun(ctx context.Context, environmentID, jobID, runID string) (st.Run, error) {
+ _, err := s.Queue.Get(ctx, environmentID, jobID, runID)
+ if environmentID != "0" && errors.Is(err, queue.ErrRunNotFound) {
+ return s.mutateAgentRunInternal(ctx, environmentID, jobID, runID, "cancel")
+ }
+ if err != nil {
+ return st.Run{}, err
+ }
+ return s.Queue.Cancel(ctx, environmentID, jobID, runID)
+}
+
+func (s *JobService) mutateAgentRunInternal(ctx context.Context, environmentID, jobID, runID, action string) (st.Run, error) {
+ environment, err := s.environment.GetEnvironmentByID(ctx, environmentID)
+ if err != nil {
+ return st.Run{}, err
+ }
+ if !environment.Enabled {
+ return st.Run{}, errors.New("environment is disabled")
+ }
+ path := "/api/environments/0/jobs/" + url.PathEscape(jobID) + "/runs/" + url.PathEscape(runID) + "/" + action
+ var run st.Run
+ // This is an explicit operator action. Never replay a lost mutation response.
+ if err := s.environment.ProxyJSONRequest(ctx, environmentID, http.MethodPost, path, nil, &run); err != nil {
+ return st.Run{}, err
+ }
+ if run.ID != runID || run.JobID != jobID {
+ return st.Run{}, errors.New("agent returned an inconsistent run identity")
+ }
+ run.EnvironmentID = environmentID
+ return run, nil
+}
diff --git a/backend/internal/job/run_routes.go b/backend/internal/job/run_routes.go
new file mode 100644
index 000000000..cc7053f2b
--- /dev/null
+++ b/backend/internal/job/run_routes.go
@@ -0,0 +1,133 @@
+package job
+
+import (
+ "context"
+ "net/http"
+ "time"
+
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/queue"
+
+ "emperror.dev/errors"
+ "github.com/danielgtaylor/huma/v2"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/authz"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"
+ "github.com/getarcaneapp/arcane/types/v2/jobschedule"
+ st "github.com/getarcaneapp/arcane/types/v2/scheduler"
+)
+
+func (h *JobSchedulesHandler) registerRunRoutesInternal(api huma.API) {
+ base := "/environments/{id}/jobs/{jobId}"
+ middleware.RegisterWithPermission(api, huma.Operation{OperationID: "list-job-runs", Method: http.MethodGet, Path: base + "/runs", Summary: "List job runs", Security: handlerutil.DefaultOperationSecurity()}, authz.PermJobsManage, h.ListRuns)
+ middleware.RegisterWithPermission(api, huma.Operation{OperationID: "get-job-run", Method: http.MethodGet, Path: base + "/runs/{runId}", Summary: "Get job run", Security: handlerutil.DefaultOperationSecurity()}, authz.PermJobsManage, h.GetRun)
+ middleware.RegisterWithPermission(api, huma.Operation{OperationID: "retry-job-run", Method: http.MethodPost, Path: base + "/runs/{runId}/retry", Summary: "Retry job run", Security: handlerutil.DefaultOperationSecurity()}, authz.PermJobsManage, h.RetryRun)
+ middleware.RegisterWithPermission(api, huma.Operation{OperationID: "cancel-job-run", Method: http.MethodPost, Path: base + "/runs/{runId}/cancel", Summary: "Cancel pending job run", Security: handlerutil.DefaultOperationSecurity()}, authz.PermJobsManage, h.CancelRun)
+ middleware.RegisterWithPermission(api, huma.Operation{OperationID: "ack-job-run", Method: http.MethodPost, Path: base + "/runs/{runId}/ack", Summary: "Acknowledge remote run completion", Security: handlerutil.DefaultOperationSecurity()}, authz.PermJobsManage, h.AcknowledgeRun)
+ middleware.RegisterWithPermission(api, huma.Operation{OperationID: "restart-job-worker", Method: http.MethodPost, Path: base + "/restart", Summary: "Restart continuous job worker", Security: handlerutil.DefaultOperationSecurity()}, authz.PermJobsManage, h.RestartWorker)
+}
+
+// ListRuns returns paginated history for the requested job and environment.
+// Remote history combines agent runs with the manager's delivery records.
+func (h *JobSchedulesHandler) ListRuns(ctx context.Context, input *jobschedule.ListRunsInput) (*jobschedule.ListRunsOutput, error) {
+ result, err := h.jobService.ListRuns(ctx, input.ID, input.JobID, input.Page, input.Limit)
+ if err != nil {
+ return nil, jobHTTPErrorInternal(err)
+ }
+ return &jobschedule.ListRunsOutput{Body: result}, nil
+}
+
+// GetRun returns a persisted run or receipt. For remote work, it queries the
+// agent when the manager has no delivery record for the requested run.
+func (h *JobSchedulesHandler) GetRun(ctx context.Context, input *jobschedule.RunInput) (*jobschedule.RunOutput, error) {
+ result, err := h.jobService.GetRun(ctx, input.ID, input.JobID, input.RunID)
+ if err != nil {
+ return nil, jobHTTPErrorInternal(err)
+ }
+ return &jobschedule.RunOutput{Body: result}, nil
+}
+
+// RetryRun requests an explicit retry while preserving the run ID and target
+// progress. Eligibility and remote-outcome checks are enforced by the service.
+func (h *JobSchedulesHandler) RetryRun(ctx context.Context, input *jobschedule.RunInput) (*jobschedule.RunOutput, error) {
+ result, err := h.jobService.RetryRun(ctx, input.ID, input.JobID, input.RunID)
+ if err != nil {
+ return nil, jobHTTPErrorInternal(err)
+ }
+ return &jobschedule.RunOutput{Body: result}, nil
+}
+
+// CancelRun cancels queued work only when it has never been attempted or sent
+// remotely. Agent-owned cancellations are forwarded to the owning environment.
+func (h *JobSchedulesHandler) CancelRun(ctx context.Context, input *jobschedule.RunInput) (*jobschedule.RunOutput, error) {
+ result, err := h.jobService.CancelRun(ctx, input.ID, input.JobID, input.RunID)
+ if err != nil {
+ return nil, jobHTTPErrorInternal(err)
+ }
+ return &jobschedule.RunOutput{Body: result}, nil
+}
+
+// AcknowledgeRun marks a terminal run's remote delivery settled so retention can
+// compact its history. Nonterminal runs cannot be acknowledged.
+func (h *JobSchedulesHandler) AcknowledgeRun(ctx context.Context, input *jobschedule.RunInput) (*jobschedule.RunOutput, error) {
+ result, err := h.jobService.GetRun(ctx, input.ID, input.JobID, input.RunID)
+ if err != nil {
+ return nil, jobHTTPErrorInternal(err)
+ }
+ err = h.jobService.Queue.UpdateRun(ctx, result, func(current *st.Run) error {
+ if !current.Status.Terminal() {
+ return errors.New("run is not terminal")
+ }
+ current.RemoteSettled = true
+ current.UpdatedAt = time.Now().UTC()
+ result = *current
+ return nil
+ })
+ if err != nil {
+ return nil, jobHTTPErrorInternal(err)
+ }
+ return &jobschedule.RunOutput{Body: result}, nil
+}
+
+// RestartWorker requests a continuous watcher's restart on the owning runtime,
+// forwarding remote requests to the agent. It does not enqueue a scheduled run.
+func (h *JobSchedulesHandler) RestartWorker(ctx context.Context, input *jobschedule.JobInput) (*RunJobOutput, error) {
+ if input.ID != "0" {
+ result, err := h.proxyRemoteJSON.JSON[jobschedule.JobRunResponse](ctx, input.ID, http.MethodPost, "/api/environments/0/jobs/"+input.JobID+"/restart", nil)
+ if err != nil {
+ return nil, err
+ }
+ return &RunJobOutput{Body: *result}, nil
+ }
+ runtime, ok := h.jobService.scheduler.(st.WorkerController)
+ if !ok {
+ return nil, huma.Error503ServiceUnavailable("Scheduler unavailable")
+ }
+ if err := runtime.RestartWatcher(ctx, input.JobID); err != nil {
+ return nil, jobHTTPErrorInternal(err)
+ }
+ return &RunJobOutput{Body: jobschedule.JobRunResponse{Success: true, Message: "Worker restart requested"}}, nil
+}
+
+func jobHTTPErrorInternal(err error) error {
+ if errors.Is(err, queue.ErrRunNotFound) {
+ return huma.Error404NotFound("Job run not found")
+ }
+ if errors.Is(err, queue.ErrRunConflict) {
+ return huma.Error409Conflict("Job run state changed")
+ }
+ return huma.Error400BadRequest(err.Error())
+}
+
+// RetryRun revalidates local job eligibility before requeuing the existing run.
+// Remote retries follow the delivery protocol and retain confirmed target progress.
+func (s *JobService) RetryRun(ctx context.Context, environmentID, jobID, runID string) (st.Run, error) {
+ if environmentID != "0" {
+ return s.RetryRemoteRun(ctx, environmentID, jobID, runID)
+ }
+ if environmentID == "0" {
+ if err := s.validateLocalJobInternal(ctx, jobID); err != nil {
+ return st.Run{}, err
+ }
+ }
+ return s.Queue.Retry(ctx, environmentID, jobID, runID)
+}
diff --git a/backend/internal/job/runtime_status.go b/backend/internal/job/runtime_status.go
new file mode 100644
index 000000000..45ae4cea4
--- /dev/null
+++ b/backend/internal/job/runtime_status.go
@@ -0,0 +1,235 @@
+package job
+
+import (
+ "context"
+ "slices"
+ "strings"
+ "time"
+
+ "emperror.dev/errors"
+ "github.com/getarcaneapp/arcane/types/v2/jobschedule"
+ st "github.com/getarcaneapp/arcane/types/v2/scheduler"
+)
+
+func (s *JobService) addRuntimeStatusInternal(ctx context.Context, environmentID string, jobs *[]jobschedule.JobStatus) error {
+ runsByJob := make(map[string][]st.Run)
+ if s.Queue != nil {
+ records, err := s.Queue.Records(ctx)
+ if err != nil {
+ return errors.WrapIf(err, "load durable job status")
+ }
+ for _, record := range records {
+ if record.EnvironmentID == environmentID {
+ runsByJob[record.JobID] = append(runsByJob[record.JobID], record.Runs...)
+ }
+ }
+ }
+ runtime, hasRuntime := s.scheduler.(st.JobScheduler)
+ if environmentID == "0" && hasRuntime {
+ appendDynamicJobStatusesInternal(ctx, runtime, jobs)
+ }
+
+ for index := range *jobs {
+ status := &(*jobs)[index]
+ applyRunStatusInternal(status, runsByJob[status.ID])
+ if environmentID == "0" && hasRuntime {
+ setRuntimeHealthInternal(ctx, runtime, status)
+ }
+ for childIndex := range status.Children {
+ child := &status.Children[childIndex]
+ applyRunStatusInternal(child, runsByJob[child.ID])
+ }
+ }
+ return nil
+}
+
+func appendDynamicJobStatusesInternal(ctx context.Context, runtime st.JobScheduler, jobs *[]jobschedule.JobStatus) {
+ parentIndices := make(map[string]int, len(*jobs))
+ for index, status := range *jobs {
+ parentIndices[status.ID] = index
+ }
+ registered := runtime.ListRegisteredJobs()
+ slices.SortFunc(registered, func(left, right st.Job) int {
+ return strings.Compare(left.Name(), right.Name())
+ })
+ for _, registeredJob := range registered {
+ parentID, _, dynamic := strings.Cut(registeredJob.Name(), ":")
+ if !dynamic {
+ continue
+ }
+ if parentID == "environment-health" {
+ if registeredJob.Name() == "environment-health:0" {
+ if index, exists := parentIndices[parentID]; exists {
+ health := dynamicJobStatusInternal(ctx, runtime, registeredJob, "monitoring", true)
+ health.Name = (*jobs)[index].Name
+ (*jobs)[index] = health
+ }
+ }
+ continue
+ }
+ category, managerOnly, supported := dynamicJobCategoryInternal(parentID)
+ if !supported {
+ continue
+ }
+ parentIndex, exists := parentIndices[parentID]
+ if !exists {
+ parentIndex = len(*jobs)
+ parentIndices[parentID] = parentIndex
+ *jobs = append(*jobs, jobschedule.JobStatus{
+ ID: parentID,
+ Name: parentID,
+ Category: category,
+ Enabled: true,
+ ManagerOnly: managerOnly,
+ Prerequisites: []jobschedule.JobPrerequisite{},
+ })
+ }
+ parent := &(*jobs)[parentIndex]
+ duplicate := slices.ContainsFunc(parent.Children, func(child jobschedule.JobStatus) bool {
+ return child.ID == registeredJob.Name()
+ })
+ if duplicate {
+ continue
+ }
+ parent.Children = append(parent.Children, dynamicJobStatusInternal(ctx, runtime, registeredJob, category, managerOnly))
+ }
+}
+
+func dynamicJobCategoryInternal(parentID string) (category string, managerOnly, supported bool) {
+ switch parentID {
+ case "environment-health":
+ return "monitoring", true, true
+ case "gitops-sync":
+ return "sync", false, true
+ case "system-backup":
+ return "maintenance", true, true
+ case "volume-backup":
+ return "maintenance", false, true
+ default:
+ return "", false, false
+ }
+}
+
+func dynamicJobStatusInternal(ctx context.Context, runtime st.JobScheduler, job st.Job, category string, managerOnly bool) jobschedule.JobStatus {
+ child := jobschedule.JobStatus{
+ ID: job.Name(),
+ Name: job.Name(),
+ Category: category,
+ Schedule: job.Schedule(ctx),
+ Enabled: true,
+ ManagerOnly: managerOnly,
+ CanRunManually: true,
+ Prerequisites: []jobschedule.JobPrerequisite{},
+ }
+ if conditional, ok := job.(st.ConditionalJob); ok {
+ child.Enabled = conditional.ShouldSchedule(ctx)
+ }
+ if state, ok := runtime.GetJobRuntimeState(child.ID); ok {
+ child.Schedule = state.Schedule
+ child.NextRun = state.NextRun
+ }
+ return child
+}
+
+func applyRunStatusInternal(status *jobschedule.JobStatus, runs []st.Run) {
+ var errorAt time.Time
+ if status.LastError != "" {
+ for _, run := range []*st.Run{status.CurrentRun, status.LastRun} {
+ if run != nil && run.UpdatedAt.After(errorAt) {
+ errorAt = run.UpdatedAt
+ }
+ }
+ }
+ for _, run := range runs {
+ updateLatestRunInternal(status, run)
+ updateLastSuccessInternal(status, run)
+ if run.UpdatedAt.After(errorAt) && run.Outcome.Message != "" && runHasErrorInternal(run.Status) {
+ status.LastError = run.Outcome.Message
+ errorAt = run.UpdatedAt
+ }
+ }
+}
+
+func updateLatestRunInternal(status *jobschedule.JobStatus, run st.Run) {
+ if !run.Status.Terminal() {
+ if status.CurrentRun == nil || preferCurrentRunInternal(run, *status.CurrentRun) {
+ status.CurrentRun = new(run)
+ }
+ return
+ }
+ current := status.CurrentRun
+ if current != nil && current.ID == run.ID && run.UpdatedAt.After(current.UpdatedAt) && (!run.RemoteAccepted || run.RemoteSettled) {
+ status.CurrentRun = nil
+ }
+ if status.LastRun == nil || run.UpdatedAt.After(status.LastRun.UpdatedAt) {
+ status.LastRun = new(run)
+ }
+}
+
+func updateLastSuccessInternal(status *jobschedule.JobStatus, run st.Run) {
+ if run.Status != st.Succeeded {
+ return
+ }
+ succeededAt := run.UpdatedAt
+ if run.FinishedAt != nil {
+ succeededAt = *run.FinishedAt
+ }
+ if status.LastSuccess == nil || succeededAt.After(*status.LastSuccess) {
+ status.LastSuccess = new(succeededAt)
+ }
+}
+
+func runHasErrorInternal(status st.RunStatus) bool {
+ switch status {
+ case st.Failed, st.Partial, st.NeedsAttention, st.Retrying, st.Waiting:
+ return true
+ case st.Queued, st.Running, st.Succeeded, st.Skipped, st.Canceled:
+ return false
+ default:
+ return false
+ }
+}
+
+func preferCurrentRunInternal(candidate, current st.Run) bool {
+ if candidate.ID == current.ID {
+ return candidate.UpdatedAt.After(current.UpdatedAt)
+ }
+ candidatePriority := currentRunPriorityInternal(candidate.Status)
+ currentPriority := currentRunPriorityInternal(current.Status)
+ if candidatePriority != currentPriority {
+ return candidatePriority < currentPriority
+ }
+ return candidate.CreatedAt.Before(current.CreatedAt)
+}
+
+func currentRunPriorityInternal(status st.RunStatus) int {
+ switch status {
+ case st.Running:
+ return 0
+ case st.NeedsAttention:
+ return 1
+ case st.Queued, st.Waiting, st.Retrying, st.Succeeded, st.Partial, st.Skipped, st.Failed, st.Canceled:
+ return 2
+ default:
+ return 2
+ }
+}
+
+func setRuntimeHealthInternal(ctx context.Context, runtime st.JobScheduler, status *jobschedule.JobStatus) {
+ if health, found := runtime.WatcherHealth(status.ID); found {
+ status.WorkerHealth = &health
+ return
+ }
+ registered, found := runtime.GetJob(status.ID)
+ if !found {
+ return
+ }
+ if conditional, ok := registered.(st.ConditionalJob); ok && !conditional.ShouldSchedule(ctx) {
+ return
+ }
+ state, found := runtime.GetJobRuntimeState(status.ID)
+ if !found || !state.Scheduled {
+ status.NextRun = nil
+ status.WorkerHealth = &st.WorkerHealth{Status: "needs_attention", LastError: "Job has no installed schedule", UpdatedAt: time.Now().UTC()}
+ }
+}
diff --git a/backend/internal/job/service.go b/backend/internal/job/service.go
index 30d7d4308..3689ef7ad 100644
--- a/backend/internal/job/service.go
+++ b/backend/internal/job/service.go
@@ -9,8 +9,13 @@ import (
"emperror.dev/errors"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/environment"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/kv"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/role"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/queue"
+ scheduleutil "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/schedule"
"github.com/getarcaneapp/arcane/types/v2/jobschedule"
"github.com/getarcaneapp/arcane/types/v2/meta"
schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
@@ -26,6 +31,10 @@ import (
// NOTE: This is intentionally separate from settings.SettingsService to keep the API
// surface job-focused and to centralize schedule validation/rescheduling.
type JobService struct {
+ Queue *queue.Queue
+ store *kv.KVService
+ environment *environment.EnvironmentService
+ roles *role.RoleService
db *database.DB
settings *settings.SettingsService
cfg *config.Config
@@ -42,12 +51,15 @@ type JobService struct {
}
func NewJobService(db *database.DB, settings *settings.SettingsService, cfg *config.Config) *JobService {
- return &JobService{
+ service := &JobService{
+ store: kv.NewKVService(db),
db: db,
settings: settings,
cfg: cfg,
location: cfg.GetLocation(),
}
+ service.Queue = queue.New(service.store, service.executeRunInternal, service.reconcileRunInternal)
+ return service
}
func (s *JobService) SetScheduler(ctx context.Context, scheduler schedulertypes.JobController) { //nolint:contextcheck // scheduler jobs must capture the app lifecycle context, not request contexts
@@ -56,6 +68,9 @@ func (s *JobService) SetScheduler(ctx context.Context, scheduler schedulertypes.
}
s.lifecycleCtx = ctx
s.scheduler = scheduler
+ if runtime, ok := scheduler.(schedulertypes.JobScheduler); ok {
+ runtime.SetDispatcher(s.Queue)
+ }
}
func (s *JobService) GetJobSchedules(ctx context.Context) jobschedule.Config {
@@ -104,7 +119,7 @@ func (s *JobService) UpdateJobSchedules(ctx context.Context, updates jobschedule
}
// Validate inputs (cron expressions)
- parser := cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow)
+ parser := scheduleutil.Parser()
for _, field := range fields {
if field.update == nil || *field.update == "" {
continue
@@ -280,8 +295,10 @@ func (s *JobService) ListJobs(ctx context.Context) (*jobschedule.JobListResponse
prerequisites := s.evaluatePrerequisitesInternal(ctx, jobMeta)
if s.scheduler != nil && jobMeta.SettingsKey != "" && jobMeta.ID != "environment-health" {
- if runtimeState, ok := s.scheduler.GetJobRuntimeState(jobMeta.ID); ok && runtimeState.Schedule != "" {
- schedule = runtimeState.Schedule
+ if runtimeState, ok := s.scheduler.GetJobRuntimeState(jobMeta.ID); ok {
+ if runtimeState.Schedule != "" {
+ schedule = runtimeState.Schedule
+ }
nextRun = runtimeState.NextRun
}
}
@@ -290,6 +307,10 @@ func (s *JobService) ListJobs(ctx context.Context) (*jobschedule.JobListResponse
jobs = append(jobs, jobStatus)
}
+ if err := s.addRuntimeStatusInternal(ctx, "0", &jobs); err != nil {
+ return nil, err
+ }
+
// Sort jobs by ID to ensure stable UI order
sort.Slice(jobs, func(i, j int) bool {
return jobs[i].ID < jobs[j].ID
@@ -298,62 +319,12 @@ func (s *JobService) ListJobs(ctx context.Context) (*jobschedule.JobListResponse
isAgent := s.cfg != nil && s.cfg.AgentMode
return &jobschedule.JobListResponse{
- Jobs: jobs,
- IsAgent: isAgent,
+ Jobs: jobs,
+ IsAgent: isAgent,
+ DurableRuns: true, ObservedAt: time.Now().UTC(),
}, nil
}
-func (s *JobService) RunJobNowInline(ctx context.Context, jobID string) error {
- // environment-health runs as per-environment dynamic jobs; "run now" fans out
- // through environment.EnvironmentService rather than a single registered job.
- if jobID == "environment-health" && s.RunEnvironmentHealthNow != nil {
- return s.RunEnvironmentHealthNow(context.WithoutCancel(ctx))
- }
- jobMeta, ok := meta.GetJobMetadata(jobID)
- if ok && jobMeta.IsContinuous && jobMeta.CanRunManually {
- if s == nil || s.scheduler == nil {
- return errors.New("job service or scheduler not initialized")
- }
- return s.scheduler.RunBusWatcherNow(context.WithoutCancel(ctx), jobID)
- }
-
- job, err := s.getRunnableJobInternal(jobID)
- if err != nil {
- return err
- }
-
- runCtx := context.WithoutCancel(ctx)
- job.Run(runCtx)
-
- return nil
-}
-
-func (s *JobService) getRunnableJobInternal(jobID string) (schedulertypes.Job, error) {
- if s == nil || s.scheduler == nil {
- return nil, errors.New("job service or scheduler not initialized")
- }
-
- jobMeta, ok := meta.GetJobMetadata(jobID)
- if !ok {
- return nil, errors.Errorf("unknown job: %s", jobID)
- }
-
- if !jobMeta.CanRunManually {
- return nil, errors.Errorf("job %s cannot be run manually", jobID)
- }
-
- if s.cfg != nil && s.cfg.AgentMode && jobMeta.ManagerOnly {
- return nil, errors.Errorf("job %s is manager-only and cannot run in agent mode", jobID)
- }
-
- job, ok := s.scheduler.GetJob(jobID)
- if !ok {
- return nil, errors.Errorf("job %s not found in scheduler", jobID)
- }
-
- return job, nil
-}
-
func (s *JobService) getJobScheduleInternal(ctx context.Context, meta meta.JobMetadata) string {
// Continuous jobs with a settings key (image-polling) are event-driven but
// also poll on that cron schedule, so surface the real expression.
@@ -411,7 +382,7 @@ func (s *JobService) calculateNextRunInternal(schedule string) *time.Time {
}
// Parse schedule and force it to use the same timezone as the scheduler.
- parser := cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow)
+ parser := scheduleutil.Parser()
sched, err := parser.Parse(schedule)
if err != nil {
return nil
diff --git a/backend/internal/job/service_test.go b/backend/internal/job/service_test.go
index fca9032c0..c769da482 100644
--- a/backend/internal/job/service_test.go
+++ b/backend/internal/job/service_test.go
@@ -2,13 +2,14 @@ package job
import (
"context"
- "errors"
+ "emperror.dev/errors"
"testing"
"time"
"github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/kv"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/types/v2/jobschedule"
schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
@@ -22,7 +23,7 @@ func setupSettingsTestDBInternal(t *testing.T) *database.DB {
t.Helper()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
- require.NoError(t, db.AutoMigrate(&settings.SettingVariable{}))
+ require.NoError(t, db.AutoMigrate(&settings.SettingVariable{}, &kv.KVEntry{}))
return &database.DB{DB: db}
}
@@ -330,17 +331,21 @@ func TestJobService_UpdateJobSchedules_DelegatesEnvironmentHealthReschedule(t *t
require.Empty(t, scheduler.rescheduled)
}
-func TestJobService_RunJobNowInline_DelegatesImageUpdateWatcherWithDetachedContext(t *testing.T) {
- ctx, cancel := context.WithCancel(context.Background())
- cancel()
-
+func TestJobService_Submit_PersistsImageUpdateWatcherRun(t *testing.T) {
+ ctx := t.Context()
+ db := setupSettingsTestDBInternal(t)
+ settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)
+ require.NoError(t, err)
scheduler := newFakeJobSchedulerInternal()
- jobSvc := &JobService{scheduler: scheduler}
-
- require.NoError(t, jobSvc.RunJobNowInline(ctx, "image-polling"))
- require.Equal(t, []string{"image-polling"}, scheduler.busWatcherRuns)
- require.Len(t, scheduler.busWatcherContexts, 1)
- require.NoError(t, scheduler.busWatcherContexts[0].Err())
+ jobSvc := NewJobService(db, settingsSvc, &config.Config{})
+ jobSvc.SetScheduler(ctx, scheduler)
+ run, err := jobSvc.Submit(ctx, schedulertypes.Request{JobID: "image-polling", EnvironmentID: "0", Trigger: "manual"})
+ require.NoError(t, err)
+ require.Equal(t, schedulertypes.Queued, run.Status)
+ persisted, err := jobSvc.Queue.Get(ctx, "0", "image-polling", run.ID)
+ require.NoError(t, err)
+ require.Equal(t, run.ID, persisted.ID)
+ require.Empty(t, scheduler.busWatcherRuns)
}
func findJobStatusByIDInternal(t *testing.T, jobs []jobschedule.JobStatus, id string) jobschedule.JobStatus {
@@ -419,4 +424,6 @@ func (j fakeJobInternal) Schedule(context.Context) string {
return "0 0 0 * * *"
}
-func (j fakeJobInternal) Run(context.Context) {}
+func (j fakeJobInternal) Run(context.Context) (schedulertypes.Outcome, error) {
+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil
+}
diff --git a/backend/internal/kv/service.go b/backend/internal/kv/service.go
index e5dece753..0b49d3c7e 100644
--- a/backend/internal/kv/service.go
+++ b/backend/internal/kv/service.go
@@ -4,6 +4,7 @@ import (
"context"
"strconv"
"strings"
+ "time"
"emperror.dev/errors"
@@ -126,3 +127,15 @@ func (s *KVService) IncrementInt64(ctx context.Context, key string, delta int64)
return nextValue, nil
}
+
+// CreateIfAbsent atomically creates an entry without replacing an existing value.
+func (s *KVService) CreateIfAbsent(ctx context.Context, key, value string) (bool, error) {
+ result := s.db.WithContext(ctx).Clauses(clause.OnConflict{DoNothing: true}).Create(&KVEntry{Key: key, Value: value})
+ return result.RowsAffected == 1, errors.WrapIf(result.Error, "create kv entry")
+}
+
+// CompareAndSwap updates only the exact value read by the caller.
+func (s *KVService) CompareAndSwap(ctx context.Context, key, previous, next string) (bool, error) {
+ result := s.db.WithContext(ctx).Model(&KVEntry{}).Where("key = ? AND value = ?", key, previous).Updates(map[string]any{"value": next, "updated_at": time.Now()})
+ return result.RowsAffected == 1, errors.WrapIf(result.Error, "compare and swap kv entry")
+}
diff --git a/backend/internal/middleware/context.go b/backend/internal/middleware/context.go
index 528b2fb5e..30d3131e2 100644
--- a/backend/internal/middleware/context.go
+++ b/backend/internal/middleware/context.go
@@ -10,6 +10,7 @@ import (
type ContextKey string
const (
+ ContextKeyApiKeyID ContextKey = "apiKeyID"
// ContextKeyUserID is the context key for the authenticated user's ID.
ContextKeyUserID ContextKey = "userID"
// ContextKeyCurrentSessionID is the context key for the authenticated session ID.
diff --git a/backend/internal/middleware/environment_middleware.go b/backend/internal/middleware/environment_middleware.go
index bb2b14810..a05374a3d 100644
--- a/backend/internal/middleware/environment_middleware.go
+++ b/backend/internal/middleware/environment_middleware.go
@@ -427,6 +427,9 @@ func (m *EnvironmentMiddleware) hasResourcePath(c *echo.Context, envID string) b
}
func isManagementPathInternal(method, suffix string) bool {
+ if suffix == "/jobs" || strings.HasPrefix(suffix, "/jobs/") {
+ return true
+ }
if isCentralSwarmManagementPathInternal(method, suffix) {
return true
}
diff --git a/backend/internal/system/system.go b/backend/internal/system/system.go
index 24beb2bc4..573147f4a 100644
--- a/backend/internal/system/system.go
+++ b/backend/internal/system/system.go
@@ -2,6 +2,8 @@ package system
import (
"github.com/getarcaneapp/arcane/backend/v2/internal/imageupdate"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
@@ -97,6 +99,12 @@ func (s *SystemService) PruneAll(ctx context.Context, environmentID string, req
defer s.finishSystemPruneInternal(environmentID)
ctx = s.activityService.Track(ctx, activityID)
+ if err := jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ID: "scheduled-prune", Status: schedulertypes.Running, ActivityID: activityID}); err != nil {
+ result.Success = false
+ result.Errors = append(result.Errors, err.Error())
+ s.completeSystemPruneActivityInternal(ctx, activityID, result)
+ return result, true, err
+ }
s.runSystemPruneInternal(ctx, req, activityID, result)
return result, true, nil
diff --git a/backend/internal/systembackup/service.go b/backend/internal/systembackup/service.go
index 35b73cc19..55cbf0bbc 100644
--- a/backend/internal/systembackup/service.go
+++ b/backend/internal/systembackup/service.go
@@ -18,6 +18,8 @@ import (
"strings"
"time"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
+
"emperror.dev/errors"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
@@ -1565,13 +1567,28 @@ func (s *SystemBackupService) SetScheduler(ctx context.Context, scheduler schedu
return s.jobs.SetScheduler(ctx, scheduler, admissionGate)
}
-func (s *SystemBackupService) runScheduledBackupInternal(ctx context.Context, policyID string) {
+func (s *SystemBackupService) runScheduledBackupInternal(ctx context.Context, policyID string) (schedulertypes.Outcome, error) {
policy, loadErr := s.loadPolicyInternal(ctx, policyID)
- if loadErr != nil || policy == nil || !policy.Enabled {
- return
+ if loadErr != nil {
+ return schedulertypes.Outcome{}, loadErr
+ }
+ if policy == nil || !policy.Enabled {
+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil
+ }
+ if previous, ok := jobcontext.Run(ctx); ok {
+ outcome := jobcontext.ConfirmedTarget(previous, policy.ID)
+ if outcome.Status == schedulertypes.Succeeded {
+ if policy.RetentionCount > 0 {
+ if err := s.applyRetentionInternal(ctx, policy.ID, policy.RetentionCount); err != nil {
+ outcome.Status = schedulertypes.Partial
+ return outcome, err
+ }
+ }
+ return outcome, nil
+ }
}
var run *SystemBackupRun
- _, runErr := activitylib.RunHandlerActivity(ctx, s.activityService, activitylib.HandlerOptions{
+ activityID, runErr := activitylib.RunHandlerActivity(ctx, s.activityService, activitylib.HandlerOptions{
EnvironmentID: "0", Type: activitytypes.TypeResourceAction, ResourceType: "system_backup",
ResourceID: policy.ID, ResourceName: "Arcane", User: &common.SystemUser,
Step: "Creating scheduled system backup", Message: "Creating scheduled Arcane system backup",
@@ -1587,18 +1604,20 @@ func (s *SystemBackupService) runScheduledBackupInternal(ctx context.Context, po
})
if errors.Is(runErr, ErrSystemBackupAlreadyRunning) {
slog.InfoContext(ctx, "Scheduled Arcane system backup skipped; another backup is running", "policyId", policy.ID)
- return
+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil
}
if runErr != nil {
slog.ErrorContext(ctx, "Scheduled Arcane system backup failed", "policyId", policy.ID, "error", runErr)
- return
+ return schedulertypes.Outcome{ActivityID: activityID}, runErr
}
if policy.RetentionCount > 0 {
if retentionErr := s.applyRetentionInternal(ctx, policy.ID, policy.RetentionCount); retentionErr != nil {
slog.ErrorContext(ctx, "System backup retention failed", "policyId", policy.ID, "error", retentionErr)
+ return schedulertypes.Outcome{Status: schedulertypes.Partial, ActivityID: activityID, Message: "Backup completed but retention failed"}, retentionErr
}
}
slog.InfoContext(ctx, "Scheduled Arcane system backup completed", "backupId", run.ID, "policyId", policy.ID)
+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded, ActivityID: activityID}, nil
}
func (s *SystemBackupService) rescheduleSystemBackupPolicyInternal(ctx context.Context, policy *SystemBackupPolicy) {
@@ -1618,8 +1637,25 @@ func (s *SystemBackupService) rescheduleSystemBackupPolicyInternal(ctx context.C
}
return current.Schedule
},
- func(ctx context.Context) {
- s.runScheduledBackupInternal(ctx, policyID)
+ func(ctx context.Context) (schedulertypes.Outcome, error) {
+ return s.runScheduledBackupInternal(ctx, policyID)
+ },
+ func(ctx context.Context, previous schedulertypes.Run) (schedulertypes.Outcome, error) {
+ outcome := jobcontext.ConfirmedTarget(previous, policy.ID)
+ if outcome.Status == schedulertypes.Succeeded {
+ current, err := s.loadPolicyInternal(ctx, policyID)
+ if err != nil {
+ return outcome, err
+ }
+ if current != nil && current.RetentionCount > 0 {
+ if err := s.applyRetentionInternal(ctx, policyID, current.RetentionCount); err != nil {
+ outcome.Status = schedulertypes.Partial
+ outcome.Message = "Backup completed but retention failed"
+ return outcome, err
+ }
+ }
+ }
+ return outcome, nil
},
)
}
diff --git a/backend/internal/systembackup/service_test.go b/backend/internal/systembackup/service_test.go
index 72c1c4178..cf08d592b 100644
--- a/backend/internal/systembackup/service_test.go
+++ b/backend/internal/systembackup/service_test.go
@@ -40,7 +40,8 @@ func newSystemBackupAdmissionGateForTestInternal(t testing.TB) *actors.Gate[acto
}
type systemBackupPolicySchedulerInternal struct {
- jobs map[string]schedulertypes.Job
+ submitted []schedulertypes.Request
+ jobs map[string]schedulertypes.Job
}
func (s *systemBackupPolicySchedulerInternal) AddJob(_ context.Context, job schedulertypes.Job) error {
@@ -293,3 +294,8 @@ func TestNormalizeSystemBackupSelectionInternal(t *testing.T) {
require.NoError(t, err)
require.Equal(t, []backuptypes.BackupFileEntry{{Path: "demo", Name: "demo", IsDirectory: true}}, selected)
}
+
+func (s *systemBackupPolicySchedulerInternal) Submit(_ context.Context, request schedulertypes.Request) (schedulertypes.Run, error) {
+ s.submitted = append(s.submitted, request)
+ return schedulertypes.Run{ID: request.RunID, JobID: request.JobID, EnvironmentID: request.EnvironmentID, Status: schedulertypes.Queued}, nil
+}
diff --git a/backend/internal/systembackup/system_volume.go b/backend/internal/systembackup/system_volume.go
index 821504e73..c29db4397 100644
--- a/backend/internal/systembackup/system_volume.go
+++ b/backend/internal/systembackup/system_volume.go
@@ -10,6 +10,9 @@ import (
"slices"
"strings"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
+
"emperror.dev/errors"
"github.com/getarcaneapp/arcane/backend/v2/internal/actors"
@@ -322,6 +325,19 @@ func (s *SystemBackupService) executeSystemVolumeBackupsInternal(ctx context.Con
S3DestinationID: policyConfig.S3DestinationID,
}
for _, candidate := range candidates {
+ if previous, ok := jobcontext.Run(ctx); ok {
+ completed := false
+ for _, target := range previous.Outcome.Targets {
+ if target.ID == candidate.Name && target.Status == schedulertypes.Succeeded {
+ completed = true
+ break
+ }
+ }
+ if completed {
+ result.Succeeded++
+ continue
+ }
+ }
if err := ctx.Err(); err != nil {
return result, err
}
@@ -340,6 +356,9 @@ func (s *SystemBackupService) executeSystemVolumeBackupsInternal(ctx context.Con
if manualPolicy {
seriesID = systemVolumeManualPolicyIDInternal(candidate.Name)
}
+ if progressErr := jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ID: candidate.Name, Status: schedulertypes.Running}); progressErr != nil {
+ return result, progressErr
+ }
_, backupErr := s.volumeService.CreateSystemManagedBackup(ctx, candidate.Name, common.SystemUser, trigger, seriesID, policy)
if errors.Is(backupErr, volume.ErrVolumeBackupAlreadyRunning) {
result.Skipped++
@@ -350,26 +369,40 @@ func (s *SystemBackupService) executeSystemVolumeBackupsInternal(ctx context.Con
result.Failures = append(result.Failures, backuptypes.SystemVolumeBackupFailure{VolumeName: candidate.Name, Error: backupErr.Error()})
continue
}
+ if progressErr := jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ID: candidate.Name, Status: schedulertypes.Succeeded}); progressErr != nil {
+ return result, progressErr
+ }
result.Succeeded++
}
return result, nil
}
-func (s *SystemBackupService) runScheduledSystemVolumeBackupInternal(ctx context.Context, policyID string) {
+func (s *SystemBackupService) runScheduledSystemVolumeBackupInternal(ctx context.Context, policyID string) (schedulertypes.Outcome, error) {
policy, err := s.systemVolumeBackupPolicyInternal(policyID)
- if err != nil || policy == nil || !policy.Enabled {
- return
+ if err != nil {
+ return schedulertypes.Outcome{}, err
+ }
+ if policy == nil || !policy.Enabled {
+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil
}
result, err := s.runSystemVolumeBackupsInternal(ctx, backuptypes.RunSystemVolumeBackupsRequest{PolicyID: policyID}, volume.VolumeBackupTriggerScheduled)
if errors.Is(err, ErrSystemBackupAlreadyRunning) {
slog.InfoContext(ctx, "Scheduled system-managed volume backups skipped; a system backup is running", "policyId", policyID)
- return
+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil
}
if err != nil {
slog.ErrorContext(ctx, "Scheduled system-managed volume backups failed", "policyId", policyID, "error", err)
- return
+ return schedulertypes.Outcome{}, err
}
slog.InfoContext(ctx, "Scheduled system-managed volume backups completed", "policyId", policyID, "matched", result.Matched, "succeeded", result.Succeeded, "failed", result.Failed, "skipped", result.Skipped)
+ outcome := schedulertypes.Outcome{Status: schedulertypes.Succeeded}
+ if result.Failed > 0 {
+ outcome.Status = schedulertypes.Partial
+ }
+ for _, failure := range result.Failures {
+ outcome.Targets = append(outcome.Targets, schedulertypes.TargetOutcome{ID: failure.VolumeName, Status: schedulertypes.Failed, Message: failure.Error})
+ }
+ return outcome, nil
}
func (s *SystemBackupService) rescheduleSystemVolumeBackupInternal(ctx context.Context, policy *backuptypes.SystemVolumeBackupPolicy) {
@@ -388,8 +421,15 @@ func (s *SystemBackupService) rescheduleSystemVolumeBackupInternal(ctx context.C
return defaultSystemVolumeSchedule
}
return current.Schedule
- }, func(ctx context.Context) {
- s.runScheduledSystemVolumeBackupInternal(ctx, policyID)
+ }, func(ctx context.Context) (schedulertypes.Outcome, error) {
+ return s.runScheduledSystemVolumeBackupInternal(ctx, policyID)
+ }, func(ctx context.Context, previous schedulertypes.Run) (schedulertypes.Outcome, error) {
+ for _, target := range previous.Outcome.Targets {
+ if target.Status != schedulertypes.Succeeded && target.Status != schedulertypes.Skipped {
+ return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Targets: previous.Outcome.Targets, Message: "A volume backup has an unconfirmed outcome"}, nil
+ }
+ }
+ return s.runScheduledSystemVolumeBackupInternal(jobcontext.WithExecution(ctx, previous, nil), policyID)
})
}
diff --git a/backend/internal/updater/service.go b/backend/internal/updater/service.go
index f8f58fa6f..34d0b249a 100644
--- a/backend/internal/updater/service.go
+++ b/backend/internal/updater/service.go
@@ -2,7 +2,6 @@ package updater
import (
"context"
- stderrors "errors"
"fmt"
"io"
"log/slog"
@@ -12,6 +11,9 @@ import (
"sync"
"time"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
+
activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
"emperror.dev/errors"
@@ -190,6 +192,10 @@ func (s *UpdaterService) ApplyPending(ctx context.Context, options arcaneupdater
s.completeAutoUpdateActivityInternal(ctx, activityID, out, err)
}()
+ if err = jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ID: "auto-update", Status: schedulertypes.Running, ActivityID: activityID}); err != nil {
+ return out, err
+ }
+
if activityID != "" && s.deps.Activity != nil {
// Bounded slot wait: an unbounded wait behind other long-running runs
// would strand the queued activity row (the completion defer above
@@ -267,7 +273,12 @@ func (s *UpdaterService) applyScopedUpdatesInternal(ctx context.Context, options
engineOpts := updater.Options{Force: options.ForceUpdate, DryRun: options.DryRun}
var engineErrs []error
for _, containerID := range containerIDs {
+ target := schedulertypes.TargetOutcome{ID: containerID, ResourceType: "container", Status: schedulertypes.Running}
+ if err := jobcontext.Progress(ctx, target); err != nil {
+ return err
+ }
moduleResult, engineErr := s.engineInternal().UpdateContainer(ctx, containerID, engineOpts)
+ target.Status = schedulertypes.NeedsAttention
if moduleResult != nil {
partial := resultFromModuleInternal(moduleResult)
out.Checked += partial.Checked
@@ -276,6 +287,12 @@ func (s *UpdaterService) applyScopedUpdatesInternal(ctx context.Context, options
out.Skipped += partial.Skipped
out.Failed += partial.Failed
out.Items = append(out.Items, partial.Items...)
+ target.Status = schedulertypes.Succeeded
+ if partial.Failed > 0 {
+ target.Status = schedulertypes.Failed
+ } else if partial.Skipped > 0 {
+ target.Status = schedulertypes.Skipped
+ }
}
if engineErr != nil {
out.Failed++
@@ -286,13 +303,18 @@ func (s *UpdaterService) applyScopedUpdatesInternal(ctx context.Context, options
Error: engineErr.Error(),
})
engineErrs = append(engineErrs, errors.WrapIff(engineErr, "%s", containerID))
+ target.Status = schedulertypes.Failed
+ target.Message = engineErr.Error()
+ }
+ if err := jobcontext.Progress(ctx, target); err != nil {
+ return errors.Combine(err, errors.Combine(engineErrs...))
}
}
s.logResultItemsInternal(ctx, out)
out.Success = out.Failed == 0
// Engine errors propagate like the unscoped path's engine error does —
// the remaining containers were still attempted and recorded above.
- return stderrors.Join(engineErrs...)
+ return errors.Combine(engineErrs...)
}
// resolveScopedContainerIDsInternal maps a scoped options payload to the
@@ -1163,7 +1185,7 @@ func (s *UpdaterService) CollectUsedImages(ctx context.Context) (map[string]stru
}
if successfulSources == 0 {
- return nil, stderrors.Join(errs...)
+ return nil, errors.Combine(errs...)
}
s.loggerInternal().DebugContext(ctx, "collectUsedImages: collected used images", "count", len(out))
diff --git a/backend/internal/volume/backup.go b/backend/internal/volume/backup.go
index f413ac767..b12487fd5 100644
--- a/backend/internal/volume/backup.go
+++ b/backend/internal/volume/backup.go
@@ -15,6 +15,9 @@ import (
"time"
"uuid"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
+
"emperror.dev/errors"
"gorm.io/gorm"
@@ -1775,13 +1778,22 @@ func (s *VolumeService) applyVolumeBackupRetentionInternal(ctx context.Context,
return s.deleteRusticBackupsInternal(ctx, entries, nil)
}
-func (s *VolumeService) runScheduledBackupInternal(ctx context.Context, policyID string) {
+func (s *VolumeService) runScheduledBackupInternal(ctx context.Context, policyID string) (schedulertypes.Outcome, error) {
var policy VolumeBackupPolicy
if err := s.db.WithContext(ctx).Where("id = ? AND enabled = ?", policyID, true).First(&policy).Error; err != nil {
- return
+ if errors.Is(err, gorm.ErrRecordNotFound) {
+ return schedulertypes.Outcome{Status: schedulertypes.Canceled, Message: "Backup policy disabled or deleted"}, nil
+ }
+ return schedulertypes.Outcome{}, err
+ }
+ if previous, ok := jobcontext.Run(ctx); ok {
+ outcome := jobcontext.ConfirmedTarget(previous, policy.VolumeName)
+ if outcome.Status == schedulertypes.Succeeded {
+ return outcome, nil
+ }
}
var entry *VolumeBackup
- _, err := activitylib.RunHandlerActivity(ctx, s.activityService, activitylib.HandlerOptions{
+ activityID, err := activitylib.RunHandlerActivity(ctx, s.activityService, activitylib.HandlerOptions{
EnvironmentID: "0",
Type: activitytypes.TypeResourceAction,
ResourceType: "volume_backup",
@@ -1809,13 +1821,14 @@ func (s *VolumeService) runScheduledBackupInternal(ctx context.Context, policyID
})
if errors.Is(err, ErrVolumeBackupAlreadyRunning) {
slog.InfoContext(ctx, "Scheduled volume backup skipped; another backup is running", "volume", policy.VolumeName)
- return
+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil
}
if err != nil {
slog.ErrorContext(ctx, "Scheduled volume backup failed", "volume", policy.VolumeName, "error", err)
- return
+ return schedulertypes.Outcome{}, err
}
slog.InfoContext(ctx, "Scheduled volume backup completed", "volume", policy.VolumeName, "backup_id", entry.ID, "remote_snapshot_id", entry.RemoteSnapshotID)
+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded, ActivityID: activityID}, nil
}
func (s *VolumeService) rescheduleVolumeBackupPolicyInternal(ctx context.Context, policy *VolumeBackupPolicy) {
@@ -1835,8 +1848,11 @@ func (s *VolumeService) rescheduleVolumeBackupPolicyInternal(ctx context.Context
}
return current.Schedule
},
- func(ctx context.Context) {
- s.runScheduledBackupInternal(ctx, policyID)
+ func(ctx context.Context) (schedulertypes.Outcome, error) {
+ return s.runScheduledBackupInternal(ctx, policyID)
+ },
+ func(_ context.Context, previous schedulertypes.Run) (schedulertypes.Outcome, error) {
+ return jobcontext.ConfirmedTarget(previous, policy.VolumeName), nil
},
)
}
diff --git a/backend/internal/volume/service_test.go b/backend/internal/volume/service_test.go
index 5f4d92224..8dc58399e 100644
--- a/backend/internal/volume/service_test.go
+++ b/backend/internal/volume/service_test.go
@@ -624,7 +624,8 @@ func TestTouchHelperInternal(t *testing.T) {
}
type volumeBackupPolicySchedulerInternal struct {
- jobs map[string]schedulertypes.Job
+ submitted []schedulertypes.Request
+ jobs map[string]schedulertypes.Job
}
func (s *volumeBackupPolicySchedulerInternal) AddJob(_ context.Context, job schedulertypes.Job) error {
@@ -1042,3 +1043,8 @@ func TestBackupPolicyDestinationLookupFailureInternal(t *testing.T) {
require.Empty(t, collection.Policies[0].S3Bucket)
}
}
+
+func (s *volumeBackupPolicySchedulerInternal) Submit(_ context.Context, request schedulertypes.Request) (schedulertypes.Run, error) {
+ s.submitted = append(s.submitted, request)
+ return schedulertypes.Run{ID: request.RunID, JobID: request.JobID, EnvironmentID: request.EnvironmentID, Status: schedulertypes.Queued}, nil
+}
diff --git a/backend/pkg/fswatch/watcher.go b/backend/pkg/fswatch/watcher.go
index 980d22697..087a806aa 100644
--- a/backend/pkg/fswatch/watcher.go
+++ b/backend/pkg/fswatch/watcher.go
@@ -143,6 +143,9 @@ func (fw *Watcher) watchLoop(ctx context.Context) {
debounceTimer := time.NewTimer(fw.debounce)
debounceTimer.Stop()
+ defer debounceTimer.Stop()
+ healthTicker := time.NewTicker(5 * time.Second)
+ defer healthTicker.Stop()
debouncePending := false
lastGoroutineLog := time.Time{}
@@ -154,15 +157,24 @@ func (fw *Watcher) watchLoop(ctx context.Context) {
return
case event, ok := <-fw.watcher.Events:
if fw.processEventInternal(ctx, event, ok, debounceTimer, &debouncePending) {
- return
+ if !fw.reconnectInternal(ctx) {
+ return
+ }
}
case <-debounceTimer.C:
if fw.fireDebounceInternal(ctx, &debouncePending, &lastGoroutineLog) {
continue
}
+ case <-healthTicker.C:
+ if len(fw.watcher.WatchList()) == 0 && !fw.reconnectInternal(ctx) {
+ return
+ }
case err, ok := <-fw.watcher.Errors:
if !ok {
- return
+ if !fw.reconnectInternal(ctx) {
+ return
+ }
+ continue
}
slog.ErrorContext(ctx, "Filesystem watcher error", "error", err)
}
@@ -430,3 +442,47 @@ func (fw *Watcher) isWatchableDirectory(path string) bool {
return resolvedInfo.IsDir()
}
+
+// reconnectInternal replaces a lost filesystem subscription without abandoning its owner.
+func (fw *Watcher) reconnectInternal(ctx context.Context) bool {
+ if err := fw.watcher.Close(); err != nil {
+ slog.DebugContext(ctx, "Closing lost filesystem subscription", "error", err)
+ }
+ delay := 5 * time.Second
+ for {
+ timer := time.NewTimer(delay)
+ select {
+ case <-ctx.Done():
+ timer.Stop()
+ return false
+ case <-fw.stopCh:
+ timer.Stop()
+ return false
+ case <-timer.C:
+ }
+ watcher, err := fsnotify.NewWatcher()
+ if err == nil {
+ err = watcher.Add(fw.watchedPath)
+ if err == nil {
+ fw.watcher = watcher
+ fw.watchAliases = make(map[string]string)
+ if err := fw.addExistingDirectories(fw.watchedPath); err != nil {
+ slog.WarnContext(ctx, "Some filesystem subscriptions could not be restored", "error", err)
+ }
+ // Changes while disconnected need a full reconciliation.
+ if fw.onChange != nil {
+ fw.onChange(ctx)
+ }
+ if fw.onChangePaths != nil {
+ fw.onChangePaths(ctx, []string{fw.watchedPath})
+ }
+ return true
+ }
+ if closeErr := watcher.Close(); closeErr != nil {
+ slog.DebugContext(ctx, "Closing failed filesystem subscription", "error", closeErr)
+ }
+ }
+ slog.WarnContext(ctx, "Filesystem subscription unavailable; retrying", "path", fw.watchedPath, "error", err)
+ delay = min(delay*2, 5*time.Minute)
+ }
+}
diff --git a/backend/pkg/libarcane/activity/handler.go b/backend/pkg/libarcane/activity/handler.go
index e2827f5a5..18ccc77b7 100644
--- a/backend/pkg/libarcane/activity/handler.go
+++ b/backend/pkg/libarcane/activity/handler.go
@@ -2,6 +2,8 @@ package activity
import (
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
@@ -190,8 +192,19 @@ func RunHandlerActivity(ctx context.Context, activityService Service, opts Handl
AwaitHandlerActivitySlot(workCtx, activityService, activityID, opts.EnvironmentID)
}
- err := action(workCtx)
+ err := jobcontext.Progress(workCtx, schedulertypes.TargetOutcome{ID: opts.ResourceID, Status: schedulertypes.Running, ActivityID: activityID})
+ if err == nil {
+ err = action(workCtx)
+ }
CompleteHandlerActivity(workCtx, activityService, activityID, opts.SuccessMessage, err)
+ target := schedulertypes.TargetOutcome{ID: opts.ResourceID, Status: schedulertypes.Succeeded, ActivityID: activityID}
+ if err != nil {
+ target.Status = schedulertypes.Failed
+ target.Message = err.Error()
+ }
+ if progressErr := jobcontext.Progress(workCtx, target); progressErr != nil && err == nil {
+ err = progressErr
+ }
return activityID, err
}
diff --git a/backend/pkg/scheduler/activity_sweep_job.go b/backend/pkg/scheduler/activity_sweep_job.go
index b73231ec5..f8c0792f0 100644
--- a/backend/pkg/scheduler/activity_sweep_job.go
+++ b/backend/pkg/scheduler/activity_sweep_job.go
@@ -4,6 +4,8 @@ import (
"context"
"log/slog"
+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
+
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
)
@@ -30,13 +32,14 @@ func (j *ActivitySweepJob) Schedule(_ context.Context) string {
return "0 */5 * * * *"
}
-func (j *ActivitySweepJob) Run(ctx context.Context) {
+func (j *ActivitySweepJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {
swept, err := j.activityService.FailAbandonedActivities(ctx)
if err != nil {
slog.WarnContext(ctx, "activity sweep failed", "jobName", ActivitySweepJobName, "swept", swept, "error", err)
- return
+ return schedulertypes.Outcome{}, err
}
if swept > 0 {
slog.InfoContext(ctx, "marked abandoned activities as failed", "jobName", ActivitySweepJobName, "count", swept)
}
+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil
}
diff --git a/backend/pkg/scheduler/actor_test.go b/backend/pkg/scheduler/actor_test.go
index 8c30e74f7..99b080a1c 100644
--- a/backend/pkg/scheduler/actor_test.go
+++ b/backend/pkg/scheduler/actor_test.go
@@ -2,6 +2,8 @@ package scheduler
import (
"context"
+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
+ "sync"
"testing"
"time"
@@ -36,6 +38,14 @@ func newJobSchedulerForTestInternal(t testing.TB, ctx context.Context, location
created, err := NewJobScheduler(ctx, runtime, location)
require.NoError(t, err)
scheduler := created.(*jobSchedulerInternal)
+ scheduler.SetDispatcher(&testDispatcherInternal{run: func(ctx context.Context, request schedulertypes.Request) error {
+ job, ok := scheduler.GetJob(request.JobID)
+ if !ok {
+ return nil
+ }
+ _, err := job.Run(ctx)
+ return err
+ }})
t.Cleanup(func() {
stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
@@ -63,3 +73,21 @@ func newSettingsServiceForTestInternal(t testing.TB, ctx context.Context, db *da
})
return settings.NewSettingsService(ctx, db, executor, effects)
}
+
+type testDispatcherInternal struct {
+ run func(context.Context, schedulertypes.Request) error
+ locks sync.Map
+}
+
+func (d *testDispatcherInternal) Submit(ctx context.Context, request schedulertypes.Request) (schedulertypes.Run, error) {
+ value, _ := d.locks.LoadOrStore(request.JobID, &sync.Mutex{})
+ lock := value.(*sync.Mutex)
+ if !lock.TryLock() {
+ return schedulertypes.Run{}, nil
+ }
+ defer lock.Unlock()
+ return schedulertypes.Run{JobID: request.JobID}, d.run(ctx, request)
+}
+func (*testDispatcherInternal) Checkpoint(context.Context, string, string, time.Time) error {
+ return nil
+}
diff --git a/backend/pkg/scheduler/analytics_job.go b/backend/pkg/scheduler/analytics_job.go
index e37bb3ef9..821ce6b8c 100644
--- a/backend/pkg/scheduler/analytics_job.go
+++ b/backend/pkg/scheduler/analytics_job.go
@@ -11,6 +11,8 @@ import (
"sync"
"time"
+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
+
"emperror.dev/errors"
"github.com/cenkalti/backoff/v5"
@@ -69,23 +71,23 @@ func (j *AnalyticsJob) Schedule(_ context.Context) string {
return analyticsHeartbeatCheckSchedule
}
-func (j *AnalyticsJob) Run(ctx context.Context) {
+func (j *AnalyticsJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {
if j.cfg.AnalyticsDisabled {
slog.DebugContext(ctx, "analytics disabled; skipping heartbeat", "analyticsDisabled", j.cfg.AnalyticsDisabled)
- return
+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil
}
if j.cfg.Environment.IsTestEnvironment() {
slog.DebugContext(ctx, "test environment; skipping heartbeat", "env", j.cfg.Environment)
- return
+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil
}
allowed, err := j.claimHeartbeatAttemptWindowInternal(ctx)
if err != nil {
slog.ErrorContext(ctx, "failed to acquire analytics heartbeat send window", "error", err)
- return
+ return schedulertypes.Outcome{}, err
}
if !allowed {
- return
+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil
}
instanceID := j.settingsService.GetStringSetting(ctx, "instanceId", "")
@@ -103,7 +105,7 @@ func (j *AnalyticsJob) Run(ctx context.Context) {
body, err := json.Marshal(payload)
if err != nil {
slog.ErrorContext(ctx, "failed to marshal analytics heartbeat body", "error", err)
- return
+ return schedulertypes.Outcome{}, err
}
slog.InfoContext(
@@ -167,7 +169,7 @@ func (j *AnalyticsJob) Run(ctx context.Context) {
)
if err != nil {
slog.ErrorContext(ctx, "analytics heartbeat failed", "error", err)
- return
+ return schedulertypes.Outcome{}, err
}
slog.InfoContext(
@@ -186,6 +188,7 @@ func (j *AnalyticsJob) Run(ctx context.Context) {
"env",
j.cfg.Environment,
)
+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil
}
func (j *AnalyticsJob) Reschedule(ctx context.Context) error {
diff --git a/backend/pkg/scheduler/analytics_job_test.go b/backend/pkg/scheduler/analytics_job_test.go
index dc255dbc2..9b570d957 100644
--- a/backend/pkg/scheduler/analytics_job_test.go
+++ b/backend/pkg/scheduler/analytics_job_test.go
@@ -67,7 +67,11 @@ func TestAnalyticsJob_Run_ManagerPayload(t *testing.T) {
job := NewAnalyticsJob(settingsService, kvService, server.Client(), cfg)
job.heartbeatURL = server.URL
- job.Run(ctx)
+ if _, err := job.Run(ctx); !assert.NoError(t, err) {
+
+ return
+
+ }
var body []byte
select {
@@ -93,7 +97,11 @@ func TestAnalyticsJob_Run_AgentPayload(t *testing.T) {
job := NewAnalyticsJob(settingsService, kvService, server.Client(), cfg)
job.heartbeatURL = server.URL
- job.Run(ctx)
+ if _, err := job.Run(ctx); !assert.NoError(t, err) {
+
+ return
+
+ }
var body []byte
select {
@@ -117,7 +125,11 @@ func TestAnalyticsJob_Run_SkipsWhenDisabled(t *testing.T) {
job := NewAnalyticsJob(settingsService, kvService, server.Client(), cfg)
job.heartbeatURL = server.URL
- job.Run(ctx)
+ if _, err := job.Run(ctx); !assert.NoError(t, err) {
+
+ return
+
+ }
select {
case <-bodyCh:
@@ -136,7 +148,11 @@ func TestAnalyticsJob_Run_SkipsWhenTestEnv(t *testing.T) {
job := NewAnalyticsJob(settingsService, kvService, server.Client(), cfg)
job.heartbeatURL = server.URL
- job.Run(ctx)
+ if _, err := job.Run(ctx); !assert.NoError(t, err) {
+
+ return
+
+ }
select {
case <-bodyCh:
@@ -157,14 +173,18 @@ func TestAnalyticsJob_Run_SkipsWithinHeartbeatWindowAfterRestart(t *testing.T) {
job := NewAnalyticsJob(settingsService, kvService, server.Client(), cfg)
job.heartbeatURL = server.URL
job.now = func() time.Time { return firstAttemptAt }
- job.Run(ctx)
+ if _, err := job.Run(ctx); !assert.NoError(t, err) {
+ return
+ }
reloadedSettingsService, err := newSettingsServiceForTestInternal(t, ctx, wrappedDB)
require.NoError(t, err)
restartedJob := NewAnalyticsJob(reloadedSettingsService, kv.NewKVService(wrappedDB), server.Client(), cfg)
restartedJob.heartbeatURL = server.URL
restartedJob.now = func() time.Time { return firstAttemptAt.Add(19 * time.Minute) }
- restartedJob.Run(ctx)
+ if _, err := restartedJob.Run(ctx); !assert.NoError(t, err) {
+ return
+ }
require.Equal(t, int32(1), requestCount.Load())
}
@@ -181,12 +201,16 @@ func TestAnalyticsJob_Run_AllowsSendAfterHeartbeatWindow(t *testing.T) {
firstJob := NewAnalyticsJob(settingsService, kvService, server.Client(), cfg)
firstJob.heartbeatURL = server.URL
firstJob.now = func() time.Time { return firstAttemptAt }
- firstJob.Run(ctx)
+ if _, err := firstJob.Run(ctx); !assert.NoError(t, err) {
+ return
+ }
secondJob := NewAnalyticsJob(settingsService, kvService, server.Client(), cfg)
secondJob.heartbeatURL = server.URL
secondJob.now = func() time.Time { return firstAttemptAt.Add(25 * time.Hour) }
- secondJob.Run(ctx)
+ if _, err := secondJob.Run(ctx); !assert.NoError(t, err) {
+ return
+ }
require.Equal(t, int32(2), requestCount.Load())
}
@@ -209,7 +233,9 @@ func TestAnalyticsJob_Run_ConcurrentRunsSendOnce(t *testing.T) {
for range 2 {
wg.Go(func() {
<-start
- job.Run(ctx)
+ if _, err := job.Run(ctx); !assert.NoError(t, err) {
+ return
+ }
})
}
diff --git a/backend/pkg/scheduler/apns_outbox_job.go b/backend/pkg/scheduler/apns_outbox_job.go
index 0c8a663af..adad24a17 100644
--- a/backend/pkg/scheduler/apns_outbox_job.go
+++ b/backend/pkg/scheduler/apns_outbox_job.go
@@ -3,6 +3,8 @@ package scheduler
import (
"context"
+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
+
"github.com/getarcaneapp/arcane/backend/v2/internal/apns"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
)
@@ -30,6 +32,9 @@ func (j *ApnsOutboxJob) ShouldSchedule(ctx context.Context) bool {
return (j.config == nil || !j.config.AgentMode) && j.service.Enabled(ctx)
}
-func (j *ApnsOutboxJob) Run(ctx context.Context) {
- j.service.DrainOutbox(ctx)
+func (j *ApnsOutboxJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {
+ if err := j.service.DrainOutbox(ctx); err != nil {
+ return schedulertypes.Outcome{}, err
+ }
+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil
}
diff --git a/backend/pkg/scheduler/auto_heal_job.go b/backend/pkg/scheduler/auto_heal_job.go
index dce30c091..ed86cc6da 100644
--- a/backend/pkg/scheduler/auto_heal_job.go
+++ b/backend/pkg/scheduler/auto_heal_job.go
@@ -2,6 +2,8 @@ package scheduler
import (
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
"context"
"log/slog"
@@ -14,9 +16,9 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/notification"
"emperror.dev/errors"
+ scheduleutil "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/schedule"
"github.com/moby/moby/api/types/container"
"github.com/moby/moby/client"
- "github.com/robfig/cron/v3"
"golang.org/x/sync/errgroup"
"github.com/getarcaneapp/arcane/backend/v2/internal/actors"
@@ -91,7 +93,7 @@ func (j *AutoHealJob) Schedule(ctx context.Context) string {
schedule = "*/30 * * * * *"
}
- parser := cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow)
+ parser := scheduleutil.Parser()
if _, err := parser.Parse(schedule); err != nil {
slog.WarnContext(ctx, "Invalid cron expression for auto-heal, using default", "invalid_schedule", schedule, "error", err)
return "*/30 * * * * *"
@@ -100,34 +102,34 @@ func (j *AutoHealJob) Schedule(ctx context.Context) string {
return schedule
}
-func (j *AutoHealJob) Run(ctx context.Context) {
+func (j *AutoHealJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {
enabled := j.settingsService.GetBoolSetting(ctx, "autoHealEnabled", false)
if !enabled {
slog.DebugContext(ctx, "auto-heal disabled; skipping run")
- return
+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil
}
lease, admitted, err := j.admissionGate.TryAcquire(ctx, actors.AdmissionKey{Scope: autoHealAdmissionScopeInternal})
if err != nil {
slog.ErrorContext(ctx, "auto-heal admission failed", "error", err)
- return
+ return schedulertypes.Outcome{}, err
}
if !admitted {
slog.WarnContext(ctx, "auto-heal run still in progress; skipping overlapping run")
- return
+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil
}
defer lease.Release()
dockerClient, err := j.getDockerClientInternal(ctx)
if err != nil {
slog.ErrorContext(ctx, "auto-heal failed to get Docker client", "error", err)
- return
+ return schedulertypes.Outcome{}, err
}
containerList, err := j.ListContainers(ctx, dockerClient)
if err != nil {
slog.ErrorContext(ctx, "auto-heal failed to list containers", "error", err)
- return
+ return schedulertypes.Outcome{}, err
}
containers := containerList
@@ -140,18 +142,30 @@ func (j *AutoHealJob) Run(ctx context.Context) {
candidates := j.filterCandidatesInternal(containers, excludedContainers, selfID)
g, groupCtx := errgroup.WithContext(ctx)
+ var resultMu sync.Mutex
+ outcome := schedulertypes.Outcome{Status: schedulertypes.Succeeded}
g.SetLimit(autoHealInspectConcurrency)
for _, candidate := range candidates {
g.Go(func() (workerErr error) {
defer utils.RecoverToError(&workerErr, "auto-heal worker")
- j.processCandidateInternal(groupCtx, dockerClient, candidate, maxRestarts, restartWindow, restartWindowMinutes)
+ target, err := j.processCandidateInternal(groupCtx, dockerClient, candidate, maxRestarts, restartWindow, restartWindowMinutes)
+ resultMu.Lock()
+ outcome.Targets = append(outcome.Targets, target)
+ if err != nil {
+ outcome.Status = schedulertypes.Partial
+ }
+ resultMu.Unlock()
+ if progressErr := jobcontext.Progress(groupCtx, target); progressErr != nil {
+ return progressErr
+ }
return nil
})
}
- _ = g.Wait()
+ err = g.Wait()
+ return outcome, err
}
// selfContainerIDInternal resolves and caches the ID (full 64-char or short
@@ -207,8 +221,16 @@ func (j *AutoHealJob) processCandidateInternal(
maxRestarts int,
restartWindow time.Duration,
restartWindowMinutes int,
-) {
+) (schedulertypes.TargetOutcome, error) {
containerID := candidate.ID
+ target := schedulertypes.TargetOutcome{ID: containerID, Status: schedulertypes.Skipped}
+ if previous, ok := jobcontext.Run(ctx); ok {
+ for _, completed := range previous.Outcome.Targets {
+ if completed.ID == containerID && completed.Status == schedulertypes.Succeeded {
+ return completed, nil
+ }
+ }
+ }
containerName := dockerutil.ContainerNameFromNames(candidate.Names)
// The daemon-side health filter already selected unhealthy containers; the
@@ -217,14 +239,16 @@ func (j *AutoHealJob) processCandidateInternal(
inspect, err := j.inspectContainerInternal(ctx, dockerClient, containerID)
if err != nil {
slog.WarnContext(ctx, "auto-heal failed to inspect container", "container", containerName, "error", err)
- return
+ target.Status = schedulertypes.Failed
+ target.Message = err.Error()
+ return target, err
}
if inspect.State == nil || inspect.State.Health == nil {
- return
+ return target, nil
}
if inspect.State.Health.Status != container.Unhealthy {
- return
+ return target, nil
}
releaseSlot, reserved := j.reserveRestartSlotInternal(containerID, maxRestarts, restartWindow)
@@ -234,18 +258,27 @@ func (j *AutoHealJob) processCandidateInternal(
"max_restarts", maxRestarts,
"window_minutes", restartWindowMinutes,
)
- return
+ return target, nil
}
+ target.Status = schedulertypes.Running
+ if err := jobcontext.Progress(ctx, target); err != nil {
+ releaseSlot()
+ return target, err
+ }
if err := j.restartContainerInternal(ctx, dockerClient, containerID); err != nil {
releaseSlot()
slog.ErrorContext(ctx, "auto-heal failed to restart container", "container", containerName, "error", err)
- return
+ target.Status = schedulertypes.Failed
+ target.Message = err.Error()
+ return target, err
}
j.postRestartActionsInternal(ctx, containerID, containerName)
slog.InfoContext(ctx, "auto-heal restarted unhealthy container", "container", containerName, "container_id", containerID)
+ target.Status = schedulertypes.Succeeded
+ return target, nil
}
func (j *AutoHealJob) postRestartActionsInternal(ctx context.Context, containerID, containerName string) {
@@ -478,3 +511,12 @@ func (j *AutoHealJob) RecordRestartAtExported(containerID string, t time.Time) {
record.timestamps = append(record.timestamps, t)
}
+
+func (j *AutoHealJob) Reconcile(ctx context.Context, previous schedulertypes.Run) (schedulertypes.Outcome, error) {
+ for _, target := range previous.Outcome.Targets {
+ if target.Status != schedulertypes.Succeeded && target.Status != schedulertypes.Skipped {
+ return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: "A container restart has an unconfirmed outcome", Targets: previous.Outcome.Targets}, nil
+ }
+ }
+ return j.Run(jobcontext.WithExecution(ctx, previous, nil))
+}
diff --git a/backend/pkg/scheduler/auto_patch_job.go b/backend/pkg/scheduler/auto_patch_job.go
index 255515ce5..fbf20d2ad 100644
--- a/backend/pkg/scheduler/auto_patch_job.go
+++ b/backend/pkg/scheduler/auto_patch_job.go
@@ -4,11 +4,13 @@ import (
"context"
"log/slog"
+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
+
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/imagepatch"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
+ scheduleutil "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/schedule"
"github.com/getarcaneapp/arcane/types/v2"
- "github.com/robfig/cron/v3"
)
const AutoPatchJobName = "auto-patch"
@@ -48,7 +50,7 @@ func (j *AutoPatchJob) Schedule(ctx context.Context) string {
schedule = "0 0 3 * * *"
}
- parser := cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow)
+ parser := scheduleutil.Parser()
if _, err := parser.Parse(schedule); err != nil {
slog.WarnContext(ctx, "Invalid cron expression for auto-patch, using default", "invalid_schedule", schedule, "error", err)
return "0 0 3 * * *"
@@ -57,10 +59,10 @@ func (j *AutoPatchJob) Schedule(ctx context.Context) string {
return schedule
}
-func (j *AutoPatchJob) Run(ctx context.Context) {
+func (j *AutoPatchJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {
if !j.settingsService.GetBoolSetting(ctx, "imageAutoPatchEnabled", false) {
slog.DebugContext(ctx, "scheduled image patching disabled; skipping run")
- return
+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil
}
slog.InfoContext(ctx, "scheduled image patching started")
@@ -68,11 +70,15 @@ func (j *AutoPatchJob) Run(ctx context.Context) {
patched, skipped, err := j.imagePatchService.PatchFlaggedImages(ctx, types.LocalDockerEnvironmentID, autoPatchSystemUser)
if err != nil {
slog.ErrorContext(ctx, "scheduled image patching failed", "error", err)
- return
+ if patched > 0 {
+ return schedulertypes.Outcome{Status: schedulertypes.Partial}, err
+ }
+ return schedulertypes.Outcome{}, err
}
slog.InfoContext(ctx, "scheduled image patching completed",
"patched", patched,
"skipped", skipped,
)
+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil
}
diff --git a/backend/pkg/scheduler/auto_update_job.go b/backend/pkg/scheduler/auto_update_job.go
index 2bd847e5d..65bf96989 100644
--- a/backend/pkg/scheduler/auto_update_job.go
+++ b/backend/pkg/scheduler/auto_update_job.go
@@ -4,6 +4,9 @@ import (
"context"
"log/slog"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
+
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/internal/updater"
@@ -55,32 +58,47 @@ func (j *AutoUpdateJob) Schedule(ctx context.Context) string {
return s
}
-func (j *AutoUpdateJob) Run(ctx context.Context) {
+func (j *AutoUpdateJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {
+ options := updatertypes.Options{}
+ unresolvedTargets := false
+ if previous, ok := jobcontext.Run(ctx); ok {
+ var priorOutcome schedulertypes.Outcome
+ options, priorOutcome, unresolvedTargets = autoUpdateRetryOptionsInternal(previous)
+ if priorOutcome.Status != "" {
+ return priorOutcome, nil
+ }
+ }
+
enabled := j.settingsService.GetBoolSetting(ctx, "autoUpdate", false)
pollingEnabled := j.settingsService.GetBoolSetting(ctx, "pollingEnabled", true)
if !enabled || !pollingEnabled {
slog.DebugContext(ctx, "auto-update disabled or polling disabled; skipping run",
"autoUpdate", enabled, "pollingEnabled", pollingEnabled)
- return
+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil
}
lease, admitted, err := j.admissionGate.TryAcquire(ctx, actors.AdmissionKey{Scope: autoUpdateAdmissionScopeInternal})
if err != nil {
slog.ErrorContext(ctx, "auto-update admission failed", "error", err)
- return
+ return schedulertypes.Outcome{}, err
}
if !admitted {
slog.WarnContext(ctx, "auto-update run still in progress; skipping overlapping run")
- return
+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil
}
defer lease.Release()
slog.InfoContext(ctx, "auto-update run started")
- result, err := j.updaterService.ApplyPending(ctx, updatertypes.Options{})
+ result, err := j.updaterService.ApplyPending(ctx, options)
if err != nil {
slog.ErrorContext(ctx, "auto-update run failed", "err", err)
- return
+ if result == nil {
+ return schedulertypes.Outcome{}, err
+ }
+ }
+ if result == nil {
+ return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: "Updater returned no operation result"}, nil
}
slog.InfoContext(ctx, "auto-update run completed",
@@ -90,9 +108,66 @@ func (j *AutoUpdateJob) Run(ctx context.Context) {
"skipped", result.Skipped,
"failed", result.Failed,
)
+ outcome := schedulertypes.Outcome{Status: schedulertypes.Succeeded}
+ if result.ActivityID != nil {
+ outcome.ActivityID = *result.ActivityID
+ }
+ for _, item := range result.Items {
+ status := schedulertypes.Succeeded
+ if item.Status == "failed" {
+ status = schedulertypes.Failed
+ }
+ if item.Status == "skipped" {
+ status = schedulertypes.Skipped
+ }
+ outcome.Targets = append(outcome.Targets, schedulertypes.TargetOutcome{ID: item.ResourceID, ResourceType: item.ResourceType, Status: status, Message: item.Error})
+ }
+ if result.Failed > 0 || err != nil || unresolvedTargets {
+ outcome.Status = schedulertypes.Partial
+ outcome.Message = "Some updates failed; review target outcomes before retrying"
+ }
+ return outcome, err
}
func (j *AutoUpdateJob) Reschedule(ctx context.Context) error {
slog.InfoContext(ctx, "rescheduling auto-update job in new scheduler; currently requires restart")
return nil
}
+
+func (j *AutoUpdateJob) Reconcile(_ context.Context, previous schedulertypes.Run) (schedulertypes.Outcome, error) {
+ return jobcontext.ConfirmedTarget(previous, "auto-update"), nil
+}
+
+func autoUpdateRetryOptionsInternal(previous schedulertypes.Run) (updatertypes.Options, schedulertypes.Outcome, bool) {
+ options := updatertypes.Options{}
+ unresolvedTargets := false
+ confirmed := jobcontext.ConfirmedTarget(previous, "auto-update")
+ if confirmed.Status == schedulertypes.Succeeded {
+ return options, confirmed, false
+ }
+ hasResourceTargets := false
+ for _, target := range previous.Outcome.Targets {
+ if target.ID == "auto-update" {
+ continue
+ }
+ hasResourceTargets = true
+ if target.Status == schedulertypes.Succeeded || target.Status == schedulertypes.Skipped {
+ continue
+ }
+ if target.ResourceType != "container" || target.Status != schedulertypes.Failed {
+ unresolvedTargets = true
+ continue
+ }
+ options.ResourceIds = append(options.ResourceIds, target.ID)
+ }
+ if hasResourceTargets && len(options.ResourceIds) == 0 {
+ if unresolvedTargets {
+ return options, schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: "Remaining image or unconfirmed targets need review before retrying", Targets: previous.Outcome.Targets}, true
+ }
+ return options, schedulertypes.Outcome{Status: schedulertypes.Succeeded, Targets: previous.Outcome.Targets}, false
+ }
+ if len(options.ResourceIds) > 0 {
+ options.Type = "container"
+ }
+ return options, schedulertypes.Outcome{}, unresolvedTargets
+}
diff --git a/backend/pkg/scheduler/docker_client_refresh_job.go b/backend/pkg/scheduler/docker_client_refresh_job.go
index b3ab4cc64..5dc5d924d 100644
--- a/backend/pkg/scheduler/docker_client_refresh_job.go
+++ b/backend/pkg/scheduler/docker_client_refresh_job.go
@@ -4,10 +4,12 @@ import (
"context"
"log/slog"
+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
+
"github.com/getarcaneapp/arcane/backend/v2/internal/docker"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
- "github.com/robfig/cron/v3"
+ scheduleutil "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/schedule"
)
const DockerClientRefreshJobName = "docker-client-refresh"
@@ -38,7 +40,7 @@ func (j *DockerClientRefreshJob) Schedule(ctx context.Context) string {
schedule = dockerClientRefreshDefaultSchedule
}
- parser := cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow)
+ parser := scheduleutil.Parser()
if _, err := parser.Parse(schedule); err != nil {
slog.WarnContext(ctx, "Invalid cron expression for Docker client refresh, using default", "invalid_schedule", schedule, "error", err)
return dockerClientRefreshDefaultSchedule
@@ -47,11 +49,12 @@ func (j *DockerClientRefreshJob) Schedule(ctx context.Context) string {
return schedule
}
-func (j *DockerClientRefreshJob) Run(ctx context.Context) {
+func (j *DockerClientRefreshJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {
if err := j.dockerClientService.RefreshClient(ctx); err != nil {
slog.WarnContext(ctx, "Docker client refresh failed", "error", err)
- return
+ return schedulertypes.Outcome{}, err
}
slog.DebugContext(ctx, "Docker client refresh completed")
+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil
}
diff --git a/backend/pkg/scheduler/entityjobs/entityjobs.go b/backend/pkg/scheduler/entityjobs/entityjobs.go
index 3c854880f..855df80a6 100644
--- a/backend/pkg/scheduler/entityjobs/entityjobs.go
+++ b/backend/pkg/scheduler/entityjobs/entityjobs.go
@@ -77,7 +77,7 @@ func (r *Registry) Context(ctx context.Context) context.Context {
func (r *Registry) JobName(entityID string) string { return r.jobPrefix + entityID }
// Register adds (or replaces) the dynamic job for entityID.
-func (r *Registry) Register(ctx context.Context, entityID string, schedule func(context.Context) string, run func(context.Context)) {
+func (r *Registry) Register(ctx context.Context, entityID string, schedule func(context.Context) string, run func(context.Context) (schedulertypes.Outcome, error), reconcile ...func(context.Context, schedulertypes.Run) (schedulertypes.Outcome, error)) {
if r.scheduler == nil {
return
}
@@ -87,6 +87,9 @@ func (r *Registry) Register(ctx context.Context, entityID string, schedule func(
ScheduleFn: schedule,
RunFn: run,
}
+ if len(reconcile) > 0 {
+ job.ReconcileFn = reconcile[0]
+ }
if err := r.scheduler.AddJob(schedulerCtx, job); err != nil {
slog.ErrorContext(schedulerCtx, "failed to register scheduled job", "job", job.JobName, "error", err)
}
diff --git a/backend/pkg/scheduler/event_cleanup_job.go b/backend/pkg/scheduler/event_cleanup_job.go
index 2444d0fa1..949cdb7d4 100644
--- a/backend/pkg/scheduler/event_cleanup_job.go
+++ b/backend/pkg/scheduler/event_cleanup_job.go
@@ -5,6 +5,8 @@ import (
"log/slog"
"time"
+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
+
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
"github.com/getarcaneapp/arcane/backend/v2/internal/event"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
@@ -38,14 +40,14 @@ func (j *EventCleanupJob) Schedule(ctx context.Context) string {
return s
}
-func (j *EventCleanupJob) Run(ctx context.Context) {
+func (j *EventCleanupJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {
slog.InfoContext(ctx, "Running event cleanup job", "jobName", EventCleanupJobName)
// Delete events older than 36 hours
olderThan := 36 * time.Hour
if err := j.eventService.DeleteOldEvents(ctx, olderThan); err != nil {
slog.ErrorContext(ctx, "Failed to delete old events", "jobName", EventCleanupJobName, "olderThan", olderThan.String(), "error", err)
- return
+ return schedulertypes.Outcome{}, err
}
slog.InfoContext(ctx, "Event cleanup job completed successfully",
@@ -58,7 +60,7 @@ func (j *EventCleanupJob) Run(ctx context.Context) {
deleted, err := j.activityService.PruneHistory(ctx, retentionDays, maxEntries)
if err != nil {
slog.ErrorContext(ctx, "Failed to prune activity history", "jobName", EventCleanupJobName, "error", err)
- return
+ return schedulertypes.Outcome{}, err
}
slog.InfoContext(ctx, "Activity history cleanup completed successfully",
@@ -67,6 +69,7 @@ func (j *EventCleanupJob) Run(ctx context.Context) {
"maxEntries", maxEntries,
"deleted", deleted)
}
+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil
}
func (j *EventCleanupJob) Reschedule(ctx context.Context) error {
diff --git a/backend/pkg/scheduler/expired_sessions_cleanup_job.go b/backend/pkg/scheduler/expired_sessions_cleanup_job.go
index feec763bc..078b92c72 100644
--- a/backend/pkg/scheduler/expired_sessions_cleanup_job.go
+++ b/backend/pkg/scheduler/expired_sessions_cleanup_job.go
@@ -5,6 +5,8 @@ import (
"log/slog"
"time"
+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
+
"github.com/getarcaneapp/arcane/backend/v2/internal/session"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
)
@@ -35,20 +37,21 @@ func (j *ExpiredSessionsCleanupJob) Schedule(ctx context.Context) string {
return s
}
-func (j *ExpiredSessionsCleanupJob) Run(ctx context.Context) {
+func (j *ExpiredSessionsCleanupJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {
slog.InfoContext(ctx, "Running expired sessions cleanup job", "jobName", ExpiredSessionsCleanupJobName)
revokedRetention := 7 * 24 * time.Hour
deleted, err := j.sessionService.DeleteExpiredSessions(ctx, revokedRetention)
if err != nil {
slog.ErrorContext(ctx, "Failed to delete expired user sessions", "jobName", ExpiredSessionsCleanupJobName, "revokedRetention", revokedRetention.String(), "error", err)
- return
+ return schedulertypes.Outcome{}, err
}
slog.InfoContext(ctx, "Expired sessions cleanup job completed successfully",
"jobName", ExpiredSessionsCleanupJobName,
"revokedRetention", revokedRetention.String(),
"deleted", deleted)
+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil
}
func (j *ExpiredSessionsCleanupJob) Reschedule(ctx context.Context) error {
diff --git a/backend/pkg/scheduler/git_clone_cleanup_job.go b/backend/pkg/scheduler/git_clone_cleanup_job.go
index 1133e1d80..2150462d5 100644
--- a/backend/pkg/scheduler/git_clone_cleanup_job.go
+++ b/backend/pkg/scheduler/git_clone_cleanup_job.go
@@ -5,6 +5,8 @@ import (
"log/slog"
"time"
+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
+
"github.com/getarcaneapp/arcane/backend/v2/internal/gitrepo"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
)
@@ -40,9 +42,9 @@ func (j *GitCloneCleanupJob) Schedule(_ context.Context) string {
return "0 30 * * * *"
}
-func (j *GitCloneCleanupJob) Run(ctx context.Context) {
+func (j *GitCloneCleanupJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {
if j.repoService == nil || j.repoService.Client == nil {
- return
+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil
}
maxAge := gitCloneScratchMinAge
@@ -54,11 +56,12 @@ func (j *GitCloneCleanupJob) Run(ctx context.Context) {
removed, err := j.repoService.PurgeScratchDirs(ctx, maxAge)
if err != nil {
slog.ErrorContext(ctx, "Failed to purge leaked git clone scratch dirs", "jobName", GitCloneCleanupJobName, "removed", removed, "error", err)
- return
+ return schedulertypes.Outcome{}, err
}
if removed > 0 {
slog.InfoContext(ctx, "Purged leaked git clone scratch dirs", "jobName", GitCloneCleanupJobName, "removed", removed)
}
+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil
}
func (j *GitCloneCleanupJob) Reschedule(_ context.Context) error {
diff --git a/backend/pkg/scheduler/image_update_watcher.go b/backend/pkg/scheduler/image_update_watcher.go
index 0cf1ee514..94e486f58 100644
--- a/backend/pkg/scheduler/image_update_watcher.go
+++ b/backend/pkg/scheduler/image_update_watcher.go
@@ -20,6 +20,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/types/v2/containerregistry"
imageupdatetypes "github.com/getarcaneapp/arcane/types/v2/imageupdate"
+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
"github.com/moby/moby/api/types/events"
"go.getarcane.app/streams/bus"
)
@@ -79,6 +80,9 @@ type ImageUpdateWatcher struct {
environmentService registryCredentialLoaderInternal
dockerService dockerEventBusProviderInternal
projectService projectImageRefsBackfillerInternal
+ dispatchMu sync.RWMutex
+ dispatcher schedulertypes.Dispatcher
+ eventDegraded atomic.Bool
actorRuntime *actors.Runtime
actorProcess atomic.Pointer[actors.Actor]
stopping atomic.Bool
@@ -147,7 +151,9 @@ func (w *ImageUpdateWatcher) Start(ctx context.Context) error {
return errors.New("docker event bus unavailable")
}
eventCh, unsubscribe := eventBus.Subscribe(events.ImageEventType, bus.WithSubscriberBuffer(16))
- defer unsubscribe()
+ defer func() { unsubscribe() }()
+ reconnect := time.NewTicker(5 * time.Second)
+ defer reconnect.Stop()
process, err := actors.NewActor(
ctx,
@@ -174,11 +180,12 @@ func (w *ImageUpdateWatcher) Start(ctx context.Context) error {
slog.InfoContext(ctx, "image update watcher started")
defer func() {
- w.actorProcess.CompareAndSwap(process, nil)
stopCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), imageUpdateWatcherStopTimeout)
defer cancel()
if err := process.Stop(stopCtx); err != nil {
slog.ErrorContext(context.WithoutCancel(ctx), "failed to stop image update watcher actor", "error", err)
+ } else {
+ w.actorProcess.CompareAndSwap(process, nil)
}
slog.InfoContext(context.WithoutCancel(ctx), "image update watcher stopped")
}()
@@ -192,10 +199,19 @@ func (w *ImageUpdateWatcher) Start(ctx context.Context) error {
return nil
}
return errors.New("image update watcher actor stopped unexpectedly")
+ case <-reconnect.C:
+ if eventCh == nil {
+ if currentBus := w.dockerService.EventBus(); currentBus != nil {
+ unsubscribe()
+ eventCh, unsubscribe = currentBus.Subscribe(events.ImageEventType, bus.WithSubscriberBuffer(16))
+ w.eventDegraded.Store(false)
+ }
+ }
case _, ok := <-eventCh:
if !ok {
slog.WarnContext(ctx, "docker image event subscription closed; scheduled image polling remains active")
eventCh = nil
+ w.eventDegraded.Store(true)
continue
}
if !w.settingsService.GetBoolSetting(ctx, "imageEventWatcherEnabled", false) {
@@ -267,12 +283,8 @@ func (w *ImageUpdateWatcher) RunNow(ctx context.Context) error {
return admission.Err
}
- select {
- case <-ctx.Done():
- return ctx.Err()
- case err := <-admission.Value.Done():
- return err
- }
+ // Cancellation reaches the scan context; retain the execution guard until its worker exits.
+ return <-admission.Value.Done()
}
func (w *ImageUpdateWatcher) executeScanInternal(ctx context.Context) error {
@@ -298,12 +310,14 @@ func (w *ImageUpdateWatcher) executeScanInternal(ctx context.Context) error {
updates := 0
scanErrors := 0
- for _, result := range results {
+ targets := make([]schedulertypes.TargetOutcome, 0, len(results))
+ for imageID, result := range results {
if result == nil {
continue
}
if result.Error != "" {
scanErrors++
+ targets = append(targets, schedulertypes.TargetOutcome{ID: imageID, Status: schedulertypes.Failed, Message: result.Error})
continue
}
if result.HasUpdate {
@@ -311,6 +325,9 @@ func (w *ImageUpdateWatcher) executeScanInternal(ctx context.Context) error {
}
}
slog.InfoContext(ctx, "image scan run completed", "checked", len(results), "updates", updates, "errors", scanErrors)
+ if scanErrors > 0 {
+ return &schedulertypes.OutcomeError{Outcome: schedulertypes.Outcome{Status: schedulertypes.Partial, Message: "Some image update checks failed", Targets: targets}}
+ }
return nil
}
@@ -331,6 +348,7 @@ func (w *ImageUpdateWatcher) backfillProjectImageRefsInternal(ctx context.Contex
type imageUpdateWatcherActorInternal struct {
watcher *ImageUpdateWatcher
scanRunning bool
+ scanSubmitting bool
automaticPending bool
triggerGeneration uint64
metadataReady bool
@@ -378,10 +396,14 @@ func (a *imageUpdateWatcherActorInternal) receiveInternal(ctx *actors.Context, r
if message.Kind != imageUpdateScanCompletedMessageInternal {
return
}
- a.scanRunning = false
if message.Value.automatic {
- a.watcher.triggerIngress.Acknowledge(message.Value.triggerGeneration)
+ a.scanSubmitting = false
+ if message.Err == nil {
+ a.watcher.triggerIngress.Acknowledge(message.Value.triggerGeneration)
+ }
a.automaticPending = a.watcher.triggerIngress.Pending()
+ } else {
+ a.scanRunning = false
}
if message.Value.automatic && message.Err != nil && !errors.Is(message.Err, context.Canceled) {
slog.ErrorContext(ctx.Context(), "image update watcher scan failed", "error", message.Err)
@@ -400,7 +422,7 @@ func (a *imageUpdateWatcherActorInternal) receiveSignalInternal(ctx *actors.Cont
triggeredAt, generation := a.watcher.triggerIngress.Begin()
a.automaticPending = true
a.triggerGeneration = generation
- if a.metadataReady && !a.scanRunning {
+ if a.metadataReady && !a.scanRunning && !a.scanSubmitting {
a.debounceTimer.Reset(ctx, imageUpdateDebounceElapsedMessageInternal, max(time.Until(triggeredAt.Add(a.watcher.debounce)), 0))
}
return
@@ -425,7 +447,7 @@ func (a *imageUpdateWatcherActorInternal) receiveSignalInternal(ctx *actors.Cont
func (a *imageUpdateWatcherActorInternal) receiveTimerInternal(ctx *actors.Context, message actors.Message[imageUpdateMessageKindInternal, uint64]) {
if message.Kind == imageUpdateDebounceElapsedMessageInternal {
- if !a.debounceTimer.Current(message.Value) || !a.automaticPending || a.scanRunning {
+ if !a.debounceTimer.Current(message.Value) || !a.automaticPending || a.scanRunning || a.scanSubmitting {
return
}
if remaining := time.Until(a.watcher.triggerIngress.Latest().Add(a.watcher.debounce)); remaining > 0 {
@@ -461,7 +483,11 @@ func (a *imageUpdateWatcherActorInternal) startBackfillInternal(ctx *actors.Cont
}
func (a *imageUpdateWatcherActorInternal) startScanInternal(scanCtx context.Context, ctx *actors.Context, automatic bool, triggerGeneration uint64, done *actors.Promise[error]) {
- a.scanRunning = true
+ if automatic {
+ a.scanSubmitting = true
+ } else {
+ a.scanRunning = true
+ }
worker := actors.Worker[imageUpdateMessageKindInternal, imageUpdateScanCompletionInternal]{
Actor: ctx,
WorkContext: scanCtx,
@@ -469,11 +495,24 @@ func (a *imageUpdateWatcherActorInternal) startScanInternal(scanCtx context.Cont
CompletionKind: imageUpdateScanCompletedMessageInternal,
PanicValue: imageUpdateScanCompletionInternal{automatic: automatic, triggerGeneration: triggerGeneration, done: done},
ActorStopped: func(_ imageUpdateScanCompletionInternal, err error) {
- done.Resolve(err)
+ if done != nil {
+ done.Resolve(err)
+ }
},
}
worker.Start(func(workerCtx context.Context) (imageUpdateScanCompletionInternal, error) {
- return imageUpdateScanCompletionInternal{automatic: automatic, triggerGeneration: triggerGeneration, done: done}, a.watcher.executeScanInternal(workerCtx)
+ completion := imageUpdateScanCompletionInternal{automatic: automatic, triggerGeneration: triggerGeneration, done: done}
+ if automatic {
+ a.watcher.dispatchMu.RLock()
+ dispatcher := a.watcher.dispatcher
+ a.watcher.dispatchMu.RUnlock()
+ if dispatcher == nil {
+ return completion, errors.New("durable job dispatcher unavailable")
+ }
+ _, err := dispatcher.Submit(workerCtx, schedulertypes.Request{JobID: a.watcher.Name(), EnvironmentID: "0", Trigger: "watcher"})
+ return completion, err
+ }
+ return completion, a.watcher.executeScanInternal(workerCtx)
})
}
@@ -489,6 +528,21 @@ func (a *imageUpdateWatcherActorInternal) resetScheduleInternal(ctx *actors.Cont
}
}
- delay := time.Until(schedule.Next(time.Now().In(a.watcher.location)))
- a.scheduleTimer.Reset(ctx, imageUpdateScheduledPollMessageInternal, delay)
+ next := schedule.Next(time.Now().In(a.watcher.location))
+ a.watcher.dispatchMu.RLock()
+ dispatcher := a.watcher.dispatcher
+ a.watcher.dispatchMu.RUnlock()
+ if dispatcher != nil {
+ if err := dispatcher.Checkpoint(ctx.Context(), a.watcher.Name(), spec, next); err != nil {
+ slog.ErrorContext(ctx.Context(), "Failed to checkpoint image polling schedule", "error", err)
+ }
+ }
+ a.scheduleTimer.Reset(ctx, imageUpdateScheduledPollMessageInternal, time.Until(next))
+}
+
+// SetDispatcher connects automatic image scans to durable admission.
+func (w *ImageUpdateWatcher) SetDispatcher(dispatcher schedulertypes.Dispatcher) {
+ w.dispatchMu.Lock()
+ defer w.dispatchMu.Unlock()
+ w.dispatcher = dispatcher
}
diff --git a/backend/pkg/scheduler/image_update_watcher_test.go b/backend/pkg/scheduler/image_update_watcher_test.go
index 9dcaa7a9e..a1f5777c0 100644
--- a/backend/pkg/scheduler/image_update_watcher_test.go
+++ b/backend/pkg/scheduler/image_update_watcher_test.go
@@ -4,6 +4,7 @@ import (
"bytes"
"context"
"fmt"
+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
"log/slog"
"sync"
"testing"
@@ -196,7 +197,7 @@ func newImageUpdateWatcherForTestInternal(t *testing.T, scanner imageUpdateScann
defer cancel()
require.NoError(t, lifecycle.Stop(stopCtx))
})
- return &ImageUpdateWatcher{
+ watcher := &ImageUpdateWatcher{
imageUpdateService: scanner,
settingsService: settings,
environmentService: registryCredentialLoaderFakeInternal{},
@@ -212,6 +213,8 @@ func newImageUpdateWatcherForTestInternal(t *testing.T, scanner imageUpdateScann
started: make(chan struct{}),
stopped: make(chan struct{}),
}
+ watcher.SetDispatcher(&testDispatcherInternal{run: func(ctx context.Context, _ schedulertypes.Request) error { return watcher.RunNow(ctx) }})
+ return watcher
}
func startImageUpdateWatcherForTestInternal(t *testing.T, watcher *ImageUpdateWatcher) (context.CancelFunc, <-chan error) {
diff --git a/backend/pkg/scheduler/job_schedule_integration_test.go b/backend/pkg/scheduler/job_schedule_integration_test.go
index f1d9a4394..3534c5032 100644
--- a/backend/pkg/scheduler/job_schedule_integration_test.go
+++ b/backend/pkg/scheduler/job_schedule_integration_test.go
@@ -8,6 +8,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/job"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/kv"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/types/v2/jobschedule"
"github.com/libtnb/sqlite"
@@ -103,7 +104,7 @@ func openJobScheduleTestDatabaseInternal(t *testing.T, ctx context.Context, data
gormDB, err := gorm.Open(sqlite.Open(databasePath), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)})
require.NoError(t, err)
- require.NoError(t, gormDB.AutoMigrate(&settings.SettingVariable{}))
+ require.NoError(t, gormDB.AutoMigrate(&settings.SettingVariable{}, &kv.KVEntry{}))
db := &database.DB{DB: gormDB}
settingsService, err := newSettingsServiceForTestInternal(t, ctx, db)
diff --git a/backend/pkg/scheduler/jobcontext/context.go b/backend/pkg/scheduler/jobcontext/context.go
new file mode 100644
index 000000000..a75662b0c
--- /dev/null
+++ b/backend/pkg/scheduler/jobcontext/context.go
@@ -0,0 +1,42 @@
+// Package jobcontext carries durable execution progress through domain operations.
+package jobcontext
+
+import (
+ "context"
+
+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
+)
+
+type progressKeyInternal struct{}
+type runKeyInternal struct{}
+
+func WithExecution(ctx context.Context, run schedulertypes.Run, persist func(schedulertypes.TargetOutcome) error) context.Context {
+ ctx = context.WithValue(ctx, runKeyInternal{}, run)
+ if persist != nil {
+ ctx = context.WithValue(ctx, progressKeyInternal{}, persist)
+ }
+ return ctx
+}
+
+func Progress(ctx context.Context, target schedulertypes.TargetOutcome) error {
+ if persist, ok := ctx.Value(progressKeyInternal{}).(func(schedulertypes.TargetOutcome) error); ok {
+ return persist(target)
+ }
+ return nil
+}
+
+func Run(ctx context.Context) (schedulertypes.Run, bool) {
+ run, ok := ctx.Value(runKeyInternal{}).(schedulertypes.Run)
+ return run, ok
+}
+
+// ConfirmedTarget returns a terminal outcome only for a target whose completion
+// was persisted or confirmed against its activity by the execution coordinator.
+func ConfirmedTarget(run schedulertypes.Run, targetID string) schedulertypes.Outcome {
+ for _, target := range run.Outcome.Targets {
+ if target.ID == targetID && target.Status == schedulertypes.Succeeded {
+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded, ActivityID: target.ActivityID, Targets: run.Outcome.Targets}
+ }
+ }
+ return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: "The interrupted operation has no confirmed completion", Targets: run.Outcome.Targets}
+}
diff --git a/backend/pkg/scheduler/queue/queue.go b/backend/pkg/scheduler/queue/queue.go
new file mode 100644
index 000000000..c6a8da88a
--- /dev/null
+++ b/backend/pkg/scheduler/queue/queue.go
@@ -0,0 +1,464 @@
+// Package queue persists scheduler runs and coordinates their execution and recovery.
+package queue
+
+import (
+ "context"
+ "crypto/rand"
+ "io"
+ "log/slog"
+ "strings"
+ "sync"
+ "time"
+
+ "emperror.dev/errors"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/kv"
+ st "github.com/getarcaneapp/arcane/types/v2/scheduler"
+ "github.com/google/uuid"
+)
+
+// Queue persists work before dispatching it to the owning domain.
+type Queue struct {
+ store *kv.KVService
+ owner string
+ mu sync.Mutex
+ active map[string]bool
+ checkpointed map[string]bool
+ ordinary int
+ health int
+ stopping bool
+ started bool
+ cancel context.CancelFunc
+ done chan struct{}
+ wake chan struct{}
+ workers sync.WaitGroup
+ execute func(context.Context, st.Run) (st.Outcome, error)
+ reconcile func(context.Context, st.Run) (st.Outcome, error)
+}
+
+// Submit persists acceptance before waking the dispatcher. Run IDs deduplicate
+// requests within a job and environment. Manual and remote requests stay separate;
+// other triggers coalesce pending work.
+// Cancellation of ctx affects admission only, not an accepted run's lifetime.
+func (q *Queue) Submit(ctx context.Context, request st.Request) (st.Run, error) {
+ q.mu.Lock()
+ defer q.mu.Unlock()
+ if q.stopping {
+ return st.Run{}, errors.New("job queue is stopping")
+ }
+ if request.EnvironmentID == "" {
+ request.EnvironmentID = "0"
+ }
+ if request.JobID == "" {
+ return st.Run{}, errors.New("job ID required")
+ }
+ if request.Trigger == "" {
+ request.Trigger = "manual"
+ }
+ if request.RunID == "" {
+ request.RunID = uuid.NewString()
+ }
+ if _, err := uuid.Parse(request.RunID); err != nil {
+ return st.Run{}, errors.New("invalid run ID")
+ }
+ now := time.Now().UTC()
+ result := st.Run{ID: request.RunID, JobID: request.JobID, EnvironmentID: request.EnvironmentID, Trigger: request.Trigger, RequestedBy: request.RequestedBy, RequestedWithKey: request.RequestedWithKey, RemoteAccepted: request.Trigger == "remote", Status: st.Queued, CreatedAt: now, UpdatedAt: now}
+ err := q.mutateInternal(ctx, request.EnvironmentID, request.JobID, func(record *st.QueueRecord) error {
+ if request.Trigger == "scheduled" || request.Trigger == "recovery" {
+ record.LastEnqueuedAt = now
+ }
+ for _, existing := range record.Runs {
+ if existing.ID == request.RunID {
+ result = existing
+ return nil
+ }
+ if request.Trigger != "manual" && request.Trigger != "remote" && existing.Trigger != "manual" && existing.Trigger != "remote" && !existing.Status.Terminal() && existing.Status != st.Running {
+ result = existing
+ return nil
+ }
+ }
+ if existing, ok := record.Receipts[request.RunID]; ok {
+ result = existing
+ return nil
+ }
+ pruneRunsInternal(record, now)
+ record.Runs = append(record.Runs, result)
+ return nil
+ })
+ if err == nil {
+ q.signalInternal()
+ }
+ return result, err
+}
+
+func (q *Queue) signalInternal() {
+ select {
+ case q.wake <- struct{}{}:
+ default:
+ }
+}
+
+// Checkpoint records the next due occurrence atomically with overdue recovery.
+func (q *Queue) Checkpoint(ctx context.Context, jobID, schedule string, next time.Time) error {
+ q.mu.Lock()
+ defer q.mu.Unlock()
+ first := !q.checkpointed[jobID]
+ recovered := false
+ err := q.mutateInternal(ctx, "0", jobID, func(record *st.QueueRecord) error {
+ now := time.Now().UTC()
+ if first && record.Schedule == schedule && !record.NextRun.IsZero() && !record.NextRun.After(now) && record.LastEnqueuedAt.Before(record.NextRun) {
+ pending := false
+ for _, run := range record.Runs {
+ if !run.Status.Terminal() {
+ pending = true
+ break
+ }
+ }
+ if !pending {
+ recovered = true
+ record.LastEnqueuedAt = now
+ record.Runs = append(record.Runs, st.Run{ID: uuid.NewString(), JobID: jobID, EnvironmentID: "0", Trigger: "recovery", Status: st.Queued, CreatedAt: now, UpdatedAt: now})
+ }
+ }
+ record.Schedule = schedule
+ record.NextRun = next
+ return nil
+ })
+ if err == nil {
+ q.checkpointed[jobID] = true
+ if recovered {
+ q.signalInternal()
+ }
+ }
+ return err
+}
+
+// Start validates persisted state and starts dispatching on ctx. Repeated calls
+// are harmless, but a stopped queue cannot be restarted; create a new instance.
+func (q *Queue) Start(ctx context.Context) error {
+ q.mu.Lock()
+ defer q.mu.Unlock()
+ if q.started {
+ return nil
+ }
+ if q.execute == nil {
+ return errors.New("job executor unavailable")
+ }
+ // Fail startup if persisted state cannot be read. Never replace it with empty state.
+ if _, err := q.Records(ctx); err != nil {
+ return err
+ }
+ runCtx, cancel := context.WithCancel(ctx)
+ q.cancel = cancel
+ q.started = true
+ go q.dispatchInternal(runCtx)
+ return nil
+}
+
+// Stop rejects new submissions, cancels execution, and waits for workers until
+// ctx expires. Unfinished claims remain persisted for startup reconciliation.
+func (q *Queue) Stop(ctx context.Context) error {
+ q.mu.Lock()
+ q.stopping = true
+ started := q.started
+ if q.cancel != nil {
+ q.cancel()
+ }
+ q.mu.Unlock()
+ if !started {
+ return nil
+ }
+ select {
+ case <-q.done:
+ return nil
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+}
+
+func (q *Queue) dispatchInternal(ctx context.Context) {
+ defer close(q.done)
+ defer q.workers.Wait()
+ nextRetention := time.Now().Add(time.Hour)
+ for ctx.Err() == nil {
+ if !time.Now().Before(nextRetention) {
+ if err := q.pruneInternal(ctx); err != nil && ctx.Err() == nil {
+ slog.ErrorContext(ctx, "Job retention failed", "error", err)
+ }
+ nextRetention = time.Now().Add(time.Hour)
+ }
+ nextRetry, err := q.dispatchReadyInternal(ctx)
+ if err != nil && ctx.Err() == nil {
+ slog.ErrorContext(ctx, "Job queue dispatch failed", "error", err)
+ nextRetry = time.Now().Add(5 * time.Second)
+ }
+ nextWake := nextRetention
+ if !nextRetry.IsZero() && nextRetry.Before(nextWake) {
+ nextWake = nextRetry
+ }
+ timer := time.NewTimer(max(time.Until(nextWake), 0))
+ select {
+ case <-ctx.Done():
+ timer.Stop()
+ return
+ case <-q.wake:
+ case <-timer.C:
+ }
+ timer.Stop()
+ }
+}
+
+func (q *Queue) dispatchReadyInternal(ctx context.Context) (time.Time, error) {
+ records, err := q.Records(ctx)
+ if err != nil {
+ return time.Time{}, err
+ }
+ var nextRetry time.Time
+ for _, record := range records {
+ key := queueKeyInternal(record.EnvironmentID, record.JobID)
+ reserved := strings.HasPrefix(record.JobID, "environment-health") || record.JobID == "docker-client-refresh" || record.JobID == "activity-sweep"
+ q.mu.Lock()
+ if q.active[key] || q.stopping || (!reserved && q.ordinary >= 4) || (reserved && q.health >= 2) {
+ q.mu.Unlock()
+ continue
+ }
+ selected, retryAt := q.nextRunInternal(record.Runs)
+ if !retryAt.IsZero() && (nextRetry.IsZero() || retryAt.Before(nextRetry)) {
+ nextRetry = retryAt
+ }
+ if selected == nil {
+ q.mu.Unlock()
+ continue
+ }
+ run := *selected
+ reconcile := run.Status == st.Running
+ err = q.UpdateRun(ctx, run, func(current *st.Run) error {
+ if current.Status != run.Status || current.Owner != run.Owner {
+ return ErrRunConflict
+ }
+ now := time.Now().UTC()
+ current.Status = st.Running
+ current.Owner = q.owner
+ current.UpdatedAt = now
+ current.StartedAt = &now
+ current.NextAttempt = nil
+ current.AttemptCount++
+ current.Attempts = append(current.Attempts, st.Attempt{Number: current.AttemptCount, StartedAt: now})
+ if len(current.Attempts) > 100 {
+ current.Attempts = current.Attempts[len(current.Attempts)-100:]
+ }
+ // Preserve the previous outcome for reconciliation and completed target filtering.
+ run = *current
+ return nil
+ })
+ if err != nil {
+ q.mu.Unlock()
+ return time.Time{}, err
+ }
+ q.active[key] = true
+ if reserved {
+ q.health++
+ } else {
+ q.ordinary++
+ }
+ q.workers.Add(1)
+ q.mu.Unlock()
+ go q.executeInternal(ctx, key, run, reserved, reconcile)
+ }
+ return nextRetry, nil
+}
+
+func (q *Queue) executeInternal(ctx context.Context, key string, run st.Run, reserved, reconcile bool) {
+ defer q.workers.Done()
+ defer func() {
+ q.mu.Lock()
+ delete(q.active, key)
+ if reserved {
+ q.health--
+ } else {
+ q.ordinary--
+ }
+ q.mu.Unlock()
+ q.signalInternal()
+ }()
+ outcome, err := q.invokeInternal(ctx, run, reconcile)
+ if outcome.Status == "" {
+ if err != nil {
+ outcome.Status = st.Failed
+ } else {
+ outcome.Status = st.Succeeded
+ }
+ }
+ if err != nil && outcome.Message == "" {
+ outcome.Message = err.Error()
+ }
+ // Shutdown leaves a running claim for startup reconciliation, never a false success.
+ if ctx.Err() != nil {
+ return
+ }
+ q.persistOutcomeInternal(ctx, run, outcome)
+}
+
+func (q *Queue) persistOutcomeInternal(ctx context.Context, run st.Run, outcome st.Outcome) {
+ // Keep the execution slot until the terminal result has been durably recorded.
+ for {
+ err := q.UpdateRun(ctx, run, func(current *st.Run) error {
+ if current.Owner != q.owner || current.Status != st.Running {
+ return ErrRunConflict
+ }
+ now := time.Now().UTC()
+ current.Status = outcome.Status
+ current.UpdatedAt = now
+ outcome = mergeTargetProgressInternal(current.Outcome, outcome)
+ current.Outcome = outcome
+ if outcome.Status == st.Retrying || outcome.Status == st.Waiting {
+ delay := 5 * time.Second * time.Duration(1<<min(run.AttemptCount-1, 6))
+ delay = min(delay, 5*time.Minute)
+ var jitter [1]byte
+ if _, jitterErr := io.ReadFull(rand.Reader, jitter[:]); jitterErr != nil {
+ // Keep the unjittered backoff when randomness is unavailable.
+ slog.WarnContext(ctx, "Retry jitter unavailable; using exponential backoff", "runId", run.ID, "error", jitterErr)
+ } else {
+ delay = time.Duration(float64(delay) * (0.8 + float64(jitter[0])/255*0.2))
+ }
+ next := now.Add(delay)
+ current.NextAttempt = &next
+ } else {
+ current.NextAttempt = nil
+ }
+ if outcome.Status.Terminal() {
+ current.FinishedAt = &now
+ }
+ if len(current.Attempts) > 0 {
+ last := &current.Attempts[len(current.Attempts)-1]
+ last.FinishedAt = &now
+ last.Outcome = outcome
+ }
+ return nil
+ })
+ if err == nil || errors.Is(err, ErrRunConflict) {
+ return
+ }
+ slog.ErrorContext(ctx, "Unable to persist job result; retaining execution guard", "runId", run.ID, "error", err)
+ timer := time.NewTimer(time.Second)
+ select {
+ case <-ctx.Done():
+ timer.Stop()
+ return
+ case <-timer.C:
+ }
+ }
+}
+
+func (q *Queue) invokeInternal(ctx context.Context, run st.Run, reconcile bool) (outcome st.Outcome, err error) {
+ defer func() {
+ if recovered := recover(); recovered != nil {
+ outcome = st.Outcome{Status: st.NeedsAttention, Message: "Job panicked; inspect logs before retrying"}
+ err = errors.Errorf("job panic: %v", recovered)
+ slog.ErrorContext(ctx, "Job panicked", "runId", run.ID, "error", err)
+ }
+ }()
+ if reconcile && q.reconcile != nil {
+ return q.reconcile(ctx, run)
+ }
+ return q.execute(ctx, run)
+}
+
+// Cancel marks an unstarted queued run canceled. Runs already attempted locally
+// or dispatched remotely must be reconciled rather than canceled as pending work.
+func (q *Queue) Cancel(ctx context.Context, environmentID, jobID, runID string) (st.Run, error) {
+ run, err := q.Get(ctx, environmentID, jobID, runID)
+ if err != nil {
+ return run, err
+ }
+ err = q.UpdateRun(ctx, run, func(current *st.Run) error {
+ if current.Status != st.Queued || current.AttemptCount != 0 || current.RemoteDeliveryAttempted {
+ return errors.New("only unstarted pending runs can be canceled")
+ }
+ now := time.Now().UTC()
+ current.Status = st.Canceled
+ current.UpdatedAt = now
+ current.FinishedAt = &now
+ run = *current
+ return nil
+ })
+ return run, err
+}
+
+// Retry requeues a failed, partial, or attention-required run without changing its
+// ID or dropping target progress. Unconfirmed remote work cannot be retried here.
+func (q *Queue) Retry(ctx context.Context, environmentID, jobID, runID string) (st.Run, error) {
+ run, err := q.Get(ctx, environmentID, jobID, runID)
+ if err != nil {
+ return run, err
+ }
+ err = q.UpdateRun(ctx, run, func(current *st.Run) error {
+ if current.Status != st.Failed && current.Status != st.NeedsAttention && current.Status != st.Partial {
+ return errors.New("run is not eligible for retry")
+ }
+ if current.RemoteAccepted && !current.RemoteSettled {
+ return errors.New("remote outcome must be confirmed before retry")
+ }
+ current.Status = st.Queued
+ current.Owner = ""
+ current.NextAttempt = nil
+ current.FinishedAt = nil
+ current.UpdatedAt = time.Now().UTC()
+ run = *current
+ return nil
+ })
+ if err == nil {
+ q.signalInternal()
+ }
+ return run, err
+}
+
+func (q *Queue) nextRunInternal(runs []st.Run) (*st.Run, time.Time) {
+ var interrupted *st.Run
+ blocked := false
+ for index := range runs {
+ run := &runs[index]
+ if run.Status == st.NeedsAttention || (run.Status == st.Running && run.Owner == q.owner) {
+ blocked = true
+ }
+ if run.Status == st.Running && run.Owner != q.owner && interrupted == nil {
+ interrupted = run
+ }
+ }
+ if interrupted != nil || blocked {
+ return interrupted, time.Time{}
+ }
+ var nextRetry time.Time
+ now := time.Now()
+ for index := range runs {
+ run := &runs[index]
+ if run.Status.Terminal() {
+ continue
+ }
+ if run.NextAttempt == nil || !run.NextAttempt.After(now) {
+ return run, time.Time{}
+ }
+ if nextRetry.IsZero() || run.NextAttempt.Before(nextRetry) {
+ nextRetry = *run.NextAttempt
+ }
+ }
+ return nil, nextRetry
+}
+
+func mergeTargetProgressInternal(previous, outcome st.Outcome) st.Outcome {
+ for _, target := range previous.Targets {
+ found := false
+ for _, updated := range outcome.Targets {
+ if updated.ID == target.ID {
+ found = true
+ break
+ }
+ }
+ if !found {
+ outcome.Targets = append(outcome.Targets, target)
+ }
+ }
+ if outcome.ActivityID == "" {
+ outcome.ActivityID = previous.ActivityID
+ }
+ return outcome
+}
diff --git a/backend/pkg/scheduler/queue/queue_store.go b/backend/pkg/scheduler/queue/queue_store.go
new file mode 100644
index 000000000..92df79e87
--- /dev/null
+++ b/backend/pkg/scheduler/queue/queue_store.go
@@ -0,0 +1,212 @@
+package queue
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json/v2"
+ "slices"
+ "sort"
+ "time"
+
+ "emperror.dev/errors"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/kv"
+ st "github.com/getarcaneapp/arcane/types/v2/scheduler"
+ "github.com/google/uuid"
+)
+
+const queuePrefixInternal = "jobs."
+
+func queueKeyInternal(environmentID, jobID string) string {
+ sum := sha256.Sum256([]byte(environmentID + "\x00" + jobID))
+ return queuePrefixInternal + hex.EncodeToString(sum[:])
+}
+
+func (q *Queue) mutateInternal(ctx context.Context, environmentID, jobID string, change func(*st.QueueRecord) error) error {
+ key := queueKeyInternal(environmentID, jobID)
+ for range 20 {
+ previous, found, err := q.store.Get(ctx, key)
+ if err != nil {
+ return err
+ }
+ record := st.QueueRecord{JobID: jobID, EnvironmentID: environmentID}
+ if found {
+ if err := json.Unmarshal([]byte(previous), &record); err != nil {
+ return errors.WrapIf(err, "decode job queue")
+ }
+ }
+ if err := change(&record); err != nil {
+ return err
+ }
+ next, err := json.Marshal(record)
+ if err != nil {
+ return err
+ }
+ if found && string(next) == previous {
+ return nil
+ }
+ var swapped bool
+ if found {
+ swapped, err = q.store.CompareAndSwap(ctx, key, previous, string(next))
+ } else {
+ swapped, err = q.store.CreateIfAbsent(ctx, key, string(next))
+ }
+ if err != nil {
+ return err
+ }
+ if swapped {
+ return nil
+ }
+ }
+ return errors.New("job queue contention; retry request with the same run ID")
+}
+
+// Records reads all persisted queue records, including run history and receipts.
+// Storage or decoding errors fail the read instead of returning an incomplete snapshot.
+func (q *Queue) Records(ctx context.Context) ([]st.QueueRecord, error) {
+ entries, err := q.store.ListByPrefix(ctx, queuePrefixInternal)
+ if err != nil {
+ return nil, err
+ }
+ records := make([]st.QueueRecord, 0, len(entries))
+ for _, entry := range entries {
+ var record st.QueueRecord
+ if err := json.Unmarshal([]byte(entry.Value), &record); err != nil {
+ return nil, errors.WrapIff(err, "decode job queue %s", entry.Key)
+ }
+ records = append(records, record)
+ }
+ return records, nil
+}
+
+// Get retrieves a run by environment, job, and run ID, including compacted
+// idempotency receipts. It returns ErrRunNotFound when no matching record exists.
+func (q *Queue) Get(ctx context.Context, environmentID, jobID, runID string) (st.Run, error) {
+ raw, found, err := q.store.Get(ctx, queueKeyInternal(environmentID, jobID))
+ if err != nil {
+ return st.Run{}, err
+ }
+ if !found {
+ return st.Run{}, ErrRunNotFound
+ }
+ var record st.QueueRecord
+ if err := json.Unmarshal([]byte(raw), &record); err != nil {
+ return st.Run{}, err
+ }
+ for _, run := range record.Runs {
+ if run.ID == runID {
+ return run, nil
+ }
+ }
+ if run, ok := record.Receipts[runID]; ok {
+ return run, nil
+ }
+ return st.Run{}, ErrRunNotFound
+}
+
+// List returns retained run history newest first, excluding compacted receipts.
+// An empty job ID includes every job in the environment; pages are one-based.
+func (q *Queue) List(ctx context.Context, environmentID, jobID string, page, limit int) (st.RunList, error) {
+ records, err := q.Records(ctx)
+ if err != nil {
+ return st.RunList{}, err
+ }
+ runs := []st.Run{}
+ for _, record := range records {
+ if record.EnvironmentID == environmentID && (jobID == "" || record.JobID == jobID) {
+ runs = append(runs, record.Runs...)
+ }
+ }
+ sort.Slice(runs, func(i, j int) bool { return runs[i].CreatedAt.After(runs[j].CreatedAt) })
+ page = max(1, page)
+ limit = min(100, max(1, limit))
+ start := len(runs)
+ if page-1 <= len(runs)/limit {
+ start = min(len(runs), (page-1)*limit)
+ }
+ end := min(len(runs), start+limit)
+ return st.RunList{Runs: runs[start:end], Total: len(runs), Page: page, Limit: limit}, nil
+}
+
+// UpdateRun atomically changes a persisted run or terminal receipt using compare
+// and swap. The callback may run more than once and must not perform external side effects.
+func (q *Queue) UpdateRun(ctx context.Context, run st.Run, change func(*st.Run) error) error {
+ wake := false
+ err := q.mutateInternal(ctx, run.EnvironmentID, run.JobID, func(record *st.QueueRecord) error {
+ for index := range record.Runs {
+ if record.Runs[index].ID == run.ID {
+ previous := record.Runs[index].Status
+ if err := change(&record.Runs[index]); err != nil {
+ return err
+ }
+ wake = previous != st.Queued && record.Runs[index].Status == st.Queued
+ return nil
+ }
+ }
+ if receipt, found := record.Receipts[run.ID]; found {
+ if err := change(&receipt); err != nil {
+ return err
+ }
+ if !receipt.Status.Terminal() {
+ return errors.New("archived runs cannot be restarted")
+ }
+ record.Receipts[run.ID] = receipt
+ return nil
+ }
+ return ErrRunNotFound
+ })
+ if err == nil && wake {
+ q.signalInternal()
+ }
+ return err
+}
+
+func (q *Queue) pruneInternal(ctx context.Context) error {
+ records, err := q.Records(ctx)
+ if err != nil {
+ return err
+ }
+ now := time.Now().UTC()
+ for _, record := range records {
+ if err := q.mutateInternal(ctx, record.EnvironmentID, record.JobID, func(current *st.QueueRecord) error {
+ pruneRunsInternal(current, now)
+ return nil
+ }); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func pruneRunsInternal(record *st.QueueRecord, now time.Time) {
+ terminal := 0
+ for index, run := range slices.Backward(record.Runs) {
+ if !run.Status.Terminal() || (run.RemoteAccepted && !run.RemoteSettled) {
+ continue
+ }
+ terminal++
+ if terminal <= 100 && now.Sub(run.UpdatedAt) < 7*24*time.Hour {
+ continue
+ }
+ if run.Trigger == "manual" || run.Trigger == "remote" {
+ if record.Receipts == nil {
+ record.Receipts = make(map[string]st.Run)
+ }
+ run.Attempts = nil
+ run.Outcome.Targets = nil
+ record.Receipts[run.ID] = run
+ }
+ record.Runs = append(record.Runs[:index], record.Runs[index+1:]...)
+ }
+}
+
+// ErrRunNotFound indicates that neither a run nor its receipt exists.
+var ErrRunNotFound = errors.Sentinel("job run not found")
+
+// ErrRunConflict indicates that execution ownership or state changed.
+var ErrRunConflict = errors.Sentinel("job run state changed")
+
+// New uses the application's existing KV store; no in-memory admission exists.
+func New(store *kv.KVService, execute, reconcile func(context.Context, st.Run) (st.Outcome, error)) *Queue {
+ return &Queue{execute: execute, reconcile: reconcile, store: store, owner: uuid.NewString(), active: make(map[string]bool), checkpointed: make(map[string]bool), wake: make(chan struct{}, 1), done: make(chan struct{})}
+}
diff --git a/backend/pkg/scheduler/schedule/cron.go b/backend/pkg/scheduler/schedule/cron.go
new file mode 100644
index 000000000..cf91062a3
--- /dev/null
+++ b/backend/pkg/scheduler/schedule/cron.go
@@ -0,0 +1,32 @@
+// Package schedule provides reusable schedule validation.
+package schedule
+
+import (
+ "strings"
+
+ "emperror.dev/errors"
+
+ "github.com/robfig/cron/v3"
+)
+
+// NormalizeSixField validates and normalizes a six-field cron schedule.
+func NormalizeSixField(value, subject string) (string, error) {
+ fields := strings.Fields(value)
+ if len(fields) == 0 {
+ return "", errors.Errorf("%s schedule is required", subject)
+ }
+ if len(fields) != 6 {
+ return "", errors.Errorf("invalid %s schedule %q: expected six fields", subject, strings.TrimSpace(value))
+ }
+ normalized := strings.Join(fields, " ")
+ parser := Parser()
+ if _, err := parser.Parse(normalized); err != nil {
+ return "", errors.WrapIff(err, "invalid %s schedule %q", subject, normalized)
+ }
+ return normalized, nil
+}
+
+// Parser is shared by schedule validation, execution, and next-run display.
+func Parser() cron.Parser {
+ return cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor)
+}
diff --git a/backend/pkg/scheduler/scheduled_prune_job.go b/backend/pkg/scheduler/scheduled_prune_job.go
index bb701b80a..7c058b407 100644
--- a/backend/pkg/scheduler/scheduled_prune_job.go
+++ b/backend/pkg/scheduler/scheduled_prune_job.go
@@ -4,12 +4,15 @@ import (
"context"
"log/slog"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
+
"github.com/getarcaneapp/arcane/backend/v2/internal/notification"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
systemtypes "github.com/getarcaneapp/arcane/types/v2/system"
"github.com/getarcaneapp/arcane/backend/v2/internal/system"
- "github.com/robfig/cron/v3"
+ scheduleutil "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/schedule"
)
const ScheduledPruneJobName = "scheduled-prune"
@@ -42,7 +45,7 @@ func (j *ScheduledPruneJob) Schedule(ctx context.Context) string {
schedule = "0 0 0 * * *"
}
- parser := cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow)
+ parser := scheduleutil.Parser()
if _, err := parser.Parse(schedule); err != nil {
slog.WarnContext(ctx, "Invalid cron expression for scheduled-prune, using default", "invalid_schedule", schedule, "error", err)
return "0 0 0 * * *"
@@ -51,18 +54,18 @@ func (j *ScheduledPruneJob) Schedule(ctx context.Context) string {
return schedule
}
-func (j *ScheduledPruneJob) Run(ctx context.Context) {
+func (j *ScheduledPruneJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {
enabled := j.settingsService.GetBoolSetting(ctx, "scheduledPruneEnabled", false)
if !enabled {
slog.DebugContext(ctx, "scheduled prune disabled; skipping run")
- return
+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil
}
req := buildScheduledPruneRequestInternal(ctx, j.settingsService)
if !hasScheduledPruneTargetsInternal(req) {
slog.InfoContext(ctx, "scheduled prune run skipped; no resource types selected")
- return
+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil
}
slog.InfoContext(ctx, "scheduled prune run started",
@@ -76,11 +79,11 @@ func (j *ScheduledPruneJob) Run(ctx context.Context) {
result, started, err := j.systemService.PruneAll(ctx, "0", req)
if err != nil {
slog.ErrorContext(ctx, "scheduled prune run failed", "error", err)
- return
+ return schedulertypes.Outcome{}, err
}
if !started {
slog.InfoContext(ctx, "scheduled prune run skipped; prune already in progress", "activityId", result.ActivityID)
- return
+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil
}
slog.InfoContext(ctx, "scheduled prune run completed",
@@ -100,6 +103,15 @@ func (j *ScheduledPruneJob) Run(ctx context.Context) {
if err := j.notificationService.SendPruneReportNotification(ctx, result); err != nil {
slog.WarnContext(ctx, "failed to send prune report notification", "error", err)
}
+ outcome := schedulertypes.Outcome{Status: schedulertypes.Succeeded}
+ if result.ActivityID != nil {
+ outcome.ActivityID = *result.ActivityID
+ }
+ if len(result.Errors) > 0 {
+ outcome.Status = schedulertypes.Partial
+ outcome.Message = "Some resources could not be pruned; see activity details"
+ }
+ return outcome, nil
}
func (j *ScheduledPruneJob) Reschedule(ctx context.Context) error {
@@ -177,3 +189,7 @@ func buildScheduledBuildCachePruneOptionsInternal(ctx context.Context, settingsS
Until: settingsService.GetStringSetting(ctx, "pruneBuildCacheUntil", ""),
}
}
+
+func (j *ScheduledPruneJob) Reconcile(_ context.Context, previous schedulertypes.Run) (schedulertypes.Outcome, error) {
+ return jobcontext.ConfirmedTarget(previous, "scheduled-prune"), nil
+}
diff --git a/backend/pkg/scheduler/scheduler.go b/backend/pkg/scheduler/scheduler.go
index de206d1ba..5c4638772 100644
--- a/backend/pkg/scheduler/scheduler.go
+++ b/backend/pkg/scheduler/scheduler.go
@@ -10,6 +10,7 @@ import (
"emperror.dev/errors"
"github.com/getarcaneapp/arcane/backend/v2/internal/actors"
+ scheduleutil "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/schedule"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
"github.com/robfig/cron/v3"
@@ -18,7 +19,7 @@ import (
// cronScheduleParser is the shared parser for all cron settings: six fields
// with seconds, plus @-descriptors. The image update watcher parses its poll
// schedule with the same spec so Jobs-UI cron values behave identically.
-var cronScheduleParser = cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor)
+var cronScheduleParser = scheduleutil.Parser()
var errJobSchedulerStoppedInternal = errors.Sentinel("job scheduler stopped")
@@ -30,7 +31,7 @@ type schedulerStateInternal struct {
runners map[string]*actors.Runner
entryIDs map[string]cron.EntryID
schedules map[string]string
- runLocks map[string]*sync.Mutex
+ supervisors map[string]*watcherSupervisorInternal
stopping bool
}
@@ -47,12 +48,15 @@ type schedulerShutdownInternal struct {
// jobSchedulerInternal serializes its control plane through one shared actor executor.
// robfig/cron remains the timing engine and invokes job bodies outside the actor.
type jobSchedulerInternal struct {
- cron *cron.Cron
- parser cron.Parser
- context context.Context
- location *time.Location
- runtime *actors.Runtime
- state *actors.State[schedulerStateInternal]
+ cron *cron.Cron
+ parser cron.Parser
+ context context.Context
+ location *time.Location
+ runtime *actors.Runtime
+ state *actors.State[schedulerStateInternal]
+ cancel context.CancelFunc
+ dispatchMu sync.RWMutex
+ dispatcher schedulertypes.Dispatcher
}
// NewJobScheduler creates an actor-owned scheduler control plane.
@@ -74,7 +78,7 @@ func NewJobScheduler(ctx context.Context, runtime *actors.Runtime, location *tim
runners: make(map[string]*actors.Runner),
entryIDs: make(map[string]cron.EntryID),
schedules: make(map[string]string),
- runLocks: make(map[string]*sync.Mutex),
+ supervisors: make(map[string]*watcherSupervisorInternal),
}
state, err := actors.NewState(ctx, runtime, "scheduler", "control-plane", 3, initial, func(value schedulerStateInternal) schedulerStateInternal {
value.jobs = slices.Clone(value.jobs)
@@ -84,17 +88,19 @@ func NewJobScheduler(ctx context.Context, runtime *actors.Runtime, location *tim
value.runners = maps.Clone(value.runners)
value.entryIDs = maps.Clone(value.entryIDs)
value.schedules = maps.Clone(value.schedules)
- value.runLocks = maps.Clone(value.runLocks)
+ value.supervisors = maps.Clone(value.supervisors)
return value
})
if err != nil {
return nil, err
}
+ lifetime, cancel := context.WithCancel(ctx)
parser := cronScheduleParser
scheduler := &jobSchedulerInternal{
cron: cron.New(cron.WithParser(parser), cron.WithLocation(location)),
parser: parser,
- context: ctx,
+ context: lifetime,
+ cancel: cancel,
location: location,
runtime: runtime,
state: state,
@@ -127,11 +133,16 @@ func (js *jobSchedulerInternal) RegisterBusWatcher(watcher schedulertypes.BusWat
if state.stopping {
return errJobSchedulerStoppedInternal
}
- runner, err := actors.NewRunner(js.context, js.runtime, "scheduler-watchers", watcher.Name(), "bus watcher "+watcher.Name(), 3, watcher.Start) //nolint:contextcheck // watcher lifetime belongs to the scheduler, not one state-actor receiver incarnation.
+ if _, exists := state.allWatchers[watcher.Name()]; exists {
+ return errors.New("watcher already registered")
+ }
+ supervisor := newWatcherSupervisorInternal(watcher)
+ runner, err := actors.NewRunner(js.context, js.runtime, "scheduler-watchers", watcher.Name(), "bus watcher "+watcher.Name(), 3, supervisor.runInternal) //nolint:contextcheck // watcher lifetime belongs to the scheduler, not one state-actor receiver incarnation.
if err != nil {
return err
}
state.runners[watcher.Name()] = runner
+ state.supervisors[watcher.Name()] = supervisor
state.allWatchers[watcher.Name()] = watcher
if canRunManually {
state.watchers[watcher.Name()] = watcher
@@ -301,6 +312,7 @@ func (js *jobSchedulerInternal) Stop(ctx context.Context) error {
}
}
+ js.cancel()
for _, registration := range shutdown.watchers {
stopErr = errors.Combine(stopErr, registration.runner.Stop(ctx))
if stopper, ok := registration.watcher.(schedulertypes.StoppableBusWatcher); ok {
@@ -344,17 +356,17 @@ func (js *jobSchedulerInternal) upsertJobInternal(ctx context.Context, state *sc
slog.DebugContext(ctx, "Job disabled; not scheduling", "name", jobName)
}
+ if shouldSchedule {
+ if err := js.checkpointInternal(ctx, jobName, schedule, parsedSchedule.Next(time.Now().In(js.location))); err != nil {
+ return err
+ }
+ }
if hadPreviousEntry {
js.cron.Remove(previousEntryID)
delete(state.entryIDs, jobName)
}
if shouldSchedule {
- runLock, ok := state.runLocks[jobName]
- if !ok {
- runLock = &sync.Mutex{}
- state.runLocks[jobName] = runLock
- }
- entryID, nextRun = js.addCronEntryInternal(job, schedule, parsedSchedule, runLock)
+ entryID, nextRun = js.addCronEntryInternal(job, schedule, parsedSchedule)
state.entryIDs[jobName] = entryID
}
state.jobsByID[jobName] = job
@@ -369,17 +381,16 @@ func (js *jobSchedulerInternal) upsertJobInternal(ctx context.Context, state *sc
return nil
}
-func (js *jobSchedulerInternal) addCronEntryInternal(job schedulertypes.Job, schedule string, parsedSchedule cron.Schedule, runLock *sync.Mutex) (cron.EntryID, *time.Time) {
+func (js *jobSchedulerInternal) addCronEntryInternal(job schedulertypes.Job, schedule string, parsedSchedule cron.Schedule) (cron.EntryID, *time.Time) {
entryID := js.cron.Schedule(parsedSchedule, cron.FuncJob(func() {
- if !runLock.TryLock() {
- slog.WarnContext(js.context, "Job skipped; previous run still in progress", "name", job.Name(), "schedule", schedule)
+ defer utils.RecoverToError(nil, "scheduled job admission", "name", job.Name())
+ if _, err := js.Submit(js.context, schedulertypes.Request{JobID: job.Name(), EnvironmentID: "0", Trigger: "scheduled"}); err != nil {
+ slog.ErrorContext(js.context, "Failed to queue scheduled job", "name", job.Name(), "error", err)
return
}
- defer runLock.Unlock()
- defer utils.RecoverToError(nil, "scheduled job", "name", job.Name(), "schedule", schedule)
- slog.InfoContext(js.context, "Job starting", "name", job.Name(), "schedule", schedule)
- job.Run(js.context)
- slog.InfoContext(js.context, "Job finished", "name", job.Name())
+ if err := js.checkpointInternal(js.context, job.Name(), schedule, parsedSchedule.Next(time.Now().In(js.location))); err != nil {
+ slog.ErrorContext(js.context, "Failed to checkpoint scheduled job", "name", job.Name(), "error", err)
+ }
}))
entry := js.cron.Entry(entryID)
diff --git a/backend/pkg/scheduler/scheduler_test.go b/backend/pkg/scheduler/scheduler_test.go
index 13fd2884f..e73c67240 100644
--- a/backend/pkg/scheduler/scheduler_test.go
+++ b/backend/pkg/scheduler/scheduler_test.go
@@ -21,10 +21,11 @@ func (j *testSchedulerJob) Name() string { return j.name }
func (j *testSchedulerJob) Schedule(context.Context) string { return j.schedule }
-func (j *testSchedulerJob) Run(ctx context.Context) {
+func (j *testSchedulerJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {
if j.run != nil {
j.run(ctx)
}
+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil
}
type conditionalTestSchedulerJob struct {
@@ -415,7 +416,7 @@ func TestJobScheduler_AddJob_GenericJobWithoutShouldRunIsScheduled(t *testing.T)
job := &schedulertypes.GenericJob{
JobName: "generic-dyn",
ScheduleFn: func(context.Context) string { return "@every 1m" },
- RunFn: func(context.Context) {},
+ RunFn: func(context.Context) (schedulertypes.Outcome, error) { return schedulertypes.Outcome{}, nil },
}
require.NoError(t, js.AddJob(context.Background(), job))
require.True(t, js.HasJob(job.Name()))
diff --git a/backend/pkg/scheduler/supervision.go b/backend/pkg/scheduler/supervision.go
new file mode 100644
index 000000000..f642279b1
--- /dev/null
+++ b/backend/pkg/scheduler/supervision.go
@@ -0,0 +1,192 @@
+package scheduler
+
+import (
+ "context"
+ "crypto/rand"
+ "math/big"
+ "sync"
+ "time"
+
+ "emperror.dev/errors"
+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
+)
+
+type watcherSupervisorInternal struct {
+ watcher schedulertypes.BusWatcher
+ mu sync.Mutex
+ health schedulertypes.WorkerHealth
+ restart chan struct{}
+ cancel context.CancelFunc
+}
+
+func newWatcherSupervisorInternal(watcher schedulertypes.BusWatcher) *watcherSupervisorInternal {
+ return &watcherSupervisorInternal{watcher: watcher, restart: make(chan struct{}, 1), health: schedulertypes.WorkerHealth{Status: "starting", UpdatedAt: time.Now()}}
+}
+
+func (s *watcherSupervisorInternal) setHealthInternal(status string, err error, next *time.Time) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ s.health = schedulertypes.WorkerHealth{Status: status, NextRetry: next, UpdatedAt: time.Now()}
+ if err != nil {
+ s.health.LastError = err.Error()
+ }
+}
+
+func (s *watcherSupervisorInternal) runInternal(ctx context.Context) error {
+ delay := 5 * time.Second
+ unexpected := 0
+ for ctx.Err() == nil {
+ runCtx, cancel := context.WithCancel(ctx)
+ s.mu.Lock()
+ s.cancel = cancel
+ s.mu.Unlock()
+ s.setHealthInternal("running", nil, nil)
+ started := time.Now()
+ panicked, err := s.startInternal(runCtx)
+ cancel()
+ if ctx.Err() != nil {
+ break
+ }
+ select {
+ case <-s.restart:
+ unexpected = 0
+ delay = 5 * time.Second
+ continue
+ default:
+ }
+ if panicked || err == nil {
+ unexpected++
+ }
+ if err == nil {
+ err = errors.New("worker exited unexpectedly")
+ }
+ if unexpected >= 3 {
+ s.setHealthInternal("needs_attention", err, nil)
+ select {
+ case <-ctx.Done():
+ case <-s.restart:
+ unexpected = 0
+ delay = 5 * time.Second
+ }
+ continue
+ }
+ if time.Since(started) > 5*time.Minute {
+ delay = 5 * time.Second
+ unexpected = 0
+ }
+ jitter, jitterErr := rand.Int(rand.Reader, big.NewInt(int64(delay/2)))
+ if jitterErr != nil {
+ return errors.WrapIf(jitterErr, "generate watcher retry delay")
+ }
+ next := time.Now().Add(delay/2 + time.Duration(jitter.Int64()))
+ s.setHealthInternal("retrying", err, &next)
+ timer := time.NewTimer(time.Until(next))
+ select {
+ case <-ctx.Done():
+ case <-s.restart:
+ unexpected = 0
+ delay = 5 * time.Second
+ case <-timer.C:
+ delay = min(2*delay, 5*time.Minute)
+ }
+ timer.Stop()
+ }
+ s.setHealthInternal("stopped", nil, nil)
+ return ctx.Err()
+}
+
+func (s *watcherSupervisorInternal) startInternal(ctx context.Context) (panicked bool, err error) {
+ defer func() {
+ if recovered := recover(); recovered != nil {
+ err = errors.Errorf("worker panic: %v", recovered)
+ panicked = true
+ }
+ }()
+ return false, s.watcher.Start(ctx)
+}
+
+func (js *jobSchedulerInternal) SetDispatcher(dispatcher schedulertypes.Dispatcher) {
+ js.dispatchMu.Lock()
+ defer js.dispatchMu.Unlock()
+ js.dispatcher = dispatcher
+}
+
+func (js *jobSchedulerInternal) dispatcherInternal() schedulertypes.Dispatcher {
+ js.dispatchMu.RLock()
+ defer js.dispatchMu.RUnlock()
+ return js.dispatcher
+}
+
+func (js *jobSchedulerInternal) Submit(ctx context.Context, request schedulertypes.Request) (schedulertypes.Run, error) {
+ if js.context.Err() != nil {
+ return schedulertypes.Run{}, errJobSchedulerStoppedInternal
+ }
+ dispatcher := js.dispatcherInternal()
+ if dispatcher == nil {
+ return schedulertypes.Run{}, errors.New("durable job dispatcher unavailable")
+ }
+ return dispatcher.Submit(ctx, request)
+}
+
+func (js *jobSchedulerInternal) checkpointInternal(ctx context.Context, jobID, schedule string, next time.Time) error {
+ dispatcher := js.dispatcherInternal()
+ if dispatcher == nil {
+ return errors.New("durable job dispatcher unavailable")
+ }
+ return dispatcher.Checkpoint(ctx, jobID, schedule, next)
+}
+
+func (js *jobSchedulerInternal) ListRegisteredJobs() []schedulertypes.Job {
+ snapshot, ok := js.state.Load()
+ if !ok {
+ return nil
+ }
+ jobs := make([]schedulertypes.Job, 0, len(snapshot.jobsByID))
+ for _, job := range snapshot.jobsByID {
+ jobs = append(jobs, job)
+ }
+ return jobs
+}
+
+func (js *jobSchedulerInternal) RestartWatcher(ctx context.Context, id string) error {
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ snapshot, ok := js.state.Load()
+ if !ok || snapshot.stopping {
+ return errJobSchedulerStoppedInternal
+ }
+ supervisor, ok := snapshot.supervisors[id]
+ if !ok {
+ return errors.New("watcher not found")
+ }
+ supervisor.mu.Lock()
+ defer supervisor.mu.Unlock()
+ select {
+ case supervisor.restart <- struct{}{}:
+ default:
+ }
+ if supervisor.cancel != nil {
+ supervisor.cancel()
+ }
+ return nil
+}
+
+func (js *jobSchedulerInternal) WatcherHealth(id string) (schedulertypes.WorkerHealth, bool) {
+ snapshot, ok := js.state.Load()
+ if !ok {
+ return schedulertypes.WorkerHealth{}, false
+ }
+ supervisor, ok := snapshot.supervisors[id]
+ if !ok {
+ return schedulertypes.WorkerHealth{}, false
+ }
+ supervisor.mu.Lock()
+ defer supervisor.mu.Unlock()
+ health := supervisor.health
+ if watcher, ok := supervisor.watcher.(*ImageUpdateWatcher); ok && watcher.eventDegraded.Load() && health.Status == "running" {
+ health.Status = "degraded"
+ health.LastError = "Docker image event subscription unavailable; polling remains active"
+ }
+ return health, true
+}
diff --git a/backend/pkg/scheduler/upload_sessions_cleanup_job.go b/backend/pkg/scheduler/upload_sessions_cleanup_job.go
index f1ecf4bff..6d4600aad 100644
--- a/backend/pkg/scheduler/upload_sessions_cleanup_job.go
+++ b/backend/pkg/scheduler/upload_sessions_cleanup_job.go
@@ -5,6 +5,8 @@ import (
"log/slog"
"time"
+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
+
"github.com/getarcaneapp/arcane/backend/v2/internal/upload"
)
@@ -34,15 +36,16 @@ func (j *UploadSessionsCleanupJob) Schedule(ctx context.Context) string {
return "0 0 * * * *"
}
-func (j *UploadSessionsCleanupJob) Run(ctx context.Context) {
+func (j *UploadSessionsCleanupJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {
removed, err := j.uploadService.PurgeExpiredSessions(ctx, uploadSessionMaxAge)
if err != nil {
slog.ErrorContext(ctx, "Failed to purge expired upload sessions", "jobName", UploadSessionsCleanupJobName, "removed", removed, "error", err)
- return
+ return schedulertypes.Outcome{}, err
}
if removed > 0 {
slog.InfoContext(ctx, "Purged expired upload sessions", "jobName", UploadSessionsCleanupJobName, "removed", removed)
}
+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil
}
func (j *UploadSessionsCleanupJob) Reschedule(ctx context.Context) error {
diff --git a/backend/pkg/scheduler/volume_helper_reaper_job.go b/backend/pkg/scheduler/volume_helper_reaper_job.go
index be664f109..eea29cd8e 100644
--- a/backend/pkg/scheduler/volume_helper_reaper_job.go
+++ b/backend/pkg/scheduler/volume_helper_reaper_job.go
@@ -5,6 +5,8 @@ import (
"log/slog"
"time"
+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
+
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/internal/volume"
)
@@ -44,9 +46,9 @@ func (j *PruningVolumeHelperJob) Schedule(ctx context.Context) string {
return volumeHelperPruningSchedule
}
-func (j *PruningVolumeHelperJob) Run(ctx context.Context) {
+func (j *PruningVolumeHelperJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {
if j.volumeService == nil {
- return
+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil
}
minutes := defaultVolumeHelperIdleTimeoutMinutes
@@ -55,13 +57,13 @@ func (j *PruningVolumeHelperJob) Run(ctx context.Context) {
}
if minutes <= 0 {
// 0 (or negative) disables idle pruning.
- return
+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil
}
removed, err := j.volumeService.ReapIdleHelpers(ctx, time.Duration(minutes)*time.Minute)
if err != nil {
slog.ErrorContext(ctx, "volume helper pruning failed", "jobName", PruningVolumeHelperJobName, "error", err)
- return
+ return schedulertypes.Outcome{}, err
}
if removed > 0 {
slog.InfoContext(ctx, "volume helper pruning completed",
@@ -69,6 +71,7 @@ func (j *PruningVolumeHelperJob) Run(ctx context.Context) {
"removed", removed,
"idleTimeoutMinutes", minutes)
}
+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil
}
func (j *PruningVolumeHelperJob) Reschedule(ctx context.Context) error {
diff --git a/backend/pkg/scheduler/vulnerability_scan_job.go b/backend/pkg/scheduler/vulnerability_scan_job.go
index 9894ddf68..4b1c008f8 100644
--- a/backend/pkg/scheduler/vulnerability_scan_job.go
+++ b/backend/pkg/scheduler/vulnerability_scan_job.go
@@ -2,14 +2,15 @@ package scheduler
import (
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
"context"
"log/slog"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability"
+ scheduleutil "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/schedule"
"github.com/getarcaneapp/arcane/types/v2"
- "github.com/robfig/cron/v3"
)
const VulnerabilityScanJobName = "vulnerability-scan"
@@ -49,7 +50,7 @@ func (j *VulnerabilityScanJob) Schedule(ctx context.Context) string {
schedule = "0 0 0 * * *"
}
- parser := cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow)
+ parser := scheduleutil.Parser()
if _, err := parser.Parse(schedule); err != nil {
slog.WarnContext(ctx, "Invalid cron expression for vulnerability-scan, using default", "invalid_schedule", schedule, "error", err)
return "0 0 0 * * *"
@@ -58,19 +59,19 @@ func (j *VulnerabilityScanJob) Schedule(ctx context.Context) string {
return schedule
}
-func (j *VulnerabilityScanJob) Run(ctx context.Context) {
+func (j *VulnerabilityScanJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {
enabled := j.settingsService.GetBoolSetting(ctx, "vulnerabilityScanEnabled", false)
if !enabled {
slog.DebugContext(ctx, "scheduled vulnerability scan disabled; skipping run")
- return
+ return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil
}
slog.InfoContext(ctx, "scheduled vulnerability scan started")
// Remove scan records for images that no longer exist (fixes e.g. "5/3 images scanned").
- deleted, err := j.vulnerabilityService.CleanupOrphanedScanRecords(ctx)
- if err != nil {
- slog.WarnContext(ctx, "orphaned vulnerability scan cleanup failed", "error", err)
+ deleted, cleanupErr := j.vulnerabilityService.CleanupOrphanedScanRecords(ctx)
+ if cleanupErr != nil {
+ slog.WarnContext(ctx, "orphaned vulnerability scan cleanup failed", "error", cleanupErr)
}
if deleted > 0 {
slog.InfoContext(ctx, "cleaned up orphaned vulnerability scan records", "deleted", deleted)
@@ -79,11 +80,15 @@ func (j *VulnerabilityScanJob) Run(ctx context.Context) {
scanned, failed, err := j.vulnerabilityService.ScanAllImages(ctx, types.LocalDockerEnvironmentID, vulnerabilityScanSystemUser, false)
if err != nil {
slog.ErrorContext(ctx, "scheduled vulnerability scan failed", "error", err)
- return
+ return schedulertypes.Outcome{}, err
}
slog.InfoContext(ctx, "scheduled vulnerability scan completed",
"scanned", scanned,
"failed", failed,
)
+ if failed > 0 || cleanupErr != nil {
+ return schedulertypes.Outcome{Status: schedulertypes.Partial, Message: "Some scans or orphaned record cleanup failed"}, cleanupErr
+ }
+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil
}
diff --git a/backend/pkg/utils/schedule/cron.go b/backend/pkg/utils/schedule/cron.go
deleted file mode 100644
index 30ac6f37e..000000000
--- a/backend/pkg/utils/schedule/cron.go
+++ /dev/null
@@ -1,26 +0,0 @@
-// Package schedule provides reusable schedule validation.
-package schedule
-
-import (
- "fmt"
- "strings"
-
- "github.com/robfig/cron/v3"
-)
-
-// NormalizeSixField validates and normalizes a six-field cron schedule.
-func NormalizeSixField(value, subject string) (string, error) {
- fields := strings.Fields(value)
- if len(fields) == 0 {
- return "", fmt.Errorf("%s schedule is required", subject)
- }
- if len(fields) != 6 {
- return "", fmt.Errorf("invalid %s schedule %q: expected six fields", subject, strings.TrimSpace(value))
- }
- normalized := strings.Join(fields, " ")
- parser := cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor)
- if _, err := parser.Parse(normalized); err != nil {
- return "", fmt.Errorf("invalid %s schedule %q: %w", subject, normalized, err)
- }
- return normalized, nil
-}
diff --git a/backend/resources/migrations/postgres/082_remove_job_key_versions.sql b/backend/resources/migrations/postgres/082_remove_job_key_versions.sql
new file mode 100644
index 000000000..c48d44bf3
--- /dev/null
+++ b/backend/resources/migrations/postgres/082_remove_job_key_versions.sql
@@ -0,0 +1,7 @@
+-- +goose Up
+UPDATE kv SET "key" = REPLACE("key", 'jobs.v1.', 'jobs.') WHERE "key" LIKE 'jobs.v1.%';
+UPDATE kv SET "key" = REPLACE("key", 'jobs-catalog.v1/', 'jobs-catalog/') WHERE "key" LIKE 'jobs-catalog.v1/%';
+
+-- +goose Down
+UPDATE kv SET "key" = REPLACE("key", 'jobs.', 'jobs.v1.') WHERE "key" LIKE 'jobs.%';
+UPDATE kv SET "key" = REPLACE("key", 'jobs-catalog/', 'jobs-catalog.v1/') WHERE "key" LIKE 'jobs-catalog/%';
diff --git a/backend/resources/migrations/sqlite/082_remove_job_key_versions.sql b/backend/resources/migrations/sqlite/082_remove_job_key_versions.sql
new file mode 100644
index 000000000..c48d44bf3
--- /dev/null
+++ b/backend/resources/migrations/sqlite/082_remove_job_key_versions.sql
@@ -0,0 +1,7 @@
+-- +goose Up
+UPDATE kv SET "key" = REPLACE("key", 'jobs.v1.', 'jobs.') WHERE "key" LIKE 'jobs.v1.%';
+UPDATE kv SET "key" = REPLACE("key", 'jobs-catalog.v1/', 'jobs-catalog/') WHERE "key" LIKE 'jobs-catalog.v1/%';
+
+-- +goose Down
+UPDATE kv SET "key" = REPLACE("key", 'jobs.', 'jobs.v1.') WHERE "key" LIKE 'jobs.%';
+UPDATE kv SET "key" = REPLACE("key", 'jobs-catalog/', 'jobs-catalog.v1/') WHERE "key" LIKE 'jobs-catalog/%';
diff --git a/cli/internal/client/client.go b/cli/internal/client/client.go
index 720549835..6d76f0ae5 100644
--- a/cli/internal/client/client.go
+++ b/cli/internal/client/client.go
@@ -29,6 +29,7 @@ import (
"encoding/json/v2"
stderrors "errors"
"io"
+ "maps"
"net"
"net/http"
"net/url"
@@ -244,7 +245,7 @@ type APIResponse[T any] struct {
// It constructs the full URL from the base URL and path, serializes the body
// as JSON (if provided), and includes authentication headers. The caller is
// responsible for closing the response body.
-func (c *Client) Request(ctx context.Context, method, path string, body any) (*http.Response, error) {
+func (c *Client) Request(ctx context.Context, method, path string, body any, headers ...http.Header) (*http.Response, error) {
fullURL, err := c.resolveURL(path)
if err != nil {
return nil, err
@@ -266,6 +267,10 @@ func (c *Client) Request(ctx context.Context, method, path string, body any) (*h
c.applyAuth(req)
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Accept", "application/json")
+ for _, values := range headers {
+ maps.Copy(req.Header, values.Clone())
+ }
+
start := time.Now()
logger.GetLogger().Debug("Sending request", "method", method, "url", fullURL, "env_id", c.envID, "streaming_body", true)
resp, err := c.httpClient.Do(req)
@@ -284,7 +289,7 @@ func (c *Client) Request(ctx context.Context, method, path string, body any) (*h
}
}
- return c.doRequest(ctx, method, fullURL, bodyBytes, true)
+ return c.doRequestInternal(ctx, method, fullURL, bodyBytes, true, headers...)
}
// RequestRaw makes an HTTP request with a raw body and custom headers.
@@ -341,7 +346,7 @@ func (c *Client) resolveURL(path string) (string, error) {
return c.baseURLParsed.ResolveReference(rel).String(), nil
}
-func (c *Client) doRequest(ctx context.Context, method, fullURL string, bodyBytes []byte, allowRefresh bool) (*http.Response, error) {
+func (c *Client) doRequestInternal(ctx context.Context, method, fullURL string, bodyBytes []byte, allowRefresh bool, headers ...http.Header) (*http.Response, error) {
attempts := max(c.maxAttempts, 1)
for attempt := 1; attempt <= attempts; attempt++ {
@@ -358,6 +363,9 @@ func (c *Client) doRequest(ctx context.Context, method, fullURL string, bodyByte
c.applyAuth(req)
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Accept", "application/json")
+ for _, values := range headers {
+ maps.Copy(req.Header, values.Clone())
+ }
start := time.Now()
logger.GetLogger().Debug("Sending request", "method", method, "url", fullURL, "env_id", c.envID, "attempt", attempt, "max_attempts", attempts, "body_bytes", len(bodyBytes))
@@ -570,9 +578,9 @@ func (c *Client) DeleteWithBody(ctx context.Context, path string, body any) (*ht
// DoJSON performs a request and decodes the response JSON, enforcing a
// successful HTTP status. Unlike GetJSON/PostJSON it does not require the
// standard API envelope or its Success flag.
-func (c *Client) DoJSON[T any](ctx context.Context, method, path string, body any) (T, error) {
+func (c *Client) DoJSON[T any](ctx context.Context, method, path string, body any, headers ...http.Header) (T, error) {
var out T
- resp, err := c.Request(ctx, method, path, body)
+ resp, err := c.Request(ctx, method, path, body, headers...)
if err != nil {
return out, err
}
diff --git a/cli/pkg/jobs/cmd.go b/cli/pkg/jobs/cmd.go
index 9a2266928..9c628957e 100644
--- a/cli/pkg/jobs/cmd.go
+++ b/cli/pkg/jobs/cmd.go
@@ -1,6 +1,7 @@
package jobs
import (
+ "crypto/rand"
"fmt"
"net/http"
"strconv"
@@ -16,7 +17,10 @@ import (
"github.com/spf13/cobra"
)
-var jsonOutput bool
+var (
+ jsonOutput bool
+ requestID string
+)
// JobsCmd is the parent command for job schedule operations.
var JobsCmd = &cobra.Command{
@@ -145,7 +149,7 @@ var listCmd = &cobra.Command{
var runCmd = &cobra.Command{
Use: "run <job-id>",
- Short: "Run a background job now",
+ Short: "Queue a background job now",
Args: cobra.ExactArgs(1),
SilenceUsage: true,
RunE: func(cmd *cobra.Command, args []string) error {
@@ -154,9 +158,17 @@ var runCmd = &cobra.Command{
return err
}
- result, err := c.DoJSON[jobschedule.JobRunResponse](cmd.Context(), http.MethodPost, types.JobRun(c.EnvID(), args[0]), nil)
+ id := requestID
+ if id == "" {
+ var value [16]byte
+ _, _ = rand.Read(value[:])
+ value[6] = (value[6] & 0x0f) | 0x40
+ value[8] = (value[8] & 0x3f) | 0x80
+ id = fmt.Sprintf("%x-%x-%x-%x-%x", value[0:4], value[4:6], value[6:8], value[8:10], value[10:16])
+ }
+ result, err := c.DoJSON[jobschedule.JobRunResponse](cmd.Context(), http.MethodPost, types.JobRun(c.EnvID(), args[0]), nil, http.Header{"Idempotency-Key": []string{id}})
if err != nil {
- return errors.WrapIf(err, "failed to run job")
+ return errors.WrapIff(err, "failed to queue job (reuse --request-id %s if delivery is uncertain)", id)
}
if jsonOutput {
@@ -167,6 +179,8 @@ var runCmd = &cobra.Command{
return errors.Errorf("job %s failed: %s", args[0], result.Message)
}
output.Success("%s", result.Message)
+ output.KeyValue("Run ID", result.RunID)
+ output.KeyValue("Status", string(result.Status))
return nil
},
}
@@ -180,6 +194,7 @@ func init() {
getCmd.Flags().BoolVar(&jsonOutput, "json", false, "Output in JSON format")
updateCmd.Flags().BoolVar(&jsonOutput, "json", false, "Output in JSON format")
listCmd.Flags().BoolVar(&jsonOutput, "json", false, "Output in JSON format")
+ runCmd.Flags().StringVar(&requestID, "request-id", "", "Stable request ID to reuse when retrying an uncertain submission")
runCmd.Flags().BoolVar(&jsonOutput, "json", false, "Output in JSON format")
updateCmd.Flags().StringVar(&environmentHealthInterval, "environment-health-interval", "", "Environment health job interval (cron expression)")
diff --git a/frontend/messages/en.json b/frontend/messages/en.json
index 41785eea4..5cd2608bd 100644
--- a/frontend/messages/en.json
+++ b/frontend/messages/en.json
@@ -3452,5 +3452,54 @@
"system_volume_backups_unavailable": "Unavailable",
"system_volume_backups_no_matching_volumes": "No matching volumes.",
"system_volume_backups_ignore_anonymous": "Ignore anonymous volumes",
- "system_volume_backups_ignore_anonymous_description": "Anonymous Docker volumes remain visible above but are skipped during backup runs."
+ "system_volume_backups_ignore_anonymous_description": "Anonymous Docker volumes remain visible above but are skipped during backup runs.",
+ "jobs_run_queued": "Run queued",
+ "jobs_submitting": "Queuing…",
+ "jobs_run_history": "Run history",
+ "jobs_run_attempts": "Attempts",
+ "jobs_last_success": "Last success",
+ "jobs_last_error": "Last error",
+ "jobs_next_retry": "Next retry",
+ "jobs_restart_worker": "Restart worker",
+ "jobs_retry_run": "Retry run",
+ "jobs_cancel_run": "Cancel pending run",
+ "jobs_last_confirmed": "Last confirmed",
+ "jobs_run_empty": "No runs recorded yet.",
+ "jobs_upgrade_required": "Upgrade this agent to use durable jobs.",
+ "jobs_offline_status": "This environment is offline. Showing its last confirmed job status.",
+ "jobs_run_id": "Run ID",
+ "jobs_run_targets": "Target outcomes",
+ "jobs_worker_health": "Worker health",
+ "jobs_run_previous": "Previous page",
+ "jobs_run_next": "Next page",
+ "jobs_run_error": "Unable to load job runs.",
+ "jobs_worker_restarted": "Worker restart requested",
+ "jobs_status_unknown": "Unknown",
+ "jobs_status_healthy": "Healthy",
+ "jobs_status_degraded": "Degraded",
+ "jobs_status_stopped": "Stopped",
+ "jobs_status_starting": "Starting",
+ "jobs_status_queued": "Queued",
+ "jobs_status_waiting": "Waiting for environment",
+ "jobs_status_running": "Running",
+ "jobs_status_retrying": "Retrying",
+ "jobs_status_succeeded": "Succeeded",
+ "jobs_status_partial": "Partial",
+ "jobs_status_skipped": "Skipped",
+ "jobs_status_failed": "Failed",
+ "jobs_status_needs_attention": "Needs attention",
+ "jobs_status_canceled": "Canceled",
+ "jobs_containers_load_error": "Failed to load containers",
+ "jobs_container_label_excluded": "Excluded by label",
+ "jobs_volume_backup_group": "Volume backups",
+ "jobs_system_backup_group": "System backups",
+ "jobs_dynamic_job_name": "{name} · {target}",
+ "jobs_capability_unknown": "This agent has not been reached yet. Job capabilities will be checked when it reconnects.",
+ "jobs_remote_status": "Agent status",
+ "jobs_target_runs": "Targets ({count})",
+ "jobs_run_all": "Run all",
+ "jobs_local_health_check": "Local environment health",
+ "jobs_health_check": "Environment health",
+ "jobs_settings_details": "Settings and details",
+ "jobs_health_scope_description": "Checks the health and connectivity of this environment."
}
diff --git a/frontend/src/lib/components/job-card/job-card.svelte b/frontend/src/lib/components/job-card/job-card.svelte
index 34e987872..c136c3332 100644
--- a/frontend/src/lib/components/job-card/job-card.svelte
+++ b/frontend/src/lib/components/job-card/job-card.svelte
@@ -3,37 +3,60 @@
import { m } from '#lib/paraglide/messages';
import { Badge } from '#lib/components/ui/badge';
import { Button } from '#lib/components/ui/button';
- import * as Card from '#lib/components/ui/card';
import { Spinner } from '#lib/components/ui/spinner';
import { jobScheduleService } from '#lib/services/job-schedule-service';
import { formatDateTimeShort, formatRelativeTime, parseInstant } from '#lib/utils/formatting';
import type { Snippet } from 'svelte';
import type { JobStatus } from '#lib/types/settings';
import JobScheduleDialog from './job-schedule-dialog.svelte';
+ import JobRunHistory from './job-run-history.svelte';
+ import { jobStatusLabel, jobNameLabel } from './job-status';
+ import { toast } from 'svelte-sonner';
import { createMutation } from '@tanstack/svelte-query';
let {
job,
environmentId = '0',
isAgent = false,
+ durableRuns = false,
onScheduleUpdate,
children,
enabledOverride,
- headerAccessory
+ headerAccessory,
+ collapsibleSettings = false
}: {
job: JobStatus;
environmentId?: string;
isAgent?: boolean;
+ durableRuns?: boolean;
onScheduleUpdate?: () => void;
children?: Snippet;
enabledOverride?: boolean;
headerAccessory?: Snippet;
+ collapsibleSettings?: boolean;
} = $props();
+ let requestId: string | undefined;
let showScheduleDialog = $state(false);
const runJobMutation = createMutation(() => ({
- mutationFn: () => jobScheduleService.runJob(job.id, environmentId)
+ mutationFn: () => {
+ requestId ??= crypto.randomUUID();
+ return jobScheduleService.runJob(job.id, environmentId, requestId);
+ },
+ onSuccess: () => {
+ requestId = undefined;
+ toast.success(m.jobs_run_queued());
+ onScheduleUpdate?.();
+ }
}));
+ const restartMutation = createMutation(() => ({
+ mutationFn: () => jobScheduleService.restartWorker(job.id, environmentId),
+ onSuccess: () => {
+ toast.success(m.jobs_worker_restarted());
+ onScheduleUpdate?.();
+ }
+ }));
+ const run = $derived(job.currentRun ?? job.lastRun);
const isRunning = $derived(runJobMutation.isPending);
const nextRunText = $derived.by(() => {
@@ -47,7 +70,7 @@
const isEnabled = $derived.by(() => enabledOverride ?? job.enabled);
- const canRun = $derived(isEnabled && job.canRunManually && !isRunning && !(isAgent && job.managerOnly));
+ const canRun = $derived(durableRuns && isEnabled && job.canRunManually && !isRunning && !(isAgent && job.managerOnly));
function runJobNow() {
if (!canRun) return;
@@ -64,76 +87,100 @@
}
</script>
-<Card.Root class="h-full rounded-xl border border-white/10 bg-background/60 shadow-none backdrop-blur-md">
- <Card.Header class="flex flex-row items-center justify-between space-y-0 px-4 py-4">
- <div class="flex flex-col gap-1.5">
- <div class="flex items-center gap-2">
- <Card.Title class="text-sm leading-none font-medium">{job.name}</Card.Title>
- {#if job.isContinuous}
+<article class="flex min-w-0 flex-col gap-3 py-5">
+ <div class="flex min-w-0 flex-wrap items-start justify-between gap-x-6 gap-y-3">
+ <div class="flex min-w-0 flex-[1_1_20rem] flex-col gap-1.5">
+ <div class="flex flex-wrap items-center gap-2">
+ {#if headerAccessory}<div class="mr-1 flex items-center">{@render headerAccessory()}</div>{/if}
+ <h4 class="text-base font-semibold">{jobNameLabel(job)}</h4>
+ {#if !isEnabled}
+ <Badge variant="secondary" size="sm">{m.common_disabled()}</Badge>
+ {:else if run}
+ <Badge variant={run.status === 'failed' || run.status === 'needs_attention' ? 'destructive' : 'outline'} size="sm"
+ >{jobStatusLabel(run.status)}</Badge
+ >
+ {:else if job.isContinuous}
<Badge variant="outline" size="sm">{m.jobs_continuous()}</Badge>
{/if}
</div>
- <Card.Description class="line-clamp-1 text-xs text-muted-foreground/80">{job.description}</Card.Description>
+ {#if job.id.startsWith('environment-health:')}
+ <p class="text-sm leading-relaxed text-muted-foreground">{m.jobs_health_scope_description()}</p>
+ {:else if job.description}<p class="text-sm leading-relaxed text-muted-foreground">{job.description}</p>{/if}
+ {#if job.lastSuccess}<p class="text-xs text-muted-foreground">
+ {m.jobs_last_success()}: {formatDateTimeShort(job.lastSuccess)}
+ </p>{/if}
+
+ <div class="mt-1 flex min-w-0 flex-wrap items-center gap-x-4 gap-y-1 text-xs text-muted-foreground">
+ {#if isEnabled}
+ {#if job.schedule && (!job.isContinuous || job.settingsKey)}
+ <div class="flex items-center gap-2">
+ <ClockIcon class="size-3.5 shrink-0" />
+ <span class="font-medium">{m.jobs_schedule()}</span>
+ <code class="break-all text-foreground">{job.schedule}</code>
+ </div>
+ {#if nextRunText}<p>{m.jobs_next_run()}: {nextRunText}</p>{/if}
+ {:else if job.isContinuous}<p>{m.jobs_continuous()}</p>{/if}
+ {#if run?.nextAttempt}<p>{m.jobs_next_retry()}: {formatDateTimeShort(run.nextAttempt)}</p>{/if}
+ {/if}
+ {#if job.workerHealth}<p>{m.jobs_worker_health()}: {jobStatusLabel(job.workerHealth.status)}</p>{/if}
+ </div>
</div>
- <div class="flex items-center gap-2">
- {#if headerAccessory}
- {@render headerAccessory()}
- {/if}
+ <div class="flex max-w-full flex-wrap items-center gap-1">
{#if isEnabled && job.settingsKey}
<Button
variant="ghost"
size="icon"
onclick={openScheduleDialog}
disabled={isAgent && job.managerOnly}
- class="size-7 opacity-75 hover:bg-white/5 hover:opacity-100"
+ aria-label={m.jobs_edit_schedule()}
+ title={m.jobs_edit_schedule()}
>
- <EditIcon class="size-3.5" />
+ <EditIcon />
</Button>
{/if}
+ {#if durableRuns && !job.children?.length}<JobRunHistory jobId={job.id} {environmentId} onUpdate={onScheduleUpdate} />{/if}
{#if isEnabled && job.canRunManually}
- <Button
- variant="outline"
- size="sm"
- onclick={runJobNow}
- disabled={!canRun}
- class="h-7 border-white/10 bg-transparent px-2.5 text-xs font-medium shadow-none hover:bg-white/5"
- >
- {#if isRunning}
- <Spinner class="mr-1.5 size-3" />
- {m.common_running()}
- {:else}
- <StartIcon class="mr-1.5 size-3.5" />
- {m.jobs_run_now()}
- {/if}
+ <Button variant="outline" size="sm" onclick={runJobNow} disabled={!canRun}>
+ {#if isRunning}<Spinner data-icon="inline-start" />{m.jobs_submitting()}
+ {:else}<StartIcon data-icon="inline-start" />{job.children?.length ? m.jobs_run_all() : m.jobs_run_now()}{/if}
</Button>
{/if}
+ {#if durableRuns && job.isContinuous && isEnabled}
+ <Button variant="ghost" size="sm" disabled={restartMutation.isPending} onclick={() => restartMutation.mutate()}
+ >{m.jobs_restart_worker()}</Button
+ >
+ {/if}
</div>
- </Card.Header>
- <Card.Content class="space-y-2 px-4 pt-0 pb-4">
- {#if isEnabled}
- <div class="grid gap-2 text-xs text-muted-foreground">
- {#if !job.isContinuous || job.settingsKey}
- <div class="flex items-center gap-2">
- <ClockIcon class="h-3.5 w-3.5" />
- <span>{m.jobs_schedule()}:</span>
- <code class="rounded bg-muted px-2 py-0.5 text-[0.65rem]">{job.schedule}</code>
- </div>
- {#if nextRunText}
- <div>
- {m.jobs_next_run()}: {nextRunText}
- </div>
- {/if}
- {:else}
- <div>{m.jobs_continuous()}</div>
- {/if}
- </div>
- {/if}
+ </div>
- {#if children}
+ {#if runJobMutation.error}<p class="text-xs text-destructive">{runJobMutation.error.message}</p>{/if}
+ {#if restartMutation.error}<p class="text-xs text-destructive">{restartMutation.error.message}</p>{/if}
+ {#if job.lastError}<p class="text-xs break-words text-destructive">{m.jobs_last_error()}: {job.lastError}</p>{/if}
+ {#if job.workerHealth?.lastError}<p class="text-xs break-words text-destructive">{job.workerHealth.lastError}</p>{/if}
+ {#if job.workerHealth?.nextRetry}<p class="text-xs text-muted-foreground">
+ {m.jobs_next_retry()}: {formatDateTimeShort(job.workerHealth.nextRetry)}
+ </p>{/if}
+ {#if run?.status === 'waiting' && run.remoteOutcome}
+ <p class="text-xs text-muted-foreground">
+ {m.jobs_remote_status()}: {jobStatusLabel(run.remoteOutcome.status)}
+ {#if run.lastConfirmedAt}<span class="ml-2">{m.jobs_last_confirmed()}: {formatDateTimeShort(run.lastConfirmedAt)}</span
+ >{/if}
+ </p>
+ {/if}
+ {#if children}
+ {#if collapsibleSettings}
+ <details class="group">
+ <summary
+ class="w-fit cursor-pointer rounded-sm text-sm text-muted-foreground outline-none hover:text-foreground focus-visible:ring-2 focus-visible:ring-ring"
+ >{m.jobs_settings_details()}</summary
+ >
+ <div class="mt-4 max-w-3xl">{@render children()}</div>
+ </details>
+ {:else}
{@render children()}
{/if}
- </Card.Content>
-</Card.Root>
+ {/if}
+</article>
{#if showScheduleDialog}
<JobScheduleDialog {job} {environmentId} bind:open={showScheduleDialog} onUpdate={handleScheduleUpdated} />
diff --git a/frontend/src/lib/components/job-card/job-run-history.svelte b/frontend/src/lib/components/job-card/job-run-history.svelte
new file mode 100644
index 000000000..d5445ec9c
--- /dev/null
+++ b/frontend/src/lib/components/job-card/job-run-history.svelte
@@ -0,0 +1,134 @@
+<script lang="ts">
+ import { queryKeys } from '#lib/query/query-keys';
+ import { createQuery, createMutation } from '@tanstack/svelte-query';
+ import { m } from '#lib/paraglide/messages';
+ import { Button } from '#lib/components/ui/button';
+ import * as Dialog from '#lib/components/ui/dialog';
+ import { jobScheduleService } from '#lib/services/job-schedule-service';
+ import { activityStore } from '#lib/stores/activity.store.svelte';
+ import { formatDateTimeShort } from '#lib/utils/formatting';
+ import { jobStatusLabel } from './job-status';
+ let { jobId, environmentId, onUpdate }: { jobId: string; environmentId: string; onUpdate?: () => void } = $props();
+ let open = $state(false);
+ let page = $state(1);
+ let selected = $state<string | undefined>();
+ const runs = createQuery(() => ({
+ queryKey: queryKeys.jobs.runs(environmentId, jobId, page),
+ queryFn: () => jobScheduleService.listRuns(jobId, environmentId, page),
+ enabled: open,
+ refetchInterval: 5000
+ }));
+ const detail = createQuery(() => ({
+ queryKey: queryKeys.jobs.run(environmentId, jobId, selected),
+ queryFn: () => jobScheduleService.getRun(jobId, selected!, environmentId),
+ enabled: open && !!selected,
+ refetchInterval: 5000
+ }));
+ const action = createMutation(() => ({
+ mutationFn: ({ runId, action }: { runId: string; action: 'retry' | 'cancel' }) =>
+ jobScheduleService.updateRun(jobId, runId, action, environmentId),
+ onSuccess: () => {
+ void runs.refetch();
+ void detail.refetch();
+ onUpdate?.();
+ }
+ }));
+</script>
+
+<Button
+ variant="ghost"
+ size="sm"
+ onclick={() => {
+ open = true;
+ page = 1;
+ selected = undefined;
+ }}>{m.jobs_run_history()}</Button
+>
+<Dialog.Root bind:open>
+ <Dialog.Content class="max-h-[85vh] overflow-y-auto sm:max-w-2xl">
+ <Dialog.Header
+ ><Dialog.Title>{m.jobs_run_history()}</Dialog.Title><Dialog.Description>{jobId}</Dialog.Description></Dialog.Header
+ >
+ {#if runs.error}<p class="text-sm text-destructive">{runs.error.message}</p>{/if}
+ {#each runs.data?.runs ?? [] as run (run.id)}
+ <Button
+ variant="outline"
+ class="h-auto w-full justify-between gap-2 text-left"
+ onclick={() => {
+ selected = run.id;
+ }}
+ >
+ <span class="truncate">{formatDateTimeShort(run.createdAt)}</span><span>{jobStatusLabel(run.status)}</span>
+ </Button>
+ {:else}
+ <p class="text-sm text-muted-foreground">{runs.isPending ? m.common_loading() : m.jobs_run_empty()}</p>
+ {/each}
+ <div class="flex justify-between gap-2">
+ <Button
+ variant="outline"
+ size="sm"
+ disabled={page <= 1 || runs.isFetching}
+ onclick={() => {
+ page--;
+ selected = undefined;
+ }}>{m.jobs_run_previous()}</Button
+ >
+ <Button
+ variant="outline"
+ size="sm"
+ disabled={!runs.data || page * runs.data.limit >= runs.data.total || runs.isFetching}
+ onclick={() => {
+ page++;
+ selected = undefined;
+ }}>{m.jobs_run_next()}</Button
+ >
+ </div>
+ {#if detail.error}<p class="text-sm text-destructive">{detail.error.message}</p>{/if}
+ {#if selected && detail.data}
+ {@const run = detail.data}
+ <div class="space-y-3 border-t pt-4 text-sm">
+ <p class="break-all"><strong>{m.jobs_run_id()}:</strong> {run.id}</p>
+ <p>{jobStatusLabel(run.status)}</p>
+ {#if run.status === 'waiting' && run.remoteOutcome}<p>
+ {m.jobs_remote_status()}: {jobStatusLabel(run.remoteOutcome.status)}
+ </p>{/if}
+ {#if run.outcome.message}<p class="break-words whitespace-pre-wrap">{run.outcome.message}</p>{/if}
+ {#if run.lastConfirmedAt}<p>{m.jobs_last_confirmed()}: {formatDateTimeShort(run.lastConfirmedAt)}</p>{/if}
+ {#if run.nextAttempt}<p>{m.jobs_next_retry()}: {formatDateTimeShort(run.nextAttempt)}</p>{/if}
+ {#if run.outcome.activityId}<Button variant="link" onclick={() => activityStore.openCenter(run.outcome.activityId)}
+ >{m.activity()}</Button
+ >{/if}
+ <h4 class="font-medium">{m.jobs_run_attempts()}: {run.attemptCount}</h4>
+ {#each run.attempts ?? [] as attempt (attempt.number)}
+ <div class="space-y-1 rounded border p-2">
+ <p>{attempt.number} · {formatDateTimeShort(attempt.startedAt)} · {jobStatusLabel(attempt.outcome.status)}</p>
+ {#if attempt.outcome.message}<p class="break-words whitespace-pre-wrap">{attempt.outcome.message}</p>{/if}
+ </div>
+ {/each}
+ {#if run.outcome.targets?.length}
+ <h4 class="font-medium">{m.jobs_run_targets()}</h4>
+ {#each run.outcome.targets as target (target.id)}
+ <div class="space-y-1 rounded border p-2">
+ <p class="break-all">{target.id} · {jobStatusLabel(target.status)}</p>
+ {#if target.message}<p class="break-words whitespace-pre-wrap">{target.message}</p>{/if}
+ {#if target.activityId}<Button variant="link" onclick={() => activityStore.openCenter(target.activityId)}
+ >{m.activity()}</Button
+ >{/if}
+ </div>
+ {/each}
+ {/if}
+ {#if action.error}<p class="text-destructive">{action.error.message}</p>{/if}
+ {#if ['failed', 'partial', 'needs_attention'].includes(run.status)}
+ <Button disabled={action.isPending} onclick={() => action.mutate({ runId: run.id, action: 'retry' })}
+ >{m.jobs_retry_run()}</Button
+ >
+ {/if}
+ {#if run.status === 'queued' && run.attemptCount === 0 && !run.remoteDeliveryAttempted}
+ <Button variant="outline" disabled={action.isPending} onclick={() => action.mutate({ runId: run.id, action: 'cancel' })}
+ >{m.jobs_cancel_run()}</Button
+ >
+ {/if}
+ </div>
+ {/if}
+ </Dialog.Content>
+</Dialog.Root>
diff --git a/frontend/src/lib/components/job-card/job-status.ts b/frontend/src/lib/components/job-card/job-status.ts
new file mode 100644
index 000000000..e47653938
--- /dev/null
+++ b/frontend/src/lib/components/job-card/job-status.ts
@@ -0,0 +1,34 @@
+import type { JobStatus } from '#lib/types/settings';
+import { m } from '#lib/paraglide/messages';
+export function jobStatusLabel(status: string): string {
+ const labels: Record<string, () => string> = {
+ queued: m.jobs_status_queued,
+ waiting: m.jobs_status_waiting,
+ running: m.jobs_status_running,
+ retrying: m.jobs_status_retrying,
+ succeeded: m.jobs_status_succeeded,
+ partial: m.jobs_status_partial,
+ skipped: m.jobs_status_skipped,
+ failed: m.jobs_status_failed,
+ needs_attention: m.jobs_status_needs_attention,
+ canceled: m.jobs_status_canceled,
+ healthy: m.jobs_status_healthy,
+ degraded: m.jobs_status_degraded,
+ stopped: m.jobs_status_stopped,
+ starting: m.jobs_status_starting
+ };
+ return labels[status]?.() ?? m.jobs_status_unknown();
+}
+
+export function jobNameLabel(job: JobStatus): string {
+ const [group = '', ...target] = job.id.split(':');
+ const labels: Record<string, () => string> = {
+ 'gitops-sync': m.gitops,
+ 'volume-backup': m.jobs_volume_backup_group,
+ 'system-backup': m.jobs_system_backup_group
+ };
+ if (group === 'environment-health' && target.length) return m.jobs_health_check();
+ const label = labels[group]?.();
+ if (!label) return job.name;
+ return target.length ? m.jobs_dynamic_job_name({ name: label, target: target.join(':') }) : label;
+}
diff --git a/frontend/src/lib/components/section-card.svelte b/frontend/src/lib/components/section-card.svelte
index d5a204b67..95edec352 100644
--- a/frontend/src/lib/components/section-card.svelte
+++ b/frontend/src/lib/components/section-card.svelte
@@ -2,10 +2,11 @@
import * as Card from '#lib/components/ui/card';
import type { IconType } from '#lib/icons';
import { cn } from '#lib/utils';
- import type { Snippet } from 'svelte';
+ import type { ComponentProps, Snippet } from 'svelte';
interface Props {
title: string;
+ variant?: ComponentProps<typeof Card.Root>['variant'];
description?: string;
icon?: IconType;
iconVariant?: 'primary' | 'emerald' | 'red' | 'amber' | 'blue' | 'purple' | 'cyan' | 'orange' | 'indigo' | 'pink';
@@ -18,6 +19,7 @@
let {
title,
+ variant = 'default',
description,
icon,
iconVariant = 'primary',
@@ -29,7 +31,7 @@
}: Props = $props();
</script>
-<Card.Root class={className}>
+<Card.Root {variant} class={className}>
<Card.Header {icon} {iconVariant} class={headerClass}>
<div class="flex min-w-0 flex-1 flex-col space-y-1.5">
<Card.Title>
diff --git a/frontend/src/lib/components/settings/image-patch-settings.svelte b/frontend/src/lib/components/settings/image-patch-settings.svelte
index d69f3bc4d..09a3265c0 100644
--- a/frontend/src/lib/components/settings/image-patch-settings.svelte
+++ b/frontend/src/lib/components/settings/image-patch-settings.svelte
@@ -29,6 +29,7 @@
</script>
<SectionCard
+ variant="transparent"
title={m.security_image_patching_heading()}
icon={SecurityIcon}
class="flex flex-col"
diff --git a/frontend/src/lib/components/settings/lifecycle-security-settings.svelte b/frontend/src/lib/components/settings/lifecycle-security-settings.svelte
index 06812a143..732d023c6 100644
--- a/frontend/src/lib/components/settings/lifecycle-security-settings.svelte
+++ b/frontend/src/lib/components/settings/lifecycle-security-settings.svelte
@@ -29,6 +29,7 @@
</script>
<SectionCard
+ variant="transparent"
title={m.security_lifecycle_hooks_heading()}
icon={CodeIcon}
class="flex flex-col"
diff --git a/frontend/src/lib/components/settings/trivy-security-settings.svelte b/frontend/src/lib/components/settings/trivy-security-settings.svelte
index aa391e9fe..c0ede60ed 100644
--- a/frontend/src/lib/components/settings/trivy-security-settings.svelte
+++ b/frontend/src/lib/components/settings/trivy-security-settings.svelte
@@ -152,6 +152,7 @@
</script>
<SectionCard
+ variant="transparent"
title={m.security_vulnerability_scanning_heading()}
icon={SecurityIcon}
class="flex flex-col"
diff --git a/frontend/src/lib/components/ui/card/card.svelte b/frontend/src/lib/components/ui/card/card.svelte
index 458af2341..4efa202d0 100644
--- a/frontend/src/lib/components/ui/card/card.svelte
+++ b/frontend/src/lib/components/ui/card/card.svelte
@@ -10,7 +10,10 @@
children,
...restProps
}: WithElementRef<
- HTMLAttributes<HTMLDivElement> & { variant?: 'default' | 'subtle' | 'outlined'; onclick?: (e: MouseEvent) => void }
+ HTMLAttributes<HTMLDivElement> & {
+ variant?: 'default' | 'subtle' | 'outlined' | 'transparent';
+ onclick?: (e: MouseEvent) => void;
+ }
> = $props();
function handleClick(e: MouseEvent) {
@@ -25,12 +28,14 @@
}
}
- function getVariantClasses(variant: 'default' | 'subtle' | 'outlined') {
+ function getVariantClasses(variant: 'default' | 'subtle' | 'outlined' | 'transparent') {
switch (variant) {
case 'default':
return 'backdrop-blur-md bg-card/60 shadow-xs dark:bg-surface/40';
case 'subtle':
return 'bg-muted/40 dark:bg-surface/30';
+ case 'transparent':
+ return 'border-border bg-transparent shadow-none';
case 'outlined':
return 'bg-card/50 backdrop-blur-md border-border/70';
default:
diff --git a/frontend/src/lib/query/query-keys.ts b/frontend/src/lib/query/query-keys.ts
index 19fbfd82f..c67bdf16a 100644
--- a/frontend/src/lib/query/query-keys.ts
+++ b/frontend/src/lib/query/query-keys.ts
@@ -13,6 +13,11 @@ function stableSerialize(value: unknown): string {
}
export const queryKeys = {
+ jobs: {
+ list: (environmentId: string) => ['environment-jobs', environmentId] as const,
+ runs: (environmentId: string, jobId: string, page: number) => ['job-runs', environmentId, jobId, page] as const,
+ run: (environmentId: string, jobId: string, runId: string | undefined) => ['job-run', environmentId, jobId, runId] as const
+ },
backupActivities: {
active: (environmentId: string, resourceKey: string, discoverFromHistory: boolean) =>
['backup-activities', environmentId, resourceKey, discoverFromHistory] as const
diff --git a/frontend/src/lib/services/job-schedule-service.ts b/frontend/src/lib/services/job-schedule-service.ts
index e4b785b99..0c156f380 100644
--- a/frontend/src/lib/services/job-schedule-service.ts
+++ b/frontend/src/lib/services/job-schedule-service.ts
@@ -1,3 +1,4 @@
+import type { JobRun, JobRunList } from '#lib/types/job';
import BaseAPIService from './api-service';
import type { JobSchedules, JobSchedulesUpdate, JobListResponse, JobRunResponse } from '#lib/types/settings';
@@ -14,8 +15,32 @@ class JobScheduleService extends BaseAPIService {
return this.handleResponse(this.api.get(`/environments/${environmentId}/jobs`));
}
- async runJob(jobId: string, environmentId: string = '0'): Promise<JobRunResponse> {
- return this.handleResponse(this.api.post(`/environments/${environmentId}/jobs/${jobId}/run`));
+ async runJob(jobId: string, environmentId: string = '0', requestId: string = crypto.randomUUID()): Promise<JobRunResponse> {
+ return this.handleResponse(
+ this.api.post(`/environments/${environmentId}/jobs/${encodeURIComponent(jobId)}/run`, undefined, {
+ headers: { 'Idempotency-Key': requestId }
+ })
+ );
+ }
+ async listRuns(jobId: string, environmentId: string, page = 1): Promise<JobRunList> {
+ return this.handleResponse(
+ this.api.get(`/environments/${environmentId}/jobs/${encodeURIComponent(jobId)}/runs`, { params: { page, limit: 20 } })
+ );
+ }
+ async getRun(jobId: string, runId: string, environmentId: string): Promise<JobRun> {
+ return this.handleResponse(
+ this.api.get(`/environments/${environmentId}/jobs/${encodeURIComponent(jobId)}/runs/${encodeURIComponent(runId)}`)
+ );
+ }
+ async updateRun(jobId: string, runId: string, action: 'retry' | 'cancel', environmentId: string): Promise<JobRun> {
+ return this.handleResponse(
+ this.api.post(
+ `/environments/${environmentId}/jobs/${encodeURIComponent(jobId)}/runs/${encodeURIComponent(runId)}/${action}`
+ )
+ );
+ }
+ async restartWorker(jobId: string, environmentId: string): Promise<{ success: boolean; message: string }> {
+ return this.handleResponse(this.api.post(`/environments/${environmentId}/jobs/${encodeURIComponent(jobId)}/restart`));
}
}
diff --git a/frontend/src/lib/types/job.ts b/frontend/src/lib/types/job.ts
new file mode 100644
index 000000000..486757bf1
--- /dev/null
+++ b/frontend/src/lib/types/job.ts
@@ -0,0 +1,37 @@
+export type JobRunStatus =
+ | 'queued'
+ | 'waiting'
+ | 'running'
+ | 'retrying'
+ | 'succeeded'
+ | 'partial'
+ | 'skipped'
+ | 'failed'
+ | 'needs_attention'
+ | 'canceled';
+export type JobTargetOutcome = { id: string; status: JobRunStatus; message?: string; activityId?: string };
+export type JobOutcome = { status: JobRunStatus; message?: string; activityId?: string; targets?: JobTargetOutcome[] };
+export type JobAttempt = { number: number; startedAt: string; finishedAt?: string; outcome: JobOutcome };
+export type JobRun = {
+ id: string;
+ jobId: string;
+ environmentId: string;
+ trigger: string;
+ requestedBy?: string;
+ status: JobRunStatus;
+ createdAt: string;
+ updatedAt: string;
+ startedAt?: string;
+ finishedAt?: string;
+ nextAttempt?: string;
+ attemptCount: number;
+ outcome: JobOutcome;
+ attempts?: JobAttempt[];
+ remoteAccepted: boolean;
+ remoteDeliveryAttempted: boolean;
+ remoteOutcome?: JobOutcome;
+ remoteSettled: boolean;
+ lastConfirmedAt?: string;
+};
+export type JobRunList = { runs: JobRun[]; total: number; page: number; limit: number };
+export type WorkerHealth = { status: string; lastError?: string; nextRetry?: string; updatedAt: string };
diff --git a/frontend/src/lib/types/settings.ts b/frontend/src/lib/types/settings.ts
index 120f54430..78d5a1159 100644
--- a/frontend/src/lib/types/settings.ts
+++ b/frontend/src/lib/types/settings.ts
@@ -1,3 +1,4 @@
+import type { JobRun, JobRunStatus, WorkerHealth } from './job';
import type { TemplateRegistryConfig } from './swarm';
// --- Application settings ---
@@ -182,6 +183,12 @@ export type JobPrerequisite = {
};
export type JobStatus = {
+ currentRun?: JobRun;
+ lastRun?: JobRun;
+ lastSuccess?: string;
+ lastError?: string;
+ workerHealth?: WorkerHealth;
+ children?: JobStatus[];
id: string;
name: string;
description: string;
@@ -197,11 +204,16 @@ export type JobStatus = {
};
export type JobListResponse = {
+ durableRuns: boolean;
+ observedAt: string;
+ offline?: boolean;
jobs: JobStatus[];
isAgent: boolean;
};
export type JobRunResponse = {
+ runId: string;
+ status: JobRunStatus;
success: boolean;
message: string;
};
diff --git a/frontend/src/routes/(app)/environments/[id]/components/ConnectionEdgeTab.svelte b/frontend/src/routes/(app)/environments/[id]/components/ConnectionEdgeTab.svelte
index 7158b0e95..91d31b7a8 100644
--- a/frontend/src/routes/(app)/environments/[id]/components/ConnectionEdgeTab.svelte
+++ b/frontend/src/routes/(app)/environments/[id]/components/ConnectionEdgeTab.svelte
@@ -48,7 +48,7 @@
{/snippet}
<div class="space-y-6">
- <Card.Root class="flex flex-col">
+ <Card.Root variant="transparent" class="flex flex-col">
<Card.Header icon={ConnectionIcon}>
<div class="flex flex-col space-y-1.5">
<Card.Title>
@@ -63,7 +63,7 @@
</Card.Root>
{#if showAgentSecurity}
- <Card.Root class="flex flex-col">
+ <Card.Root variant="transparent" class="flex flex-col">
<Card.Header icon={SecurityIcon}>
<div class="flex flex-col space-y-1.5">
<Card.Title>
diff --git a/frontend/src/routes/(app)/environments/[id]/components/DockerTab.svelte b/frontend/src/routes/(app)/environments/[id]/components/DockerTab.svelte
index 22033b948..f5402e717 100644
--- a/frontend/src/routes/(app)/environments/[id]/components/DockerTab.svelte
+++ b/frontend/src/routes/(app)/environments/[id]/components/DockerTab.svelte
@@ -50,7 +50,7 @@
];
</script>
-<Card.Root class="flex flex-col">
+<Card.Root variant="transparent" class="flex flex-col">
<Card.Header icon={DockerBrandIcon}>
<div class="flex flex-col space-y-1.5">
<Card.Title>
diff --git a/frontend/src/routes/(app)/environments/[id]/components/JobsTab.svelte b/frontend/src/routes/(app)/environments/[id]/components/JobsTab.svelte
index e880d69fb..87bbc6c18 100644
--- a/frontend/src/routes/(app)/environments/[id]/components/JobsTab.svelte
+++ b/frontend/src/routes/(app)/environments/[id]/components/JobsTab.svelte
@@ -1,4 +1,6 @@
<script lang="ts">
+ import { queryKeys } from '#lib/query/query-keys';
+ import { createQuery } from '@tanstack/svelte-query';
import { SvelteSet } from 'svelte/reactivity';
import { jobScheduleService } from '#lib/services/job-schedule-service';
import { containerService } from '#lib/services/container-service';
@@ -7,7 +9,6 @@
import { Spinner } from '#lib/components/ui/spinner';
import { m } from '#lib/paraglide/messages';
import * as Alert from '#lib/components/ui/alert';
- import * as Card from '#lib/components/ui/card';
import { Label } from '#lib/components/ui/label';
import { Switch } from '#lib/components/ui/switch';
import { Input } from '#lib/components/ui/input';
@@ -21,29 +22,25 @@
let { formInputs, environmentId }: JobsTabProps = $props();
- let refreshSignal = $state(0);
-
- const jobsPromise = $derived.by(async () => {
- refreshSignal; // trigger dependency
- if (!environmentId) return null;
-
- const result = await tryCatch(jobScheduleService.listJobs(environmentId));
-
- if (result.error) {
- throw result.error;
- }
-
- return {
- ...result.data,
- jobs: result.data.jobs.map((job) => ({
- ...job,
- prerequisites: job.prerequisites.map((prereq) => ({
- ...prereq,
- settingsUrl: resolveSettingsUrl(job, prereq)
+ const jobsQuery = createQuery(() => ({
+ queryKey: queryKeys.jobs.list(environmentId),
+ queryFn: async () => {
+ const response = await jobScheduleService.listJobs(environmentId);
+ return {
+ ...response,
+ jobs: response.jobs.map((job) => ({
+ ...job,
+ prerequisites: (job.prerequisites ?? []).map((prereq) => ({ ...prereq, settingsUrl: resolveSettingsUrl(job, prereq) }))
}))
- }))
- };
- });
+ };
+ },
+ enabled: !!environmentId,
+ refetchInterval: 5000
+ }));
+ const jobsResponse = $derived(jobsQuery.data);
+ const capabilityUnknown = $derived(
+ !!jobsResponse?.offline && (!jobsResponse.observedAt || jobsResponse.observedAt.startsWith('0001-'))
+ );
const containersPromise = $derived.by(async () => {
if (!environmentId) return [];
@@ -95,7 +92,7 @@
}
function loadJobs() {
- refreshSignal++;
+ void jobsQuery.refetch();
}
function toggleContainerExclusion(containerName: string) {
@@ -132,11 +129,8 @@
{ id: 'telemetry', label: m.jobs_telemetry_heading() }
];
- const hiddenJobIds = new Set(['analytics-heartbeat', 'filesystem-watcher']);
-
function getJobsByCategory(categoryId: string, jobs: JobStatus[]): JobStatus[] {
return jobs.filter((j) => {
- if (hiddenJobIds.has(j.id)) return false;
if (j.category !== categoryId) return false;
// Only show manager-only jobs on the local environment (ID "0")
if (j.managerOnly && environmentId !== '0') return false;
@@ -173,7 +167,7 @@
value: name,
label: name,
disabled: labelExcluded,
- hint: labelExcluded ? '(Label)' : undefined,
+ hint: labelExcluded ? m.jobs_container_label_excluded() : undefined,
selected: excludedContainers.has(name)
};
}
@@ -230,178 +224,206 @@
{/if}
{:catch error}
<div class="p-2 text-sm text-destructive">
- {(error instanceof Error ? error.message : '') || 'Failed to load containers'}
+ {(error instanceof Error ? error.message : '') || m.jobs_containers_load_error()}
</div>
{/await}
</div>
</ScrollArea.Root>
{/snippet}
-<div class="space-y-6">
- <Card.Root>
- <Card.Header icon={JobsIcon}>
- <div class="flex flex-col space-y-1.5">
- <Card.Title>
- <h2>{m.automations()}</h2>
- </Card.Title>
- <Card.Description>{m.jobs_environment_scope_description()}</Card.Description>
+<section class="flex w-full min-w-0 flex-col gap-6">
+ <header class="flex items-start gap-3">
+ <JobsIcon class="mt-0.5 size-5 text-muted-foreground" />
+ <div class="flex flex-col gap-1.5">
+ <h2 class="text-lg font-semibold">{m.automations()}</h2>
+ <p class="text-sm text-muted-foreground">{m.jobs_environment_scope_description()}</p>
+ </div>
+ </header>
+ <div>
+ {#if jobsQuery.isPending}
+ <div class="flex h-32 items-center justify-center">
+ <Spinner class="size-8" />
</div>
- </Card.Header>
- <Card.Content class="p-4 sm:p-6">
- {#await jobsPromise}
- <div class="flex h-32 items-center justify-center">
- <Spinner class="size-8" />
- </div>
- {:then jobsResponse}
- {#if jobsResponse}
- <div class="space-y-8">
- {#each categories as category (category.id)}
- {@const categoryJobs = getJobsByCategory(category.id, jobsResponse.jobs)}
- {#if categoryJobs.length > 0}
- <div class="space-y-4">
- <h3 class="text-sm font-semibold tracking-tight text-muted-foreground uppercase">
- {category.label}
- </h3>
- <div class="grid gap-4 sm:grid-cols-2 lg:grid-cols-2">
- {#each categoryJobs as job (job.id)}
- <JobCard
- {job}
- {environmentId}
- isAgent={jobsResponse.isAgent}
- onScheduleUpdate={loadJobs}
- enabledOverride={getEnabledOverride(job)}
- >
- {#snippet headerAccessory()}
- {#if job.id === 'image-polling'}
- <Switch bind:checked={$formInputs.pollingEnabled.value} />
- {:else if job.id === 'auto-update'}
- <Switch bind:checked={$formInputs.autoUpdate.value} disabled={!$formInputs.pollingEnabled.value} />
- {:else if job.id === 'scheduled-prune'}
- <Switch bind:checked={$formInputs.scheduledPruneEnabled.value} />
- {:else if job.id === 'vulnerability-scan'}
- <Switch bind:checked={$formInputs.vulnerabilityScanEnabled.value} />
- {:else if job.id === 'auto-heal'}
- <Switch bind:checked={$formInputs.autoHealEnabled.value} />
- {/if}
- {/snippet}
-
- {#if job.id === 'auto-update' && $formInputs.autoUpdate.value}
- <div class="space-y-3 border-t border-border/20 pt-3">
- <div class="space-y-1">
- <Label class="text-sm font-medium">
- {m.excluded_containers()}
- {#await containersPromise then containers}
- <span class="ml-1 font-normal text-muted-foreground">
- ({containers.filter((c) => excludedContainers.has(getContainerName(c))).length})
- </span>
- {/await}
- </Label>
- <p class="text-xs text-muted-foreground">{m.auto_update_exclude_description()}</p>
- </div>
-
- <div class="space-y-2">
- <Input type="search" placeholder={m.jobs_search_containers()} class="h-8" bind:value={searchTerm} />
- {@render ContainerExclusionList({
- term: searchTerm,
- mapItem: mapContainerToItem,
- idPrefix: 'container-',
- onToggle: toggleContainerExclusion
- })}
- </div>
- </div>
+ {:else}
+ {#if jobsResponse}
+ {#if capabilityUnknown}<p class="mb-4 text-sm text-muted-foreground">{m.jobs_capability_unknown()}</p>
+ {:else if !jobsResponse.durableRuns}<p class="mb-4 text-sm text-muted-foreground">{m.jobs_upgrade_required()}</p>{/if}
+ {#if jobsResponse.offline}<p class="mb-4 text-sm text-muted-foreground">
+ {m.jobs_offline_status()}
+ {#if !capabilityUnknown}{m.jobs_last_confirmed()}: {jobsResponse.observedAt}{/if}
+ </p>{/if}
+ <div class="grid items-start gap-x-12 gap-y-10 2xl:grid-cols-2">
+ {#each categories as category (category.id)}
+ {@const categoryJobs = getJobsByCategory(category.id, jobsResponse.jobs)}
+ {#if categoryJobs.length > 0}
+ <div
+ class="grid min-w-0 gap-x-6 gap-y-2 rounded-xl border border-border bg-transparent px-4 py-2 sm:px-5 lg:grid-cols-[7rem_minmax(0,1fr)]"
+ >
+ <h3 class="text-sm font-semibold text-foreground lg:pt-5">
+ {category.label}
+ </h3>
+ <div class="min-w-0 divide-y divide-border/50">
+ {#each categoryJobs as job (job.id)}
+ <JobCard
+ {job}
+ {environmentId}
+ isAgent={jobsResponse.isAgent}
+ durableRuns={jobsResponse.durableRuns || capabilityUnknown}
+ onScheduleUpdate={loadJobs}
+ enabledOverride={capabilityUnknown ? undefined : getEnabledOverride(job)}
+ collapsibleSettings={job.id === 'image-polling' ||
+ (job.id === 'auto-update' && $formInputs.autoUpdate.value) ||
+ (job.id === 'auto-heal' && $formInputs.autoHealEnabled.value)}
+ >
+ {#snippet headerAccessory()}
+ {#if job.id === 'image-polling'}
+ <Switch aria-label={job.name} bind:checked={$formInputs.pollingEnabled.value} />
+ {:else if job.id === 'auto-update'}
+ <Switch
+ aria-label={job.name}
+ bind:checked={$formInputs.autoUpdate.value}
+ disabled={!$formInputs.pollingEnabled.value}
+ />
+ {:else if job.id === 'scheduled-prune'}
+ <Switch aria-label={job.name} bind:checked={$formInputs.scheduledPruneEnabled.value} />
+ {:else if job.id === 'vulnerability-scan'}
+ <Switch aria-label={job.name} bind:checked={$formInputs.vulnerabilityScanEnabled.value} />
+ {:else if job.id === 'auto-heal'}
+ <Switch aria-label={job.name} bind:checked={$formInputs.autoHealEnabled.value} />
{/if}
+ {/snippet}
+
+ {#if job.id === 'auto-update' && $formInputs.autoUpdate.value}
+ <div class="space-y-3 border-t border-border/20 pt-3">
+ <div class="space-y-1">
+ <Label class="text-sm font-medium">
+ {m.excluded_containers()}
+ {#await containersPromise then containers}
+ <span class="ml-1 font-normal text-muted-foreground">
+ ({containers.filter((c) => excludedContainers.has(getContainerName(c))).length})
+ </span>
+ {/await}
+ </Label>
+ <p class="text-xs text-muted-foreground">{m.auto_update_exclude_description()}</p>
+ </div>
- {#if job.id === 'image-polling'}
- <div class="space-y-3 border-t border-border/20 pt-3">
- <div class="flex items-center justify-between gap-3">
- <div class="space-y-1">
- <Label class="text-sm font-medium">{m.jobs_image_event_watcher_label()}</Label>
- <p class="text-xs text-muted-foreground">{m.jobs_image_event_watcher_description()}</p>
- </div>
- <Switch
- id="image-event-watcher-enabled"
- bind:checked={$formInputs.imageEventWatcherEnabled.value}
- />
- </div>
- <Alert.Root variant="warning" class="py-2 [&>svg]:top-2">
- <AlertTriangleIcon class="size-4" />
- <Alert.Description class="text-xs">{m.jobs_image_event_watcher_warning()}</Alert.Description>
- </Alert.Root>
+ <div class="space-y-2">
+ <Input type="search" placeholder={m.jobs_search_containers()} class="h-8" bind:value={searchTerm} />
+ {@render ContainerExclusionList({
+ term: searchTerm,
+ mapItem: mapContainerToItem,
+ idPrefix: 'container-',
+ onToggle: toggleContainerExclusion
+ })}
</div>
- {/if}
+ </div>
+ {/if}
- {#if job.id === 'auto-heal' && $formInputs.autoHealEnabled.value}
- <div class="space-y-3 border-t border-border/20 pt-3">
- <div class="grid gap-3 sm:grid-cols-2">
- <div class="space-y-1">
- <Label for="auto-heal-max-restarts" class="text-sm font-medium"
- >{m.auto_heal_max_restarts_label()}</Label
- >
- <p class="text-xs text-muted-foreground">{m.auto_heal_max_restarts_description()}</p>
- <Input
- id="auto-heal-max-restarts"
- type="number"
- min="1"
- class="h-8 w-full"
- bind:value={$formInputs.autoHealMaxRestarts.value}
- />
- </div>
- <div class="space-y-1">
- <Label for="auto-heal-restart-window" class="text-sm font-medium"
- >{m.auto_heal_restart_window_label()}</Label
- >
- <p class="text-xs text-muted-foreground">{m.auto_heal_restart_window_description()}</p>
- <Input
- id="auto-heal-restart-window"
- type="number"
- min="1"
- class="h-8 w-full"
- bind:value={$formInputs.autoHealRestartWindow.value}
- />
- </div>
+ {#if job.id === 'image-polling'}
+ <div class="space-y-3 border-t border-border/20 pt-3">
+ <div class="flex items-center justify-between gap-3">
+ <div class="space-y-1">
+ <Label class="text-sm font-medium">{m.jobs_image_event_watcher_label()}</Label>
+ <p class="text-xs text-muted-foreground">{m.jobs_image_event_watcher_description()}</p>
</div>
-
+ <Switch id="image-event-watcher-enabled" bind:checked={$formInputs.imageEventWatcherEnabled.value} />
+ </div>
+ <Alert.Root variant="warning" class="py-2 [&>svg]:top-2">
+ <AlertTriangleIcon class="size-4" />
+ <Alert.Description class="text-xs">{m.jobs_image_event_watcher_warning()}</Alert.Description>
+ </Alert.Root>
+ </div>
+ {/if}
+
+ {#if job.id === 'auto-heal' && $formInputs.autoHealEnabled.value}
+ <div class="space-y-3 border-t border-border/20 pt-3">
+ <div class="grid gap-3 sm:grid-cols-2">
<div class="space-y-1">
- <Label class="text-sm font-medium">
- {m.excluded_containers()}
- {#await containersPromise then containers}
- <span class="ml-1 font-normal text-muted-foreground">
- ({containers.filter((c) => autoHealExcludedContainers.has(getContainerName(c))).length})
- </span>
- {/await}
- </Label>
- <p class="text-xs text-muted-foreground">{m.auto_heal_exclude_description()}</p>
+ <Label for="auto-heal-max-restarts" class="text-sm font-medium"
+ >{m.auto_heal_max_restarts_label()}</Label
+ >
+ <p class="text-xs text-muted-foreground">{m.auto_heal_max_restarts_description()}</p>
+ <Input
+ id="auto-heal-max-restarts"
+ type="number"
+ min="1"
+ class="h-8 w-full"
+ bind:value={$formInputs.autoHealMaxRestarts.value}
+ />
</div>
-
- <div class="space-y-2">
+ <div class="space-y-1">
+ <Label for="auto-heal-restart-window" class="text-sm font-medium"
+ >{m.auto_heal_restart_window_label()}</Label
+ >
+ <p class="text-xs text-muted-foreground">{m.auto_heal_restart_window_description()}</p>
<Input
- type="search"
- placeholder={m.jobs_search_containers()}
- class="h-8"
- bind:value={autoHealSearchTerm}
+ id="auto-heal-restart-window"
+ type="number"
+ min="1"
+ class="h-8 w-full"
+ bind:value={$formInputs.autoHealRestartWindow.value}
/>
- {@render ContainerExclusionList({
- term: autoHealSearchTerm,
- mapItem: mapContainerToAutoHealItem,
- idPrefix: 'auto-heal-container-',
- onToggle: toggleAutoHealContainerExclusion
- })}
</div>
</div>
- {/if}
- </JobCard>
- {/each}
- </div>
+
+ <div class="space-y-1">
+ <Label class="text-sm font-medium">
+ {m.excluded_containers()}
+ {#await containersPromise then containers}
+ <span class="ml-1 font-normal text-muted-foreground">
+ ({containers.filter((c) => autoHealExcludedContainers.has(getContainerName(c))).length})
+ </span>
+ {/await}
+ </Label>
+ <p class="text-xs text-muted-foreground">{m.auto_heal_exclude_description()}</p>
+ </div>
+
+ <div class="space-y-2">
+ <Input
+ type="search"
+ placeholder={m.jobs_search_containers()}
+ class="h-8"
+ bind:value={autoHealSearchTerm}
+ />
+ {@render ContainerExclusionList({
+ term: autoHealSearchTerm,
+ mapItem: mapContainerToAutoHealItem,
+ idPrefix: 'auto-heal-container-',
+ onToggle: toggleAutoHealContainerExclusion
+ })}
+ </div>
+ </div>
+ {/if}
+ {#if job.children?.length}
+ <details class="mt-1">
+ <summary
+ class="w-fit cursor-pointer rounded-sm text-sm text-muted-foreground outline-none hover:text-foreground focus-visible:ring-2 focus-visible:ring-ring"
+ >{m.jobs_target_runs({ count: job.children.length })}</summary
+ >
+ <div class="mt-3 divide-y divide-border border-l border-border pl-4 sm:pl-6">
+ {#each job.children ?? [] as child (child.id)}
+ <JobCard
+ job={child}
+ {environmentId}
+ isAgent={jobsResponse.isAgent}
+ durableRuns={jobsResponse.durableRuns || capabilityUnknown}
+ onScheduleUpdate={loadJobs}
+ />
+ {/each}
+ </div>
+ </details>
+ {/if}
+ </JobCard>
+ {/each}
</div>
- {/if}
- {/each}
- </div>
- {/if}
- {:catch error}
- <div class="rounded-lg border border-destructive/50 bg-destructive/10 p-4 text-destructive">
- {(error instanceof Error ? error.message : '') || String(error)}
+ </div>
+ {/if}
+ {/each}
</div>
- {/await}
- </Card.Content>
- </Card.Root>
-</div>
+ {/if}
+ {/if}
+ {#if jobsQuery.error}<div class="rounded-lg border border-destructive/50 bg-destructive/10 p-4 text-destructive">
+ {jobsQuery.error.message}
+ </div>{/if}
+ </div>
+</section>
diff --git a/frontend/src/routes/(app)/environments/[id]/components/StorageTab.svelte b/frontend/src/routes/(app)/environments/[id]/components/StorageTab.svelte
index 0bc35e864..ed2b8f4c9 100644
--- a/frontend/src/routes/(app)/environments/[id]/components/StorageTab.svelte
+++ b/frontend/src/routes/(app)/environments/[id]/components/StorageTab.svelte
@@ -11,7 +11,7 @@
</script>
<div class="space-y-6">
- <Card.Root class="flex flex-col">
+ <Card.Root variant="transparent" class="flex flex-col">
<Card.Header icon={FolderOpenIcon}>
<div class="flex flex-col space-y-1.5">
<Card.Title>
@@ -64,7 +64,7 @@
</Card.Content>
</Card.Root>
- <Card.Root class="flex flex-col">
+ <Card.Root variant="transparent" class="flex flex-col">
<Card.Header icon={UploadIcon}>
<div class="flex flex-col space-y-1.5">
<Card.Title>
diff --git a/types/jobschedule/jobschedule.go b/types/jobschedule/jobschedule.go
index 9cb551f92..6e16362ca 100644
--- a/types/jobschedule/jobschedule.go
+++ b/types/jobschedule/jobschedule.go
@@ -1,11 +1,12 @@
package jobschedule
-import "time"
+import (
+ "time"
-// Config represents the configured intervals (in minutes) for Arcane background jobs.
-//
-// All fields are in minutes.
-// This makes conversion to time.Duration straightforward in the backend.
+ "github.com/getarcaneapp/arcane/types/v2/scheduler"
+)
+
+// Config contains cron schedules for Arcane background jobs.
type Config struct {
EnvironmentHealthInterval string `json:"environmentHealthInterval"`
EventCleanupInterval string `json:"eventCleanupInterval"`
@@ -19,7 +20,7 @@ type Config struct {
AutoHealInterval string `json:"autoHealInterval"`
}
-// Update is used to update job schedule intervals (in minutes).
+// Update changes job cron schedules.
//
// Any nil field is ignored.
type Update struct {
@@ -37,18 +38,24 @@ type Update struct {
// JobStatus represents the current status and metadata for a background job.
type JobStatus struct {
- ID string `json:"id"`
- Name string `json:"name"`
- Description string `json:"description"`
- Category string `json:"category"`
- Schedule string `json:"schedule"`
- NextRun *time.Time `json:"nextRun,omitempty"`
- Enabled bool `json:"enabled"`
- ManagerOnly bool `json:"managerOnly"`
- IsContinuous bool `json:"isContinuous"`
- CanRunManually bool `json:"canRunManually"`
- Prerequisites []JobPrerequisite `json:"prerequisites"`
- SettingsKey string `json:"settingsKey,omitempty"`
+ CurrentRun *scheduler.Run `json:"currentRun,omitempty"`
+ LastRun *scheduler.Run `json:"lastRun,omitempty"`
+ LastSuccess *time.Time `json:"lastSuccess,omitempty"`
+ LastError string `json:"lastError,omitempty"`
+ WorkerHealth *scheduler.WorkerHealth `json:"workerHealth,omitempty"`
+ Children []JobStatus `json:"children,omitempty"`
+ ID string `json:"id"`
+ Name string `json:"name"`
+ Description string `json:"description"`
+ Category string `json:"category"`
+ Schedule string `json:"schedule"`
+ NextRun *time.Time `json:"nextRun,omitempty"`
+ Enabled bool `json:"enabled"`
+ ManagerOnly bool `json:"managerOnly"`
+ IsContinuous bool `json:"isContinuous"`
+ CanRunManually bool `json:"canRunManually"`
+ Prerequisites []JobPrerequisite `json:"prerequisites"`
+ SettingsKey string `json:"settingsKey,omitempty"`
}
// JobPrerequisite represents a requirement that must be met for a job to run.
@@ -61,8 +68,11 @@ type JobPrerequisite struct {
// JobListResponse contains all jobs and system mode information.
type JobListResponse struct {
- Jobs []JobStatus `json:"jobs"`
- IsAgent bool `json:"isAgent"`
+ DurableRuns bool `json:"durableRuns"`
+ ObservedAt time.Time `json:"observedAt"`
+ Offline bool `json:"offline,omitempty"`
+ Jobs []JobStatus `json:"jobs"`
+ IsAgent bool `json:"isAgent"`
}
// JobRunRequest is the request to manually run a job.
@@ -72,6 +82,13 @@ type JobRunRequest struct {
// JobRunResponse is the response after manually triggering a job.
type JobRunResponse struct {
- Success bool `json:"success"`
- Message string `json:"message"`
+ RunID string `json:"runId,omitempty"`
+ Status scheduler.RunStatus `json:"status,omitempty"`
+ Success bool `json:"success"`
+ Message string `json:"message"`
+}
+
+// SubmitRunInput preserves identity across manager-to-agent delivery retries.
+type SubmitRunInput struct {
+ RunID string `json:"runId"`
}
diff --git a/types/jobschedule/runs.go b/types/jobschedule/runs.go
new file mode 100644
index 000000000..72d6c4c74
--- /dev/null
+++ b/types/jobschedule/runs.go
@@ -0,0 +1,21 @@
+package jobschedule
+
+import st "github.com/getarcaneapp/arcane/types/v2/scheduler"
+
+type JobInput struct {
+ ID string `path:"id"`
+ JobID string `path:"jobId" minLength:"1"`
+}
+type ListRunsInput struct {
+ ID string `path:"id"`
+ JobID string `path:"jobId" minLength:"1"`
+ Page int `query:"page" default:"1" minimum:"1"`
+ Limit int `query:"limit" default:"20" minimum:"1" maximum:"100"`
+}
+type RunInput struct {
+ ID string `path:"id"`
+ JobID string `path:"jobId" minLength:"1"`
+ RunID string `path:"runId" format:"uuid"`
+}
+type RunOutput struct{ Body st.Run }
+type ListRunsOutput struct{ Body st.RunList }
diff --git a/types/scheduler/job.go b/types/scheduler/job.go
index 80872402c..049793c67 100644
--- a/types/scheduler/job.go
+++ b/types/scheduler/job.go
@@ -16,7 +16,7 @@ type JobRuntimeState struct {
type Job interface {
Name() string
Schedule(ctx context.Context) string
- Run(ctx context.Context)
+ Run(ctx context.Context) (Outcome, error)
}
// ConditionalJob allows a job to opt out of cron registration when it is disabled.
@@ -41,6 +41,7 @@ type StoppableBusWatcher interface {
// DynamicScheduler owns jobs registered and removed while the application is running.
type DynamicScheduler interface {
+ Submit(ctx context.Context, request Request) (Run, error)
AddJob(ctx context.Context, job Job) error
RemoveJob(ctx context.Context, name string)
HasJob(name string) bool
@@ -54,9 +55,18 @@ type JobController interface {
RunBusWatcherNow(ctx context.Context, watcherID string) error
}
+// WorkerController exposes continuous-worker health and recovery operations.
+type WorkerController interface {
+ RestartWatcher(ctx context.Context, watcherID string) error
+ WatcherHealth(watcherID string) (WorkerHealth, bool)
+}
+
// JobScheduler is the shared public contract for the actor-owned scheduler.
// Concrete cron and actor state remains private to the backend implementation.
type JobScheduler interface {
+ SetDispatcher(dispatcher Dispatcher)
+ ListRegisteredJobs() []Job
+ WorkerController
DynamicScheduler
JobController
@@ -79,7 +89,8 @@ type JobScheduler interface {
type GenericJob struct {
JobName string
ScheduleFn func(ctx context.Context) string
- RunFn func(ctx context.Context)
+ RunFn func(ctx context.Context) (Outcome, error)
+ ReconcileFn func(ctx context.Context, previous Run) (Outcome, error)
ShouldRunFn func(ctx context.Context) bool
}
@@ -87,7 +98,7 @@ func (g *GenericJob) Name() string { return g.JobName }
func (g *GenericJob) Schedule(ctx context.Context) string { return g.ScheduleFn(ctx) }
-func (g *GenericJob) Run(ctx context.Context) { g.RunFn(ctx) }
+func (g *GenericJob) Run(ctx context.Context) (Outcome, error) { return g.RunFn(ctx) }
// ShouldSchedule satisfies ConditionalJob. A GenericJob without a ShouldRunFn is
// always scheduled; the scheduler treats a ConditionalJob returning false as "do
@@ -98,3 +109,10 @@ func (g *GenericJob) ShouldSchedule(ctx context.Context) bool {
}
return g.ShouldRunFn(ctx)
}
+
+func (g *GenericJob) Reconcile(ctx context.Context, previous Run) (Outcome, error) {
+ if g.ReconcileFn != nil {
+ return g.ReconcileFn(ctx, previous)
+ }
+ return Outcome{Status: NeedsAttention, Message: "The interrupted operation has no confirmed result; review its domain records before retrying", Targets: previous.Outcome.Targets}, nil
+}
diff --git a/types/scheduler/queue.go b/types/scheduler/queue.go
new file mode 100644
index 000000000..fdd27e015
--- /dev/null
+++ b/types/scheduler/queue.go
@@ -0,0 +1,14 @@
+package scheduler
+
+import "time"
+
+// QueueRecord holds atomic admission, claims, and checkpoints for one job and target.
+type QueueRecord struct {
+ JobID string `json:"jobId"`
+ EnvironmentID string `json:"environmentId"`
+ Schedule string `json:"schedule"`
+ LastEnqueuedAt time.Time `json:"lastEnqueuedAt"`
+ NextRun time.Time `json:"nextRun"`
+ Runs []Run `json:"runs"`
+ Receipts map[string]Run `json:"receipts,omitempty"`
+}
diff --git a/types/scheduler/run.go b/types/scheduler/run.go
new file mode 100644
index 000000000..99de46ec4
--- /dev/null
+++ b/types/scheduler/run.go
@@ -0,0 +1,127 @@
+package scheduler
+
+import (
+ "context"
+ "time"
+)
+
+// RunStatus describes durable execution, independently of the configured schedule.
+type RunStatus string
+
+const (
+ Queued RunStatus = "queued"
+ Waiting RunStatus = "waiting"
+ Running RunStatus = "running"
+ Retrying RunStatus = "retrying"
+ Succeeded RunStatus = "succeeded"
+ Partial RunStatus = "partial"
+ Skipped RunStatus = "skipped"
+ Failed RunStatus = "failed"
+ NeedsAttention RunStatus = "needs_attention"
+ Canceled RunStatus = "canceled"
+)
+
+func (s RunStatus) Terminal() bool {
+ return s == Succeeded || s == Partial || s == Skipped || s == Failed || s == Canceled
+}
+
+type TargetOutcome struct {
+ ResourceType string `json:"resourceType,omitempty"`
+ ID string `json:"id"`
+ Status RunStatus `json:"status"`
+ Message string `json:"message,omitempty"`
+ ActivityID string `json:"activityId,omitempty"`
+}
+
+type Outcome struct {
+ Status RunStatus `json:"status"`
+ Message string `json:"message,omitempty"`
+ ActivityID string `json:"activityId,omitempty"`
+ Targets []TargetOutcome `json:"targets,omitempty"`
+}
+
+type Request struct {
+ RunID string `json:"runId,omitempty"`
+ JobID string `json:"jobId"`
+ EnvironmentID string `json:"environmentId"`
+ Trigger string `json:"trigger"`
+ RequestedWithKey string `json:"requestedWithKey,omitempty"`
+ RequestedBy string `json:"requestedBy,omitempty"`
+}
+
+type Attempt struct {
+ Number int `json:"number"`
+ StartedAt time.Time `json:"startedAt"`
+ FinishedAt *time.Time `json:"finishedAt,omitempty"`
+ Outcome Outcome `json:"outcome"`
+}
+
+type Run struct {
+ RequestedWithKey string `json:"requestedWithKey,omitempty"`
+ ID string `json:"id"`
+ JobID string `json:"jobId"`
+ EnvironmentID string `json:"environmentId"`
+ Trigger string `json:"trigger"`
+ RequestedBy string `json:"requestedBy,omitempty"`
+ Status RunStatus `json:"status"`
+ CreatedAt time.Time `json:"createdAt"`
+ UpdatedAt time.Time `json:"updatedAt"`
+ StartedAt *time.Time `json:"startedAt,omitempty"`
+ FinishedAt *time.Time `json:"finishedAt,omitempty"`
+ NextAttempt *time.Time `json:"nextAttempt,omitempty"`
+ AttemptCount int `json:"attemptCount"`
+ Owner string `json:"owner,omitempty"`
+ Outcome Outcome `json:"outcome"`
+ Attempts []Attempt `json:"attempts,omitempty"`
+ RemoteDeliveryAttempted bool `json:"remoteDeliveryAttempted"`
+ RemoteOutcome *Outcome `json:"remoteOutcome,omitempty"`
+ RemoteRetryRequested bool `json:"remoteRetryRequested,omitempty"`
+ RemoteRetryAttempted bool `json:"remoteRetryAttempted,omitempty"`
+ RemoteAttemptCount int `json:"remoteAttemptCount,omitempty"`
+ RemoteAccepted bool `json:"remoteAccepted"`
+ RemoteSettled bool `json:"remoteSettled"`
+ LastConfirmedAt *time.Time `json:"lastConfirmedAt,omitempty"`
+}
+
+type RunList struct {
+ Runs []Run `json:"runs"`
+ Total int `json:"total"`
+ Page int `json:"page"`
+ Limit int `json:"limit"`
+}
+
+type WorkerHealth struct {
+ Status string `json:"status"`
+ LastError string `json:"lastError,omitempty"`
+ NextRetry *time.Time `json:"nextRetry,omitempty"`
+ UpdatedAt time.Time `json:"updatedAt"`
+}
+
+// Dispatcher owns durable admission and execution. Cron only submits work.
+type Dispatcher interface {
+ Submit(ctx context.Context, request Request) (Run, error)
+ Checkpoint(ctx context.Context, jobID, schedule string, nextRun time.Time) error
+}
+
+// Reconciler consults domain state before repeating interrupted work.
+type Reconciler interface {
+ Reconcile(ctx context.Context, previous Run) (Outcome, error)
+}
+
+// OutcomeError carries partial results through error-only watcher contracts.
+type OutcomeError struct {
+ Outcome Outcome
+ Cause error
+}
+
+func (e *OutcomeError) Error() string {
+ if e.Outcome.Message != "" {
+ return e.Outcome.Message
+ }
+ if e.Cause != nil {
+ return e.Cause.Error()
+ }
+ return string(e.Outcome.Status)
+}
+
+func (e *OutcomeError) Unwrap() error { return e.Cause }