Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
refactor: migrate jobs to use fan out workflows (#4356)
Kyle Mendell1cb757731a43
diff --git a/AGENTS.md b/AGENTS.mdindex dead06019..48cb1c5df 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -253,6 +253,8 @@ Determine global admin status through `PermissionSet.IsGlobalAdmin()` or the useBackground jobs implement the scheduler job contract. Wire them through `internal/di` and register them in `internal/bootstrap/jobs_bootstrap.go`.+Multi-step, fan-out, or resumable jobs use `backend/pkg/scheduler/flow`: Francis workflows that run inside coordinator runs. Follow the migration recipe in its package doc. Steps are idempotent service methods, the last step returns `scheduler.Outcome`, the job registers a `flow.Job`, and `flowtest.AssertDefinitions` guards version bumps. Keep single-operation jobs as ordinary jobs.+## Validation### Testsdiff --git a/backend/internal/activity/service.go b/backend/internal/activity/service.goindex c3cc1f943..8378f5c0f 100644--- a/backend/internal/activity/service.go+++ b/backend/internal/activity/service.go@@ -19,6 +19,7 @@ import ("github.com/samber/mo""go.getarcane.app/kit/pkg""gorm.io/gorm"+ "gorm.io/gorm/clause""github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/internal/database"@@ -309,6 +310,18 @@ func (s *ActivityService) StartActivity(ctx context.Context, req StartActivityRereturn nil, err}+ if req.ID != "" {+ var existing Activity+ err := s.db.WithContext(ctx).Where("id = ?", req.ID).Limit(1).Find(&existing).Error+ if err != nil {+ return nil, fmt.Errorf("failed to load activity: %w", err)+ }+ if existing.ID != "" {+ dto := activityToDTOInternal(&existing)+ return &dto, nil+ }+ }+now := time.Now()environmentID := cmp.Or(strings.TrimSpace(req.EnvironmentID), "0")@@ -361,15 +374,30 @@ func (s *ActivityService) StartActivity(ctx context.Context, req StartActivityReMetadata: cloneJSONInternal(req.Metadata),CreatedAt: now,}+ model.ID = req.IDif model.Type == "" {model.Type = activity.TypeAutoUpdate}- if err := s.db.WithContext(ctx).Create(model).Error; err != nil {+ create := s.db.WithContext(ctx)+ if req.ID != "" {+ create = create.Clauses(clause.OnConflict{DoNothing: true})+ }+ created := create.Create(model)+ if created.Error != nil || created.RowsAffected == 0 {if slotRelease != nil {slotRelease()}- return nil, fmt.Errorf("failed to create activity: %w", err)+ if created.Error != nil {+ return nil, fmt.Errorf("failed to create activity: %w", created.Error)+ }+ // A concurrent start with the same ID created it first.+ var existing Activity+ if err := s.db.WithContext(ctx).First(&existing, "id = ?", req.ID).Error; err != nil {+ return nil, fmt.Errorf("failed to load activity: %w", err)+ }+ dto := activityToDTOInternal(&existing)+ return &dto, nil}if slotRelease != nil {s.registerSlotReleaseInternal(model.ID, slotRelease)diff --git a/backend/internal/activity/service_test.go b/backend/internal/activity/service_test.goindex 3b16ffe57..3fa42d4e3 100644--- a/backend/internal/activity/service_test.go+++ b/backend/internal/activity/service_test.go@@ -1064,3 +1064,21 @@ func TestSyncResourcesToEnvironmentOutcomes(t *testing.T) {})}}++func TestActivityService_StartActivityWithIDIsIdempotent(t *testing.T) {+ db := setupActivityServiceTestDBInternal(t)+ service := NewActivityService(db, nil)++ first, err := service.StartActivity(t.Context(), StartActivityRequest{ID: "workflow-activity", Type: activity.TypeAutoUpdate, LatestMessage: "first"})+ require.NoError(t, err)+ require.Equal(t, "workflow-activity", first.ID)++ again, err := service.StartActivity(t.Context(), StartActivityRequest{ID: "workflow-activity", Type: activity.TypeAutoUpdate, LatestMessage: "redelivered"})+ require.NoError(t, err)+ require.Equal(t, first.ID, again.ID)+ require.Equal(t, "first", again.LatestMessage)++ var count int64+ require.NoError(t, db.Model(&Activity{}).Where("id = ?", "workflow-activity").Count(&count).Error)+ require.Equal(t, int64(1), count)+}diff --git a/backend/internal/apikey/handler.go b/backend/internal/apikey/handler.goindex 339bab30c..3a0f77292 100644--- a/backend/internal/apikey/handler.go+++ b/backend/internal/apikey/handler.go@@ -148,14 +148,7 @@ func (h *ApiKeyHandler) DeleteApiKey(ctx context.Context, input *DeleteApiKeyInpreturn nil, huma.Error500InternalServerError("Failed to delete API key")}- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{- Message: "API key deleted successfully",- },- },- }, nil+ return handlerutil.MessageOutput("API key deleted successfully", ""), nil}// ListMyApiKeys lists API keys owned by the current user (self-service).@@ -239,12 +232,5 @@ func (h *ApiKeyHandler) DeleteMyApiKey(ctx context.Context, input *DeleteApiKeyIreturn nil, huma.Error500InternalServerError("Failed to delete API key")}- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{- Message: "API key deleted successfully",- },- },- }, nil+ return handlerutil.MessageOutput("API key deleted successfully", ""), nil}diff --git a/backend/internal/apns/handler.go b/backend/internal/apns/handler.goindex 2113e7f52..35bab8895 100644--- a/backend/internal/apns/handler.go+++ b/backend/internal/apns/handler.go@@ -105,7 +105,7 @@ func (h *ApnsHandler) DeleteDevice(ctx context.Context, input *DeviceIDInput) (*if deleteDeviceErr := h.service.DeleteDevice(ctx, user.ID, input.ID); deleteDeviceErr != nil {return nil, apnsHTTPErrorInternal(deleteDeviceErr)}- return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Device removed"}}}, nil+ return handlerutil.MessageOutput("Device removed", ""), nil}func (h *ApnsHandler) TestDevice(ctx context.Context, input *DeviceIDInput) (*handlerutil.Out[base.MessageResponse], error) {@@ -116,5 +116,5 @@ func (h *ApnsHandler) TestDevice(ctx context.Context, input *DeviceIDInput) (*haif testDeviceErr := h.service.TestDevice(ctx, user.ID, input.ID); testDeviceErr != nil {return nil, apnsHTTPErrorInternal(testDeviceErr)}- return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Test notification queued"}}}, nil+ return handlerutil.MessageOutput("Test notification queued", ""), nil}diff --git a/backend/internal/auth/handler.go b/backend/internal/auth/handler.goindex 66cdb548c..11c8640da 100644--- a/backend/internal/auth/handler.go+++ b/backend/internal/auth/handler.go@@ -25,7 +25,7 @@ import ("github.com/getarcaneapp/arcane/types/v2/user""github.com/labstack/echo/v5""github.com/samber/mo"- "go.getarcane.app/kit/pkg"+ kit "go.getarcane.app/kit/pkg""github.com/getarcaneapp/arcane/backend/v2/internal/apikey""github.com/getarcaneapp/arcane/backend/v2/internal/common"@@ -272,14 +272,7 @@ func (h *AuthHandler) ChangePassword(ctx context.Context, input *ChangePasswordI}}- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{- Message: "Password changed successfully",- },- },- }, nil+ return handlerutil.MessageOutput("Password changed successfully", ""), nil}// LogoutAllOtherSessions revokes every active session for the current user@@ -295,14 +288,7 @@ func (h *AuthHandler) LogoutAllOtherSessions(ctx context.Context, input *struct{return nil, huma.Error500InternalServerError("failed to revoke sessions: " + logoutAllOtherSessionsErr.Error())}- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{- Message: "All other sessions signed out",- },- },- }, nil+ return handlerutil.MessageOutput("All other sessions signed out", ""), nil}// validatePreferencesInternal checks the preference values Huma's enum tagsdiff --git a/backend/internal/backup/model.go b/backend/internal/backup/model.goindex 6112c21fb..fd6afd886 100644--- a/backend/internal/backup/model.go+++ b/backend/internal/backup/model.go@@ -9,4 +9,5 @@ type SystemBackupRecoveryConfig struct {EncryptedRecoveryKey string `gorm:"column:encrypted_recovery_key;type:text;not null"`}+// TableName is the recovery key table.func (SystemBackupRecoveryConfig) TableName() string { return "system_backup_recovery_config" }diff --git a/backend/internal/backup/service.go b/backend/internal/backup/service.goindex 023a35a88..cabd3c168 100644--- a/backend/internal/backup/service.go+++ b/backend/internal/backup/service.go@@ -1,10 +1,9 @@// Package backup owns the shared Rustic backup engine used by volume and// system backups: typed repository operations, per-repository serialization-// and durable run admission.+// and run admission.package backupimport (- "bytes""cmp""context""crypto/rand"@@ -16,7 +15,6 @@ import ("fmt""io""log/slog"- "maps""path""regexp""slices"@@ -28,33 +26,33 @@ import ("github.com/getarcaneapp/arcane/types/v2/backup""github.com/getarcaneapp/arcane/types/v2/scheduler""github.com/getarcaneapp/arcane/types/v2/user"- "github.com/italypaleale/francis/actor"- "github.com/italypaleale/francis/host/local""github.com/moby/moby/api/types/container""github.com/moby/moby/api/types/mount""github.com/moby/moby/client"- "go.getarcane.app/kit/pkg"+ kit "go.getarcane.app/kit/pkg""go.getarcane.app/sys/crypto""gorm.io/gorm""github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/image"+ "github.com/getarcaneapp/arcane/backend/v2/internal/middleware""github.com/getarcaneapp/arcane/backend/v2/internal/s3""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/rustic""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/volumehelper"- "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/schedule"- "github.com/getarcaneapp/arcane/backend/v2/pkg/utils"- "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis"s3util "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/s3")const (VolumeAdmissionScope = "volume-backup"SystemAdmissionScope = "system-backup"- RecoveryKeyConfigID = "system-recovery"+ // SystemAdmissionID is the one admission key system and system-managed volume backups share.+ SystemAdmissionID = "system"+ RecoveryKeyConfigID = "system-recovery"+ // VolumeRoot is the S3 prefix under which each instance keeps its volume backup repository.+ VolumeRoot = "arcane-volume-backups"// rusticRepositoryMissingMessage is how Rustic reports an uninitialized repository.rusticRepositoryMissingMessage = "No repository config file found"@@ -71,7 +69,7 @@ type Snapshot struct {Size int64 `json:"-"`}-type rusticSnapshotOutputInternal struct {+type rusticSnapshotOutput struct {ID string `json:"id"`Summary struct {TotalBytesProcessed int64 `json:"total_bytes_processed"`@@ -107,7 +105,7 @@ func RootSnapshotInput(source mount.Mount, tags ...string) CreateSnapshotInput {return CreateSnapshotInput{Mounts: []mount.Mount{source}, Sources: []string{source.Target}, AsPath: "/", Tags: tags}}-func snapshotCommandInternal(label string, input CreateSnapshotInput) ([]string, error) {+func snapshotCommand(label string, input CreateSnapshotInput) ([]string, error) {if len(input.Sources) == 0 {return nil, errors.New("at least one snapshot source is required")}@@ -128,58 +126,40 @@ func snapshotCommandInternal(label string, input CreateSnapshotInput) ([]string,return append(command, input.Sources...), nil}-// Engine owns repository serialization and application-owned backup workers.+// Engine owns repository serialization and backup admission.type Engine struct {- imageService *image.ImageService- admission *runs.Admission- lifecycleCtx context.Context- cancel context.CancelFunc- workers sync.WaitGroup- stopping bool- mu sync.Mutex- repositories map[string]*sync.Mutex- service *actor.Service- handlers map[string]func(context.Context, string, []byte, bool) error- failures map[string]func(context.Context, string, []byte, error) error- leases map[string]*runs.Lease- unresolved map[string]backup.DurableRunCommand- executionReady func() bool- authorize func(context.Context, backup.DurableRunCommand) error+ imageService *image.ImageService+ admission *runs.Admission+ mu sync.Mutex+ repositories map[string]*sync.Mutex+ // held parks admission a manual start won until the run that won it takes it over.+ held map[scheduler.AdmissionKey]heldLease+ authorize func(context.Context, backup.Requester) error}-func NewEngine(ctx context.Context, admission *runs.Admission, imageService *image.ImageService) *Engine {- runCtx, cancel := context.WithCancel(ctx)+// heldLease is a parked lease and the run entitled to take it.+type heldLease struct {+ run string+ lease *runs.Lease+}++// NewRequester captures the user, and API key if any, behind a manual backup request.+func NewRequester(ctx context.Context, requestedBy user.Actor, environmentID, permission string) backup.Requester {+ keyID, _ := ctx.Value(middleware.ContextKeyApiKeyID).(string)+ return backup.Requester{UserID: requestedBy.ID, APIKeyID: keyID, EnvironmentID: environmentID, Permission: permission}+}++// NewEngine builds the backup engine on the shared admission gate.+func NewEngine(admission *runs.Admission, imageService *image.ImageService) *Engine {return &Engine{- cancel: cancel,imageService: imageService,admission: admission,- lifecycleCtx: runCtx,- repositories: make(- map[string]*sync.Mutex,- ),- handlers: make(- map[string]func(- context.Context,- string,- []byte,- bool,- ) error,- ),- failures: make(- map[string]func(- context.Context,- string,- []byte,- error,- ) error,- ),- leases: make(- map[string]*runs.Lease,- ),- unresolved: make(map[string]backup.DurableRunCommand),+ repositories: make(map[string]*sync.Mutex),+ held: make(map[scheduler.AdmissionKey]heldLease),}}+// TryAcquireRun takes the admission lease for one backup resource, reporting false while another run holds it.func (e *Engine) TryAcquireRun(ctx context.Context, scope, id string) (*runs.Lease, bool, error) {if e == nil || e.admission == nil {return nil, false, errors.New("backup engine is unavailable")@@ -187,46 +167,79 @@ func (e *Engine) TryAcquireRun(ctx context.Context, scope, id string) (*runs.Leareturn e.admission.TryAcquire(ctx, scheduler.AdmissionKey{Scope: scope, ID: id})}-func (e *Engine) Stop(ctx context.Context) error {- if e == nil {- return nil- }+// Hold parks a lease won by a manual start for run until that run's task takes it over with AcquireRun.+// The returned func releases the lease if the run never took it, such as a start that failed to submit.+func (e *Engine) Hold(scope, id, run string, lease *runs.Lease) func(context.Context) {+ key := scheduler.AdmissionKey{Scope: scope, ID: id}e.mu.Lock()- e.stopping = true- e.cancel()+ e.held[key] = heldLease{run: run, lease: lease}e.mu.Unlock()- done := make(chan struct{})- go func() {- e.workers.Wait()+ return func(ctx context.Context) {e.mu.Lock()- leases := e.leases- e.leases = make(map[string]*runs.Lease)+ untaken := e.held[key].lease == lease+ if untaken {+ delete(e.held, key)+ }e.mu.Unlock()- for _, lease := range leases {+ if untaken {lease.Release(ctx)}- close(done)- }()- select {- case <-done:+ }+}++// AcquireRun takes over the lease held for run, or acquires admission otherwise, such as after a restart.+// A lease parked for another run stays parked, so that run's task still finds its admission.+func (e *Engine) AcquireRun(ctx context.Context, scope, id, run string) (*runs.Lease, bool, error) {+ key := scheduler.AdmissionKey{Scope: scope, ID: id}+ e.mu.Lock()+ held, ok := e.held[key]+ if ok = ok && run != "" && held.run == run; ok {+ delete(e.held, key)+ }+ e.mu.Unlock()+ if ok {+ return held.lease, true, nil+ }+ return e.TryAcquireRun(ctx, scope, id)+}++// SetAuthorize sets the check Authorize applies to manual backup requesters.+func (e *Engine) SetAuthorize(authorize func(context.Context, backup.Requester) error) {+ e.authorize = authorize+}++// Authorize checks that the requester may still run the backup.+func (e *Engine) Authorize(ctx context.Context, requester backup.Requester) error {+ if e.authorize == nil {return nil- case <-ctx.Done():- return ctx.Err()}+ return e.authorize(ctx, requester)+}++// Stop releases leases that no workflow task took over.+func (e *Engine) Stop(ctx context.Context) error {+ e.mu.Lock()+ held := e.held+ e.held = make(map[scheduler.AdmissionKey]heldLease)+ e.mu.Unlock()+ for _, held := range held {+ held.lease.Release(ctx)+ }+ return nil}// CreateSnapshot backs the input's sources up into the repository as one// snapshot. The repository is initialized on first use.func (e *Engine) CreateSnapshot(ctx context.Context, dockerClient *client.Client, repository Repository, password, label string, input CreateSnapshotInput) (Snapshot, error) {- command, err := snapshotCommandInternal(label, input)+ command, err := snapshotCommand(label, input)if err != nil {return Snapshot{}, err}- output, err := e.runInternal(ctx, dockerClient, repository, password, command, input.Mounts...)+ output, err := e.run(ctx, dockerClient, repository, password, command, input.Mounts...)if err != nil {return Snapshot{}, err}- var decoded rusticSnapshotOutputInternal+ var decoded rusticSnapshotOutputif unmarshalErr := json.Unmarshal([]byte(output), &decoded); unmarshalErr != nil {return Snapshot{}, fmt.Errorf("failed to decode Rustic snapshot: %w", unmarshalErr)}@@ -242,14 +255,10 @@ func (e *Engine) RestoreSnapshot(ctx context.Context, dockerClient *client.Clienif options.DeleteExtra {command = append(command, "--delete")}- source := snapshotID + ":/"- if options.SourcePath != "" {- source = snapshotID + ":/" + strings.TrimPrefix(options.SourcePath, "/")- }- destination := cmp.Or(options.DestinationPath, target.Target)- command = append(command, "--", source, destination)+ source := snapshotID + ":/" + strings.TrimPrefix(options.SourcePath, "/")+ command = append(command, "--", source, cmp.Or(options.DestinationPath, target.Target))mounts := append([]mount.Mount{target}, options.ExtraMounts...)- _, err := e.runInternal(ctx, dockerClient, repository, password, command, mounts...)+ _, err := e.run(ctx, dockerClient, repository, password, command, mounts...)return err}@@ -260,13 +269,14 @@ func (e *Engine) ListSnapshotFiles(ctx context.Context, dockerClient *client.Cliif recursive {command = append(command, "--recursive")}- cleanedPath := strings.Trim(strings.TrimSpace(filePath), "/")+ filePath = strings.TrimSpace(filePath)+ cleanedPath := strings.Trim(filePath, "/")source := snapshotID + ":/" + cleanedPath- if cleanedPath != "" && strings.HasSuffix(strings.TrimSpace(filePath), "/") {+ if cleanedPath != "" && strings.HasSuffix(filePath, "/") {source += "/"}command = append(command, "--", source)- output, err := e.runInternal(ctx, dockerClient, repository, password, command)+ output, err := e.run(ctx, dockerClient, repository, password, command)if err != nil {return nil, fmt.Errorf("failed to list Rustic snapshot: %w", err)}@@ -278,21 +288,21 @@ func (e *Engine) ListSnapshotFiles(ctx context.Context, dockerClient *client.Cli// Rustic's JSON listing omits node types; the stable long listing restores them.longCommand := slices.Clone(command)longCommand[1] = "--long"- longOutput, runErr := e.runInternal(ctx, dockerClient, repository, password, longCommand)+ longOutput, runErr := e.run(ctx, dockerClient, repository, password, longCommand)if runErr != nil {return nil, fmt.Errorf("failed to list Rustic snapshot metadata: %w", runErr)}- files, runErr = markSnapshotDirectoriesInternal(files, longOutput)+ files, runErr = markSnapshotDirectories(files, longOutput)if runErr != nil {return nil, runErr}}- return qualifySnapshotListingInternal(files, cleanedPath), nil+ return qualifySnapshotListing(files, cleanedPath), nil}-func qualifySnapshotListingInternal(files []string, snapshotPath string) []string {+// qualifySnapshotListing prefixes listed files with the cleaned snapshot path they were listed under.+func qualifySnapshotListing(files []string, prefix string) []string {qualified := slices.Clone(files)- prefix := strings.Trim(strings.TrimSpace(snapshotPath), "/")if prefix == "" {return qualified}@@ -307,7 +317,7 @@ func qualifySnapshotListingInternal(files []string, snapshotPath string) []strinreturn qualified}-func markSnapshotDirectoriesInternal(files []string, longOutput string) ([]string, error) {+func markSnapshotDirectories(files []string, longOutput string) ([]string, error) {if len(files) == 0 {if strings.TrimSpace(longOutput) != "" {return nil, errors.New("rustic file and metadata listings have different lengths")@@ -332,7 +342,7 @@ func markSnapshotDirectoriesInternal(files []string, longOutput string) ([]strin// ReadSnapshotTextFile returns one text file from a snapshot.func (e *Engine) ReadSnapshotTextFile(ctx context.Context, dockerClient *client.Client, repository Repository, password, snapshotID, filePath string) (string, error) {- output, err := e.runInternal(ctx, dockerClient, repository, password, []string{+ output, err := e.run(ctx, dockerClient, repository, password, []string{"dump", "--archive", "content", "--", snapshotID + ":/" + strings.TrimPrefix(filePath, "/"),})if err != nil {@@ -357,11 +367,11 @@ func RunSnapshotTag(runID string) string { return "arcane-run:" + runID }// ConfirmRunSnapshot makes a validated snapshot discoverable by run ID.func (e *Engine) ConfirmRunSnapshot(ctx context.Context, dockerClient *client.Client, repository Repository, password, runID, snapshotID string) (Snapshot, error) {- if !fullSnapshotIDInternal(snapshotID) {+ if !fullSnapshotID(snapshotID) {return Snapshot{}, errors.New("a full snapshot ID is required")}command := []string{"tag", "--add", RunSnapshotTag(runID), "--", snapshotID}- if _, err := e.runInternal(ctx, dockerClient, repository, password, command); err != nil {+ if _, err := e.run(ctx, dockerClient, repository, password, command); err != nil {return Snapshot{}, err}// Tagging rewrites the snapshot and changes its ID.@@ -403,14 +413,14 @@ func (e *Engine) FindRunSnapshot(ctx context.Context, dockerClient *client.Clien// ListSnapshots enumerates every snapshot in the repository.func (e *Engine) ListSnapshots(ctx context.Context, dockerClient *client.Client, repository Repository, password string) ([]DiscoveredSnapshot, error) {- output, err := e.runInternal(ctx, dockerClient, repository, password, []string{"snapshots", "--json"})+ output, err := e.run(ctx, dockerClient, repository, password, []string{"snapshots", "--json"})if err != nil {return nil, err}- return decodeSnapshotsInternal(output)+ return decodeSnapshots(output)}-func decodeSnapshotsInternal(output string) ([]DiscoveredSnapshot, error) {+func decodeSnapshots(output string) ([]DiscoveredSnapshot, error) {trimmedOutput := strings.TrimSpace(output)if trimmedOutput == "" || trimmedOutput[0] != '[' {return nil, errors.New("invalid Rustic snapshot listing: expected an array")@@ -439,25 +449,34 @@ func decodeSnapshotsInternal(output string) ([]DiscoveredSnapshot, error) {return nil, errors.New("invalid Rustic snapshot listing: mixed flat and grouped entries")}grouped = hasSnapshots- if !hasSnapshots {- snapshot, err := decodeSnapshotInternal(entry)- if err != nil {- return nil, fmt.Errorf("invalid Rustic snapshot listing entry %d: %w", index, err)+ members := []jsontext.Value{entry}+ if hasSnapshots {+ if groupEntries.Kind() != '[' {+ return nil, fmt.Errorf("invalid Rustic snapshot group %d: expected a snapshots array", index)+ }+ members = nil+ if err := json.Unmarshal(groupEntries, &members); err != nil {+ return nil, fmt.Errorf("failed to decode Rustic snapshot group %d: %w", index, err)}- snapshots = append(snapshots, snapshot)- continue- }- if groupEntries.Kind() != '[' {- return nil, fmt.Errorf("invalid Rustic snapshot group %d: expected a snapshots array", index)- }- var members []jsontext.Value- if err := json.Unmarshal(groupEntries, &members); err != nil {- return nil, fmt.Errorf("failed to decode Rustic snapshot group %d: %w", index, err)}for memberIndex, member := range members {- snapshot, err := decodeSnapshotInternal(member)- if err != nil {- return nil, fmt.Errorf("invalid Rustic snapshot group %d entry %d: %w", index, memberIndex, err)+ location := kit.Ternary(hasSnapshots, fmt.Sprintf("group %d entry %d", index, memberIndex), fmt.Sprintf("listing entry %d", index))+ var memberFields map[string]jsontext.Value+ if member.Kind() != '{' {+ return nil, fmt.Errorf("invalid Rustic snapshot %s: expected a snapshot object", location)+ }+ if err := json.Unmarshal(member, &memberFields); err != nil {+ return nil, fmt.Errorf("invalid Rustic snapshot %s: failed to decode Rustic snapshot fields: %w", location, err)+ }+ if _, nested := memberFields["snapshots"]; nested {+ return nil, fmt.Errorf("invalid Rustic snapshot %s: expected a flat snapshot object", location)+ }+ var snapshot DiscoveredSnapshot+ if err := json.Unmarshal(member, &snapshot); err != nil {+ return nil, fmt.Errorf("invalid Rustic snapshot %s: failed to decode Rustic snapshot payload: %w", location, err)+ }+ if !fullSnapshotID(snapshot.ID) {+ return nil, fmt.Errorf("invalid Rustic snapshot %s: invalid full snapshot ID", location)}snapshots = append(snapshots, snapshot)}@@ -465,28 +484,7 @@ func decodeSnapshotsInternal(output string) ([]DiscoveredSnapshot, error) {return snapshots, nil}-func decodeSnapshotInternal(raw jsontext.Value) (DiscoveredSnapshot, error) {- if raw.Kind() != '{' {- return DiscoveredSnapshot{}, errors.New("expected a snapshot object")- }- var fields map[string]jsontext.Value- if err := json.Unmarshal(raw, &fields); err != nil {- return DiscoveredSnapshot{}, fmt.Errorf("failed to decode Rustic snapshot fields: %w", err)- }- if _, grouped := fields["snapshots"]; grouped {- return DiscoveredSnapshot{}, errors.New("expected a flat snapshot object")- }- var snapshot DiscoveredSnapshot- if err := json.Unmarshal(raw, &snapshot); err != nil {- return DiscoveredSnapshot{}, fmt.Errorf("failed to decode Rustic snapshot payload: %w", err)- }- if !fullSnapshotIDInternal(snapshot.ID) {- return DiscoveredSnapshot{}, errors.New("invalid full snapshot ID")- }- return snapshot, nil-}--func fullSnapshotIDInternal(id string) bool {+func fullSnapshotID(id string) bool {if len(id) != 64 {return false}@@ -499,30 +497,25 @@ func fullSnapshotIDInternal(id string) bool {func (e *Engine) ForgetSnapshots(ctx context.Context, dockerClient *client.Client, repository Repository, password string, snapshotIDs []string) error {requested := make([]string, 0, len(snapshotIDs))for _, id := range snapshotIDs {- if !fullSnapshotIDInternal(id) {+ if !fullSnapshotID(id) {return errors.New("a full snapshot ID is required")}requested = append(requested, strings.ToLower(id))}requested = kit.Unique(requested)- if e == nil {- return errors.New("backup engine is unavailable")- }- if strings.TrimSpace(repository.ID) == "" {- return errors.New("backup repository ID is required")- }- defer e.lockRepositoryInternal(repository.ID)()- if err := ctx.Err(); err != nil {+ unlock, err := e.lockRepository(ctx, repository.ID)+ if err != nil {return err}- output, err := e.runContainerInternal(ctx, dockerClient, repository, password, []string{"snapshots", "--json"})+ defer unlock()+ output, err := e.runContainer(ctx, dockerClient, repository, password, []string{"snapshots", "--json"})if len(snapshotIDs) == 0 && err != nil && strings.Contains(err.Error(), rusticRepositoryMissingMessage) {return nil}if err != nil {return err}- snapshots, err := decodeSnapshotsInternal(output)+ snapshots, err := decodeSnapshots(output)if err != nil {return err}@@ -542,7 +535,7 @@ func (e *Engine) ForgetSnapshots(ctx context.Context, dockerClient *client.Clienif len(command) == 2 {command = []string{"prune"}}- _, err = e.runContainerInternal(ctx, dockerClient, repository, password, command)+ _, err = e.runContainer(ctx, dockerClient, repository, password, command)return err}@@ -551,16 +544,12 @@ func (e *Engine) ChangeRepositoryPassword(ctx context.Context, dockerClient *cliif strings.TrimSpace(newPassword) == "" {return errors.New("new repository password is required")}- _, err := e.runInternal(ctx, dockerClient, repository, currentPassword, []string{"key", "password", "--new-password", newPassword})+ _, err := e.run(ctx, dockerClient, repository, currentPassword, []string{"key", "password", "--new-password", newPassword})return err}-// Replicate copies one snapshot between repositories by restoring it into a-// temporary volume and backing that volume up into the target repository. The-// source is read once from the repository, never from the live data. Rustic's-// native `copy` would move only missing packs, but it addresses the target via-// a TOML config profile, which the env-only Repository cannot express yet —-// the materialize-and-rebackup here trades disk and I/O for that simplicity.+// Replicate copies one snapshot into another repository through a temporary volume, never the live data.+// Rustic's native `copy` needs a TOML profile for the target, which the env-only Repository cannot express.func (e *Engine) Replicate(ctx context.Context, dockerClient *client.Client, from Repository, fromSnapshotID string, to Repository, password, label string, tags ...string) (Snapshot, error) {temporaryVolume := "arcane-rustic-copy-" + uuid.New().String()if _, err := dockerClient.VolumeCreate(ctx, client.VolumeCreateOptions{Name: temporaryVolume, Labels: volumehelper.Labels()}); err != nil {@@ -581,58 +570,47 @@ func (e *Engine) Replicate(ctx context.Context, dockerClient *client.Client, froreturn snapshot, nil}-func (e *Engine) runInternal(ctx context.Context, dockerClient *client.Client, repository Repository, password string, command []string, extraMounts ...mount.Mount) (string, error) {- if e == nil {- return "", errors.New("backup engine is unavailable")- }- if strings.TrimSpace(repository.ID) == "" {- return "", errors.New("backup repository ID is required")- }- defer e.lockRepositoryInternal(repository.ID)()- if err := ctx.Err(); err != nil {+func (e *Engine) run(ctx context.Context, dockerClient *client.Client, repository Repository, password string, command []string, extraMounts ...mount.Mount) (string, error) {+ unlock, err := e.lockRepository(ctx, repository.ID)+ if err != nil {return "", err}- return e.runContainerInternal(ctx, dockerClient, repository, password, command, extraMounts...)+ defer unlock()+ return e.runContainer(ctx, dockerClient, repository, password, command, extraMounts...)}-func (e *Engine) ensureImageInternal(ctx context.Context, dockerClient *client.Client) error {- if _, err := dockerClient.ImageInspect(ctx, rustic.DefaultImage); err == nil {- return nil- }- if e.imageService == nil {- return errors.New("image service is unavailable")- }- if err := e.imageService.PullImage(ctx, rustic.DefaultImage, io.Discard, user.SystemUser, nil); err != nil {- return fmt.Errorf("failed to pull official Rustic image: %w", err)+// runContainer runs one Rustic command; callers hold the repository lock.+func (e *Engine) runContainer(ctx context.Context, dockerClient *client.Client, repository Repository, password string, command []string, extraMounts ...mount.Mount) (string, error) {+ if _, err := dockerClient.ImageInspect(ctx, rustic.DefaultImage); err != nil {+ if e.imageService == nil {+ return "", errors.New("image service is unavailable")+ }+ if pullErr := e.imageService.PullImage(ctx, rustic.DefaultImage, io.Discard, user.SystemUser, nil); pullErr != nil {+ return "", fmt.Errorf("failed to pull official Rustic image: %w", pullErr)+ }}- return nil-}--func arcaneNetworkModeInternal(ctx context.Context, dockerClient *client.Client) container.NetworkMode {+ var networkMode container.NetworkModearcane, err := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient)if err != nil || arcane == nil || arcane.ID == "" {slog.DebugContext(ctx, "backup engine: running Rustic on the default network", "error", err)- return ""- }- return container.NetworkMode("container:" + arcane.ID)-}--func (e *Engine) runContainerInternal(ctx context.Context, dockerClient *client.Client, repository Repository, password string, command []string, extraMounts ...mount.Mount) (string, error) {- if err := e.ensureImageInternal(ctx, dockerClient); err != nil {- return "", err+ } else {+ networkMode = container.NetworkMode("container:" + arcane.ID)}- mounts := append([]mount.Mount{}, repository.Mounts...)- mounts = append(mounts, extraMounts...)- return rustic.Run(ctx, dockerClient, password, command, repository.Environment, mounts, arcaneNetworkModeInternal(ctx, dockerClient))+ mounts := append(slices.Clone(repository.Mounts), extraMounts...)+ return rustic.Run(ctx, dockerClient, password, command, repository.Environment, mounts, networkMode)}var (ErrRecoveryKeyNotConfigured = errors.New("recovery key is not configured")+ // ErrBackupSettled reports a checkpointed backup whose own attempt already failed it, so there is nothing to+ // resume; a retry starts a new backup instead.+ ErrBackupSettled = errors.New("the checkpointed backup already failed")// RecoveryKeyFormat is 8 hyphenated groups of 6 base32 characters, used verbatim as the Rustic password.RecoveryKeyFormat = regexp.MustCompile(`^[A-Z2-7]{6}(-[A-Z2-7]{6}){7}$`))+// ValidateRecoveryKey checks that a key has the generated recovery key format.func ValidateRecoveryKey(key string) error {if !RecoveryKeyFormat.MatchString(strings.TrimSpace(key)) {return errors.New("enter the generated recovery key (8 groups of 6 characters)")@@ -640,6 +618,7 @@ func ValidateRecoveryKey(key string) error {return nil}+// GenerateRecoveryKey creates a random recovery key in RecoveryKeyFormat.func GenerateRecoveryKey() (string, error) {raw := make([]byte, 30)if _, err := rand.Read(raw); err != nil {@@ -658,10 +637,12 @@ type RecoveryKeyStore struct {db *database.DB}+// NewRecoveryKeyStore builds the recovery key store.func NewRecoveryKeyStore(db *database.DB) *RecoveryKeyStore {return &RecoveryKeyStore{db: db}}+// Configured reports whether a recovery key is stored.func (s *RecoveryKeyStore) Configured(ctx context.Context) (bool, error) {var count int64if err := s.db.WithContext(ctx).Model(&SystemBackupRecoveryConfig{}).@@ -673,10 +654,40 @@ func (s *RecoveryKeyStore) Configured(ctx context.Context) (bool, error) {// Get returns the decrypted recovery key or ErrRecoveryKeyNotConfigured.func (s *RecoveryKeyStore) Get(ctx context.Context) (string, error) {+ return s.load(ctx, RecoveryKeyConfigID)+}++// Set validates and stores the recovery key, encrypted.+func (s *RecoveryKeyStore) Set(ctx context.Context, recoveryKey string) error {+ return s.save(ctx, RecoveryKeyConfigID, recoveryKey)+}++// HoldRunKey keeps a recovery key entered for one backup run, encrypted, until ReleaseRunKey, so the run can+// resume after a restart without it ever entering the run's workflow payload.+func (s *RecoveryKeyStore) HoldRunKey(ctx context.Context, runID, recoveryKey string) error {+ return s.save(ctx, "run:"+runID, recoveryKey)+}++// RunKey returns the key held for a run, or "" when none is.+func (s *RecoveryKeyStore) RunKey(ctx context.Context, runID string) (string, error) {+ key, err := s.load(ctx, "run:"+runID)+ if errors.Is(err, ErrRecoveryKeyNotConfigured) {+ return "", nil+ }+ return key, err+}++// ReleaseRunKey forgets the key held for a run.+func (s *RecoveryKeyStore) ReleaseRunKey(ctx context.Context, runID string) error {+ if err := s.db.WithContext(ctx).Where("id = ?", "run:"+runID).Delete(&SystemBackupRecoveryConfig{}).Error; err != nil {+ return fmt.Errorf("failed to release backup run recovery key: %w", err)+ }+ return nil+}++func (s *RecoveryKeyStore) load(ctx context.Context, id string) (string, error) {var config SystemBackupRecoveryConfig- if err := s.db.WithContext(ctx).- Where("id = ? AND encrypted_recovery_key <> ''", RecoveryKeyConfigID).- First(&config).Error; err != nil {+ if err := s.db.WithContext(ctx).Where("id = ? AND encrypted_recovery_key <> ''", id).First(&config).Error; err != nil {if errors.Is(err, gorm.ErrRecordNotFound) {return "", ErrRecoveryKeyNotConfigured}@@ -689,7 +700,7 @@ func (s *RecoveryKeyStore) Get(ctx context.Context) (string, error) {return key, nil}-func (s *RecoveryKeyStore) Set(ctx context.Context, recoveryKey string) error {+func (s *RecoveryKeyStore) save(ctx context.Context, id, recoveryKey string) error {if err := ValidateRecoveryKey(recoveryKey); err != nil {return err}@@ -697,14 +708,40 @@ func (s *RecoveryKeyStore) Set(ctx context.Context, recoveryKey string) error {if err != nil {return fmt.Errorf("failed to encrypt recovery key: %w", err)}- config := SystemBackupRecoveryConfig{ID: RecoveryKeyConfigID, EncryptedRecoveryKey: encrypted}- if saveRecoveryKeyErr := s.db.WithContext(ctx).Save(&config).Error; saveRecoveryKeyErr != nil {- return fmt.Errorf("failed to save recovery key: %w", saveRecoveryKeyErr)+ if saveErr := s.db.WithContext(ctx).Save(&SystemBackupRecoveryConfig{BaseModel: database.BaseModel{ID: id}, EncryptedRecoveryKey: encrypted}).Error; saveErr != nil {+ return fmt.Errorf("failed to save recovery key: %w", saveErr)}return nil}-func (e *Engine) lockRepositoryInternal(id string) func() {+// AcceptedTarget is the progress a manual backup's workflow starts with, so startup recovery protects+// its record from the moment the backup is accepted.+func AcceptedTarget(backupID string) scheduler.TargetOutcome {+ data, _ := json.Marshal(struct {+ BackupID string `json:"backupId"`+ }{backupID})+ return scheduler.TargetOutcome{ResourceType: "backup", ID: "accepted:" + backupID, Status: scheduler.Queued, RecoveryData: data}+}++// AcceptedBackupID returns the backup a manual backup workflow's run owns, from its AcceptedTarget.+func AcceptedBackupID(run scheduler.Run) string {+ index := slices.IndexFunc(run.Outcome.Targets, func(target scheduler.TargetOutcome) bool {+ return strings.HasPrefix(target.ID, "accepted:")+ })+ if index < 0 {+ return ""+ }+ return strings.TrimPrefix(run.Outcome.Targets[index].ID, "accepted:")+}++// lockRepository serializes commands against one repository and returns its unlock func.+func (e *Engine) lockRepository(ctx context.Context, id string) (func(), error) {+ if e == nil {+ return nil, errors.New("backup engine is unavailable")+ }+ if strings.TrimSpace(id) == "" {+ return nil, errors.New("backup repository ID is required")+ }e.mu.Lock()lock := e.repositories[id]if lock == nil {@@ -713,7 +750,11 @@ func (e *Engine) lockRepositoryInternal(id string) func() {}e.mu.Unlock()lock.Lock()- return lock.Unlock+ if err := ctx.Err(); err != nil {+ lock.Unlock()+ return nil, err+ }+ return lock.Unlock, nil}// PolicyReconciliation reconciles a full set of policy updates against the@@ -737,47 +778,70 @@ type PolicyReconciliation[P, U any] struct {Reschedule func(ctx context.Context, policy *P)}+// Run applies updates as the full policy set: it saves them, removes the rest, and reschedules their jobs.func (r PolicyReconciliation[P, U]) Run(ctx context.Context, updates []U) error {- policies, kept, err := r.buildInternal(ctx, updates)- if err != nil {- return err- }- if persistErr := r.persistInternal(ctx, policies, kept); persistErr != nil {- return fmt.Errorf("failed to save %s backup policies: %w", r.Domain, persistErr)- }+ byID := make(map[string]P, len(r.Existing))for i := range r.Existing {- if _, ok := kept[r.ID(&r.Existing[i])]; !ok {- r.Unregister(ctx, r.ID(&r.Existing[i]))- }+ byID[r.ID(&r.Existing[i])] = r.Existing[i]}- for i := range policies {- r.Reschedule(ctx, &policies[i])+ policies := make([]P, 0, len(updates))+ kept := make(map[string]struct{}, len(updates))+ for _, update := range updates {+ policy := r.New()+ if updateID := r.UpdateID(update); updateID != "" {+ var ok bool+ policy, ok = byID[updateID]+ if !ok {+ return fmt.Errorf("%s backup policy not found", r.Domain)+ }+ if _, duplicate := kept[updateID]; duplicate {+ return fmt.Errorf("duplicate %s backup policy", r.Domain)+ }+ }+ if err := r.Build(ctx, &policy, update); err != nil {+ return err+ }+ if policyID := r.ID(&policy); policyID != "" {+ kept[policyID] = struct{}{}+ }+ policies = append(policies, policy)}- return nil-}+ removed := slices.DeleteFunc(slices.Clone(r.Existing), func(policy P) bool {+ _, ok := kept[r.ID(&policy)]+ return ok+ })-func (r PolicyReconciliation[P, U]) persistInternal(ctx context.Context, policies []P, kept map[string]struct{}) error {- if r.Persist != nil {- return r.Persist(ctx, policies)- }- if r.DB == nil {- return errors.New("backup policy database is unavailable")- }- return r.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {- for i := range policies {- if saveErr := tx.Save(&policies[i]).Error; saveErr != nil {- return saveErr+ var persistErr error+ switch {+ case r.Persist != nil:+ persistErr = r.Persist(ctx, policies)+ case r.DB == nil:+ persistErr = errors.New("backup policy database is unavailable")+ default:+ persistErr = r.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ for i := range policies {+ if saveErr := tx.Save(&policies[i]).Error; saveErr != nil {+ return saveErr+ }}- }- for i := range r.Existing {- if _, ok := kept[r.ID(&r.Existing[i])]; !ok {- if deleteErr := tx.Delete(&r.Existing[i]).Error; deleteErr != nil {+ for i := range removed {+ if deleteErr := tx.Delete(&removed[i]).Error; deleteErr != nil {return deleteErr}}- }- return nil- })+ return nil+ })+ }+ if persistErr != nil {+ return fmt.Errorf("failed to save %s backup policies: %w", r.Domain, persistErr)+ }+ for i := range removed {+ r.Unregister(ctx, r.ID(&removed[i]))+ }+ for i := range policies {+ r.Reschedule(ctx, &policies[i])+ }+ return nil}// ValidatePolicyUpdate applies the shared cron, retention, and destination rules used by backup policies.@@ -809,53 +873,20 @@ func ValidatePolicyUpdate(ctx context.Context, domain string, update backup.Updareturn update, nil}-// buildInternal maps the updates onto existing or new policy rows and reports-// which existing IDs survive the reconciliation.-func (r PolicyReconciliation[P, U]) buildInternal(ctx context.Context, updates []U) ([]P, map[string]struct{}, error) {- byID := make(map[string]P, len(r.Existing))- for i := range r.Existing {- byID[r.ID(&r.Existing[i])] = r.Existing[i]- }- policies := make([]P, 0, len(updates))- kept := make(map[string]struct{}, len(updates))- for _, update := range updates {- policy := r.New()- updateID := r.UpdateID(update)- if updateID != "" {- var ok bool- policy, ok = byID[updateID]- if !ok {- return nil, nil, fmt.Errorf("%s backup policy not found", r.Domain)- }- if _, duplicate := kept[updateID]; duplicate {- return nil, nil, fmt.Errorf("duplicate %s backup policy", r.Domain)- }- }- if err := r.Build(ctx, &policy, update); err != nil {- return nil, nil, err- }- if policyID := r.ID(&policy); policyID != "" {- kept[policyID] = struct{}{}- }- policies = append(policies, policy)- }- return policies, kept, nil-}-var ErrRemoteRepositoryMissing = errors.New("S3 backup storage is missing")const RemoteDisabledMessage = "S3 backup storage is missing. Remote backups were disabled; edit the policy to resume."-// RemoteSnapshotChecker memoizes targeted checks within one response budget.-// A nil result means the remote copy could not be verified.-func RemoteSnapshotChecker(ctx context.Context, destinations *s3.S3DestinationService, root string) func(string, string) *bool {+// RemoteSnapshotChecker memoizes targeted checks within one response budget; each lookup names the+// repository root that holds the snapshot. A nil result means the remote copy could not be verified.+func RemoteSnapshotChecker(ctx context.Context, destinations *s3.S3DestinationService) func(destinationID, root, snapshotID string) *bool {checked := make(map[string]*backup.RepositoryObservation)deadline := time.Now().Add(10 * time.Second)- return func(destinationID, snapshotID string) *bool {+ return func(destinationID, root, snapshotID string) *bool {if destinations == nil || root == "" || destinationID == "" || snapshotID == "" {return nil}- key := destinationID + ":" + snapshotID+ key := destinationID + ":" + root + ":" + snapshotIDobservation, seen := checked[key]if !seen {checked[key] = nil@@ -894,6 +925,34 @@ func CheckRemoteRepository(ctx context.Context, destinations *s3.S3DestinationSereturn s3util.CheckRepository(checkCtx, configuration, root, "")}+// DisableMissingRemote stops a scheduled policy from using S3 storage that went missing: a policy that also backs up+// locally loses S3, an S3-only one is disabled. save clears that field on the stored policy unless it changed since it+// was loaded, and reports whether it did; the loaded flags then follow. It reports whether S3 was turned off.+func DisableMissingRemote(remoteErr error, localEnabled bool, s3Enabled, enabled *bool, save func(field string) (bool, error)) (bool, error) {+ if !errors.Is(remoteErr, ErrRemoteRepositoryMissing) {+ return false, nil+ }+ field, flag := "enabled", enabled+ if localEnabled {+ field, flag = "s3_enabled", s3Enabled+ }+ saved, err := save(field)+ if err != nil || !saved {+ return false, err+ }+ *flag = false+ return true, nil+}++// DisableStoredRemote is DisableMissingRemote's save for a policy table row, which it changes only while the row still+// matches the loaded policy.+func DisableStoredRemote(ctx context.Context, db *database.DB, model any, id, destinationID string, localEnabled bool, field string) (bool, error) {+ result := db.WithContext(ctx).Model(model).+ Where("id = ? AND s3_destination_id = ? AND enabled = ? AND s3_enabled = ? AND local_enabled = ?", id, destinationID, true, true, localEnabled).+ Update(field, false)+ return result.Error == nil && result.RowsAffected > 0, result.Error+}+// CheckScheduledRemote permits first-use initialization, but detects lost repositories.func CheckScheduledRemote(ctx context.Context, db *database.DB, destinations *s3.S3DestinationService, table, destinationID, root string) error {result, err := CheckRemoteRepository(ctx, destinations, destinationID, root)@@ -914,371 +973,6 @@ func CheckScheduledRemote(ctx context.Context, db *database.DB, destinations *s3return fmt.Errorf("%w: %s", ErrRemoteRepositoryMissing, result.Reason)}-const backupRunTypeInternal = "backup-run"--type backupRunActorInternal struct {- id string- engine *Engine- service *actor.Service-}--// RegisterRunKind binds a domain handler before the actor host starts.-func (e *Engine) RegisterRunKind(kind string, execute func(context.Context, string, []byte, bool) error, failures ...func(context.Context, string, []byte, error) error) {- e.mu.Lock()- defer e.mu.Unlock()- e.handlers[kind] = execute- if len(failures) > 0 {- e.failures[kind] = failures[0]- }-}--func (e *Engine) SetAuthorize(authorize func(context.Context, backup.DurableRunCommand) error) {- e.authorize = authorize-}--func (e *Engine) SetExecutionReady(ready func() bool) { e.executionReady = ready }--func (e *Engine) Register(runtime *francis.Runtime) error {- e.service = runtime.Service()- return runtime.RegisterActor(backupRunTypeInternal, func(id string, service *actor.Service) actor.Actor {- return &backupRunActorInternal{id: id, engine: e, service: service}- }, local.WithCapacityGroup("jobs", 4), local.WithCompletedJobRetention(7*24*time.Hour))-}--// SubmitDurableRun persists the accepted command before dispatching its work.-// A nil error means the run was accepted or its outcome is still unresolved;-// an error means the intent was confirmed absent and admission is the caller's.-func (e *Engine) SubmitDurableRun(ctx context.Context, command backup.DurableRunCommand, lease *runs.Lease) error {- if e == nil || e.service == nil {- return errors.New("backup actor host is unavailable")- }- if e.authorize != nil {- if err := e.authorize(ctx, command); err != nil {- return err- }- }- e.mu.Lock()- if e.stopping {- e.mu.Unlock()- return errors.New("backup engine is stopping")- }- e.leases[command.RunID] = lease- e.mu.Unlock()- ctx = context.WithoutCancel(ctx)- _, err := e.service.Invoke(ctx, backupRunTypeInternal, command.RunID, "submit", command)- if err == nil {- return nil- }- var state backup.DurableRunState- readErr := e.service.GetState(ctx, backupRunTypeInternal, command.RunID, &state)- switch {- case readErr == nil && sameDurableCommandInternal(state.Command, command):- slog.WarnContext(ctx, "backup run accepted after submit error", "runId", command.RunID, "error", err)- return nil- case readErr == nil || errors.Is(readErr, actor.ErrStateNotFound):- e.mu.Lock()- delete(e.leases, command.RunID)- e.mu.Unlock()- return err- default:- // Keep admission parked until reconciliation can read the intent back.- slog.WarnContext(ctx, "backup run outcome unresolved", "runId", command.RunID, "error", err, "readError", readErr)- e.mu.Lock()- e.unresolved[command.RunID] = command- e.mu.Unlock()- return nil- }-}--// AcquireDurableRun keeps the original admission, or reacquires it after a restart.-func (e *Engine) AcquireDurableRun(ctx context.Context, runID, scope, resourceID string) (*runs.Lease, bool, error) {- e.mu.Lock()- lease := e.leases[runID]- delete(e.leases, runID)- e.mu.Unlock()- if lease != nil {- return lease, true, nil- }- return e.TryAcquireRun(ctx, scope, resourceID)-}--func (a *backupRunActorInternal) Invoke(ctx context.Context, _ string, data actor.Envelope) (any, error) {- var command backup.DurableRunCommand- if err := data.Decode(&command); err != nil {- return nil, err- }- var state backup.DurableRunState- err := a.service.GetState(ctx, backupRunTypeInternal, a.id, &state)- switch {- case errors.Is(err, actor.ErrStateNotFound):- state = backup.DurableRunState{Command: command, Status: scheduler.Queued}- if setStateErr := a.service.SetState(ctx, backupRunTypeInternal, a.id, state, nil); setStateErr != nil {- return nil, setStateErr- }- case err != nil:- return nil, err- case !sameDurableCommandInternal(state.Command, command):- return nil, errors.New("backup run already exists with a different command")- }- if state.Status == scheduler.Succeeded || state.Status == scheduler.Failed || state.Status == scheduler.NeedsAttention {- return nil, nil- }- // The intent is persisted, so ReconcileDispatches repairs a failed dispatch.- if _, _, dispatchErr := a.service.Dispatch(ctx, backupRunTypeInternal, a.id, "execute", nil, actor.WithIdempotencyKey(a.id)); dispatchErr != nil {- slog.WarnContext(ctx, "dispatch accepted backup run", "runId", a.id, "error", dispatchErr)- }- return nil, nil-}--func sameDurableCommandInternal(a, b backup.DurableRunCommand) bool {- return a.UserID == b.UserID &&- a.EnvironmentID == b.EnvironmentID &&- a.Permission == b.Permission &&- a.RequestedWithKey == b.RequestedWithKey &&- a.Kind == b.Kind &&- a.RunID == b.RunID &&- a.ActivityID == b.ActivityID &&- bytes.Equal(a.Payload, b.Payload)-}--func (a *backupRunActorInternal) Job(ctx context.Context, _ string, _ actor.Envelope) error {- if a.engine.executionReady != nil && !a.engine.executionReady() {- return actor.ErrJobRejected- }- a.engine.mu.Lock()- if a.engine.stopping {- a.engine.mu.Unlock()- return actor.ErrJobRejected- }- a.engine.workers.Add(1)- a.engine.mu.Unlock()- defer a.engine.workers.Done()- jobCtx, cancel := context.WithCancel(ctx)- stop := context.AfterFunc(a.engine.lifecycleCtx, cancel) //nolint:contextcheck // Cancels the inherited job context when the engine stops.- defer stop()- defer cancel()- ctx = jobCtx-- var state backup.DurableRunState- if err := a.service.GetState(ctx, backupRunTypeInternal, a.id, &state); err != nil {- return err- }- if state.Status == scheduler.Succeeded || state.Status == scheduler.Failed || state.Status == scheduler.NeedsAttention {- return nil- }- a.engine.mu.Lock()- pendingLease := a.engine.leases[a.id]- a.engine.mu.Unlock()- defer pendingLease.Release(ctx)- interrupted := state.Started && len(state.Targets) > 0- state.Started = true- state.Status = scheduler.Running- if err := a.service.SetState(ctx, backupRunTypeInternal, a.id, state, nil); err != nil {- return err- }- var progressMu sync.Mutex- ctx = jobcontext.WithExecution(ctx, scheduler.Run{ID: a.id, EnvironmentID: "0", Outcome: scheduler.Outcome{Targets: state.Targets}}, func(target scheduler.TargetOutcome) error {- progressMu.Lock()- defer progressMu.Unlock()- updated := false- for i := range state.Targets {- if state.Targets[i].ID == target.ID {- if len(target.RecoveryData) == 0 {- target.RecoveryData = state.Targets[i].RecoveryData- }- state.Targets[i] = target- updated = true- break- }- }- if !updated {- state.Targets = append(state.Targets, target)- }- return a.service.SetState(ctx, backupRunTypeInternal, a.id, state, nil)- })- runErr := a.executeInternal(ctx, state.Command, interrupted)- if ctx.Err() != nil {- return ctx.Err()- }- state.Status = scheduler.Succeeded- if runErr != nil {- state.Status = scheduler.NeedsAttention- state.Error = runErr.Error()- }- if err := a.service.SetState(ctx, backupRunTypeInternal, a.id, state, nil); err != nil {- return err- }- if runErr != nil {- return fmt.Errorf("%s: %w", runErr.Error(), actor.ErrJobPermanentFailure)- }- return nil-}--func (a *backupRunActorInternal) executeInternal(ctx context.Context, command backup.DurableRunCommand, interrupted bool) (err error) {- defer utils.RecoverToError(&err, "durable backup")- a.engine.mu.Lock()- handler := a.engine.handlers[command.Kind]- failure := a.engine.failures[command.Kind]- a.engine.mu.Unlock()- if handler == nil {- return errors.New("backup run kind is unavailable")- }- if a.engine.authorize != nil {- if authorizeErr := a.engine.authorize(ctx, command); authorizeErr != nil {- a.engine.mu.Lock()- lease := a.engine.leases[a.id]- delete(a.engine.leases, a.id)- a.engine.mu.Unlock()- defer lease.Release(ctx)- if failure != nil {- authorizeErr = errors.Join(authorizeErr, failure(ctx, a.id, command.Payload, authorizeErr))- }- return authorizeErr- }- }- return handler(ctx, a.id, command.Payload, interrupted)-}--func (e *Engine) activeRunsInternal(ctx context.Context) ([]backup.DurableRunState, error) {- result := []backup.DurableRunState{}- for cursor := ""; ; {- page, err := e.service.ListStates(ctx, backupRunTypeInternal, &actor.ListStatesOpts{IncludeData: true, After: cursor, Limit: 100})- if err != nil {- return nil, err- }- for _, item := range page.States {- if item.Data == nil {- continue- }- var state backup.DurableRunState- if decodeErr := item.Data.Decode(&state); decodeErr != nil {- return nil, decodeErr- }- if state.Status == scheduler.Queued || state.Status == scheduler.Running || state.Status == scheduler.NeedsAttention {- result = append(result, state)- }- }- cursor = page.AfterID()- if cursor == "" {- break- }- }- return result, nil-}--func (e *Engine) ActiveRunIDs(ctx context.Context) ([]string, error) {- states, err := e.activeRunsInternal(ctx)- if err != nil {- return nil, err- }- ids := make([]string, 0, len(states))- for _, state := range states {- ids = append(ids, state.Command.RunID)- for _, target := range state.Targets {- var checkpoint struct {- BackupID string `json:"backupId"`- }- if len(target.RecoveryData) > 0 && json.Unmarshal(target.RecoveryData, &checkpoint) == nil && checkpoint.BackupID != "" {- ids = append(ids, checkpoint.BackupID)- }- }- }- return ids, nil-}--func (e *Engine) ActiveActivityIDs(ctx context.Context) ([]string, error) {- states, err := e.activeRunsInternal(ctx)- if err != nil {- return nil, err- }- ids := make([]string, 0, len(states))- for _, state := range states {- if state.Command.ActivityID != "" {- ids = append(ids, state.Command.ActivityID)- }- }- return ids, nil-}--// ReconcileDispatches resolves unresolved submissions and repairs accepted-// commands whose dispatch was interrupted.-func (e *Engine) ReconcileDispatches(ctx context.Context) error {- failures := e.resolveUnresolvedRunsInternal(ctx)- states, err := e.activeRunsInternal(ctx)- if err != nil {- return errors.Join(append(failures, err)...)- }- for _, state := range states {- if state.Status == scheduler.NeedsAttention {- continue- }- if repairErr := e.repairDispatchInternal(ctx, state.Command.RunID); repairErr != nil {- failures = append(failures, fmt.Errorf("repair backup run %s: %w", state.Command.RunID, repairErr))- }- }- return errors.Join(failures...)-}--func (e *Engine) repairDispatchInternal(ctx context.Context, runID string) error {- jobs, err := e.service.ListJobs(ctx, backupRunTypeInternal, runID)- if err != nil {- return err- }- for _, job := range jobs {- if !job.Status.IsTerminal() {- return nil- }- if job.Status == actor.JobStatusDeadLettered {- _, retryJobErr := e.service.RetryJob(ctx, job.JobID)- return retryJobErr- }- if deleteJobErr := e.service.DeleteJob(ctx, backupRunTypeInternal, runID, job.JobID); deleteJobErr != nil && !errors.Is(deleteJobErr, actor.ErrJobNotFound) {- return deleteJobErr- }- }- _, _, err = e.service.Dispatch(ctx, backupRunTypeInternal, runID, "execute", nil, actor.WithIdempotencyKey(runID))- return err-}--// resolveUnresolvedRunsInternal reads back submissions whose outcome was-// unknown. Found intents continue as accepted; confirmed absence releases-// admission and fails the run through its kind's failure handler.-func (e *Engine) resolveUnresolvedRunsInternal(ctx context.Context) []error {- e.mu.Lock()- pending := maps.Clone(e.unresolved)- e.mu.Unlock()- var failures []error- for runID, command := range pending {- var state backup.DurableRunState- readErr := e.service.GetState(ctx, backupRunTypeInternal, runID, &state)- if readErr != nil && !errors.Is(readErr, actor.ErrStateNotFound) {- failures = append(failures, fmt.Errorf("read back backup run %s: %w", runID, readErr))- continue- }- e.mu.Lock()- lease := e.leases[runID]- failure := e.failures[command.Kind]- if readErr != nil {- delete(e.leases, runID)- }- e.mu.Unlock()- if readErr != nil {- lease.Release(ctx)- // Stay unresolved until the failure is recorded, so a later tick retries it.- if failure != nil {- if failErr := failure(ctx, runID, command.Payload, errors.New("backup run was not accepted")); failErr != nil {- failures = append(failures, fmt.Errorf("fail backup run %s: %w", runID, failErr))- continue- }- }- }- e.mu.Lock()- delete(e.unresolved, runID)- e.mu.Unlock()- }- return failures-}-// ExpiredRunIDs returns the IDs of succeeded runs with snapshots that fall// outside the newest keep entries for one policy, oldest last. Callers delete// each run through their own delete path so snapshots are forgotten too.diff --git a/backend/internal/backup/service_test.go b/backend/internal/backup/service_test.goindex 4b363f31f..c30d2edab 100644--- a/backend/internal/backup/service_test.go+++ b/backend/internal/backup/service_test.go@@ -1,14 +1,8 @@package backupimport (- "context"- "path/filepath"- "sync/atomic""testing"- "time"- "github.com/getarcaneapp/arcane/types/v2/backup"- "github.com/getarcaneapp/arcane/types/v2/scheduler""github.com/libtnb/sqlite""github.com/moby/moby/api/types/mount""github.com/stretchr/testify/require"@@ -16,29 +10,26 @@ import ("gorm.io/gorm""github.com/getarcaneapp/arcane/backend/v2/internal/database"- "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"- "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"- francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing")-func TestMarkSnapshotDirectoriesInternal(t *testing.T) {+func TestMarkSnapshotDirectories(t *testing.T) {files := []string{"/volume", "/volume/folder", "/volume/file.txt", "/volume/link"}longOutput := "drwxr-xr-x root root 0 1 Jan 2026 00:00 \"/volume\"\r\ndrwxr-xr-x root root 0 1 Jan 2026 00:00 \"/volume/" +"folder\"\r\n-rw-r--r-- root root 5 1 Jan 2026 00:00 \"/volume/file.txt\"\r\nlrwxrwxrwx root root 4 1 Jan 20" +"26 00:00 \"/volume/link\" -> \"file.txt\""- marked, err := markSnapshotDirectoriesInternal(files, longOutput)+ marked, err := markSnapshotDirectories(files, longOutput)require.NoError(t, err)require.Equal(t, []string{"/volume/", "/volume/folder/", "/volume/file.txt", "/volume/link"}, marked)require.Equal(t, []string{"/volume", "/volume/folder", "/volume/file.txt", "/volume/link"}, files)}-func TestMarkSnapshotDirectoriesRejectsMismatchedListingsInternal(t *testing.T) {- _, err := markSnapshotDirectoriesInternal([]string{"/volume", "/volume/file.txt"}, "drwxr-xr-x root root 0 1 Jan 2026 00:00 \"/volume\"")+func TestMarkSnapshotDirectoriesRejectsMismatchedListings(t *testing.T) {+ _, err := markSnapshotDirectories([]string{"/volume", "/volume/file.txt"}, "drwxr-xr-x root root 0 1 Jan 2026 00:00 \"/volume\"")require.ErrorContains(t, err, "different lengths")}-func TestQualifySnapshotListingInternal(t *testing.T) {+func TestQualifySnapshotListing(t *testing.T) {tests := []struct {name stringfiles []string@@ -60,19 +51,19 @@ func TestQualifySnapshotListingInternal(t *testing.T) {{name: "nested listing",files: []string{"nested/", "file.txt", "./link"},- snapshotPath: "folder/",+ snapshotPath: "folder",expected: []string{"folder/nested/", "folder/file.txt", "folder/link"},},{name: "legacy system project listing",files: []string{"demo/compose.yaml"},- snapshotPath: "/app/data/projects/",+ snapshotPath: "app/data/projects",expected: []string{"app/data/projects/demo/compose.yaml"},},}for _, test := range tests {t.Run(test.name, func(t *testing.T) {- qualified := qualifySnapshotListingInternal(test.files, test.snapshotPath)+ qualified := qualifySnapshotListing(test.files, test.snapshotPath)require.Equal(t, test.expected, qualified)if len(test.files) > 0 {require.NotSame(t, &test.files[0], &qualified[0])@@ -119,12 +110,12 @@ func TestRecoveryKeyValidation(t *testing.T) {require.NoError(t, ValidateRecoveryKey(key))}-func TestSnapshotCommandInternal(t *testing.T) {- single, err := snapshotCommandInternal("volume", RootSnapshotInput(mount.Mount{Type: mount.TypeVolume, Source: "data", Target: "/volume"}))+func TestSnapshotCommand(t *testing.T) {+ single, err := snapshotCommand("volume", RootSnapshotInput(mount.Mount{Type: mount.TypeVolume, Source: "data", Target: "/volume"}))require.NoError(t, err)require.Equal(t, []string{"backup", "--init", "--json", "--host", "arcane", "--label", "volume", "--as-path", "/", "--", "/volume"}, single)- multi, err := snapshotCommandInternal("arcane-system-recovery", CreateSnapshotInput{Sources: []string{"/data", "/projects"}, Globs: []string{"!/data/arcane.db-wal"}})+ multi, err := snapshotCommand("arcane-system-recovery", CreateSnapshotInput{Sources: []string{"/data", "/projects"}, Globs: []string{"!/data/arcane.db-wal"}})require.NoError(t, err)require.Equal(t,@@ -145,111 +136,8 @@ func TestSnapshotCommandInternal(t *testing.T) {multi,)- _, err = snapshotCommandInternal("x", CreateSnapshotInput{Sources: []string{"/data", "/projects"}, AsPath: "/"})+ _, err = snapshotCommand("x", CreateSnapshotInput{Sources: []string{"/data", "/projects"}, AsPath: "/"})require.ErrorContains(t, err, "single source")- _, err = snapshotCommandInternal("x", CreateSnapshotInput{})+ _, err = snapshotCommand("x", CreateSnapshotInput{})require.ErrorContains(t, err, "at least one snapshot source")}--func TestDurableBackupRestartKeepsCompletionInternal(t *testing.T) {- url := "file:" + filepath.Join(t.TempDir(), "actors.db")- var calls atomic.Int32- runtime := francistest.New(t, url)- engine := NewEngine(t.Context(), nil, nil)- engine.RegisterRunKind("test", func(ctx context.Context, _ string, _ []byte, interrupted bool) error {- calls.Add(1)- require.False(t, interrupted)- return jobcontext.Progress(ctx, scheduler.TargetOutcome{ID: "snapshot", Status: scheduler.Succeeded})- })- require.NoError(t, engine.Register(runtime))- francistest.Start(t, runtime)- command := backup.DurableRunCommand{Kind: "test", RunID: "run", ActivityID: "activity"}- require.NoError(t, engine.SubmitDurableRun(t.Context(), command, nil))- require.Eventually(t, func() bool {- states, err := engine.activeRunsInternal(t.Context())- return err == nil && len(states) == 0 && calls.Load() == 1- }, 10*time.Second, 20*time.Millisecond)- stopCtx, cancel := context.WithTimeout(t.Context(), 10*time.Second)- defer cancel()- require.NoError(t, runtime.Stop(stopCtx))- restarted := francistest.New(t, url)- next := NewEngine(t.Context(), nil, nil)- next.RegisterRunKind("test", func(context.Context, string, []byte, bool) error { calls.Add(1); return nil })- require.NoError(t, next.Register(restarted))- francistest.Start(t, restarted)- require.NoError(t, next.SubmitDurableRun(t.Context(), command, nil))- require.NoError(t, next.ReconcileDispatches(t.Context()))- require.EqualValues(t, 1, calls.Load())-}--func TestDurableBackupShutdownKeepsInterruptedEvidenceInternal(t *testing.T) {- url := "file:" + filepath.Join(t.TempDir(), "actors.db")- runtime := francistest.New(t, url)- admission := runs.NewAdmission(runtime.Service(), t.Name())- require.NoError(t, admission.Register(runtime))- engine := NewEngine(t.Context(), admission, nil)- entered := make(chan struct{})- drain := make(chan struct{})- t.Cleanup(func() {- select {- case <-drain:- default:- close(drain)- }- })- engine.RegisterRunKind("test", func(ctx context.Context, _ string, _ []byte, _ bool) error {- if err := jobcontext.Progress(ctx, scheduler.TargetOutcome{ID: "snapshot", Status: scheduler.Running, RecoveryData: []byte(`{"backupId":"frozen"}`)}); err != nil {- return err- }- close(entered)- <-ctx.Done()- <-drain- return ctx.Err()- })- require.NoError(t, engine.Register(runtime))- francistest.Start(t, runtime)- lease, acquired, err := engine.TryAcquireRun(t.Context(), "backup", "data")- require.NoError(t, err)- require.True(t, acquired)- require.NoError(t, engine.SubmitDurableRun(t.Context(), backup.DurableRunCommand{Kind: "test", RunID: "interrupted"}, lease))- select {- case <-entered:- case <-time.After(10 * time.Second):- t.Fatal("backup did not start")- }- stopCtx, cancel := context.WithTimeout(t.Context(), 10*time.Second)- defer cancel()- stopped := make(chan error, 1)- go func() { stopped <- engine.Stop(stopCtx) }()- require.Eventually(t, func() bool { return engine.lifecycleCtx.Err() != nil }, time.Second, time.Millisecond)- _, acquired, err = engine.TryAcquireRun(t.Context(), "backup", "data")- require.NoError(t, err)- require.False(t, acquired)- close(drain)- require.NoError(t, <-stopped)- lease, acquired, err = engine.TryAcquireRun(t.Context(), "backup", "data")- require.NoError(t, err)- require.True(t, acquired)- lease.Release(t.Context())- require.NoError(t, runtime.Stop(stopCtx))- restarted := francistest.New(t, url)- next := NewEngine(t.Context(), nil, nil)- resumed := make(chan struct{})- next.RegisterRunKind("test", func(ctx context.Context, _ string, _ []byte, interrupted bool) error {- require.True(t, interrupted)- previous, ok := jobcontext.Run(ctx)- require.True(t, ok)- require.Len(t, previous.Outcome.Targets, 1)- require.JSONEq(t, `{"backupId":"frozen"}`, string(previous.Outcome.Targets[0].RecoveryData))- close(resumed)- return nil- })- require.NoError(t, next.Register(restarted))- francistest.Start(t, restarted)- require.NoError(t, next.ReconcileDispatches(t.Context()))- select {- case <-resumed:- case <-time.After(15 * time.Second):- t.Fatal("durable backup did not resume")- }-}diff --git a/backend/internal/bootstrap/bootstrap_test.go b/backend/internal/bootstrap/bootstrap_test.goindex dc6e756ee..25e245226 100644--- a/backend/internal/bootstrap/bootstrap_test.go+++ b/backend/internal/bootstrap/bootstrap_test.go@@ -33,6 +33,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/swarm""github.com/getarcaneapp/arcane/backend/v2/internal/system""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"@@ -55,7 +56,7 @@ func (w *blockingBusWatcherInternal) Start(ctx context.Context) error {func (w *blockingBusWatcherInternal) RunNow(context.Context) error { return nil }-func TestNormalizeTunnelGRPCRequestPathInternal(t *testing.T) {+func TestNormalizeTunnelGRPCRequestPath(t *testing.T) {fullMethodPath := tunnelpb.TunnelService_Connect_FullMethodNamet.Run("nil request", func(t *testing.T) {@@ -111,7 +112,7 @@ func TestNormalizeTunnelGRPCRequestPathInternal(t *testing.T) {})}-func TestIsTunnelGRPCRequestInternal(t *testing.T) {+func TestIsTunnelGRPCRequest(t *testing.T) {fullMethodPath := tunnelpb.TunnelService_Connect_FullMethodNamet.Run("detects by grpc content-type", func(t *testing.T) {@@ -160,7 +161,7 @@ func TestIsTunnelGRPCRequestInternal(t *testing.T) {})}-func TestConfigureHTTPProtocolsInternal(t *testing.T) {+func TestConfigureHTTPProtocols(t *testing.T) {handler := http.NewServeMux()t.Run("tls enables http1 and http2", func(t *testing.T) {@@ -186,7 +187,7 @@ func TestConfigureHTTPProtocolsInternal(t *testing.T) {})}-func TestHTTP2APIResponsesDoNotUseAPIGzipInternal(t *testing.T) {+func TestHTTP2APIResponsesDoNotUseAPIGzip(t *testing.T) {cfg := &config.Config{AgentMode: true,AppUrl: "http://localhost:3552",@@ -199,7 +200,7 @@ func TestHTTP2APIResponsesDoNotUseAPIGzipInternal(t *testing.T) {Project: project.New(&project.ProjectService{}, nil),Container: container.New(&container.ContainerService{}, nil, nil, nil),Swarm: swarm.New(&swarm.SwarmService{}, nil, nil, nil),- System: system.New(&system.SystemService{}, nil, nil, nil, nil),+ System: system.New(&system.SystemService{}, nil, nil, nil),},AuthMiddleware: auth.NewAuthMiddleware(nil, cfg),TunnelRegistry: edge.NewTunnelRegistry(),@@ -254,7 +255,7 @@ func TestHTTP2APIResponsesDoNotUseAPIGzipInternal(t *testing.T) {}}-func TestH2CStreamSurvivesPastReadHeaderTimeoutInternal(t *testing.T) {+func TestH2CStreamSurvivesPastReadHeaderTimeout(t *testing.T) {// Regression guard for the go1.26.6 CVE-2026-56853 backport, which arms// ReadHeaderTimeout on the raw conn before the h2c preface sniff and never// clears it on the HTTP/2 handoff. Without ClearReadDeadline in the h2c@@ -310,7 +311,7 @@ func TestH2CStreamSurvivesPastReadHeaderTimeoutInternal(t *testing.T) {require.Equal(t, "done", string(body))}-func TestHTTPServerStopCancelsStreamingRequestContextsInternal(t *testing.T) {+func TestHTTPServerStopCancelsStreamingRequestContexts(t *testing.T) {appCtx := t.Context()handlerEntered := make(chan struct{})@@ -350,7 +351,7 @@ func TestHTTPServerStopCancelsStreamingRequestContextsInternal(t *testing.T) {require.Less(t, time.Since(start), time.Second)}-func TestNewHTTPServerRejectsInvalidTLSCertificateInternal(t *testing.T) {+func TestNewHTTPServerRejectsInvalidTLSCertificate(t *testing.T) {cfg := &config.Config{AgentMode: true,TLSEnabled: true,@@ -384,7 +385,7 @@ func TestRegisterAppCancelHookRunsAfterLaterStopHooks(t *testing.T) {require.ErrorIs(t, appCtx.Err(), context.Canceled)}-func TestJobSchedulerStopCancelsItsPrivateContextInternal(t *testing.T) {+func TestJobSchedulerStopCancelsItsPrivateContext(t *testing.T) {appCtx := t.Context()lifecycle := fxtest.NewLifecycle(t)@@ -400,6 +401,8 @@ func TestJobSchedulerStopCancelsItsPrivateContextInternal(t *testing.T) {require.NoError(t, coordinator.Register(runtime))admission := runs.NewAdmission(runtime.Service(), t.Name())require.NoError(t, admission.Register(runtime))+ workflows, err := flow.New(appCtx, runtime, coordinator, nil)+ require.NoError(t, err)lifecycle.Append(fx.Hook{OnStart: func(ctx context.Context) error { return runtime.Start(ctx, appCtx, nil) },OnStop: runtime.Stop,@@ -409,7 +412,7 @@ func TestJobSchedulerStopCancelsItsPrivateContextInternal(t *testing.T) {gitopsSync := gitops.NewGitOpsSyncService(&database.DB{DB: db}, nil, nil, nil, nil, nil)require.NoError(t, db.Create(&environment.Environment{ID: "0", Name: "Local", Enabled: true}).Error)require.NoError(t, db.Create(&project.GitOpsSync{ID: "overdue", EnvironmentID: "0", AutoSync: true, SyncInterval: 1}).Error)- jobScheduler, err := newJobScheduler(appCtx, lifecycle, &config.Config{}, coordinator, admission, nil, nil, nil, jobService, nil, nil, nil, gitopsSync)+ jobScheduler, err := newJobScheduler(appCtx, lifecycle, &config.Config{}, coordinator, admission, nil, nil, nil, jobService, workflows, nil, gitopsSync, runtime)require.NoError(t, err)require.NoError(t, registerDynamicJobs(dynamicJobsParams{AppCtx: appCtx, Config: &config.Config{}, Scheduler: jobScheduler,@@ -507,7 +510,7 @@ func TestPrepareServerTLSInternal_AllowsExternalMTLSTermination(t *testing.T) {require.FileExists(t, assetsDir+"/ca.key")}-func TestIsWeakProductionEncryptionKeyInternal(t *testing.T) {+func TestIsWeakProductionEncryptionKey(t *testing.T) {assert.True(t, isWeakProductionEncryptionKeyInternal("short", "production", false))assert.False(t, isWeakProductionEncryptionKeyInternal("test-encryption-key-for-edge-mtls-32bytes-min", "production", false))assert.False(t, isWeakProductionEncryptionKeyInternal("hex:abc", "production", false))diff --git a/backend/internal/bootstrap/jobs_bootstrap.go b/backend/internal/bootstrap/jobs_bootstrap.goindex b7148f540..19143e001 100644--- a/backend/internal/bootstrap/jobs_bootstrap.go+++ b/backend/internal/bootstrap/jobs_bootstrap.go@@ -1,6 +1,7 @@package bootstrapimport (+ "cmp""context""encoding/json/v2""errors"@@ -9,26 +10,26 @@ import ("net/http""slices""sync"- "time""github.com/getarcaneapp/arcane/types/v2/features"schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"+ "github.com/italypaleale/francis/actor""go.uber.org/fx""github.com/getarcaneapp/arcane/backend/v2/internal/activity""github.com/getarcaneapp/arcane/backend/v2/internal/apns"- "github.com/getarcaneapp/arcane/backend/v2/internal/backup""github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/environment""github.com/getarcaneapp/arcane/backend/v2/internal/gitops""github.com/getarcaneapp/arcane/backend/v2/internal/job""github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/backend/v2/internal/system"- "github.com/getarcaneapp/arcane/backend/v2/internal/updater""github.com/getarcaneapp/arcane/backend/v2/internal/volume""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis")func newJobScheduler(@@ -41,10 +42,10 @@ func newJobScheduler(analytics *scheduler.AnalyticsJob,systemService *system.SystemService,jobService *job.JobService,- backupEngine *backup.Engine,- updaterService *updater.UpdaterService,+ workflows *flow.Engine,volumes *volume.VolumeService,gitopsSync *gitops.GitOpsSyncService,+ actors *francis.Runtime,) (schedulertypes.JobScheduler,error,@@ -56,7 +57,6 @@ func newJobScheduler(return nil, err}jobService.SetScheduler(schedulerCtx, jobScheduler)- var repairDone chan struct{}var resumeDone chan struct{}lc.Append(fx.Hook{OnStart: func(ctx context.Context) error {@@ -70,9 +70,10 @@ func newJobScheduler(if startErr := jobService.Coordinator().Start(ctx, schedulerCtx); startErr != nil {return fmt.Errorf("initialize scheduler: %w", startErr)}- if reconcileCoordinatorStartupErr := reconcileCoordinatorStartupInternal(ctx, jobService, backupEngine, updaterService, volumes, systemService, gitopsSync); reconcileCoordinatorStartupErr != nil {+ if reconcileCoordinatorStartupErr := reconcileCoordinatorStartup(ctx, jobService, workflows, volumes, systemService, gitopsSync); reconcileCoordinatorStartupErr != nil {return fmt.Errorf("reconcile scheduler startup: %w", reconcileCoordinatorStartupErr)}+ settleLegacyActorWork(ctx, actors.Service())if gitopsSync != nil {gitopsSync.RegisterAutoSyncJobsOnStartup(ctx)gitopsSync.SubscribeProjectFileChanges(schedulerCtx)@@ -87,21 +88,22 @@ func newJobScheduler(return fmt.Errorf("start scheduler: %w", startSchedulerErr)}jobService.Coordinator().Activate()- if backupEngine != nil {- repairDone = make(chan struct{})- go func() {- defer close(repairDone)- repairBackupDispatchesInternal(schedulerCtx, backupEngine)- }()- }if analytics != nil {if _, submitErr := jobService.Coordinator().Submit(ctx, schedulertypes.Request{JobID: analytics.Name(), EnvironmentID: "0", Trigger: "startup"}); submitErr != nil {return fmt.Errorf("submit startup job %q: %w", analytics.Name(), submitErr)}}+ // Standalone tasks wait until update-all recovery has read its job, so a resumed step's new checkpoint+ // is never mistaken for one the previous process left.if !cfg.AgentMode && systemService != nil {resumeDone = make(chan struct{})- go func() { defer close(resumeDone); systemService.ResumeUpdateAllOnStartup(schedulerCtx) }()+ go func() {+ defer close(resumeDone)+ systemService.ResumeUpdateAllOnStartup(schedulerCtx)+ workflows.Ready()+ }()+ } else {+ workflows.Ready()}if startupCancellationErr := ctx.Err(); startupCancellationErr != nil {return fmt.Errorf("complete scheduler startup: %w", startupCancellationErr)@@ -110,138 +112,125 @@ func newJobScheduler(return nil},OnStop: func(ctx context.Context) error {- return stopJobSchedulerInternal(ctx, cancelScheduler, []chan struct{}{repairDone, resumeDone}, jobService.Coordinator(), jobScheduler)+ cancelScheduler()+ if resumeDone != nil {+ select {+ case <-resumeDone:+ case <-ctx.Done():+ return ctx.Err()+ }+ }+ if stopErr := errors.Join(jobService.Coordinator().Stop(ctx), jobScheduler.Stop(ctx)); stopErr != nil {+ slog.ErrorContext(ctx, "Job scheduler exited with error", "error", stopErr)+ return stopErr+ }+ slog.InfoContext(ctx, "Scheduler stopped")+ return nil},})return jobScheduler, nil}-func stopJobSchedulerInternal(ctx context.Context, cancelScheduler context.CancelFunc, workers []chan struct{}, coordinator *runs.Coordinator, jobScheduler schedulertypes.JobScheduler) error {- cancelScheduler()- for _, done := range workers {- if done != nil {- select {- case <-done:- case <-ctx.Done():- return ctx.Err()+// legacyActorWork names the state and job actor types of the actors workflows replaced.+// TODO(v3): remove once no supported upgrade path can carry their saved work.+var legacyActorWork = []struct{ stateType, jobType, jobActorID string }{+ {stateType: "backup-run", jobType: "backup-run"},+ {stateType: "single-container-update-state", jobType: "single-container-update", jobActorID: "single-container"},+}++// settleLegacyActorWork removes the saved commands and pending jobs of the replaced backup and container-update+// actors. Nothing delivers them any more; the startup reconcile above has already failed their records and activities.+func settleLegacyActorWork(ctx context.Context, service *actor.Service) {+ for _, legacy := range legacyActorWork {+ settled := 0+ for cursor := ""; ; {+ page, err := service.ListStates(ctx, legacy.stateType, &actor.ListStatesOpts{After: cursor, Limit: 100})+ if err != nil {+ slog.WarnContext(ctx, "Failed to list work saved by a replaced actor", "actorType", legacy.stateType, "error", err)+ break+ }+ for _, state := range page.States {+ jobs, listErr := service.ListJobs(ctx, legacy.jobType, cmp.Or(legacy.jobActorID, state.ActorID))+ for _, job := range jobs {+ listErr = errors.Join(listErr, service.DeleteJob(ctx, legacy.jobType, cmp.Or(legacy.jobActorID, state.ActorID), job.JobID))+ }+ if settleErr := errors.Join(listErr, service.DeleteState(ctx, legacy.stateType, state.ActorID)); settleErr != nil && !errors.Is(settleErr, actor.ErrStateNotFound) {+ slog.WarnContext(ctx, "Failed to settle work saved by a replaced actor", "actorType", legacy.stateType, "actorId", state.ActorID, "error", settleErr)+ continue+ }+ settled+++ }+ if cursor = page.AfterID(); cursor == "" {+ break}}+ if settled > 0 {+ slog.InfoContext(ctx, "Settled work saved by a replaced actor; interrupted backups and updates were marked failed", "actorType", legacy.stateType, "count", settled)+ }}- err := errors.Join(coordinator.Stop(ctx), jobScheduler.Stop(ctx))- if err != nil {- slog.ErrorContext(ctx, "Job scheduler exited with error", "error", err)- return err- }- slog.InfoContext(ctx, "Scheduler stopped")- return nil}-func reconcileCoordinatorStartupInternal(+// reconcileCoordinatorStartup fails backups and activities the previous process left running,+// except those that unfinished runs and standalone workflows still own.+func reconcileCoordinatorStartup(ctx context.Context,jobs *job.JobService,- engine *backup.Engine,- updates *updater.UpdaterService,+ workflows *flow.Engine,volumes *volume.VolumeService,systemService *system.SystemService,syncService *gitops.GitOpsSyncService,) error {- activities, backups, err := startupProtectionInternal(ctx, jobs, engine, updates)- if err != nil {- return err- }- if volumes != nil {- if reconcileInterruptedBackupsErr := volumes.ReconcileInterruptedBackups(ctx, backups...); reconcileInterruptedBackupsErr != nil {- return reconcileInterruptedBackupsErr- }- }- if systemService != nil {- if reconcileSystemBackupsErr := systemService.ReconcileInterruptedBackups(ctx, backups...); reconcileSystemBackupsErr != nil {- return reconcileSystemBackupsErr- }- }- if syncService != nil {- if reconcileInterruptedBackupsOnStartupErr := syncService.ReconcileInterruptedBackupsOnStartup(ctx, backups...); reconcileInterruptedBackupsOnStartupErr != nil {- return reconcileInterruptedBackupsOnStartupErr- }- }- return jobs.ReconcileStartupActivities(ctx, activities...)-}--func startupProtectionInternal(ctx context.Context, jobs *job.JobService, engine *backup.Engine, updates *updater.UpdaterService) ([]string, []string, error) {records, err := jobs.Coordinator().Records(ctx)if err != nil {- return nil, nil, err+ return err}- scheduled, err := scheduledBackupProtectionInternal(records)+ standalone, err := workflows.Standalone(ctx)if err != nil {- return nil, nil, err- }-- activities := []string{}- backups := make([]string, 0, len(scheduled))- if engine != nil {- var loadActivityIDsErr error- activities, loadActivityIDsErr = engine.ActiveActivityIDs(ctx)- if loadActivityIDsErr != nil {- return nil, nil, loadActivityIDsErr- }- backups, loadActivityIDsErr = engine.ActiveRunIDs(ctx)- if loadActivityIDsErr != nil {- return nil, nil, loadActivityIDsErr- }- if reconcileDispatchesErr := engine.ReconcileDispatches(ctx); reconcileDispatchesErr != nil {- return nil, nil, reconcileDispatchesErr- }- }- if updates != nil {- ids, activeUpdateActivityIDsErr := updates.ActiveUpdateActivityIDs(ctx)- if activeUpdateActivityIDsErr != nil {- return nil, nil, activeUpdateActivityIDsErr- }- activities = append(activities, ids...)+ return err}- return activities, append(backups, scheduled...), nil-}--func scheduledBackupProtectionInternal(records []schedulertypes.QueueRecord) ([]string, error) {- ids := []string{}- for _, record := range records {+ activities, backups := []string{}, []string{}+ for _, record := range append(records, schedulertypes.QueueRecord{Runs: standalone}) {for _, run := range record.Runs {if run.Status.Terminal() {continue}for _, target := range run.Outcome.Targets {- if len(target.RecoveryData) == 0 {- continue- }var checkpoint struct {BackupID string `json:"backupId"`}- if err := json.Unmarshal(target.RecoveryData, &checkpoint); err != nil {- return nil, fmt.Errorf("decode startup recovery evidence: %w", err)+ if len(target.RecoveryData) == 0 {+ continue+ }+ if decodeErr := json.Unmarshal(target.RecoveryData, &checkpoint); decodeErr != nil {+ return fmt.Errorf("decode startup recovery evidence: %w", decodeErr)}if checkpoint.BackupID != "" {- ids = append(ids, checkpoint.BackupID)+ backups = append(backups, checkpoint.BackupID)}}}}- return ids, nil-}--func repairBackupDispatchesInternal(ctx context.Context, engine *backup.Engine) {- timer := time.NewTicker(15 * time.Second)- defer timer.Stop()- for {- select {- case <-ctx.Done():- return- case <-timer.C:- if err := engine.ReconcileDispatches(ctx); err != nil && ctx.Err() == nil {- slog.ErrorContext(ctx, "Backup dispatch reconciliation failed", "error", err)- }+ for _, run := range standalone {+ if run.ActivityID != "" {+ activities = append(activities, run.ActivityID)+ }+ }+ if volumes != nil {+ if reconcileInterruptedBackupsErr := volumes.ReconcileInterruptedBackups(ctx, backups...); reconcileInterruptedBackupsErr != nil {+ return reconcileInterruptedBackupsErr+ }+ }+ if systemService != nil {+ if reconcileSystemBackupsErr := systemService.ReconcileInterruptedBackups(ctx, backups...); reconcileSystemBackupsErr != nil {+ return reconcileSystemBackupsErr+ }+ }+ if syncService != nil {+ if reconcileInterruptedBackupsOnStartupErr := syncService.ReconcileInterruptedBackupsOnStartup(ctx, backups...); reconcileInterruptedBackupsOnStartupErr != nil {+ return reconcileInterruptedBackupsOnStartupErr}}+ return jobs.ReconcileStartupActivities(ctx, activities...)}type registerJobsParams struct {@@ -262,7 +251,7 @@ type registerJobsParams struct {Admission *runs.AdmissionApns *apns.ApnsService- AutoUpdate *scheduler.AutoUpdateJob+ AutoUpdate *flow.Job `name:"auto-update"`ImageUpdateWatcher *scheduler.ImageUpdateWatcherDockerClientRefresh *scheduler.DockerClientRefreshJobAnalytics *scheduler.AnalyticsJob@@ -272,9 +261,9 @@ type registerJobsParams struct {ExpiredSessionsCleanup *scheduler.ExpiredSessionsCleanupJobScheduledPrune *scheduler.ScheduledPruneJobFilesystemWatcher *scheduler.FilesystemWatcherJob- VulnerabilityScan *scheduler.VulnerabilityScanJob+ VulnerabilityScan *flow.Job `name:"vulnerability-scan"`VulnerabilityRisk *scheduler.VulnerabilityRiskJob- AutoPatch *scheduler.AutoPatchJob+ AutoPatch *flow.Job `name:"auto-patch"`AutoHeal *scheduler.AutoHealJobActivitySweep *scheduler.ActivitySweepJobUploadSessionsCleanup *scheduler.UploadSessionsCleanupJob@@ -330,7 +319,7 @@ func registerJobs(params registerJobsParams) error {return err}- if err := setupSettingsSubscriptionsInternal(settingsSubscriptionsParams{+ if err := setupSettingsSubscriptions(settingsSubscriptionsParams{Lifecycle: params.Lifecycle,LifecycleCtx: params.AppCtx,Config: params.Config,@@ -413,36 +402,36 @@ type settingsSubscriptionsParams struct {Lifecycle fx.LifecycleLifecycleCtx context.ContextConfig *config.Config- Scheduler settingsEffectsSchedulerInternal- Settings settingsChangeSubscriberInternal- Environment timeoutSettingsEnvironmentInternal+ Scheduler settingsEffectsScheduler+ Settings settingsChangeSubscriber+ Environment timeoutSettingsEnvironment- AutoUpdate *scheduler.AutoUpdateJob+ AutoUpdate *flow.Job `name:"auto-update"`ImageUpdateWatcher *scheduler.ImageUpdateWatcherFilesystemWatcher *scheduler.FilesystemWatcherJobScheduledPrune *scheduler.ScheduledPruneJob- VulnerabilityScan *scheduler.VulnerabilityScanJob+ VulnerabilityScan *flow.Job `name:"vulnerability-scan"`VulnerabilityRisk *scheduler.VulnerabilityRiskJob- AutoPatch *scheduler.AutoPatchJob+ AutoPatch *flow.Job `name:"auto-patch"`AutoHeal *scheduler.AutoHealJobApns *apns.ApnsServiceApnsOutbox *scheduler.ApnsOutboxJob}-type settingsChangeSubscriberInternal interface {+type settingsChangeSubscriber interface {SubscribeSettingsChanges(keys []string, callback func([]libarcane.SettingUpdate)) func()}-type settingsEffectsSchedulerInternal interface {+type settingsEffectsScheduler interface {RescheduleJob(ctx context.Context, job schedulertypes.Job) error}-type timeoutSettingsEnvironmentInternal interface {+type timeoutSettingsEnvironment interface {ListRemoteEnvironments(ctx context.Context) ([]environment.Environment, error)ProxyRequest(ctx context.Context, envID, method, path string, body []byte) ([]byte, int, error)}-func setupSettingsSubscriptionsInternal(params settingsSubscriptionsParams) error {+func setupSettingsSubscriptions(params settingsSubscriptionsParams) error {unsubscribes := make([]func(), 0, 8)subscribe := func(keys []string, callback func([]libarcane.SettingUpdate)) {unsubscribes = append(unsubscribes, params.Settings.SubscribeSettingsChanges(keys, callback))@@ -454,7 +443,7 @@ func setupSettingsSubscriptionsInternal(params settingsSubscriptionsParams) erro}})}- timeoutSyncExecutor, cancelTimeoutSync := setupTimeoutSettingsSubscriptionInternal(params, subscribe)+ timeoutSyncExecutor, cancelTimeoutSync := setupTimeoutSettingsSubscription(params, subscribe)subscribe([]string{"pollingEnabled", "pollingInterval"}, func(_ []libarcane.SettingUpdate) {if params.ImageUpdateWatcher != nil {@@ -513,7 +502,7 @@ func setupSettingsSubscriptionsInternal(params settingsSubscriptionsParams) erroreturn nil}-type timeoutSyncWorkerInternal struct {+type timeoutSyncWorker struct {mu sync.Mutexupdates [][]libarcane.SettingUpdatewake chan struct{}@@ -523,7 +512,7 @@ type timeoutSyncWorkerInternal struct {closed bool}-func (w *timeoutSyncWorkerInternal) Stop(ctx context.Context) error {+func (w *timeoutSyncWorker) Stop(ctx context.Context) error {if w == nil {return nil}@@ -539,12 +528,12 @@ func (w *timeoutSyncWorkerInternal) Stop(ctx context.Context) error {}}-func setupTimeoutSettingsSubscriptionInternal(params settingsSubscriptionsParams, subscribe func([]string, func([]libarcane.SettingUpdate))) (*timeoutSyncWorkerInternal, context.CancelFunc) {+func setupTimeoutSettingsSubscription(params settingsSubscriptionsParams, subscribe func([]string, func([]libarcane.SettingUpdate))) (*timeoutSyncWorker, context.CancelFunc) {if params.Config.AgentMode {return nil, nil}ctx, cancel := context.WithCancel(params.LifecycleCtx)- worker := &timeoutSyncWorkerInternal{wake: make(chan struct{}, 1), done: make(chan struct{}), ctx: ctx, cancel: cancel}+ worker := &timeoutSyncWorker{wake: make(chan struct{}, 1), done: make(chan struct{}), ctx: ctx, cancel: cancel}go func() {defer close(worker.done)for {@@ -562,7 +551,7 @@ func setupTimeoutSettingsSubscriptionInternal(params settingsSubscriptionsParamsupdates := worker.updates[0]worker.updates = worker.updates[1:]worker.mu.Unlock()- syncTimeoutSettingsToAgentsInternal(ctx, params.Environment, updates)+ syncTimeoutSettingsToAgents(ctx, params.Environment, updates)}}}()@@ -581,8 +570,8 @@ func setupTimeoutSettingsSubscriptionInternal(params settingsSubscriptionsParamsreturn worker, cancel}-// syncTimeoutSettingsToAgentsInternal syncs timeout settings to all connected remote environments-func syncTimeoutSettingsToAgentsInternal(ctx context.Context, localEnvironment timeoutSettingsEnvironmentInternal, timeoutSettings []libarcane.SettingUpdate) {+// syncTimeoutSettingsToAgents syncs timeout settings to all connected remote environments+func syncTimeoutSettingsToAgents(ctx context.Context, localEnvironment timeoutSettingsEnvironment, timeoutSettings []libarcane.SettingUpdate) {envs, err := localEnvironment.ListRemoteEnvironments(ctx)if err != nil {slog.WarnContext(ctx, "Failed to list remote environments for timeout sync", "error", err)diff --git a/backend/internal/bootstrap/jobs_bootstrap_test.go b/backend/internal/bootstrap/jobs_bootstrap_test.goindex b1a7db187..eb2d2df26 100644--- a/backend/internal/bootstrap/jobs_bootstrap_test.go+++ b/backend/internal/bootstrap/jobs_bootstrap_test.go@@ -13,8 +13,11 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/environment"+ "github.com/getarcaneapp/arcane/backend/v2/internal/image"+ "github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow")type settingsSubscriptionStubInternal struct {@@ -74,19 +77,20 @@ func (s *timeoutSyncEnvironmentStubInternal) ProxyRequest(ctx context.Context, _return nil, 0, ctx.Err()}-func TestSettingsTimeoutSyncDoesNotBlockOtherEffectsInternal(t *testing.T) {+func TestSettingsTimeoutSyncDoesNotBlockOtherEffects(t *testing.T) {lifecycle := fxtest.NewLifecycle(t)settings := &settingsSubscriptionStubInternal{}localScheduler := &settingsSubscriptionSchedulerStubInternal{rescheduled: make(chan struct{})}localEnvironment := &timeoutSyncEnvironmentStubInternal{started: make(chan struct{})}- require.NoError(t, setupSettingsSubscriptionsInternal(settingsSubscriptionsParams{+ require.NoError(t, setupSettingsSubscriptions(settingsSubscriptionsParams{Lifecycle: lifecycle,LifecycleCtx: t.Context(),Config: &config.Config{},Scheduler: localScheduler,Settings: settings,Environment: localEnvironment,+ AutoUpdate: &flow.Job{JobName: "auto-update"},}))timeoutCallbackDone := make(chan struct{})@@ -118,19 +122,19 @@ func TestSettingsTimeoutSyncDoesNotBlockOtherEffectsInternal(t *testing.T) {require.NoError(t, lifecycle.Stop(stopCtx))}-func TestFeatureChangeReschedulesScanAndPatchJobsInternal(t *testing.T) {+func TestFeatureChangeReschedulesScanAndPatchJobs(t *testing.T) {lifecycle := fxtest.NewLifecycle(t)settings := &settingsSubscriptionStubInternal{}schedulerStub := &settingsSubscriptionSchedulerStubInternal{rescheduled: make(chan struct{})}- require.NoError(t, setupSettingsSubscriptionsInternal(settingsSubscriptionsParams{+ require.NoError(t, setupSettingsSubscriptions(settingsSubscriptionsParams{Lifecycle: lifecycle,LifecycleCtx: t.Context(),Config: &config.Config{},Scheduler: schedulerStub,Settings: settings,- VulnerabilityScan: scheduler.NewVulnerabilityScanJob(nil, nil),+ VulnerabilityScan: scheduler.NewVulnerabilityScanJob(nil, &vulnerability.VulnerabilityService{}, nil),VulnerabilityRisk: scheduler.NewVulnerabilityRiskJob(nil, nil),- AutoPatch: scheduler.NewAutoPatchJob(nil, nil),+ AutoPatch: scheduler.NewAutoPatchJob(nil, &image.ImageService{}, nil),}))require.Len(t, settings.featureCallbacks, 3)for _, callback := range settings.featureCallbacks {diff --git a/backend/internal/container/handler.go b/backend/internal/container/handler.goindex cf23f9710..0c1188612 100644--- a/backend/internal/container/handler.go+++ b/backend/internal/container/handler.go@@ -19,7 +19,7 @@ import ("github.com/moby/moby/api/types/network""github.com/samber/mo""go.getarcane.app/docker/types"- "go.getarcane.app/kit/pkg"+ kit "go.getarcane.app/kit/pkg""github.com/getarcaneapp/arcane/backend/v2/internal/activity""github.com/getarcaneapp/arcane/backend/v2/internal/common"@@ -658,12 +658,7 @@ func (h *ContainerHandler) runContainerActionInternal(ctx context.Context, input}activitylib.CompleteHandlerActivity(runtimeCtx, h.activityService, activityID, cfg.CompleteMessage, nil)- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{Message: cfg.SuccessMessage, ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()},- },- }, nil+ return handlerutil.MessageOutput(cfg.SuccessMessage, activityID), nil}func (h *ContainerHandler) CommitContainer(ctx context.Context, input *CommitContainerInput) (*handlerutil.Out[container.CommitResult], error) {@@ -876,12 +871,7 @@ func (h *ContainerHandler) DeleteContainer(ctx context.Context, input *DeleteCon}activitylib.CompleteHandlerActivity(runtimeCtx, h.activityService, activityID, "Container deleted", nil)- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{Message: "Container deleted successfully", ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()},- },- }, nil+ return handlerutil.MessageOutput("Container deleted successfully", activityID), nil}func (h *ContainerHandler) SetAutoUpdate(ctx context.Context, input *SetAutoUpdateInput) (*handlerutil.Out[base.MessageResponse], error) {@@ -898,10 +888,5 @@ func (h *ContainerHandler) SetAutoUpdate(ctx context.Context, input *SetAutoUpdamsg := kit.Ternary(excluded, "Auto-update disabled", "Auto-update enabled")- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{Message: msg},- },- }, nil+ return handlerutil.MessageOutput(msg, ""), nil}diff --git a/backend/internal/dashboard/service_test.go b/backend/internal/dashboard/service_test.goindex 0dfb08e04..7a676e062 100644--- a/backend/internal/dashboard/service_test.go+++ b/backend/internal/dashboard/service_test.go@@ -197,7 +197,7 @@ func TestDashboardService_GetSnapshot_ReturnsDashboardSnapshot(t *testing.T) {}).Error)imageSvc := image.NewImageService(db, nil, nil, nil, nil, nil, nil, nil)projectSvc := project.NewProjectService(db, settingsSvc, nil, imageSvc, nil, nil, nil, nil, config.Load(), nil, nil)- svc := NewDashboardService(db, dockerSvc, nil, projectSvc, imageSvc, settingsSvc, nil, nil, nil, volume.NewVolumeService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil))+ svc := NewDashboardService(db, dockerSvc, nil, projectSvc, imageSvc, settingsSvc, nil, nil, nil, volume.NewVolumeService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil))snapshot, err := svc.GetSnapshot(t.Context(), DashboardActionItemsOptions{}, true)require.NoError(t, err)diff --git a/backend/internal/di/di.go b/backend/internal/di/di.goindex 1631ac484..776c1e6d0 100644--- a/backend/internal/di/di.go+++ b/backend/internal/di/di.go@@ -24,7 +24,6 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/gitops""github.com/getarcaneapp/arcane/backend/v2/internal/gitrepo""github.com/getarcaneapp/arcane/backend/v2/internal/image"- "github.com/getarcaneapp/arcane/backend/v2/internal/imageupdate""github.com/getarcaneapp/arcane/backend/v2/internal/job""github.com/getarcaneapp/arcane/backend/v2/internal/kv""github.com/getarcaneapp/arcane/backend/v2/internal/network"@@ -56,6 +55,7 @@ import (var ActorOptions = fx.Options(fx.Provide(provideActorRuntimeInternal, provideRunCoordinatorInternal),fx.Provide(provideAdmissionGateInternal),+ fx.Provide(provideWorkflowEngineInternal),fx.Provide(provideTunnelRegistryInternal),)@@ -81,9 +81,9 @@ var ServiceOptions = fx.Options(environment.NewEnvironmentService,apns.NewApnsService,notification.NewNotificationService,- vulnerability.NewVulnerabilityService,- imageupdate.NewImageUpdateService,- image.NewImageService,+ provideVulnerabilityServiceInternal,+ provideImageUpdateServiceInternal,+ provideImageServiceInternal,build.NewBuildService,project.NewLifecycleService,container.NewContainerService,@@ -93,16 +93,16 @@ var ServiceOptions = fx.Options(swarm.NewSwarmService,template.NewTemplateService,oidc.NewOidcService,- system.NewSystemService,+ provideSystemServiceInternal,diagnostics.NewDiagnosticsService,- gitops.NewGitOpsSyncService,+ provideGitOpsSyncServiceInternal,variable.NewVariableService,backup.NewRecoveryKeyStore,upload.NewUploadService,auth.NewAuthService,settings.NewSettingsSearchService,fx.Annotate(settings.NewSettingsService, fx.OnStop(func(ctx context.Context, service *settings.SettingsService) error { return service.Stop(ctx) })),- fx.Annotate(volume.NewVolumeService, fx.OnStop(func(ctx context.Context, service *volume.VolumeService) { service.CleanupHelperContainers(ctx) })),+ provideVolumeServiceInternal,fx.Annotate(webhook.NewWebhookService,fx.OnStart(func(ctx context.Context, service *webhook.WebhookService) error { return service.LoadTokenHashes(ctx) }),fx.OnStop(func(ctx context.Context, service *webhook.WebhookService) {@@ -163,7 +163,7 @@ var ServiceOptions = fx.Options(// remain bootstrap concerns because their ordering is application-specific.var JobOptions = fx.Options(fx.Provide(- scheduler.NewAutoUpdateJob,+ fx.Annotate(scheduler.NewAutoUpdateJob, fx.ResultTags(`name:"auto-update"`)),scheduler.NewImageUpdateWatcher,scheduler.NewDockerClientRefreshJob,scheduler.NewAnalyticsJob,@@ -172,9 +172,9 @@ var JobOptions = fx.Options(scheduler.NewExpiredSessionsCleanupJob,scheduler.NewScheduledPruneJob,provideFilesystemWatcherJobInternal,- scheduler.NewVulnerabilityScanJob,+ fx.Annotate(scheduler.NewVulnerabilityScanJob, fx.ResultTags(`name:"vulnerability-scan"`)),scheduler.NewVulnerabilityRiskJob,- scheduler.NewAutoPatchJob,+ fx.Annotate(scheduler.NewAutoPatchJob, fx.ResultTags(`name:"auto-patch"`)),scheduler.NewAutoHealJob,scheduler.NewActivitySweepJob,scheduler.NewUploadSessionsCleanupJob,diff --git a/backend/internal/di/di_test.go b/backend/internal/di/di_test.goindex d5d91aee5..62067af9a 100644--- a/backend/internal/di/di_test.go+++ b/backend/internal/di/di_test.go@@ -60,6 +60,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability""github.com/getarcaneapp/arcane/backend/v2/internal/webhook""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/oidcjwk")@@ -110,7 +111,7 @@ type graphParams struct {AuthMiddleware *auth.AuthMiddlewareJWKSetManager *oidcjwk.KeySetManager- AutoUpdate *scheduler.AutoUpdateJob+ AutoUpdate *flow.Job `name:"auto-update"`ImageUpdateWatcher *scheduler.ImageUpdateWatcherDockerClientRefresh *scheduler.DockerClientRefreshJobAnalytics *scheduler.AnalyticsJob@@ -119,7 +120,7 @@ type graphParams struct {ExpiredSessionsCleanup *scheduler.ExpiredSessionsCleanupJobScheduledPrune *scheduler.ScheduledPruneJobFilesystemWatcher *scheduler.FilesystemWatcherJob- VulnerabilityScan *scheduler.VulnerabilityScanJob+ VulnerabilityScan *flow.Job `name:"vulnerability-scan"`AutoHeal *scheduler.AutoHealJob}diff --git a/backend/internal/di/providers.go b/backend/internal/di/providers.goindex 5f8851245..332473f86 100644--- a/backend/internal/di/providers.go+++ b/backend/internal/di/providers.go@@ -19,14 +19,17 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/backup""github.com/getarcaneapp/arcane/backend/v2/internal/build""github.com/getarcaneapp/arcane/backend/v2/internal/config"+ "github.com/getarcaneapp/arcane/backend/v2/internal/container""github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/docker""github.com/getarcaneapp/arcane/backend/v2/internal/environment""github.com/getarcaneapp/arcane/backend/v2/internal/event"+ "github.com/getarcaneapp/arcane/backend/v2/internal/gitops""github.com/getarcaneapp/arcane/backend/v2/internal/gitrepo""github.com/getarcaneapp/arcane/backend/v2/internal/image""github.com/getarcaneapp/arcane/backend/v2/internal/imageupdate""github.com/getarcaneapp/arcane/backend/v2/internal/kv"+ "github.com/getarcaneapp/arcane/backend/v2/internal/network""github.com/getarcaneapp/arcane/backend/v2/internal/notification""github.com/getarcaneapp/arcane/backend/v2/internal/passkey""github.com/getarcaneapp/arcane/backend/v2/internal/project"@@ -34,14 +37,18 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/role"s3domain "github.com/getarcaneapp/arcane/backend/v2/internal/s3""github.com/getarcaneapp/arcane/backend/v2/internal/settings"+ "github.com/getarcaneapp/arcane/backend/v2/internal/swarm""github.com/getarcaneapp/arcane/backend/v2/internal/system""github.com/getarcaneapp/arcane/backend/v2/internal/template""github.com/getarcaneapp/arcane/backend/v2/internal/updater""github.com/getarcaneapp/arcane/backend/v2/internal/user""github.com/getarcaneapp/arcane/backend/v2/internal/version"+ "github.com/getarcaneapp/arcane/backend/v2/internal/volume"+ "github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability""github.com/getarcaneapp/arcane/backend/v2/pkg/authz""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/concurrency""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis"@@ -66,45 +73,29 @@ func provideSettingsModuleInternal(service *settings.SettingsService, search *sereturn settings.New(service, search, localEnvironment.ProxyJSONRequest, cfg)}-func provideBackupEngineInternal(- ctx context.Context,- lc fx.Lifecycle,- admission *runs.Admission,- imageService *image.ImageService,- runtime *francis.Runtime,- coordinator *runs.Coordinator,- roles *role.RoleService,- cfg *config.Config,-) (- *backup.Engine,- error,-) {- engine := backup.NewEngine(ctx, admission, imageService)- if err := engine.Register(runtime); err != nil {- return nil, err- }- engine.SetExecutionReady(coordinator.Active)- engine.SetAuthorize(func(ctx context.Context, command backuptypes.DurableRunCommand) error {- if cfg.AgentMode && command.UserID == "agent" && command.RequestedWithKey == "" {+func provideBackupEngineInternal(lc fx.Lifecycle, admission *runs.Admission, imageService *image.ImageService, roles *role.RoleService, cfg *config.Config) *backup.Engine {+ engine := backup.NewEngine(admission, imageService)+ engine.SetAuthorize(func(ctx context.Context, requester backuptypes.Requester) error {+ if cfg.AgentMode && requester.UserID == "agent" && requester.APIKeyID == "" {return nil}- if command.UserID == "" {+ if requester.UserID == "" {return errors.New("requesting user unavailable")}- permissions, err := roles.ResolveExecutionPermissions(ctx, command.UserID, command.RequestedWithKey)+ permissions, err := roles.ResolveExecutionPermissions(ctx, requester.UserID, requester.APIKeyID)if err != nil {return err}- if !permissions.Allows(command.Permission, command.EnvironmentID) {+ if !permissions.Allows(requester.Permission, requester.EnvironmentID) {return errors.New("requesting user no longer has permission for this backup")}- if command.Permission == authz.PermSystemBackupsManage && !permissions.IsGlobalAdmin() {+ if requester.Permission == authz.PermSystemBackupsManage && !permissions.IsGlobalAdmin() {return errors.New("system backups require a global administrator")}return nil})lc.Append(fx.Hook{OnStop: engine.Stop})- return engine, nil+ return engine}func provideActorRuntimeInternal(@@ -143,6 +134,17 @@ func provideRunCoordinatorInternal(runtime *francis.Runtime, store *kv.KVServicereturn coordinator, nil}+// provideWorkflowEngineInternal starts monitoring after the host is ready; its+// hook follows the runtime's because the engine depends on it.+func provideWorkflowEngineInternal(appCtx context.Context, runtime *francis.Runtime, coordinator *runs.Coordinator, activities *activity.ActivityService, lc fx.Lifecycle) (*flow.Engine, error) {+ engine, err := flow.New(appCtx, runtime, coordinator, activities)+ if err != nil {+ return nil, err+ }+ lc.Append(fx.Hook{OnStart: engine.Start, OnStop: engine.Stop})+ return engine, nil+}+func provideAdmissionGateInternal(runtime *francis.Runtime) (*runs.Admission, error) {admission := runs.NewAdmission(runtime.Service(), uuid.New().String())if err := admission.Register(runtime); err != nil {@@ -273,10 +275,8 @@ func provideProjectServiceInternal(type updaterServiceParams struct {fx.In- Context context.ContextConfig *config.Config- Lifecycle fx.Lifecycle- ActorRuntime *francis.Runtime+ Workflows *flow.EngineCoordinator *runs.CoordinatorAdmission *runs.AdmissionRoles *role.RoleService@@ -293,6 +293,64 @@ type updaterServiceParams struct {Activity *activity.ActivityService}+// provideImageUpdateServiceInternal registers the image-check workflow while the host is still unstarted.+func provideImageUpdateServiceInternal(+ db *database.DB,+ settingsService *settings.SettingsService,+ registryService *registry.ContainerRegistryService,+ dockerService *docker.DockerClientService,+ eventService *event.EventService,+ notificationService *notification.NotificationService,+ activityService *activity.ActivityService,+ engine *flow.Engine,+) (*imageupdate.ImageUpdateService, error) {+ service := imageupdate.NewImageUpdateService(db, settingsService, registryService, dockerService, eventService, notificationService, activityService)+ if err := service.RegisterWorkflows(engine); err != nil {+ return nil, err+ }+ return service, nil+}++// provideImageServiceInternal registers the auto-patch workflow while the host is still unstarted.+func provideImageServiceInternal(+ db *database.DB,+ dockerService *docker.DockerClientService,+ registryService *registry.ContainerRegistryService,+ imageUpdateService *imageupdate.ImageUpdateService,+ vulnerabilityService *vulnerability.VulnerabilityService,+ eventService *event.EventService,+ settingsService *settings.SettingsService,+ activityService *activity.ActivityService,+ engine *flow.Engine,+) (*image.ImageService, error) {+ service := image.NewImageService(db, dockerService, registryService, imageUpdateService, vulnerabilityService, eventService, settingsService, activityService)+ if err := service.RegisterWorkflows(engine); err != nil {+ return nil, err+ }+ return service, nil+}++// provideVulnerabilityServiceInternal registers the scheduled scan workflow while the host is still unstarted.+func provideVulnerabilityServiceInternal(+ db *database.DB,+ cfg *config.Config,+ dockerService *docker.DockerClientService,+ eventService *event.EventService,+ settingsService *settings.SettingsService,+ notificationService *notification.NotificationService,+ activityService *activity.ActivityService,+ registryService *registry.ContainerRegistryService,+ kvService *kv.KVService,+ httpClient *http.Client,+ engine *flow.Engine,+) (*vulnerability.VulnerabilityService, error) {+ service := vulnerability.NewVulnerabilityService(db, cfg, dockerService, eventService, settingsService, notificationService, activityService, registryService, kvService, httpClient)+ if err := service.RegisterWorkflows(engine); err != nil {+ return nil, err+ }+ return service, nil+}+func provideUpdaterServiceInternal(p updaterServiceParams) (*updater.UpdaterService, error) {service, err := updater.NewUpdaterService(p.DB,@@ -314,21 +372,9 @@ func provideUpdaterServiceInternal(p updaterServiceParams) (*updater.UpdaterServif err != nil {return nil, err}- if registerActorsErr := service.RegisterActors(p.ActorRuntime); registerActorsErr != nil {- return nil, registerActorsErr+ if registerWorkflowsErr := service.RegisterWorkflows(p.Workflows); registerWorkflowsErr != nil {+ return nil, registerWorkflowsErr}- p.Lifecycle.Append(- fx.Hook{- OnStart: func(- context.Context,- ) error {- return service.Start(- p.Context,- )- },- OnStop: service.Stop,- },- ) //nolint:contextcheck // Workers inherit the application lifetime after startup returns.return service, nil}@@ -410,3 +456,75 @@ func provideFilesystemWatcherJobInternal(})return job, nil}++// provideGitOpsSyncServiceInternal registers the gitops-sync workflow while the host is still unstarted.+func provideGitOpsSyncServiceInternal(+ db *database.DB,+ repoService *gitrepo.GitRepositoryService,+ projectService *project.ProjectService,+ swarmService *swarm.SwarmService,+ eventService *event.EventService,+ settingsService *settings.SettingsService,+ engine *flow.Engine,+) (*gitops.GitOpsSyncService, error) {+ service := gitops.NewGitOpsSyncService(db, repoService, projectService, swarmService, eventService, settingsService)+ if err := service.RegisterWorkflows(engine); err != nil {+ return nil, err+ }+ return service, nil+}++// provideVolumeServiceInternal registers the volume backup workflows while the host is still unstarted.+func provideVolumeServiceInternal(+ lc fx.Lifecycle,+ db *database.DB,+ dockerService *docker.DockerClientService,+ eventService *event.EventService,+ settingsService *settings.SettingsService,+ containerService *container.ContainerService,+ imageService *image.ImageService,+ backupEngine *backup.Engine,+ s3Destinations *s3domain.S3DestinationService,+ cfg *config.Config,+ recoveryKeys *backup.RecoveryKeyStore,+ engine *flow.Engine,+) (*volume.VolumeService, error) {+ service := volume.NewVolumeService(db, dockerService, eventService, settingsService, containerService, imageService, backupEngine, s3Destinations, cfg, recoveryKeys)+ if err := service.RegisterWorkflows(engine); err != nil {+ return nil, err+ }+ lc.Append(fx.Hook{OnStop: func(ctx context.Context) error { service.CleanupHelperContainers(ctx); return nil }})+ return service, nil+}++// provideSystemServiceInternal registers the system backup workflows while the host is still unstarted.+func provideSystemServiceInternal(+ db *database.DB,+ dockerService *docker.DockerClientService,+ containerService *container.ContainerService,+ imageUpdateService *imageupdate.ImageUpdateService,+ volumeService *volume.VolumeService,+ networkService *network.NetworkService,+ settingsService *settings.SettingsService,+ activityService *activity.ActivityService,+ versionService *version.VersionService,+ eventService *event.EventService,+ projectService *project.ProjectService,+ backupEngine *backup.Engine,+ s3Destinations *s3domain.S3DestinationService,+ recoveryKeys *backup.RecoveryKeyStore,+ actors *francis.Runtime,+ cfg *config.Config,+ environments *environment.EnvironmentService,+ engine *flow.Engine,+ roles *role.RoleService,+) (*system.SystemService, error) {+ service := system.NewSystemService(+ db, dockerService, containerService, imageUpdateService, volumeService, networkService, settingsService, activityService,+ versionService, eventService, projectService, backupEngine, s3Destinations, recoveryKeys, actors, cfg,+ )+ if err := service.RegisterWorkflows(engine, environments, roles); err != nil {+ return nil, err+ }+ return service, nil+}diff --git a/backend/internal/environment/handler.go b/backend/internal/environment/handler.goindex 4bc89e3de..856440a48 100644--- a/backend/internal/environment/handler.go+++ b/backend/internal/environment/handler.go@@ -26,8 +26,7 @@ import ("github.com/getarcaneapp/arcane/types/v2/environment"usertypes "github.com/getarcaneapp/arcane/types/v2/user""github.com/getarcaneapp/arcane/types/v2/version"- "github.com/samber/mo"- "go.getarcane.app/kit/pkg"+ kit "go.getarcane.app/kit/pkg""go.getarcane.app/kit/pkg/mapping""go.getarcane.app/streams/agg"@@ -568,14 +567,7 @@ func (h *EnvironmentHandler) DeleteEnvironment(ctx context.Context, input *Deletreturn nil, huma.Error500InternalServerError("Failed to delete environment: " + err.Error())}- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{- Message: "Environment deleted successfully",- },- },- }, nil+ return handlerutil.MessageOutput("Environment deleted successfully", ""), nil}// TestConnection tests connectivity to an environment.@@ -622,14 +614,7 @@ func (h *EnvironmentHandler) UpdateHeartbeat(ctx context.Context, input *UpdateHreturn nil, huma.Error500InternalServerError("Failed to update heartbeat")}- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{- Message: "Heartbeat updated successfully",- },- },- }, nil+ return handlerutil.MessageOutput("Heartbeat updated successfully", ""), nil}// PairAgent generates or rotates the local agent pairing token.@@ -670,15 +655,7 @@ func (h *EnvironmentHandler) SyncEnvironment(ctx context.Context, input *SyncEnvreturn nil, huma.Error500InternalServerError(err.Error())}- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{- Message: "Environment synced successfully",- ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer(),- },- },- }, nil+ return handlerutil.MessageOutput("Environment synced successfully", activityID), nil}func (h *EnvironmentHandler) buildUpdateMapInternal(req *environment.Update, isLocalEnv bool) map[string]any {@@ -808,14 +785,7 @@ func (h *EnvironmentHandler) PairEnvironment(ctx context.Context, input *PairEnvslog.InfoContext(ctx, "Environment pairing completed", "environmentId", *envID, "environmentName", env.Name)h.triggerEnvironmentResourceSyncInternal(ctx, *envID, env.Name, "environment pairing")- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{- Message: "Environment pairing completed successfully",- },- },- }, nil+ return handlerutil.MessageOutput("Environment pairing completed successfully", ""), nil}// GetDeploymentSnippets returns deployment snippets for an environment.diff --git a/backend/internal/event/handler.go b/backend/internal/event/handler.goindex 701c58e81..73ff7a464 100644--- a/backend/internal/event/handler.go+++ b/backend/internal/event/handler.go@@ -121,12 +121,5 @@ func (h *EventHandler) DeleteEvent(ctx context.Context, input *DeleteEventInput)return nil, huma.Error500InternalServerError("Failed to delete event: " + err.Error())}- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{- Message: "Event deleted successfully",- },- },- }, nil+ return handlerutil.MessageOutput("Event deleted successfully", ""), nil}diff --git a/backend/internal/federated/handler.go b/backend/internal/federated/handler.goindex fd7de51a7..dfa3981dc 100644--- a/backend/internal/federated/handler.go+++ b/backend/internal/federated/handler.go@@ -122,14 +122,7 @@ func (h *FederatedCredentialHandler) DeleteFederatedCredential(ctx context.Contereturn nil, federatedCredentialManagementErrorInternal(err)}- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{- Message: "Federated credential deleted successfully",- },- },- }, nil+ return handlerutil.MessageOutput("Federated credential deleted successfully", ""), nil}func writeFederatedTokenExchangeErrorInternal(c *echo.Context, err error) error {diff --git a/backend/internal/gitops/children/sync/service.go b/backend/internal/gitops/children/sync/service.goindex 17a5593d6..8c3375199 100644--- a/backend/internal/gitops/children/sync/service.go+++ b/backend/internal/gitops/children/sync/service.go@@ -3,6 +3,7 @@package syncimport (+ "cmp""context""encoding/json/v2""errors"@@ -24,6 +25,7 @@ import ("github.com/getarcaneapp/arcane/types/v2/user""github.com/go-git/go-git/v5""github.com/go-git/go-git/v5/plumbing"+ "github.com/go-git/go-git/v5/plumbing/transport""go.getarcane.app/acfs""go.getarcane.app/kit/pkg""gorm.io/gorm"@@ -39,18 +41,62 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/utils")-// Service runs deployment syncs. Shared sync limits, error events and job-// registration stay with the parent and arrive as callbacks.-type Service struct {- db *database.DB- repoService *gitrepo.GitRepositoryService- projectService *projectpkg.ProjectService- swarmService *swarm.SwarmService- eventService *event.EventService- limits func(context.Context, *projectpkg.GitOpsSync) (int, int64, int64)- logError func(context.Context, *projectpkg.GitOpsSync, user.Actor, string)- unregisterJob func(context.Context, string)-}+// reservedRootEnvFiles are the project-root env files the override merge owns; git never writes them raw.+var reservedRootEnvFiles = []string{projects.EffectiveEnvFileName, projects.GitSourceEnvFileName, projects.OverrideEnvFileName, projects.GlobalEnvFileName}++type (+ // Service runs deployment syncs. Shared sync limits, error events and job+ // registration stay with the parent and arrive as callbacks.+ Service struct {+ db *database.DB+ repoService *gitrepo.GitRepositoryService+ projectService *projectpkg.ProjectService+ swarmService *swarm.SwarmService+ eventService *event.EventService+ limits func(context.Context, *projectpkg.GitOpsSync) (int, int64, int64)+ logError func(context.Context, *projectpkg.GitOpsSync, user.Actor, string)+ unregisterJob func(context.Context, string)+ }++ // syncRevision is the recovery data a run records so a retry replays the same source revision.+ syncRevision struct {+ Commit string `json:"commit"`+ }++ // preparedSyncSource is the cloned and validated source a sync applies.+ preparedSyncSource struct {+ repoPath string+ commitHash string+ composeContent string+ envContent *string+ overrideContent *string+ overrideFileName string+ // replay marks a retry of a pinned revision, which redeploys even when the files are unchanged.+ replay bool+ }++ // stagedDirectorySync holds the fully prepared directory-sync result before it+ // is promoted into the live project path.+ stagedDirectorySync struct {+ stagePath string+ // stageLogical is stagePath relative to the projects directory, the root every staging mutation is confined to.+ stageLogical string+ projectsDir string+ composeFileName string+ project *projectpkg.Project+ // syncFiles are the repo files written raw; syncedFiles are their paths.+ syncFiles []projects.SyncFile+ syncedFiles []string+ // oldSyncedFiles are the paths the previous sync tracked.+ oldSyncedFiles []string+ // gitEnvContent is the repo's project-root .env, routed through the env merge.+ gitEnvContent *string+ serviceCount int+ contentsChanged bool+ // backupScope names every live path promotion may create, rewrite or delete.+ backupScope projects.ProjectUpdateBackupScope+ }+)func New(db *database.DB,@@ -74,138 +120,176 @@ func New(}}-// Run clones the sync source and applies it to its project or Swarm stack.+// Run clones the sync source and applies it to its project or Swarm stack. A retry replays the+// revision its run pinned, and redeploys it unless the record already confirms that revision.func (s *Service) Run(ctx context.Context, sync *projectpkg.GitOpsSync, actor user.Actor, result *gitops.SyncResult) (*gitops.SyncResult, error) {- source, err := s.prepareSyncSource(ctx, sync, result, actor)- if source != nil && source.repoPath != "" {+ pinned := ""+ if run, ok := jobcontext.Run(ctx); ok {+ index := slices.IndexFunc(run.Outcome.Targets, func(target scheduler.TargetOutcome) bool {+ return target.ID == sync.ID && len(target.RecoveryData) > 0+ })+ var revision syncRevision+ if index >= 0 {+ if err := json.Unmarshal(run.Outcome.Targets[index].RecoveryData, &revision); err != nil {+ return result, err+ }+ }+ pinned = revision.Commit+ }+ if pinned != "" && kit.FromPtr(sync.LastSyncCommit) == pinned && kit.FromPtr(sync.LastSyncStatus) == "success" {+ result.Success = true+ result.Message = "Revision " + pinned + " is already synced"+ return result, nil+ }++ source, err := s.prepareSyncSource(ctx, sync, result, actor, pinned)+ if source != nil {defer s.repoService.Discard(ctx, source.repoPath)}if err != nil {return result, err}- if source == nil {- return result, errors.New("prepared sync source is missing")- }-- if sync.TargetType == "swarm_stack" {- return s.performSwarmStackSyncInternal(ctx, sync, sync.ID, actor, result, source)- }+ var syncFiles []projects.SyncFileif sync.SyncDirectory {- return s.performDirectorySync(ctx, sync, sync.ID, actor, result, source)- }-- return s.performSingleFileSyncInternal(ctx, sync, sync.ID, actor, result, source)-}--// DirectoryProject resolves the linked project for a directory sync,-// relinking or recovering it when the stored reference is stale.-func (s *Service) DirectoryProject(ctx context.Context, sync *projectpkg.GitOpsSync) (*projectpkg.Project, error) {- return s.getDirectorySyncProjectInternal(ctx, sync)-}--// ReconcileRevision confirms an interrupted run when the record already holds-// the source revision that run pinned.-func ReconcileRevision(ctx context.Context, previous scheduler.Run, record *projectpkg.GitOpsSync, outcome scheduler.Outcome) (scheduler.Outcome, error) {- target, commit, err := recordedRevision(previous, record.ID)- if err != nil {- return outcome, err+ if syncFiles, err = s.walkAndParseSyncDirectory(ctx, sync, source.repoPath); err != nil {+ return result, s.failSync(ctx, result, sync, actor, "Failed to walk directory", err)+ }}- if commit == "" || kit.FromPtr(record.LastSyncCommit) != commit || kit.FromPtr(record.LastSyncStatus) != "success" {- return outcome, nil+ var project *projectpkg.Project+ var syncedFiles []string+ changed := false+ switch {+ case sync.TargetType == "swarm_stack":+ if s.swarmService == nil {+ return result, s.failSync(ctx, result, sync, actor, "Swarm service is unavailable", errors.New("swarm service is unavailable"))+ }+ swarmFiles := make([]swarmtypes.SyncFile, len(syncFiles))+ for i, file := range syncFiles {+ swarmFiles[i] = swarmtypes.SyncFile{RelativePath: file.RelativePath, Content: file.Content}+ syncedFiles = append(syncedFiles, file.RelativePath)+ }+ // Swarm tasks pull with only the auth embedded in the service spec, so registry auth is always resolved.+ request := swarmtypes.StackDeployRequest{+ Name: sync.ProjectName,+ ComposeContent: source.composeContent,+ OverrideContent: kit.FromPtr(source.overrideContent),+ EnvContent: kit.FromPtr(source.envContent),+ Files: swarmFiles,+ Prune: true,+ WithRegistryAuth: true,+ WorkingDir: filepath.Dir(filepath.Join(source.repoPath, sync.ComposePath)),+ }+ if _, deployErr := s.swarmService.DeployStack(ctx, sync.EnvironmentID, request); deployErr != nil {+ return result, s.failSync(ctx, result, sync, actor, "Failed to deploy swarm stack", deployErr)+ }+ result.Message = fmt.Sprintf("Successfully deployed swarm stack %s from %s", sync.ProjectName, sync.ComposePath)+ case sync.SyncDirectory:+ if project, syncedFiles, _, changed, err = s.syncProjectDirectory(ctx, sync, syncFiles, actor); err != nil {+ message := kit.Ternary(errors.Is(err, common.ErrGitOpsSyncProjectBindingBroken), "GitOps project binding broken", "Failed to sync project directory")+ return result, s.failSync(ctx, result, sync, actor, message, err)+ }+ result.Message = fmt.Sprintf("Successfully synced directory with %d files to project %s", len(syncedFiles), project.Name)+ default:+ if project, changed, err = s.syncComposeFile(ctx, sync, actor, result, source); err != nil {+ return result, err+ }+ result.Message = fmt.Sprintf("Successfully synced compose file from %s to project %s", sync.ComposePath, project.Name)}- target.Status = scheduler.Succeeded- if progressErr := jobcontext.Progress(ctx, target); progressErr != nil {- return outcome, progressErr+ if syncedFiles == nil {+ syncedFiles = []string{filepath.Base(sync.ComposePath)}+ if source.overrideFileName != "" {+ syncedFiles = append(syncedFiles, source.overrideFileName)+ }}- return scheduler.Outcome{Status: scheduler.Succeeded, Targets: []scheduler.TargetOutcome{target}}, nil-}-// recordedRevision returns the target and source revision an earlier run recorded for syncID.-func recordedRevision(run scheduler.Run, syncID string) (scheduler.TargetOutcome, string, error) {- idx := slices.IndexFunc(run.Outcome.Targets, func(target scheduler.TargetOutcome) bool {- return target.ID == syncID && len(target.RecoveryData) > 0- })- if idx < 0 {- return scheduler.TargetOutcome{}, "", nil- }- var revision syncRevision- if err := json.Unmarshal(run.Outcome.Targets[idx].RecoveryData, &revision); err != nil {- return scheduler.TargetOutcome{}, "", err+ if project != nil && (changed || source.replay) {+ if err = s.deployOnce(ctx, sync, project, actor, source.replay); err != nil {+ // The files reached disk, so the row keeps their revision and list.+ errMsg := err.Error()+ result.Message = "Sync wrote files but redeploy failed"+ result.Error = &errMsg+ s.updateSyncStatus(ctx, sync.ID, "failed", errMsg, source.commitHash, syncedFiles)+ s.logError(ctx, sync, actor, errMsg)+ return result, err+ }}- return run.Outcome.Targets[idx], revision.Commit, nil-}--// syncRevision is the recovery data a sync run records so a retry replays the same source revision.-type syncRevision struct {- Commit string `json:"commit"`-}-// preparedSyncSource captures the repository data needed by the sync execution-// paths after the source repository has been cloned and validated.-type preparedSyncSource struct {- repoPath string- commitHash string- composeContent string- envContent *string- overrideContent *string- overrideFileName string+ s.updateSyncStatus(ctx, sync.ID, "success", "", source.commitHash, syncedFiles)+ result.Success = true+ targetKind, targetName := "swarm stack", sync.ProjectName+ if project != nil {+ targetKind, targetName = "project", project.Name+ }+ _, _ = s.eventService.CreateEvent(ctx, event.CreateEventRequest{+ Type: event.EventTypeGitSyncRun,+ Severity: event.EventSeveritySuccess,+ Title: kit.Ternary(project == nil, "Git sync completed for stack", "Git sync completed"),+ Description: fmt.Sprintf("Successfully synced '%s' to %s '%s'", sync.Name, targetKind, targetName),+ ResourceType: new("git_sync"),+ ResourceID: new(sync.ID),+ ResourceName: new(sync.Name),+ UserID: new(actor.ID),+ Username: new(actor.Username),+ EnvironmentID: new(sync.EnvironmentID),+ })+ slog.InfoContext(ctx, "GitOps sync completed", "syncId", sync.ID, "target", targetName)+ return result, nil}-// stagedDirectorySync holds the fully prepared directory-sync result before it-// is promoted into the live project path.-type stagedDirectorySync struct {- stagePath string- // stageLogical is stagePath expressed relative to the projects directory,- // which is the confinement root every staging mutation runs against.- stageLogical string- projectsDir string- composeFileName string- project *projectpkg.Project- // syncFiles are the repo files written raw; syncedFiles are their paths.- syncFiles []projects.SyncFile- syncedFiles []string- // oldSyncedFiles are the paths the previous sync tracked.- oldSyncedFiles []string- // gitEnvContent is the repo's project-root .env, routed through the env merge.- gitEnvContent *string- serviceCount int- contentsChanged bool- // backupScope names every live path promotion may create, rewrite or delete.- backupScope projects.ProjectUpdateBackupScope+// deployOnce redeploys the synced project, recording the deploy on the run. The deploy runs lifecycle hooks, so a+// replay skips a deploy the run confirmed and refuses one that started without a recorded result.+func (s *Service) deployOnce(ctx context.Context, sync *projectpkg.GitOpsSync, project *projectpkg.Project, actor user.Actor, replay bool) error {+ target := scheduler.TargetOutcome{ResourceType: "gitops_deploy", ID: sync.ID + ":deploy", Status: scheduler.Running}+ previous, _ := jobcontext.Run(ctx)+ if index := slices.IndexFunc(previous.Outcome.Targets, func(recorded scheduler.TargetOutcome) bool { return recorded.ID == target.ID }); replay && index >= 0 {+ switch previous.Outcome.Targets[index].Status {+ case scheduler.Succeeded:+ return nil+ case scheduler.Running:+ return errors.New("the interrupted deploy may already have run its lifecycle hooks; redeploy the project to finish it")+ default:+ }+ }+ if err := jobcontext.Progress(ctx, target); err != nil {+ return err+ }+ deployErr := s.redeployIfRunningAfterSync(ctx, sync, project, actor)+ // Interrupted mid-deploy, its effects are unknown, so the deploy stays running for the replay to refuse.+ if ctx.Err() != nil {+ return cmp.Or(deployErr, ctx.Err())+ }+ target.Status = kit.Ternary(deployErr == nil, scheduler.Succeeded, scheduler.Failed)+ return errors.Join(deployErr, jobcontext.Progress(ctx, target))}-// prepareSyncSource clones the source repository, validates that the configured-// compose file exists, and reads the compose/env inputs for the sync flow.-func (s *Service) prepareSyncSource(ctx context.Context, sync *projectpkg.GitOpsSync, result *gitops.SyncResult, actor user.Actor) (*preparedSyncSource, error) {+// prepareSyncSource clones the source at the pinned revision, or the branch head, records that revision on+// the run, and reads the compose, env and override inputs.+func (s *Service) prepareSyncSource(ctx context.Context, sync *projectpkg.GitOpsSync, result *gitops.SyncResult, actor user.Actor, pinned string) (*preparedSyncSource, error) {repository := sync.Repositoryif repository == nil {- return nil, s.failSync(ctx, sync.ID, result, sync, actor, "Repository not found", "repository not found")+ return nil, s.failSync(ctx, result, sync, actor, "Repository not found", errors.New("repository not found"))}-authConfig, err := s.repoService.GetAuthConfig(ctx, repository)if err != nil {- return nil, s.failSync(ctx, sync.ID, result, sync, actor, "Failed to get authentication config", err.Error())- }-- // A retry replays the revision its interrupted predecessor recorded, which needs full history.- pinnedCommit := ""- if previous, ok := jobcontext.Run(ctx); ok {- if _, pinnedCommit, err = recordedRevision(previous, sync.ID); err != nil {- return nil, err- }+ return nil, s.failSync(ctx, result, sync, actor, "Failed to get authentication config", err)}- repoPath, err := s.repoService.Clone(ctx, repository.URL, sync.Branch, authConfig, kit.Ternary(pinnedCommit == "", 1, 0))+ // Replaying a pinned revision needs full history.+ repoPath, err := s.repoService.Clone(ctx, repository.URL, sync.Branch, authConfig, kit.Ternary(pinned == "", 1, 0))if err != nil {- return nil, s.failSync(ctx, sync.ID, result, sync, actor, "Failed to clone repository", err.Error())+ // Anything but a rejected or missing repository is worth another attempt.+ if !errors.Is(err, transport.ErrAuthenticationRequired) && !errors.Is(err, transport.ErrAuthorizationFailed) && !errors.Is(err, transport.ErrRepositoryNotFound) {+ err = common.Classify(common.ErrUnavailable, err)+ }+ return nil, s.failSync(ctx, result, sync, actor, "Failed to clone repository", err)}- source := &preparedSyncSource{repoPath: repoPath}+ source := &preparedSyncSource{repoPath: repoPath, replay: pinned != ""}source.commitHash, err = s.repoService.GetCurrentCommit(ctx, repoPath)if err != nil {slog.WarnContext(ctx, "Failed to get commit hash", "error", err)}- if pinnedCommit != "" && pinnedCommit != source.commitHash {+ if pinned != "" && pinned != source.commitHash {repo, openErr := git.PlainOpen(repoPath)if openErr != nil {return source, openErr@@ -214,10 +298,10 @@ func (s *Service) prepareSyncSource(ctx context.Context, sync *projectpkg.GitOpsif treeErr != nil {return source, treeErr}- if checkoutErr := tree.Checkout(&git.CheckoutOptions{Hash: plumbing.NewHash(pinnedCommit)}); checkoutErr != nil {+ if checkoutErr := tree.Checkout(&git.CheckoutOptions{Hash: plumbing.NewHash(pinned)}); checkoutErr != nil {return source, checkoutErr}- source.commitHash = pinnedCommit+ source.commitHash = pinned}if source.commitHash == "" {return source, errors.New("GitOps source revision is unavailable")@@ -232,31 +316,25 @@ func (s *Service) prepareSyncSource(ctx context.Context, sync *projectpkg.GitOps}if !s.repoService.FileExists(ctx, repoPath, sync.ComposePath) {- errMsg := "compose file not found: " + sync.ComposePath- return source, s.failSync(ctx, sync.ID, result, sync, actor, "Compose file not found at "+sync.ComposePath, errMsg)+ return source, s.failSync(ctx, result, sync, actor, "Compose file not found at "+sync.ComposePath, errors.New("compose file not found: "+sync.ComposePath))}source.composeContent, err = s.repoService.ReadFile(ctx, repoPath, sync.ComposePath)if err != nil {- return source, s.failSync(ctx, sync.ID, result, sync, actor, "Failed to read compose file", err.Error())+ return source, s.failSync(ctx, result, sync, actor, "Failed to read compose file", err)}- envPath := filepath.Join(filepath.Dir(sync.ComposePath), ".env")+ composeDir := filepath.Dir(sync.ComposePath)+ envPath := filepath.Join(composeDir, ".env")if s.repoService.FileExists(ctx, repoPath, envPath) {- content, readFileErr := s.repoService.ReadFile(ctx, repoPath, envPath)- if readFileErr != nil {- slog.WarnContext(ctx, "Failed to read .env file", "path", envPath, "error", readFileErr)+ if content, readErr := s.repoService.ReadFile(ctx, repoPath, envPath); readErr != nil {+ slog.WarnContext(ctx, "Failed to read .env file", "path", envPath, "error", readErr)} else {source.envContent = &content}}- // Detect a Docker Compose override file (compose.override.yaml, etc.) sitting- // beside the compose file in the repo, mirroring `docker compose` behavior.- // Only auto-load an override when the compose file was referenced by a- // standard filename; an explicit custom path is the `-f` case, where docker- // compose does not auto-load overrides.+ // Like docker compose, only a standard compose filename auto-loads a sibling override; a custom path is the -f case.if slices.Contains(projects.ComposeFileCandidates(), filepath.Base(sync.ComposePath)) {- composeDir := filepath.Dir(sync.ComposePath)overrideName, overrideContent, overrideFound, overrideErr := projects.ResolveComposeOverride(func(name string) bool {return s.repoService.FileExists(ctx, repoPath, filepath.Join(composeDir, name))@@ -265,1022 +343,447 @@ func (s *Service) prepareSyncSource(ctx context.Context, sync *projectpkg.GitOpsreturn s.repoService.ReadFile(ctx, repoPath, filepath.Join(composeDir, name))},)- // Fail the sync on a read error instead of degrading: WriteComposeOverrideFile- // with nil content removes the project's existing override, so a transient- // read failure here would delete the override and redeploy without it.+ // Degrading would delete the project's existing override on a transient read failure.if overrideErr != nil {- return source, s.failSync(ctx, sync.ID, result, sync, actor, "Failed to read compose override file", overrideErr.Error())+ return source, s.failSync(ctx, result, sync, actor, "Failed to read compose override file", overrideErr)}if overrideFound {source.overrideContent = &overrideContentsource.overrideFileName = overrideName}}-return source, nil}-// performDirectorySync runs the directory-sync path and only triggers a-// redeploy when an already running project's synced contents changed.-func (- s *Service,-) performDirectorySync(- ctx context.Context,- sync *projectpkg.GitOpsSync,- id string,- actor user.Actor,- result *gitops.SyncResult,- source *preparedSyncSource,-) (- *gitops.SyncResult,- error,-) {- slog.InfoContext(ctx, "Using directory sync mode", "syncId", id, "composePath", sync.ComposePath)-- syncFiles, err := s.walkAndParseSyncDirectory(ctx, sync, source.repoPath)- if err != nil {- return result, s.failSync(ctx, id, result, sync, actor, "Failed to walk directory", err.Error())+// syncComposeFile applies a single-file sync, creating the project on the first sync. It reports whether the+// project files changed.+func (s *Service) syncComposeFile(ctx context.Context, sync *projectpkg.GitOpsSync, actor user.Actor, result *gitops.SyncResult, source *preparedSyncSource) (*projectpkg.Project, bool, error) {+ slog.InfoContext(ctx, "Using single file sync mode", "syncId", sync.ID, "composePath", sync.ComposePath)+ // Files referenced through COMPOSE_FILE or COMPOSE_ENV_FILES never reach a single-file project.+ if projects.ComposeEnvRequiresDirectorySync(sync.ComposePath, source.overrideFileName, source.envContent) {+ return nil, false, s.failSync(ctx, result, sync, actor, "COMPOSE_FILE or COMPOSE_ENV_FILES references additional files",+ errors.New("the synced .env references additional files via COMPOSE_FILE or COMPOSE_ENV_FILES; enable \"Sync entire directory\" for this sync"))}- project, syncedFiles, _, contentsChanged, err := s.syncProjectDirectoryInternal(ctx, sync, syncFiles, actor)- if err != nil {- if errors.Is(err, common.ErrGitOpsSyncProjectBindingBroken) {- errMsg := err.Error()- result.Message = "GitOps project binding broken"- result.Error = new(errMsg)- return result, err+ if projectID := kit.FromPtr(sync.ProjectID); projectID != "" {+ project, found, err := s.projectService.FindProjectByID(ctx, projectID)+ if err != nil {+ return nil, false, s.failSync(ctx, result, sync, actor, "Failed to load existing project", fmt.Errorf("failed to get project %s: %w", projectID, err))}- return result, s.failSync(ctx, id, result, sync, actor, "Failed to sync project directory", err.Error())- }-- if contentsChanged {- if redeployErr := s.redeployIfRunningAfterSync(ctx, sync, project, actor, "directory"); redeployErr != nil {- if errors.Is(redeployErr, common.ErrRedeployAfterSyncFailed) {- s.markSyncRedeployFailedInternal(ctx, sync, id, source.commitHash, syncedFiles, redeployErr, actor, result)- }- return result, redeployErr+ if !found {+ slog.WarnContext(ctx, "Existing project not found; GitOps project binding is broken", "projectId", projectID, "syncId", sync.ID)+ bindingErr := common.Classify(common.ErrGitOpsSyncProjectBindingBroken, fmt.Errorf("GitOps sync project binding broken: sync %s references missing project %s", sync.ID, projectID))+ return nil, false, s.failSync(ctx, result, sync, actor, "GitOps project binding broken", bindingErr)}- }-- s.updateSyncStatusWithFiles(ctx, id, "success", "", source.commitHash, syncedFiles)- result.Success = true- result.Message = fmt.Sprintf("Successfully synced directory with %d files to project %s", len(syncedFiles), project.Name)- s.logSyncSuccess(ctx, sync, project, actor)- slog.InfoContext(ctx, "GitOps sync completed", "syncId", id, "project", project.Name)-- return result, nil-}--// singleFileSyncedFilesInternal builds the tracked-file list for a single-file-// sync: the compose file's base name plus the override file name when one was-// resolved. Shared by the compose-only and swarm-stack single-file paths so the-// two never disagree on what a single-file sync tracked.-func singleFileSyncedFilesInternal(sync *projectpkg.GitOpsSync, source *preparedSyncSource) []string {- syncedFiles := []string{filepath.Base(sync.ComposePath)}- if source.overrideFileName != "" {- syncedFiles = append(syncedFiles, source.overrideFileName)- }- return syncedFiles-}--// performSingleFileSyncInternal preserves the legacy compose-only Git sync behavior.-func (- s *Service,-) performSingleFileSyncInternal(- ctx context.Context,- sync *projectpkg.GitOpsSync,- id string,- actor user.Actor,- result *gitops.SyncResult,- source *preparedSyncSource,-) (- *gitops.SyncResult,- error,-) {- slog.InfoContext(ctx, "Using single file sync mode", "syncId", id, "composePath", sync.ComposePath)-- // Single-file sync copies only the one compose file (plus .env and a sibling- // override). If the synced .env references additional files via COMPOSE_FILE- // or COMPOSE_ENV_FILES, those files never reach the project, so direct the- // user to directory sync instead of silently deploying an incomplete- // selection.- if projects.ComposeEnvRequiresDirectorySync(sync.ComposePath, source.overrideFileName, source.envContent) {- return result, s.failSync(ctx, id, result, sync, actor,- "COMPOSE_FILE or COMPOSE_ENV_FILES references additional files",- "the synced .env references additional files via COMPOSE_FILE or COMPOSE_ENV_FILES; enable \"Sync entire directory\" for this sync")- }-- syncedFiles := singleFileSyncedFilesInternal(sync, source)-- project, err := s.getOrCreateProjectInternal(ctx, sync, id, source.composeContent, source.envContent, source.overrideContent, source.overrideFileName, result, actor)- if err != nil {- if errors.Is(err, common.ErrRedeployAfterSyncFailed) {- s.markSyncRedeployFailedInternal(ctx, sync, id, source.commitHash, syncedFiles, err, actor, result)+ _, changed, err := s.projectService.ApplyGitSyncProjectFiles(ctx, project.ID, source.composeContent, source.envContent, source.overrideContent, source.overrideFileName, actor)+ if err != nil {+ return nil, false, s.failSync(ctx, result, sync, actor, "Failed to update project files", err)}- return result, err+ slog.InfoContext(ctx, "Updated project files", "projectName", project.Name, "projectId", project.ID, "changed", changed)+ return project, changed, nil}- s.updateSyncStatusWithFiles(ctx, id, "success", "", source.commitHash, syncedFiles)- result.Success = true- result.Message = fmt.Sprintf("Successfully synced compose file from %s to project %s", sync.ComposePath, project.Name)- s.logSyncSuccess(ctx, sync, project, actor)- slog.InfoContext(ctx, "GitOps sync completed", "syncId", id, "project", project.Name)-- return result, nil-}--// buildSwarmStackDeployRequestInternal assembles the deploy request for a Git Sync that targets-// a Swarm stack. WithRegistryAuth is always set so the Git Sync path resolves stored registry-// credentials the same way a direct stack deploy does. Resolution happens per image at deploy-// time and yields nothing unless a configured container registry matches that image's host, so-// stacks built only from public images are unaffected.-func buildSwarmStackDeployRequestInternal(sync *projectpkg.GitOpsSync, source *preparedSyncSource, overrideContent, envContent string, swarmFiles []swarmtypes.SyncFile) swarmtypes.StackDeployRequest {- return swarmtypes.StackDeployRequest{- Name: sync.ProjectName,- ComposeContent: source.composeContent,- OverrideContent: overrideContent,- EnvContent: envContent,- Files: swarmFiles,- Prune: true,- WithRegistryAuth: true,- WorkingDir: filepath.Dir(filepath.Join(source.repoPath, sync.ComposePath)),+ // The non-suffixing create: a name collision means the binding is broken, never a "-N" duplicate.+ project, err := s.projectService.CreateProject(ctx, sync.ProjectName, source.composeContent, source.envContent, projecttypes.CreateProjectWorkspaceManifest{}, nil, nil, nil, actor, false)+ if errors.Is(err, projects.ErrProjectDirExists) {+ err = common.Classify(common.ErrGitOpsSyncProjectBindingBroken, fmt.Errorf("GitOps sync project binding broken: sync %s cannot create project %q: a directory with that name "++ "already exists; refusing to create a duplicate", sync.ID, projects.SanitizeProjectName(sync.ProjectName)))+ return nil, false, s.failSync(ctx, result, sync, actor, "GitOps project binding broken", err)}-}--// performSwarmStackSyncInternal executes a single file sync targeted at a Swarm Stack-func (- s *Service,-) performSwarmStackSyncInternal(- ctx context.Context,- sync *projectpkg.GitOpsSync,- id string,- actor user.Actor,- result *gitops.SyncResult,- source *preparedSyncSource,-) (- *gitops.SyncResult,- error,-) {- slog.InfoContext(ctx, "Deploying Swarm Stack from GitOps sync", "syncId", id, "stackName", sync.ProjectName)-- if s.swarmService == nil {- return result, s.failSync(ctx, id, result, sync, actor, "Swarm service is unavailable", "swarm service is unavailable")- }-- envContent := ""- if source.envContent != nil {- envContent = *source.envContent+ if err != nil {+ return nil, false, s.failSync(ctx, result, sync, actor, "Failed to create project", err)}-- overrideContent := ""- if source.overrideContent != nil {- overrideContent = *source.overrideContent+ if err = s.db.WithContext(ctx).Model(&projectpkg.GitOpsSync{}).Where("id = ?", sync.ID).Update("project_id", project.ID).Error; err != nil {+ return nil, false, s.failSync(ctx, result, sync, actor, "Failed to update sync with project ID", err)}-- var syncFiles []projects.SyncFile- if sync.SyncDirectory {- files, err := s.walkAndParseSyncDirectory(ctx, sync, source.repoPath)- if err != nil {- return result, s.failSync(ctx, id, result, sync, actor, "Failed to walk directory", err.Error())- }- syncFiles = files+ if err = s.db.WithContext(ctx).Model(&projectpkg.Project{}).Where("id = ?", project.ID).Update("gitops_managed_by", sync.ID).Error; err != nil {+ return nil, false, s.failSync(ctx, result, sync, actor, "Failed to mark project as GitOps-managed", err)}-- swarmFiles := make([]swarmtypes.SyncFile, len(syncFiles))- syncedFiles := make([]string, 0, len(syncFiles))- for i, f := range syncFiles {- swarmFiles[i] = swarmtypes.SyncFile{- RelativePath: f.RelativePath,- Content: f.Content,- }- syncedFiles = append(syncedFiles, f.RelativePath)+ if _, _, err = s.projectService.ApplyGitSyncProjectFiles(ctx, project.ID, source.composeContent, source.envContent, source.overrideContent, source.overrideFileName, actor); err != nil {+ return nil, false, s.failSync(ctx, result, sync, actor, "Failed to sync project env files", err)}-- req := buildSwarmStackDeployRequestInternal(sync, source, overrideContent, envContent, swarmFiles)-- if _, err := s.swarmService.DeployStack(ctx, sync.EnvironmentID, req); err != nil {- return result, s.failSync(ctx, id, result, sync, actor, "Failed to deploy swarm stack", err.Error())- }-- if len(syncedFiles) == 0 {- syncedFiles = singleFileSyncedFilesInternal(sync, source)- }- s.updateSyncStatusWithFiles(ctx, id, "success", "", source.commitHash, syncedFiles)- result.Success = true- result.Message = fmt.Sprintf("Successfully deployed swarm stack %s from %s", sync.ProjectName, sync.ComposePath)-- // Log event- _, _ = s.eventService.CreateEvent(ctx, event.CreateEventRequest{- Type: event.EventTypeGitSyncRun,- Severity: event.EventSeveritySuccess,- Title: "Git sync completed for stack",- Description: fmt.Sprintf("Successfully synced '%s' to swarm stack '%s'", sync.Name, sync.ProjectName),- ResourceType: new("git_sync"),- ResourceID: new(sync.ID),- ResourceName: new(sync.Name),- UserID: new(actor.ID),- Username: new(actor.Username),- EnvironmentID: new(sync.EnvironmentID),- })-- slog.InfoContext(ctx, "GitOps swarm stack sync completed", "syncId", id, "stack", sync.ProjectName)- return result, nil+ slog.InfoContext(ctx, "Created project for GitOps sync", "projectName", sync.ProjectName, "projectId", project.ID)+ return project, true, nil}-// redeployIfRunningAfterSync redeploys a project when it is already running,-// or when the sync has RedeployAfterSync enabled (in which case a stopped-// project is redeployed too, mirroring Portainer's "Re-pull image" stack-// option). Returns a common.ErrRedeployAfterSyncFailed when the redeploy-// itself fails; callers surface that on the sync row's LastSyncError.-//-// When neither condition holds — project stopped and RedeployAfterSync off —-// RedeployProject is never called (Arcane does not start a stopped project on-// its behalf by default), which also means the image pull that RedeployProject-// performs as a side effect never happens. Left alone, that can leave a-// stopped container referencing an image tag that was since re-pointed or-// pruned upstream, with no local copy to fall back on at next start. If the-// sync opted into PullImageAfterSync, pull the image here instead so it stays-// available even while the project is stopped.-func (s *Service) redeployIfRunningAfterSync(ctx context.Context, sync *projectpkg.GitOpsSync, project *projectpkg.Project, actor user.Actor, syncMode string) error {+// redeployIfRunningAfterSync redeploys a running project, or any project when RedeployAfterSync is on. A project+// left stopped still gets its images pulled when PullImageAfterSync is on, so a re-pointed tag stays available.+func (s *Service) redeployIfRunningAfterSync(ctx context.Context, sync *projectpkg.GitOpsSync, project *projectpkg.Project, actor user.Actor) error {details, err := s.projectService.GetProjectDetails(ctx, project.ID, projecttypes.DetailsOptions{})running := err == nil && (details.Status == string(projectpkg.ProjectStatusRunning) || details.Status == string(projectpkg.ProjectStatusPartiallyRunning))-- if !running && !sync.RedeployAfterSync {- return s.pullImageAfterSyncIfConfiguredInternal(ctx, sync, project, actor)- }-- slog.InfoContext(ctx, "Redeploying project due to content change from Git sync", "syncMode", syncMode, "projectName", project.Name, "projectId", project.ID, "wasRunning", running)- if redeployProjectErr := s.projectService.RedeployProject(ctx, project.ID, actor, nil); redeployProjectErr != nil {- slog.ErrorContext(ctx, "Failed to redeploy project after Git sync", "syncMode", syncMode, "error", redeployProjectErr, "projectId", project.ID)- return common.Classify(common.ErrRedeployAfterSyncFailed, fmt.Errorf("redeploy failed: %w", redeployProjectErr))- }- return nil-}--// pullImageAfterSyncIfConfiguredInternal pulls each service's image for a-// project that Git sync changed but did not redeploy (because it wasn't-// running), when the sync has PullImageAfterSync enabled. Best-effort: a pull-// failure is logged but never fails the sync itself, mirroring how-// RedeployProject treats its own pre-deploy pull.-func (s *Service) pullImageAfterSyncIfConfiguredInternal(ctx context.Context, sync *projectpkg.GitOpsSync, project *projectpkg.Project, actor user.Actor) error {- if !sync.PullImageAfterSync {- return nil- }-- credentials, cerr := s.projectService.ResolveRegistryCredentials(ctx)- if cerr != nil {- slog.WarnContext(ctx, "failed to resolve registry credentials for post-sync pull", "error", cerr, "projectId", project.ID)- }- slog.InfoContext(ctx, "Pulling project images after Git sync (project not running)", "projectName", project.Name, "projectId", project.ID)- if err := s.projectService.PullProjectImages(ctx, project.ID, io.Discard, actor, credentials); err != nil {- slog.ErrorContext(ctx, "failed to pull project images after Git sync", "error", err, "projectId", project.ID)- return common.Classify(common.ErrRedeployAfterSyncFailed, fmt.Errorf("post-sync image pull failed: %w", err))+ switch {+ case running || sync.RedeployAfterSync:+ slog.InfoContext(ctx, "Redeploying project after Git sync", "projectName", project.Name, "projectId", project.ID, "wasRunning", running)+ if redeployErr := s.projectService.RedeployProject(ctx, project.ID, actor, nil); redeployErr != nil {+ slog.ErrorContext(ctx, "Failed to redeploy project after Git sync", "error", redeployErr, "projectId", project.ID)+ return common.Classify(common.ErrRedeployAfterSyncFailed, fmt.Errorf("redeploy failed: %w", redeployErr))+ }+ case sync.PullImageAfterSync:+ credentials, credentialsErr := s.projectService.ResolveRegistryCredentials(ctx)+ if credentialsErr != nil {+ slog.WarnContext(ctx, "failed to resolve registry credentials for post-sync pull", "error", credentialsErr, "projectId", project.ID)+ }+ slog.InfoContext(ctx, "Pulling project images after Git sync (project not running)", "projectName", project.Name, "projectId", project.ID)+ if pullErr := s.projectService.PullProjectImages(ctx, project.ID, io.Discard, actor, credentials); pullErr != nil {+ slog.ErrorContext(ctx, "failed to pull project images after Git sync", "error", pullErr, "projectId", project.ID)+ return common.Classify(common.ErrRedeployAfterSyncFailed, fmt.Errorf("post-sync image pull failed: %w", pullErr))+ }}return nil}-// logSyncSuccess records the Git sync completion event once the filesystem and-// sync-status updates have already succeeded.-func (s *Service) logSyncSuccess(ctx context.Context, sync *projectpkg.GitOpsSync, project *projectpkg.Project, actor user.Actor) {- _, _ = s.eventService.CreateEvent(ctx, event.CreateEventRequest{- Type: event.EventTypeGitSyncRun,- Severity: event.EventSeveritySuccess,- Title: "Git sync completed",- Description: fmt.Sprintf("Successfully synced '%s' to project '%s'", sync.Name, project.Name),- ResourceType: new("git_sync"),- ResourceID: new(sync.ID),- ResourceName: new(sync.Name),- UserID: new(actor.ID),- Username: new(actor.Username),- EnvironmentID: new(sync.EnvironmentID),- })-}--func (s *Service) updateSyncStatus(ctx context.Context, id, status, errorMsg, commitHash string) {- now := time.Now()+// updateSyncStatus stores a run's result on the row. An empty commit keeps the recorded one, and nil+// syncedFiles keeps the tracked list.+func (s *Service) updateSyncStatus(ctx context.Context, id, status, errorMsg, commitHash string, syncedFiles []string) {updates := map[string]any{- "last_sync_at": now,+ "last_sync_at": time.Now(),"last_sync_status": status,+ "last_sync_error": kit.Ternary[any](errorMsg != "", errorMsg, nil),}-- updates["last_sync_error"] = kit.Ternary[any](errorMsg != "", errorMsg, nil)-if commitHash != "" {updates["last_sync_commit"] = commitHash}-+ if syncedFiles != nil {+ updates["synced_files"] = projectpkg.EncodeSyncedFiles(syncedFiles)+ }if err := s.db.WithContext(ctx).Model(&projectpkg.GitOpsSync{}).Where("id = ?", id).Updates(updates).Error; err != nil {slog.ErrorContext(ctx, "Failed to update sync status", "error", err, "syncId", id)}}-func (s *Service) failSync(ctx context.Context, id string, result *gitops.SyncResult, sync *projectpkg.GitOpsSync, actor user.Actor, message, errMsg string) error {- result.Message = message- result.Error = new(errMsg)- s.updateSyncStatus(ctx, id, "failed", errMsg, "")- s.logError(ctx, sync, actor, errMsg)- return fmt.Errorf("%s", errMsg)-}--// markSyncRedeployFailedInternal records a sync where the file sync wrote-// cleanly but the auto-redeploy that follows failed (typically a pre-deploy-// lifecycle hook returning non-zero). The synced-files list and commit hash-// are preserved on the row so operators can see what reached disk; the-// error message surfaces the redeploy failure on LastSyncError.-func (- s *Service,-) markSyncRedeployFailedInternal(- ctx context.Context,- sync *projectpkg.GitOpsSync,- id, commitHash string,- syncedFiles []string,- redeployErr error,- actor user.Actor,- result *gitops.SyncResult,-) {- errMsg := redeployErr.Error()- result.Success = false- result.Message = "Sync wrote files but redeploy failed"- result.Error = new(errMsg)- s.updateSyncStatusWithFiles(ctx, id, "failed", errMsg, commitHash, syncedFiles)- s.logError(ctx, sync, actor, errMsg)-}--func (s *Service) disableAutoSyncForBrokenBindingInternal(ctx context.Context, sync *projectpkg.GitOpsSync) {- if sync == nil || sync.ID == "" {- return- }- if err := s.db.WithContext(ctx).- Model(&projectpkg.GitOpsSync{}).- Where("id = ?", sync.ID).- Update("auto_sync", false).Error; err != nil {- slog.ErrorContext(ctx, "Failed to disable GitOps auto-sync after broken project binding", "syncId", sync.ID, "error", err)- }- sync.AutoSync = false- s.unregisterJob(ctx, sync.ID)-}--func (- s *Service,-) failSyncAndDisableAutoSyncInternal(- ctx context.Context,- id string,- result *gitops.SyncResult,- sync *projectpkg.GitOpsSync,- actor user.Actor,- message string,- failure error,-) error {- errMsg := failure.Error()- _ = s.failSync(ctx, id, result, sync, actor, message, errMsg)- s.disableAutoSyncForBrokenBindingInternal(ctx, sync)- return failure-}--func (s *Service) recordBrokenProjectBindingInternal(ctx context.Context, sync *projectpkg.GitOpsSync, actor user.Actor, err error) {- if !errors.Is(err, common.ErrGitOpsSyncProjectBindingBroken) || sync == nil {- return- }+// failSync records a failed sync on the result, the row and an event, and returns err. A broken project+// binding also turns auto-sync off so the sync stops retrying.+func (s *Service) failSync(ctx context.Context, result *gitops.SyncResult, sync *projectpkg.GitOpsSync, actor user.Actor, message string, err error) error {errMsg := err.Error()- s.updateSyncStatus(ctx, sync.ID, "failed", errMsg, "")+ result.Message = message+ result.Error = &errMsg+ s.updateSyncStatus(ctx, sync.ID, "failed", errMsg, "", nil)s.logError(ctx, sync, actor, errMsg)- s.disableAutoSyncForBrokenBindingInternal(ctx, sync)-}--func (- s *Service,-) createProjectForSyncInternal(- ctx context.Context,- sync *projectpkg.GitOpsSync,- id, composeContent string,- envContent, overrideContent *string,- overrideFileName string,- result *gitops.SyncResult,- actor user.Actor,-) (- *projectpkg.Project,- error,-) {- // Use the non-suffixing create: a GitOps sync must never mint a "-N" duplicate.- // A name collision means a project directory already exists for this name, so the- // binding is broken — fail loudly and disable auto-sync instead of duplicating.- project, err := s.projectService.CreateProject(ctx, sync.ProjectName, composeContent, envContent, projecttypes.CreateProjectWorkspaceManifest{}, nil, nil, nil, actor, false)- if err != nil {- if errors.Is(err, projects.ErrProjectDirExists) {- bindingErr := common.Classify(- common.ErrGitOpsSyncProjectBindingBroken,- fmt.Errorf(- "GitOps sync project binding broken: sync %s cannot create project %q: a directory with that name "+- "already exists; refusing to create a duplicate",- sync.ID,- projects.SanitizeProjectName(- sync.ProjectName,- ),- ),- )-- return nil, s.failSyncAndDisableAutoSyncInternal(ctx, id, result, sync, actor, "GitOps project binding broken", bindingErr)- }- return nil, s.failSync(ctx, id, result, sync, actor, "Failed to create project", err.Error())- }-- // Update sync with project ID- if linkSyncProjectErr := s.db.WithContext(ctx).Model(&projectpkg.GitOpsSync{}).Where("id = ?", id).Updates(map[string]any{- "project_id": project.ID,- }).Error; linkSyncProjectErr != nil {- return nil, s.failSync(ctx, id, result, sync, actor, "Failed to update sync with project ID", linkSyncProjectErr.Error())- }-- // Mark project as GitOps-managed- if markProjectManagedErr := s.db.WithContext(ctx).Model(&projectpkg.Project{}).Where("id = ?", project.ID).Update("gitops_managed_by", id).Error; markProjectManagedErr != nil {- return nil, s.failSync(ctx, id, result, sync, actor, "Failed to mark project as GitOps-managed", markProjectManagedErr.Error())- }-- if _, _, applyGitSyncProjectFilesErr := s.projectService.ApplyGitSyncProjectFiles(- ctx, project.ID, composeContent, envContent, overrideContent, overrideFileName, actor,- ); applyGitSyncProjectFilesErr != nil {- return nil, s.failSync(ctx, id, result, sync, actor, "Failed to sync project env files", applyGitSyncProjectFilesErr.Error())- }-- slog.InfoContext(ctx, "Created project for GitOps sync", "projectName", sync.ProjectName, "projectId", project.ID)-- // A freshly created project has no containers yet, so it's always- // "stopped" from RedeployProject's point of view — mirror- // updateProjectForSyncInternal's post-sync behavior here instead of- // silently leaving the new project undeployed with no image pulled.- if sync.RedeployAfterSync {- slog.InfoContext(ctx, "Redeploying newly created project from Git sync", "projectName", project.Name, "projectId", project.ID)- if redeployProjectErr := s.projectService.RedeployProject(ctx, project.ID, actor, nil); redeployProjectErr != nil {- slog.ErrorContext(ctx, "Failed to redeploy newly created project after Git sync", "error", redeployProjectErr, "projectId", project.ID)- return nil, common.Classify(common.ErrRedeployAfterSyncFailed, fmt.Errorf("redeploy failed: %w", redeployProjectErr))- }- } else if pullImageAfterSyncIfConfiguredErr := s.pullImageAfterSyncIfConfiguredInternal(ctx, sync, project, actor); pullImageAfterSyncIfConfiguredErr != nil {- return nil, pullImageAfterSyncIfConfiguredErr- }-- return project, nil-}--func (- s *Service,-) getOrCreateProjectInternal(- ctx context.Context,- sync *projectpkg.GitOpsSync,- id, composeContent string,- envContent, overrideContent *string,- overrideFileName string,- result *gitops.SyncResult,- actor user.Actor,-) (- *projectpkg.Project,- error,-) {- var project *projectpkg.Project-- if sync.ProjectID != nil && *sync.ProjectID != "" {- var found bool- var lookupErr error- project, found, lookupErr = s.projectService.FindProjectByID(ctx, *sync.ProjectID)- if lookupErr != nil {- lookupErr = fmt.Errorf("failed to get project %s: %w", *sync.ProjectID, lookupErr)- return nil, s.failSync(ctx, id, result, sync, actor, "Failed to load existing project", lookupErr.Error())- }- if !found {- err := common.Classify(common.ErrGitOpsSyncProjectBindingBroken, fmt.Errorf("GitOps sync project binding broken: sync %s references missing project %s", sync.ID, *sync.ProjectID))-- slog.WarnContext(ctx, "Existing project not found; GitOps project binding is broken", "projectId", *sync.ProjectID, "syncId", sync.ID)- return nil, s.failSyncAndDisableAutoSyncInternal(ctx, id, result, sync, actor, "GitOps project binding broken", err)- }- }-- if project == nil {- return s.createProjectForSyncInternal(ctx, sync, id, composeContent, envContent, overrideContent, overrideFileName, result, actor)- }-- if err := s.updateProjectForSyncInternal(ctx, sync, id, project, composeContent, envContent, overrideContent, overrideFileName, result, actor); err != nil {- return nil, err- }- return project, nil-}--func (- s *Service,-) updateProjectForSyncInternal(- ctx context.Context,- sync *projectpkg.GitOpsSync,- id string,- project *projectpkg.Project,- composeContent string,- envContent, overrideContent *string,- overrideFileName string,- result *gitops.SyncResult,- actor user.Actor,-) error {- _, changed, err := s.projectService.ApplyGitSyncProjectFiles(- ctx, project.ID, composeContent, envContent, overrideContent, overrideFileName, actor,- )- if err != nil {- return s.failSync(ctx, id, result, sync, actor, "Failed to update project files", err.Error())- }- slog.InfoContext(ctx, "Updated project files", "projectName", project.Name, "projectId", project.ID, "changed", changed)- if !changed {- return nil- }- return s.redeployIfRunningAfterSync(ctx, sync, project, actor, "single-file")-}--// walkAndParseSyncDirectory walks the repository directory and returns all files with their contents.-// Returns the list of SyncFile entries and an error if any; it fails if the compose file is missing.-func (s *Service) walkAndParseSyncDirectory(ctx context.Context, sync *projectpkg.GitOpsSync, repoPath string) ([]projects.SyncFile, error) {- slog.InfoContext(ctx, "Starting directory walk", "syncId", sync.ID, "composePath", sync.ComposePath)-- // Walk the directory to get all files- maxFiles, maxTotalSize, maxBinarySize := s.limits(ctx, sync)-- walkResult, err := s.repoService.WalkDirectory(ctx, repoPath, sync.ComposePath, maxFiles, maxTotalSize, maxBinarySize)- if err != nil {- return nil, fmt.Errorf("failed to walk directory: %w", err)- }-- slog.InfoContext(ctx, "Directory walk complete",- "syncId", sync.ID,- "totalFiles", walkResult.TotalFiles,- "totalSize", walkResult.TotalSize,- "skippedBinaries", walkResult.SkippedBinaries)-- // WalkDirectory roots the walk at filepath.Dir(sync.ComposePath), so the- // compose file is always emitted at the top level as filepath.Base(sync.ComposePath).- composeFileName := filepath.Base(sync.ComposePath)- composeFound := false-- // Convert walked files to SyncFile format- syncFiles := make([]projects.SyncFile, len(walkResult.Files))- for i, f := range walkResult.Files {- syncFiles[i] = projects.SyncFile{- RelativePath: f.RelativePath,- Content: f.Content,- Executable: f.Executable,- }- if f.RelativePath == composeFileName {- composeFound = true- }- }-- if !composeFound {- return nil, fmt.Errorf("compose file %s not found in walked directory", composeFileName)- }-- return syncFiles, nil-}--// syncProjectDirectoryInternal runs the new directory-sync path end to end:-// stage files, validate the staged tree, then create or update the project.-func (- s *Service,-) syncProjectDirectoryInternal(- ctx context.Context,- sync *projectpkg.GitOpsSync,- syncFiles []projects.SyncFile,- actor user.Actor,-) (- *projectpkg.Project,- []string,- bool,- bool,- error,-) {- stage, err := s.stageDirectorySyncInternal(ctx, sync, syncFiles)- if err != nil {- s.recordBrokenProjectBindingInternal(ctx, sync, actor, err)- return nil, nil, false, false, err- }- // The stage must go even after ctx was cancelled: acfs refuses work on a- // cancelled context.- defer func() {- if stage.stagePath != "" {- _ = acfs.RemoveAll(context.WithoutCancel(ctx), stage.projectsDir, stage.stageLogical)- }- }()-- if stage.project == nil {- project, createDirectorySyncProjectErr := s.createDirectorySyncProjectInternal(ctx, sync, stage, actor)- if createDirectorySyncProjectErr != nil {- // A name-collision on create surfaces as a broken-binding error; disable- // auto-sync so it cannot keep retrying (no-op for other error kinds).- s.recordBrokenProjectBindingInternal(ctx, sync, actor, createDirectorySyncProjectErr)- return nil, nil, false, false, createDirectorySyncProjectErr+ if errors.Is(err, common.ErrGitOpsSyncProjectBindingBroken) {+ if disableErr := s.db.WithContext(ctx).Model(&projectpkg.GitOpsSync{}).Where("id = ?", sync.ID).Update("auto_sync", false).Error; disableErr != nil {+ slog.ErrorContext(ctx, "Failed to disable GitOps auto-sync after broken project binding", "syncId", sync.ID, "error", disableErr)}- return project, stage.syncedFiles, true, true, nil- }-- project, err := s.updateDirectorySyncProjectInternal(ctx, sync, stage)- if err != nil {- return nil, nil, false, false, err+ sync.AutoSync = false+ s.unregisterJob(ctx, sync.ID)}- return project, stage.syncedFiles, false, stage.contentsChanged, nil+ return err}-// stageDirectorySyncInternal builds a temporary project tree that reflects the exact-// repo layout after sync, including cleanup of files removed from the repo. For an-// existing project the tree is sparse: only paths the sync touches are materialized-// and the rest is linked to the live project, so validation never copies it.-func (s *Service) stageDirectorySyncInternal(ctx context.Context, sync *projectpkg.GitOpsSync, syncFiles []projects.SyncFile) (stage *stagedDirectorySync, err error) {+// syncProjectDirectory stages and validates the synced tree, then creates or updates the project. It reports+// the synced paths, whether it created the project, and whether the contents changed.+func (s *Service) syncProjectDirectory(ctx context.Context, sync *projectpkg.GitOpsSync, syncFiles []projects.SyncFile, actor user.Actor) (*projectpkg.Project, []string, bool, bool, error) {projectsDir, err := s.projectService.GetProjectsDirectory(ctx)if err != nil {- return nil, fmt.Errorf("failed to get projects directory: %w", err)+ return nil, nil, false, false, fmt.Errorf("failed to get projects directory: %w", err)}-- // The project-root env files are reserved for the three-file override- // merge applied below by the Project domain rather than a raw- // overwrite — a raw .env write would silently wipe edits made in Arcane- // on every sync.- filteredSyncFiles, gitEnvContent := partitionReservedRootEnvFiles(ctx, syncFiles)-stageLogical, err := acfs.MkdirTemp(ctx, projectsDir, "/", ".gitops-sync-stage-*")if err != nil {- return nil, fmt.Errorf("failed to create staging directory: %w", err)+ return nil, nil, false, false, fmt.Errorf("failed to create staging directory: %w", err)}- defer func() {- if err != nil {- _ = acfs.RemoveAll(context.WithoutCancel(ctx), projectsDir, stageLogical)- }- }()-- stage = &stagedDirectorySync{+ stage := &stagedDirectorySync{stagePath: filepath.Join(projectsDir, filepath.FromSlash(strings.TrimPrefix(stageLogical, "/"))),stageLogical: stageLogical,projectsDir: projectsDir,composeFileName: filepath.Base(sync.ComposePath),- syncFiles: filteredSyncFiles,- syncedFiles: make([]string, len(filteredSyncFiles)),- gitEnvContent: gitEnvContent,+ // Older syncs may still track .env, which cleanup would delete from the stage before the env merge reads it.+ oldSyncedFiles: slices.DeleteFunc(sync.SyncedFileList(), func(path string) bool { return slices.Contains(reservedRootEnvFiles, path) }),contentsChanged: true,}- for i, file := range filteredSyncFiles {- stage.syncedFiles[i] = file.RelativePath- }-- // Syncs created before this fix may still have .env recorded as a tracked- // file. Drop reserved root env files here too, or CleanupRemovedFiles would- // treat .env as removed-by-git and delete the stage .env before- // ApplyGitSyncEnvToDirectory gets a chance to read it for the merge.- stage.oldSyncedFiles = filterReservedRootEnvFiles(sync.SyncedFileList())+ // The stage must go even after ctx was cancelled: acfs refuses work on a cancelled context.+ defer func() {+ if stage.stagePath != "" {+ _ = acfs.RemoveAll(context.WithoutCancel(ctx), projectsDir, stageLogical)+ }+ }()- stage.project, err = s.getDirectorySyncProjectInternal(ctx, sync)- if err != nil {- return nil, err+ // Reserved root env files are Arcane's bookkeeping and only come from the env merge, which takes a committed .env as its git source.+ for _, file := range syncFiles {+ switch {+ case !slices.Contains(reservedRootEnvFiles, file.RelativePath):+ stage.syncFiles = append(stage.syncFiles, file)+ stage.syncedFiles = append(stage.syncedFiles, file.RelativePath)+ case file.RelativePath == projects.EffectiveEnvFileName:+ stage.gitEnvContent = new(string(file.Content))+ default:+ slog.WarnContext(ctx, "dropping reserved Arcane env file from git payload; will be re-derived from override merge", "path", file.RelativePath)+ }}+ if stage.project, err = s.DirectoryProject(ctx, sync); err != nil {+ return nil, nil, false, false, err+ }if stage.project != nil {staleFiles, staleErr := projects.StaleComposeFiles(ctx, stage.project.Path, stage.composeFileName, stage.syncedFiles)if staleErr != nil {- return nil, fmt.Errorf("failed to detect stale compose files: %w", staleErr)+ return nil, nil, false, false, fmt.Errorf("failed to detect stale compose files: %w", staleErr)}stage.backupScope.Paths = slices.Concat(stage.syncedFiles, stage.oldSyncedFiles, staleFiles,[]string{projects.EffectiveEnvFileName, projects.GitSourceEnvFileName, projects.OverrideEnvFileName})- if linkProjectIntoStageErr := linkProjectIntoStage(stage.project.Path, stage.stagePath, stage.backupScope.Paths); linkProjectIntoStageErr != nil {- return nil, fmt.Errorf("failed to stage current project files: %w", linkProjectIntoStageErr)+ if linkErr := linkProjectIntoStage(stage.project.Path, stage.stagePath, stage.backupScope.Paths); linkErr != nil {+ return nil, nil, false, false, fmt.Errorf("failed to stage current project files: %w", linkErr)}- if _, seedStageEnvFromDirErr := seedStageEnvFromDir(ctx, stage.project.Path, projectsDir, stage.stagePath); seedStageEnvFromDirErr != nil {- return nil, seedStageEnvFromDirErr+ if _, seedErr := seedStageEnvFromDir(ctx, stage.project.Path, projectsDir, stage.stagePath); seedErr != nil {+ return nil, nil, false, false, seedErr}- stage.contentsChanged, err = projects.DirectorySyncContentsChanged(ctx, stage.project.Path, filteredSyncFiles, stage.oldSyncedFiles, stage.composeFileName)- if err != nil {- return nil, fmt.Errorf("failed to compare staged directory changes: %w", err)+ if stage.contentsChanged, err = projects.DirectorySyncContentsChanged(ctx, stage.project.Path, stage.syncFiles, stage.oldSyncedFiles, stage.composeFileName); err != nil {+ return nil, nil, false, false, fmt.Errorf("failed to compare staged directory changes: %w", err)}- } else if seedStageEnvFromCandidateDirErr := s.seedStageEnvFromCandidateDirInternal(ctx, sync, projectsDir, stage.stagePath); seedStageEnvFromCandidateDirErr != nil {- return nil, seedStageEnvFromCandidateDirErr+ } else if seedErr := s.seedStageEnvFromCandidateDir(ctx, sync, projectsDir, stage.stagePath); seedErr != nil {+ return nil, nil, false, false, seedErr}- preEnvContent, postEnvContent, err := s.applyDirectorySyncInternal(ctx, stage.stagePath, stage)+ preEnvContent, postEnvContent, err := s.applyDirectorySync(ctx, stage.stagePath, stage)if err != nil {- return nil, err+ return nil, nil, false, false, err}if stage.project != nil && projects.EnvContentChanged(preEnvContent, postEnvContent) {stage.contentsChanged = true}-- stage.serviceCount, err = s.projectService.ValidateComposeDirectory(ctx, sync.ProjectName, stage.stagePath, stage.composeFileName)- if err != nil {- return nil, fmt.Errorf("invalid compose file: %w", err)+ if stage.serviceCount, err = s.projectService.ValidateComposeDirectory(ctx, sync.ProjectName, stage.stagePath, stage.composeFileName); err != nil {+ return nil, nil, false, false, fmt.Errorf("invalid compose file: %w", err)}- return stage, nil-}--// applyDirectorySyncInternal applies the sync to targetPath in the order-// validation saw it: drop files git no longer tracks and stale compose files,-// write the repo files, then run the project-root env merge. It returns the-// effective .env content before and after the merge.-func (s *Service) applyDirectorySyncInternal(ctx context.Context, targetPath string, stage *stagedDirectorySync) (string, string, error) {- if len(stage.oldSyncedFiles) > 0 {- if err := projects.CleanupRemovedFiles(ctx, stage.projectsDir, targetPath, stage.oldSyncedFiles, stage.syncedFiles); err != nil {- return "", "", fmt.Errorf("failed to clean removed synced files: %w", err)+ if stage.project == nil {+ project, createErr := s.createDirectorySyncProject(ctx, sync, stage, actor)+ if createErr != nil {+ return nil, nil, false, false, createErr}+ return project, stage.syncedFiles, true, true, nil}-- if err := projects.RemoveStaleComposeFiles(ctx, targetPath, stage.composeFileName, stage.syncedFiles); err != nil {- return "", "", fmt.Errorf("failed to remove stale compose files: %w", err)- }-- // Write the repo files (excluding reserved root env files, handled below)- // after cleanup so validation sees the final on-disk tree exactly as it- // will exist in the managed project.- if _, err := projects.WriteSyncedDirectory(ctx, stage.projectsDir, targetPath, stage.syncFiles); err != nil {- return "", "", fmt.Errorf("failed to write staged sync files: %w", err)+ project, err := s.updateDirectorySyncProject(ctx, sync, stage)+ if err != nil {+ return nil, nil, false, false, err}-- // Route the project-root .env through the same three-file override merge- // single-file git sync uses: git is source-of-truth, edits made in Arcane- // become an override that wins, and new git-introduced keys still flow in.- return s.projectService.ApplyGitSyncEnvToDirectory(ctx, targetPath, stage.projectsDir, stage.gitEnvContent)+ return project, stage.syncedFiles, false, stage.contentsChanged, nil}-// seedStageEnvFromCandidateDirInternal copies env files from a pre-existing project-// directory at the conventional path (projectsDir/<sanitized-sync-name>/) into the-// staging directory before initial-sync validation. This lets users pre-seed-// ${VAR} substitutions via a server-side .env when the compose file in git-// expects values that the git repo intentionally does not provide. Only env-// files are touched — other files would conflict with what WriteSyncedDirectory-// is about to lay down from git.-func (s *Service) seedStageEnvFromCandidateDirInternal(ctx context.Context, sync *projectpkg.GitOpsSync, projectsDir, stagePath string) error {+// seedStageEnvFromCandidateDir copies the env files of a pre-existing directory at the conventional project+// path into a first-sync stage, so a server-side .env can supply ${VAR} values the repo leaves out.+func (s *Service) seedStageEnvFromCandidateDir(ctx context.Context, sync *projectpkg.GitOpsSync, projectsDir, stagePath string) error {candidatePath := filepath.Join(projectsDir, projects.SanitizeProjectName(sync.ProjectName))info, err := os.Stat(candidatePath)+ if errors.Is(err, os.ErrNotExist) {+ return nil+ }if err != nil {- if errors.Is(err, os.ErrNotExist) {- return nil- }return fmt.Errorf("inspect pre-existing project directory %s: %w", candidatePath, err)}if !info.IsDir() {return nil}-state, err := seedStageEnvFromDir(ctx, candidatePath, projectsDir, stagePath)- if err != nil {+ if err != nil || state.HasEffective || (!state.HasGitSource && !state.HasOverride) {return err}- if !state.HasEffective && !state.HasGitSource && !state.HasOverride {- return nil+ // Only .env.git and/or project.env exist, so derive .env from them.+ merged, err := projects.BuildEffectiveEnvContent(state.GitContent, state.OverrideContent)+ if err != nil {+ return fmt.Errorf("build effective env from pre-existing project: %w", err)}-- if !state.HasEffective {- // Only .env.git and/or project.env exist, so derive .env from them.- merged, mergeErr := projects.BuildEffectiveEnvContent(state.GitContent, state.OverrideContent)- if mergeErr != nil {- return fmt.Errorf("build effective env from pre-existing project: %w", mergeErr)- }- if writeProjectFileErr := projects.WriteProjectFile(ctx, projectsDir, stagePath, projects.EffectiveEnvFileName, merged); writeProjectFileErr != nil {- return fmt.Errorf("seed stage .env: %w", writeProjectFileErr)- }+ if writeErr := projects.WriteProjectFile(ctx, projectsDir, stagePath, projects.EffectiveEnvFileName, merged); writeErr != nil {+ return fmt.Errorf("seed stage .env: %w", writeErr)}-- slog.DebugContext(ctx, "Seeded GitOps stage with pre-existing project env files",- "candidatePath", candidatePath,- "hasEffective", state.HasEffective,- "hasGit", state.HasGitSource,- "hasOverride", state.HasOverride,- )return nil}-func (s *Service) lookupProjectByPathInternal(ctx context.Context, projectPath string) (*projectpkg.Project, bool, error) {- var project projectpkg.Project- if err := s.db.WithContext(ctx).Where("path = ?", projectPath).First(&project).Error; err != nil {- if errors.Is(err, gorm.ErrRecordNotFound) {- return nil, false, nil+// DirectoryProject resolves the linked project for a directory sync, relinking or recovering it when the+// stored reference is stale. A sync whose project is gone and cannot be recovered has a broken binding.+func (s *Service) DirectoryProject(ctx context.Context, sync *projectpkg.GitOpsSync) (*projectpkg.Project, error) {+ projectID := kit.FromPtr(sync.ProjectID)+ if projectID != "" {+ project, found, err := s.projectService.FindProjectByID(ctx, projectID)+ if err != nil {+ return nil, fmt.Errorf("failed to get project %s: %w", projectID, err)}- return nil, false, fmt.Errorf("failed to get project by path %s: %w", projectPath, err)+ if found {+ if rootErr := s.projectService.EnsureProjectPathUnderRoot(ctx, project, true); rootErr != nil {+ return nil, rootErr+ }+ return project, s.projectService.EnsureGitOpsProjectLinked(ctx, sync, project)+ }+ slog.WarnContext(ctx, "Existing project not found, attempting recovery", "projectId", projectID, "syncId", sync.ID)}- return &project, true, nil+ project, err := s.recoverProject(ctx, sync)+ if projectID != "" && project == nil {+ cause := cmp.Or(err, errors.New("no unique recovery candidate was found"))+ return nil, common.Classify(common.ErrGitOpsSyncProjectBindingBroken, fmt.Errorf("GitOps sync project binding broken: sync %s references missing project %s: %w", sync.ID, projectID, cause))+ }+ return project, err}-func (s *Service) findRecoverableManagedProjectInternal(ctx context.Context, sync *projectpkg.GitOpsSync) (*projectpkg.Project, error) {- var managedProjects []projectpkg.Project- if err := s.db.WithContext(ctx).- Where("gitops_managed_by = ?", sync.ID).- Find(&managedProjects).Error; err != nil {+// recoverProject finds the one project still managed by the sync, or else adopts the one projects-directory+// entry that matches the sync's name and holds its compose file.+func (s *Service) recoverProject(ctx context.Context, sync *projectpkg.GitOpsSync) (*projectpkg.Project, error) {+ var managed []projectpkg.Project+ if err := s.db.WithContext(ctx).Where("gitops_managed_by = ?", sync.ID).Find(&managed).Error; err != nil {return nil, fmt.Errorf("failed to list GitOps-managed projects for sync %s: %w", sync.ID, err)}-- matches := make([]projectpkg.Project, 0, len(managedProjects))- for i := range managedProjects {- project := managedProjects[i]- if err := s.projectService.EnsureProjectPathUnderRoot(ctx, &project, true); err != nil {+ var recovered []projectpkg.Project+ for i := range managed {+ if err := s.projectService.EnsureProjectPathUnderRoot(ctx, &managed[i], true); err != nil {return nil, err}- if _, err := s.projectService.ResolveProjectComposeFile(ctx, &project); err != nil {- if errors.Is(err, common.ErrProjectComposeFileNotFound) {- continue- }+ if _, err := s.projectService.ResolveProjectComposeFile(ctx, &managed[i]); err == nil {+ recovered = append(recovered, managed[i])+ } else if !errors.Is(err, common.ErrProjectComposeFileNotFound) {return nil, err}- matches = append(matches, project)}-- switch len(matches) {+ switch len(recovered) {case 0:- return nil, nilcase 1:- return &matches[0], nil+ return &recovered[0], s.projectService.EnsureGitOpsProjectLinked(ctx, sync, &recovered[0])default:return nil, fmt.Errorf("multiple GitOps-managed projects match sync %s; refusing automatic relink", sync.ID)}-}-func (s *Service) findUniqueProjectDirectoryCandidateInternal(ctx context.Context, sync *projectpkg.GitOpsSync) (string, error) {projectsDir, err := s.projectService.GetProjectsDirectory(ctx)if err != nil {- return "", err+ return nil, err}-entries, err := acfs.List(ctx, projectsDir, "/")if err != nil {- return "", fmt.Errorf("failed to list projects directory %s: %w", projectsDir, err)+ return nil, fmt.Errorf("failed to list projects directory %s: %w", projectsDir, err)}-composeFileName := strings.TrimSpace(filepath.Base(sync.ComposePath))if composeFileName == "" || composeFileName == "." {- return "", nil+ return nil, nil}prefix := projects.SanitizeProjectName(sync.ProjectName)- matches := make([]string, 0, 1)+ var matches []stringfor _, entry := range entries {- // Adoption never follows symlinks, and acfs reports a symlink as a- // non-directory, so a plain directory is the only candidate kind.- if !entry.IsDirectory {+ // acfs reports a symlink as a non-directory, so adoption never follows one; Arcane's own scratch dirs never qualify.+ if !entry.IsDirectory || projects.IsInternalScratchDirName(entry.Name) || (prefix != "" && entry.Name != prefix && !strings.HasPrefix(entry.Name, prefix+"-")) {continue}- // Never adopt one of Arcane's own scratch dirs as a recovery candidate.- if projects.IsInternalScratchDirName(entry.Name) {- continue- }- if prefix != "" && entry.Name != prefix && !strings.HasPrefix(entry.Name, prefix+"-") {- continue- }-candidatePath := filepath.Join(projectsDir, entry.Name)- composePath := filepath.Join(candidatePath, composeFileName)- if composeEntry, statErr := acfs.Stat(ctx, projectsDir, path.Join(entry.Path, composeFileName), true); statErr == nil {- if !composeEntry.IsDirectory {- matches = append(matches, candidatePath)- }- } else if !errors.Is(statErr, fs.ErrNotExist) {- return "", fmt.Errorf("failed to inspect recovery candidate %s: %w", composePath, statErr)+ composeEntry, statErr := acfs.Stat(ctx, projectsDir, path.Join(entry.Path, composeFileName), true)+ switch {+ case statErr == nil && !composeEntry.IsDirectory:+ matches = append(matches, candidatePath)+ case statErr != nil && !errors.Is(statErr, fs.ErrNotExist):+ return nil, fmt.Errorf("failed to inspect recovery candidate %s: %w", filepath.Join(candidatePath, composeFileName), statErr)}}-switch len(matches) {case 0:- return "", nil+ return nil, nilcase 1:- return matches[0], nildefault:- return "", fmt.Errorf("multiple candidate project directories match sync %s; refusing automatic relink", sync.ID)+ return nil, fmt.Errorf("multiple candidate project directories match sync %s; refusing automatic relink", sync.ID)}-}-func (s *Service) createRecoveredProjectFromDirectoryInternal(ctx context.Context, sync *projectpkg.GitOpsSync, projectPath string) (*projectpkg.Project, error) {- project := &projectpkg.Project{+ var project projectpkg.Project+ err = s.db.WithContext(ctx).Where("path = ?", matches[0]).First(&project).Error+ if err == nil {+ if ensureErr := s.projectService.EnsureProjectPathUnderRoot(ctx, &project, true); ensureErr != nil {+ return nil, ensureErr+ }+ return &project, s.projectService.EnsureGitOpsProjectLinked(ctx, sync, &project)+ }+ if !errors.Is(err, gorm.ErrRecordNotFound) {+ return nil, fmt.Errorf("failed to get project by path %s: %w", matches[0], err)+ }++ project = projectpkg.Project{Name: sync.ProjectName,- DirName: new(filepath.Base(projectPath)),- Path: projectPath,+ DirName: new(filepath.Base(matches[0])),+ Path: matches[0],Status: projectpkg.ProjectStatusUnknown,StatusReason: new("Project recovered from existing GitOps-managed directory"),- ServiceCount: 0,- RunningCount: 0,GitOpsManagedBy: &sync.ID,}-- if serviceCount, err := s.projectService.CountServicesFromCompose(ctx, *project); err == nil {- project.ServiceCount = serviceCount- } else {- slog.WarnContext(ctx, "Failed to count services while recovering GitOps project", "syncId", sync.ID, "path", projectPath, "error", err)+ if project.ServiceCount, err = s.projectService.CountServicesFromCompose(ctx, project); err != nil {+ slog.WarnContext(ctx, "Failed to count services while recovering GitOps project", "syncId", sync.ID, "path", matches[0], "error", err)}-- if err := s.projectService.CreateGitOpsManagedProject(ctx, sync, project, user.Actor{}, false); err != nil {- return nil, err+ if createErr := s.projectService.CreateGitOpsManagedProject(ctx, sync, &project, user.Actor{}, false); createErr != nil {+ return nil, createErr}-- return project, nil+ return &project, nil}-func (s *Service) recoverProjectFromDirectoryCandidateInternal(ctx context.Context, sync *projectpkg.GitOpsSync) (*projectpkg.Project, error) {- projectPath, err := s.findUniqueProjectDirectoryCandidateInternal(ctx, sync)- if err != nil || projectPath == "" {- return nil, err- }+// walkAndParseSyncDirectory walks the repository directory and returns all files with their contents.+// Returns the list of SyncFile entries and an error if any; it fails if the compose file is missing.+func (s *Service) walkAndParseSyncDirectory(ctx context.Context, sync *projectpkg.GitOpsSync, repoPath string) ([]projects.SyncFile, error) {+ slog.InfoContext(ctx, "Starting directory walk", "syncId", sync.ID, "composePath", sync.ComposePath)++ // Walk the directory to get all files+ maxFiles, maxTotalSize, maxBinarySize := s.limits(ctx, sync)- project, found, err := s.lookupProjectByPathInternal(ctx, projectPath)+ walkResult, err := s.repoService.WalkDirectory(ctx, repoPath, sync.ComposePath, maxFiles, maxTotalSize, maxBinarySize)if err != nil {- return nil, err- }- if found {- if ensureProjectPathUnderRootErr := s.projectService.EnsureProjectPathUnderRoot(ctx, project, true); ensureProjectPathUnderRootErr != nil {- return nil, ensureProjectPathUnderRootErr- }- if ensureGitOpsProjectLinkedErr := s.projectService.EnsureGitOpsProjectLinked(ctx, sync, project); ensureGitOpsProjectLinkedErr != nil {- return nil, ensureGitOpsProjectLinkedErr- }- return project, nil+ return nil, fmt.Errorf("failed to walk directory: %w", err)}- return s.createRecoveredProjectFromDirectoryInternal(ctx, sync, projectPath)-}+ slog.InfoContext(ctx, "Directory walk complete",+ "syncId", sync.ID,+ "totalFiles", walkResult.TotalFiles,+ "totalSize", walkResult.TotalSize,+ "skippedBinaries", walkResult.SkippedBinaries)-// getDirectorySyncProjectInternal resolves the linked project for a sync when one-// exists, while tolerating deleted/stale project references.-func (s *Service) getDirectorySyncProjectInternal(ctx context.Context, sync *projectpkg.GitOpsSync) (*projectpkg.Project, error) {- if sync == nil {- return nil, nil- }+ // WalkDirectory roots the walk at filepath.Dir(sync.ComposePath), so the+ // compose file is always emitted at the top level as filepath.Base(sync.ComposePath).+ composeFileName := filepath.Base(sync.ComposePath)+ composeFound := false- establishedBinding := hasEstablishedProjectBinding(sync)- if establishedBinding {- project, found, err := s.projectService.FindProjectByID(ctx, *sync.ProjectID)- if err != nil {- return nil, fmt.Errorf("failed to get project %s: %w", *sync.ProjectID, err)+ // Convert walked files to SyncFile format+ syncFiles := make([]projects.SyncFile, len(walkResult.Files))+ for i, f := range walkResult.Files {+ syncFiles[i] = projects.SyncFile{+ RelativePath: f.RelativePath,+ Content: f.Content,+ Executable: f.Executable,}- if found {- if ensureProjectPathUnderRootErr := s.projectService.EnsureProjectPathUnderRoot(ctx, project, true); ensureProjectPathUnderRootErr != nil {- return nil, ensureProjectPathUnderRootErr- }- if ensureGitOpsProjectLinkedErr := s.projectService.EnsureGitOpsProjectLinked(ctx, sync, project); ensureGitOpsProjectLinkedErr != nil {- return nil, ensureGitOpsProjectLinkedErr- }- return project, nil+ if f.RelativePath == composeFileName {+ composeFound = true}-- slog.WarnContext(ctx, "Existing project not found, attempting recovery", "projectId", *sync.ProjectID, "syncId", sync.ID)}- project, findRecoverableManagedProjectErr := s.findRecoverableManagedProjectInternal(ctx, sync)- if findRecoverableManagedProjectErr != nil {- if establishedBinding {- return nil, common.Classify(- common.ErrGitOpsSyncProjectBindingBroken,- fmt.Errorf(- "GitOps sync project binding broken: sync %s references missing project %s: %w",- sync.ID,- *sync.ProjectID,- findRecoverableManagedProjectErr,- ),- )- }- return nil, findRecoverableManagedProjectErr- }- if project != nil {- if err := s.projectService.EnsureGitOpsProjectLinked(ctx, sync, project); err != nil {- return nil, err- }- return project, nil+ if !composeFound {+ return nil, fmt.Errorf("compose file %s not found in walked directory", composeFileName)}- project, findRecoverableManagedProjectErr = s.recoverProjectFromDirectoryCandidateInternal(ctx, sync)- if findRecoverableManagedProjectErr != nil {- if establishedBinding {- return nil, common.Classify(- common.ErrGitOpsSyncProjectBindingBroken,- fmt.Errorf(- "GitOps sync project binding broken: sync %s references missing project %s: %w",- sync.ID,- *sync.ProjectID,- findRecoverableManagedProjectErr,- ),- )+ return syncFiles, nil+}++// applyDirectorySync drops untracked and stale compose files, writes the repo files, then runs the env merge, the+// order validation saw. It returns the effective .env content before and after the merge.+func (s *Service) applyDirectorySync(ctx context.Context, targetPath string, stage *stagedDirectorySync) (string, string, error) {+ if len(stage.oldSyncedFiles) > 0 {+ if err := projects.CleanupRemovedFiles(ctx, stage.projectsDir, targetPath, stage.oldSyncedFiles, stage.syncedFiles); err != nil {+ return "", "", fmt.Errorf("failed to clean removed synced files: %w", err)}- return nil, findRecoverableManagedProjectErr}- if project != nil {- return project, nil++ if err := projects.RemoveStaleComposeFiles(ctx, targetPath, stage.composeFileName, stage.syncedFiles); err != nil {+ return "", "", fmt.Errorf("failed to remove stale compose files: %w", err)}- if establishedBinding {- return nil, common.Classify(- common.ErrGitOpsSyncProjectBindingBroken,- fmt.Errorf(- "GitOps sync project binding broken: sync %s references missing project %s: no unique recovery candidate was found",- sync.ID,- *sync.ProjectID,- ),- )+ // Write the repo files (excluding reserved root env files, handled below)+ // after cleanup so validation sees the final on-disk tree exactly as it+ // will exist in the managed project.+ if _, err := projects.WriteSyncedDirectory(ctx, stage.projectsDir, targetPath, stage.syncFiles); err != nil {+ return "", "", fmt.Errorf("failed to write staged sync files: %w", err)}- return nil, nil+ // Route the project-root .env through the same three-file override merge+ // single-file git sync uses: git is source-of-truth, edits made in Arcane+ // become an override that wins, and new git-introduced keys still flow in.+ return s.projectService.ApplyGitSyncEnvToDirectory(ctx, targetPath, stage.projectsDir, stage.gitEnvContent)}-// createDirectorySyncProjectInternal promotes a validated staged tree into a new+// createDirectorySyncProject promotes a validated staged tree into a new// managed project directory and links it back to the Git sync record.-func (s *Service) createDirectorySyncProjectInternal(ctx context.Context, sync *projectpkg.GitOpsSync, stage *stagedDirectorySync, actor user.Actor) (*projectpkg.Project, error) {+func (s *Service) createDirectorySyncProject(ctx context.Context, sync *projectpkg.GitOpsSync, stage *stagedDirectorySync, actor user.Actor) (*projectpkg.Project, error) {projectsDir, err := s.projectService.GetProjectsDirectory(ctx)if err != nil {return nil, fmt.Errorf("failed to get projects directory: %w", err)}// Non-suffixing create: a GitOps sync must never mint a "-N" duplicate. Any- // adoptable existing directory was already resolved by getDirectorySyncProjectInternal,+ // adoptable existing directory was already resolved by getDirectorySyncProject,// so a collision here means the name is taken by an unrelated/unrecoverable dir;// treat it as a broken binding rather than creating a duplicate.basePath := filepath.Join(projectsDir, projects.SanitizeProjectName(sync.ProjectName))@@ -1337,10 +840,9 @@ func (s *Service) createDirectorySyncProjectInternal(ctx context.Context, sync *return project, nil}-// updateDirectorySyncProjectInternal applies a validated stage to the existing-// project path in place so running containers keep their bind-mount inodes; a-// backup scoped to the touched paths allows rollback if promotion fails.-func (s *Service) updateDirectorySyncProjectInternal(ctx context.Context, sync *projectpkg.GitOpsSync, stage *stagedDirectorySync) (*projectpkg.Project, error) {+// updateDirectorySyncProject applies a validated stage in place, so running containers keep their bind-mount+// inodes; a backup scoped to the touched paths rolls a failed promotion back.+func (s *Service) updateDirectorySyncProject(ctx context.Context, sync *projectpkg.GitOpsSync, stage *stagedDirectorySync) (*projectpkg.Project, error) {project := stage.projectprojectPath := filepath.Clean(project.Path)existed := true@@ -1400,7 +902,7 @@ func (s *Service) updateDirectorySyncProjectInternal(ctx context.Context, sync *return errors.Join(cause, fmt.Errorf("rollback project directory: %w", restoreErr))}- if _, _, err := s.applyDirectorySyncInternal(ctx, projectPath, stage); err != nil {+ if _, _, err := s.applyDirectorySync(ctx, projectPath, stage); err != nil {return nil, restore(fmt.Errorf("failed to promote staged project directory: %w", err))}@@ -1415,30 +917,8 @@ func (s *Service) updateDirectorySyncProjectInternal(ctx context.Context, sync *return project, nil}-// updateSyncStatusWithFiles updates sync status including the list of synced files-func (s *Service) updateSyncStatusWithFiles(ctx context.Context, id, status, errorMsg, commitHash string, syncedFiles []string) {- now := time.Now()- updates := map[string]any{- "last_sync_at": now,- "last_sync_status": status,- "synced_files": projectpkg.EncodeSyncedFiles(syncedFiles),- }-- updates["last_sync_error"] = kit.Ternary[any](errorMsg != "", errorMsg, nil)-- if commitHash != "" {- updates["last_sync_commit"] = commitHash- }-- if err := s.db.WithContext(ctx).Model(&projectpkg.GitOpsSync{}).Where("id = ?", id).Updates(updates).Error; err != nil {- slog.ErrorContext(ctx, "Failed to update sync status with files", "error", err, "syncId", id)- }-}--// seedStageEnvFromDir copies the readable project-root env files from-// sourceDir into the stage verbatim and returns the env state it read. A-// permission-locked file is skipped rather than aborting the whole staging-// attempt, matching how the env merge leaves such files untouched.+// seedStageEnvFromDir copies sourceDir's readable project-root env files into the stage and returns the state+// it read; like the env merge, it skips a permission-locked file.func seedStageEnvFromDir(ctx context.Context, sourceDir, projectsDir, stagePath string) (projects.ProjectEnvState, error) {state, err := projects.ReadProjectEnvState(sourceDir)if err != nil {@@ -1462,74 +942,8 @@ func seedStageEnvFromDir(ctx context.Context, sourceDir, projectsDir, stagePathreturn state, nil}-// partitionReservedRootEnvFiles splits syncFiles into the set safe to-// write directly (everything except the reserved root env files) and the-// git-sourced root .env content, if the repo has one. A committed-// .env.git/project.env/.env.global at the project root is dropped rather than-// raw-written: those are Arcane-owned bookkeeping files that must only ever be-// produced by the override merge, never by an untrusted git payload.-func partitionReservedRootEnvFiles(ctx context.Context, syncFiles []projects.SyncFile) ([]projects.SyncFile, *string) {- filtered := make([]projects.SyncFile, 0, len(syncFiles))- var gitEnvContent *string- for _, file := range syncFiles {- if !isReservedRootEnvFile(file.RelativePath) {- filtered = append(filtered, file)- continue- }- switch file.RelativePath {- case projects.EffectiveEnvFileName:- content := string(file.Content)- gitEnvContent = &content- case projects.GlobalEnvFileName:- slog.WarnContext(ctx, "dropping .env.global from git payload; project-root env files are not raw-written from git and .env.global is not project-scoped",- "path", file.RelativePath,- )- default:- slog.WarnContext(ctx, "dropping reserved Arcane env file from git payload; will be re-derived from override merge",- "path", file.RelativePath,- )- }- }- return filtered, gitEnvContent-}--// isReservedRootEnvFile reports whether relPath is one of the-// project-root env bookkeeping files the override-merge system owns (.env,-// .env.git, project.env, .env.global). These are excluded from the raw-// directory-sync write and routed through ProjectService.ApplyGitSyncEnvToDirectory-// instead. Exact-matching RelativePath intentionally leaves nested env files-// (e.g. svc/.env) in the raw write set — only the project root is-// merge-managed.-func isReservedRootEnvFile(relPath string) bool {- switch relPath {- case projects.EffectiveEnvFileName, projects.GitSourceEnvFileName, projects.OverrideEnvFileName, projects.GlobalEnvFileName:- return true- default:- return false- }-}--// filterReservedRootEnvFiles drops reserved root env file paths from a-// tracked synced-file list. Syncs created before the override-merge migration-// may still have .env recorded from an earlier sync; leaving it in would make-// CleanupRemovedFiles treat .env as removed-by-git and delete the live-copied-// stage .env before ApplyGitSyncEnvToDirectory can read it for the merge.-func filterReservedRootEnvFiles(paths []string) []string {- filtered := make([]string, 0, len(paths))- for _, p := range paths {- if !isReservedRootEnvFile(p) {- filtered = append(filtered, p)- }- }- return filtered-}--// linkProjectIntoStage builds the sparse stage view of an existing-// project: paths in scope are left for the apply step, directories on the way-// to them are recreated, and everything else is exposed through a symlink to-// its live location so validation sees the whole tree without copying it.-// The links are validation-only and never promoted. acfs refuses to create-// symlinks by design, so this is the one place the stage is built with os.+// linkProjectIntoStage symlinks every live path outside scopePaths into the stage, so validation sees the whole+// project without copying it. The links are never promoted; acfs refuses symlinks, so this uses os.func linkProjectIntoStage(livePath, stagePath string, scopePaths []string) error {scope := make(map[string]struct{}, len(scopePaths))ancestors := make(map[string]struct{})@@ -1570,7 +984,3 @@ func linkProjectIntoStage(livePath, stagePath string, scopePaths []string) error}return link("")}--func hasEstablishedProjectBinding(sync *projectpkg.GitOpsSync) bool {- return sync != nil && sync.ProjectID != nil && strings.TrimSpace(*sync.ProjectID) != ""-}diff --git a/backend/internal/gitops/children/sync/service_test.go b/backend/internal/gitops/children/sync/service_test.goindex cd6a2f9c9..6c8ccf7c6 100644--- a/backend/internal/gitops/children/sync/service_test.go+++ b/backend/internal/gitops/children/sync/service_test.go@@ -4,15 +4,12 @@ import ("context""encoding/json""errors"- "fmt""os""path/filepath""runtime""strings""testing"- "github.com/getarcaneapp/arcane/types/v2/gitops"- "github.com/getarcaneapp/arcane/types/v2/swarm""github.com/getarcaneapp/arcane/types/v2/user""github.com/libtnb/sqlite""github.com/stretchr/testify/assert"@@ -31,7 +28,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/projects")-func setupSyncTestServiceInternal(t *testing.T) (*Service, *database.DB, string) {+func setupSyncTestService(t *testing.T) (*Service, *database.DB, string) {t.Helper()ctx := t.Context()@@ -61,7 +58,7 @@ func setupSyncTestServiceInternal(t *testing.T) (*Service, *database.DB, string)return New(arcaneDB, nil, projectService, nil, eventService, limits, logError, unregisterJob), arcaneDB, projectsDir}-func writeFileInternal(t *testing.T, rootDir, relativePath string, content []byte) {+func writeFile(t *testing.T, rootDir, relativePath string, content []byte) {t.Helper()targetPath := filepath.Join(rootDir, relativePath)@@ -71,10 +68,10 @@ func writeFileInternal(t *testing.T, rootDir, relativePath string, content []byt// TestGitOpsSyncService_SyncProjectDirectory_RefusesDuplicateOnNameCollision verifies a// directory sync refuses to create a "-N" sibling when its target name is already taken-// by a non-adoptable directory; instead it errors as a broken binding and disables auto-sync.+// by a non-adoptable directory; instead it errors as a broken binding.func TestGitOpsSyncService_SyncProjectDirectory_RefusesDuplicateOnNameCollision(t *testing.T) {ctx := t.Context()- svc, db, projectsDir := setupSyncTestServiceInternal(t)+ svc, db, projectsDir := setupSyncTestService(t)// Occupy the target name with a dir that is NOT an adoptable GitOps project// (it has no matching compose file).@@ -97,20 +94,16 @@ func TestGitOpsSyncService_SyncProjectDirectory_RefusesDuplicateOnNameCollision({RelativePath: "docker-compose.yaml", Content: []byte("services:\n app:\n image: nginx:alpine\n")},}- _, _, _, _, err := svc.syncProjectDirectoryInternal(ctx, sync, syncFiles, user.Actor{})+ _, _, _, _, err := svc.syncProjectDirectory(ctx, sync, syncFiles, user.Actor{})require.ErrorIs(t, err, common.ErrGitOpsSyncProjectBindingBroken)_, statErr := os.Stat(filepath.Join(projectsDir, "Dozzle-1"))require.ErrorIs(t, statErr, os.ErrNotExist, "must not mint a -N duplicate")-- var got projectpkg.GitOpsSync- require.NoError(t, db.Where("id = ?", sync.ID).First(&got).Error)- assert.False(t, got.AutoSync, "auto-sync should be disabled on broken binding")}func TestGitOpsSyncService_SyncProjectDirectory_CreatesProjectPreservingRepoLayout(t *testing.T) {ctx := t.Context()- svc, db, _ := setupSyncTestServiceInternal(t)+ svc, db, _ := setupSyncTestService(t)sync := &projectpkg.GitOpsSync{ID: "sync-directory-create",@@ -149,7 +142,7 @@ services:},}- project, syncedFiles, created, changed, err := svc.syncProjectDirectoryInternal(ctx, sync, syncFiles, user.Actor{})+ project, syncedFiles, created, changed, err := svc.syncProjectDirectory(ctx, sync, syncFiles, user.Actor{})require.NoError(t, err)require.NotNil(t, project)require.True(t, created)@@ -189,7 +182,7 @@ services:func TestGitOpsSyncService_SyncProjectDirectory_UpdatesProjectAndCleansOldSyncedFiles(t *testing.T) {ctx := t.Context()- svc, db, projectsDir := setupSyncTestServiceInternal(t)+ svc, db, projectsDir := setupSyncTestService(t)projectPath := filepath.Join(projectsDir, "demo-project")require.NoError(t, os.MkdirAll(filepath.Join(projectPath, "nested"), 0o755))@@ -254,7 +247,7 @@ services:keepBefore, err := os.Stat(filepath.Join(projectPath, "keep.txt"))require.NoError(t, err)- updatedProject, syncedFiles, created, changed, err := svc.syncProjectDirectoryInternal(ctx, sync, syncFiles, user.Actor{})+ updatedProject, syncedFiles, created, changed, err := svc.syncProjectDirectory(ctx, sync, syncFiles, user.Actor{})require.NoError(t, err)require.NotNil(t, updatedProject)require.False(t, created)@@ -293,7 +286,7 @@ services:// for https://github.com/getarcaneapp/arcane/issues/2476.func TestGitOpsSyncService_SyncProjectDirectory_PreservesEnvOverrideAndAddsNewGitKey(t *testing.T) {ctx := t.Context()- svc, db, projectsDir := setupSyncTestServiceInternal(t)+ svc, db, projectsDir := setupSyncTestService(t)projectPath := filepath.Join(projectsDir, "demo-project")require.NoError(t, os.MkdirAll(projectPath, 0o755))@@ -345,7 +338,7 @@ func TestGitOpsSyncService_SyncProjectDirectory_PreservesEnvOverrideAndAddsNewGi},}- updatedProject, syncedFiles, created, changed, err := svc.syncProjectDirectoryInternal(ctx, sync, syncFiles, user.Actor{})+ updatedProject, syncedFiles, created, changed, err := svc.syncProjectDirectory(ctx, sync, syncFiles, user.Actor{})require.NoError(t, err)require.NotNil(t, updatedProject)require.False(t, created)@@ -386,7 +379,7 @@ func TestGitOpsSyncService_SyncProjectDirectory_PreservesEnvOverrideAndAddsNewGi// of migrated into project.env.func TestGitOpsSyncService_SyncProjectDirectory_MigratesLegacyTrackedEnvOnFirstSyncAfterUpgrade(t *testing.T) {ctx := t.Context()- svc, db, projectsDir := setupSyncTestServiceInternal(t)+ svc, db, projectsDir := setupSyncTestService(t)projectPath := filepath.Join(projectsDir, "demo-project")require.NoError(t, os.MkdirAll(projectPath, 0o755))@@ -436,7 +429,7 @@ func TestGitOpsSyncService_SyncProjectDirectory_MigratesLegacyTrackedEnvOnFirstS},}- updatedProject, _, created, _, err := svc.syncProjectDirectoryInternal(ctx, sync, syncFiles, user.Actor{})+ updatedProject, _, created, _, err := svc.syncProjectDirectory(ctx, sync, syncFiles, user.Actor{})require.NoError(t, err)require.NotNil(t, updatedProject)require.False(t, created)@@ -457,7 +450,7 @@ func TestGitOpsSyncService_SyncProjectDirectory_MigratesLegacyTrackedEnvOnFirstS// .env.git/project.env on disk are still produced solely by the merge.func TestGitOpsSyncService_SyncProjectDirectory_IgnoresCommittedReservedEnvFiles(t *testing.T) {ctx := t.Context()- svc, db, _ := setupSyncTestServiceInternal(t)+ svc, db, _ := setupSyncTestService(t)sync := &projectpkg.GitOpsSync{ID: "sync-directory-reserved-env",@@ -492,7 +485,7 @@ func TestGitOpsSyncService_SyncProjectDirectory_IgnoresCommittedReservedEnvFiles},}- project, syncedFiles, created, _, err := svc.syncProjectDirectoryInternal(ctx, sync, syncFiles, user.Actor{})+ project, syncedFiles, created, _, err := svc.syncProjectDirectory(ctx, sync, syncFiles, user.Actor{})require.NoError(t, err)require.NotNil(t, project)require.True(t, created)@@ -515,11 +508,11 @@ func TestGitOpsSyncService_SyncProjectDirectory_IgnoresCommittedReservedEnvFilesfunc TestGitOpsSyncService_DirectorySync_RealWalkWithNestedConfig(t *testing.T) {ctx := t.Context()- svc, db, _ := setupSyncTestServiceInternal(t)+ svc, db, _ := setupSyncTestService(t)svc.repoService = &gitrepo.GitRepositoryService{Client: git.NewClient("")}repoPath := t.TempDir()- writeFileInternal(t, repoPath, "traefik (nl10)/docker-compose.yml", []byte(`services:+ writeFile(t, repoPath, "traefik (nl10)/docker-compose.yml", []byte(`services:traefik:image: traefik:v3.4volumes:@@ -527,7 +520,7 @@ func TestGitOpsSyncService_DirectorySync_RealWalkWithNestedConfig(t *testing.T)- ./logs:/var/log/traefik- ./config/dynamic_config.yml:/etc/traefik/dynamic_config.yml:ro`))- writeFileInternal(t, repoPath, "traefik (nl10)/config/dynamic_config.yml", []byte("http:\n routers:\n dashboard:\n rule: Host(`traefik.example.com`)\n"))+ writeFile(t, repoPath, "traefik (nl10)/config/dynamic_config.yml", []byte("http:\n routers:\n dashboard:\n rule: Host(`traefik.example.com`)\n"))sync := &projectpkg.GitOpsSync{ID: "sync-directory-real-walk",@@ -550,7 +543,7 @@ func TestGitOpsSyncService_DirectorySync_RealWalkWithNestedConfig(t *testing.T)}assert.Contains(t, composeContent, "./config/dynamic_config.yml")- project, syncedFiles, created, changed, err := svc.syncProjectDirectoryInternal(ctx, sync, syncFiles, user.Actor{})+ project, syncedFiles, created, changed, err := svc.syncProjectDirectory(ctx, sync, syncFiles, user.Actor{})require.NoError(t, err)require.NotNil(t, project)require.True(t, created)@@ -577,11 +570,11 @@ func TestGitOpsSyncService_DirectorySync_RealWalkWithNestedConfig(t *testing.T)func TestGitOpsSyncService_DirectorySync_OverwritesExistingDirectoryAtFilePath(t *testing.T) {ctx := t.Context()- svc, db, projectsDir := setupSyncTestServiceInternal(t)+ svc, db, projectsDir := setupSyncTestService(t)svc.repoService = &gitrepo.GitRepositoryService{Client: git.NewClient("")}repoPath := t.TempDir()- writeFileInternal(t, repoPath, "traefik (nl10)/docker-compose.yml", []byte(`services:+ writeFile(t, repoPath, "traefik (nl10)/docker-compose.yml", []byte(`services:traefik:image: traefik:v3.4volumes:@@ -589,7 +582,7 @@ func TestGitOpsSyncService_DirectorySync_OverwritesExistingDirectoryAtFilePath(t- ./logs:/var/log/traefik- ./config/dynamic_config.yml:/etc/traefik/dynamic_config.yml:ro`))- writeFileInternal(t, repoPath, "traefik (nl10)/config/dynamic_config.yml", []byte("http:\n routers:\n dashboard:\n rule: Host(`traefik.example.com`)\n"))+ writeFile(t, repoPath, "traefik (nl10)/config/dynamic_config.yml", []byte("http:\n routers:\n dashboard:\n rule: Host(`traefik.example.com`)\n"))projectPath := filepath.Join(projectsDir, "traefik-project")require.NoError(t, os.MkdirAll(filepath.Join(projectPath, "config", "dynamic_config.yml"), 0o755))@@ -621,7 +614,7 @@ func TestGitOpsSyncService_DirectorySync_OverwritesExistingDirectoryAtFilePath(tsyncFiles, err := svc.walkAndParseSyncDirectory(ctx, sync, repoPath)require.NoError(t, err)- updatedProject, syncedFiles, created, changed, err := svc.syncProjectDirectoryInternal(ctx, sync, syncFiles, user.Actor{})+ updatedProject, syncedFiles, created, changed, err := svc.syncProjectDirectory(ctx, sync, syncFiles, user.Actor{})require.NoError(t, err)require.NotNil(t, updatedProject)require.False(t, created)@@ -650,9 +643,9 @@ func TestGitOpsSyncService_DirectorySync_OverwritesExistingDirectoryAtFilePath(tassert.True(t, dockerArtifactInfo.IsDir())}-func TestGitOpsSyncService_CreateDirectorySyncProjectInternal_RollsBackProjectOnUpdateFailure(t *testing.T) {+func TestGitOpsSyncService_CreateDirectorySyncProject_RollsBackProjectOnUpdateFailure(t *testing.T) {ctx := t.Context()- svc, db, projectsDir := setupSyncTestServiceInternal(t)+ svc, db, projectsDir := setupSyncTestService(t)sync := &projectpkg.GitOpsSync{ID: "sync-directory-tx-rollback",@@ -687,7 +680,7 @@ func TestGitOpsSyncService_CreateDirectorySyncProjectInternal_RollsBackProjectOn_ = db.Callback().Update().Remove(callbackName)}()- project, err := svc.createDirectorySyncProjectInternal(ctx, sync, stage, user.Actor{})+ project, err := svc.createDirectorySyncProject(ctx, sync, stage, user.Actor{})require.Error(t, err)require.Nil(t, project)assert.Contains(t, err.Error(), "failed to mark project as GitOps-managed")@@ -704,21 +697,21 @@ func TestGitOpsSyncService_CreateDirectorySyncProjectInternal_RollsBackProjectOnrequire.ErrorIs(t, statErr, os.ErrNotExist)}-// TestGitOpsSyncService_UpdateDirectorySyncProjectInternal_RollsBackScopedChangesOnUpdateFailure+// TestGitOpsSyncService_UpdateDirectorySyncProject_RollsBackScopedChangesOnUpdateFailure// forces the project metadata update to fail after the synced files were written// to the live project, and verifies the scoped rollback restores exactly what the// sync touched (in place, keeping inodes) while unrelated files are never copied,// rewritten or pruned.-func TestGitOpsSyncService_UpdateDirectorySyncProjectInternal_RollsBackScopedChangesOnUpdateFailure(t *testing.T) {+func TestGitOpsSyncService_UpdateDirectorySyncProject_RollsBackScopedChangesOnUpdateFailure(t *testing.T) {ctx := t.Context()- svc, db, projectsDir := setupSyncTestServiceInternal(t)+ svc, db, projectsDir := setupSyncTestService(t)projectPath := filepath.Join(projectsDir, "demo-project")- writeFileInternal(t, projectPath, "docker-compose.yaml", []byte("services:\n app:\n image: nginx:1.26-alpine\n"))- writeFileInternal(t, projectPath, "old.txt", []byte("remove me\n"))- writeFileInternal(t, projectPath, "keep.txt", []byte("keep me\n"))- writeFileInternal(t, projectPath, ".env", []byte("A=1\n"))- writeFileInternal(t, projectPath, "data/blob.bin", []byte("bind-mount data\n"))+ writeFile(t, projectPath, "docker-compose.yaml", []byte("services:\n app:\n image: nginx:1.26-alpine\n"))+ writeFile(t, projectPath, "old.txt", []byte("remove me\n"))+ writeFile(t, projectPath, "keep.txt", []byte("keep me\n"))+ writeFile(t, projectPath, ".env", []byte("A=1\n"))+ writeFile(t, projectPath, "data/blob.bin", []byte("bind-mount data\n"))// A directory sits where git now ships a file.require.NoError(t, os.MkdirAll(filepath.Join(projectPath, "config", "dynamic.yml"), 0o755))@@ -795,7 +788,7 @@ func TestGitOpsSyncService_UpdateDirectorySyncProjectInternal_RollsBackScopedCha_ = db.Callback().Update().Remove(callbackName)}()- updatedProject, _, _, _, err := svc.syncProjectDirectoryInternal(ctx, sync, syncFiles, user.Actor{})+ updatedProject, _, _, _, err := svc.syncProjectDirectory(ctx, sync, syncFiles, user.Actor{})require.Error(t, err)require.Nil(t, updatedProject)assert.Contains(t, err.Error(), "forced project update failure")@@ -846,9 +839,9 @@ func TestGitOpsSyncService_UpdateDirectorySyncProjectInternal_RollsBackScopedCha}}-func TestGitOpsSyncService_GetDirectorySyncProjectInternal_RelinksManagedProjectWhenProjectIDStale(t *testing.T) {+func TestGitOpsSyncService_GetDirectorySyncProject_RelinksManagedProjectWhenProjectIDStale(t *testing.T) {ctx := t.Context()- svc, db, projectsDir := setupSyncTestServiceInternal(t)+ svc, db, projectsDir := setupSyncTestService(t)projectPath := filepath.Join(projectsDir, "Radarr-3")require.NoError(t, os.MkdirAll(projectPath, 0o755))@@ -878,7 +871,7 @@ func TestGitOpsSyncService_GetDirectorySyncProjectInternal_RelinksManagedProject}require.NoError(t, db.Create(project).Error)- recovered, err := svc.getDirectorySyncProjectInternal(ctx, sync)+ recovered, err := svc.DirectoryProject(ctx, sync)require.NoError(t, err)require.NotNil(t, recovered)assert.Equal(t, project.ID, recovered.ID)@@ -889,9 +882,9 @@ func TestGitOpsSyncService_GetDirectorySyncProjectInternal_RelinksManagedProjectassert.Equal(t, project.ID, *storedSync.ProjectID)}-func TestGitOpsSyncService_GetDirectorySyncProjectInternal_RecoversUniqueDirectoryCandidate(t *testing.T) {+func TestGitOpsSyncService_GetDirectorySyncProject_RecoversUniqueDirectoryCandidate(t *testing.T) {ctx := t.Context()- svc, db, projectsDir := setupSyncTestServiceInternal(t)+ svc, db, projectsDir := setupSyncTestService(t)projectPath := filepath.Join(projectsDir, "Radarr-3")require.NoError(t, os.MkdirAll(projectPath, 0o755))@@ -910,7 +903,7 @@ func TestGitOpsSyncService_GetDirectorySyncProjectInternal_RecoversUniqueDirecto}require.NoError(t, db.Create(sync).Error)- recovered, err := svc.getDirectorySyncProjectInternal(ctx, sync)+ recovered, err := svc.DirectoryProject(ctx, sync)require.NoError(t, err)require.NotNil(t, recovered)assert.Equal(t, projectPath, recovered.Path)@@ -928,120 +921,6 @@ func TestGitOpsSyncService_GetDirectorySyncProjectInternal_RecoversUniqueDirectoassert.Equal(t, 1, storedProject.ServiceCount)}-// TestBuildSwarmStackDeployRequestInternal guards the Git Sync swarm deploy request.-// WithRegistryAuth must always be set: swarm tasks pull with only the auth embedded in-// the service spec, so a request without it makes every private image fail to pull.-func TestBuildSwarmStackDeployRequestInternal(t *testing.T) {- t.Parallel()-- files := []swarm.SyncFile{{RelativePath: "configs/app.conf", Content: []byte("k=v")}}-- cases := []struct {- name string- sync *projectpkg.GitOpsSync- source *preparedSyncSource- overrideContent string- envContent string- files []swarm.SyncFile- want swarm.StackDeployRequest- }{- {- name: "populates all fields and enables registry auth",- sync: &projectpkg.GitOpsSync{ProjectName: "descent", ComposePath: "deploy/swarm/compose.yaml"},- source: &preparedSyncSource{repoPath: "/tmp/repo", composeContent: "services: {}\n"},- overrideContent: "override",- envContent: "A=1",- files: files,- want: swarm.StackDeployRequest{- Name: "descent",- ComposeContent: "services: {}\n",- OverrideContent: "override",- EnvContent: "A=1",- Files: files,- Prune: true,- WithRegistryAuth: true,- WorkingDir: filepath.Join(string(filepath.Separator), "tmp", "repo", "deploy", "swarm"),- },- },- {- name: "enables registry auth with empty optional content and a root-level compose path",- sync: &projectpkg.GitOpsSync{ProjectName: "ptest", ComposePath: "compose.yaml"},- source: &preparedSyncSource{repoPath: "/tmp/repo", composeContent: "services: {}\n"},- want: swarm.StackDeployRequest{- Name: "ptest",- ComposeContent: "services: {}\n",- Prune: true,- WithRegistryAuth: true,- WorkingDir: "/tmp/repo",- },- },- {- name: "passes an empty file list through unchanged",- sync: &projectpkg.GitOpsSync{ProjectName: "ptest", ComposePath: "stacks/compose.yaml"},- source: &preparedSyncSource{repoPath: "/tmp/repo", composeContent: "services: {}\n"},- // performSwarmStackSyncInternal always passes a non-nil slice; the builder must not reshape it.- files: []swarm.SyncFile{},- want: swarm.StackDeployRequest{- Name: "ptest",- ComposeContent: "services: {}\n",- Files: []swarm.SyncFile{},- Prune: true,- WithRegistryAuth: true,- WorkingDir: filepath.Join(string(filepath.Separator), "tmp", "repo", "stacks"),- },- },- }-- for _, tc := range cases {- t.Run(tc.name, func(t *testing.T) {- got := buildSwarmStackDeployRequestInternal(tc.sync, tc.source, tc.overrideContent, tc.envContent, tc.files)- assert.Equal(t, tc.want, got)- })- }-}--func TestMarkSyncRedeployFailedInternal_PersistsErrorOnSyncRow(t *testing.T) {- ctx := t.Context()- svc, db, _ := setupSyncTestServiceInternal(t)- // Event logging requires a real event.EventService; the shared setup leaves it- // nil since most tests don't exercise the event path.- require.NoError(t, db.AutoMigrate(&event.Event{}))- svc.eventService = event.NewEventService(db, config.Load(), nil)-- sync := &projectpkg.GitOpsSync{- ID: "sync-1",- Name: "redeploy-fail",- EnvironmentID: "0",- RepositoryID: "repo-1",- Branch: "main",- ComposePath: "compose.yml",- TargetType: "project",- }- require.NoError(t, db.WithContext(ctx).Select("*").Omit("Environment", "Repository", "Project").Create(sync).Error)-- result := &gitops.SyncResult{Success: true}- syncedFiles := []string{"compose.yml", "scripts/pre-deploy.sh"}- hookErr := common.Classify(common.ErrRedeployAfterSyncFailed, fmt.Errorf("redeploy failed: %w", errors.New("pre-deploy hook failed: exit 1")))-- svc.markSyncRedeployFailedInternal(ctx, sync, sync.ID, "abc123", syncedFiles, hookErr, user.Actor{ID: "user", Username: "tester"}, result)-- require.False(t, result.Success)- require.NotNil(t, result.Error)- require.Contains(t, *result.Error, "pre-deploy hook failed")-- var stored projectpkg.GitOpsSync- require.NoError(t, db.WithContext(ctx).First(&stored, "id = ?", sync.ID).Error)- require.NotNil(t, stored.LastSyncStatus)- require.Equal(t, "failed", *stored.LastSyncStatus)- require.NotNil(t, stored.LastSyncError)- require.Contains(t, *stored.LastSyncError, "pre-deploy hook failed")- // The synced-files list should still be populated so operators can see- // what reached disk before the redeploy died.- require.NotNil(t, stored.SyncedFiles)- require.Contains(t, *stored.SyncedFiles, "compose.yml")- require.Contains(t, *stored.SyncedFiles, "scripts/pre-deploy.sh")-}-// TestGitOpsSyncService_SyncProjectDirectory_PreservesUnreadableBindMountData verifies// that re-syncing an existing GitOps project tolerates a foreign-owned unreadable file// inside the project directory (e.g. data a container wrote through a relative bind@@ -1056,10 +935,10 @@ func TestGitOpsSyncService_SyncProjectDirectory_PreservesUnreadableBindMountData}ctx := t.Context()- svc, db, projectsDir := setupSyncTestServiceInternal(t)+ svc, db, projectsDir := setupSyncTestService(t)projectPath := filepath.Join(projectsDir, "demo-project")- writeFileInternal(t, projectPath, "docker-compose.yaml", []byte(`services:+ writeFile(t, projectPath, "docker-compose.yaml", []byte(`services:app:image: nginx:1.26-alpine`))@@ -1107,7 +986,7 @@ func TestGitOpsSyncService_SyncProjectDirectory_PreservesUnreadableBindMountData},}- updatedProject, _, created, _, err := svc.syncProjectDirectoryInternal(ctx, sync, syncFiles, user.Actor{})+ updatedProject, _, created, _, err := svc.syncProjectDirectory(ctx, sync, syncFiles, user.Actor{})require.NoError(t, err)require.NotNil(t, updatedProject)require.False(t, created)diff --git a/backend/internal/gitops/handler.go b/backend/internal/gitops/handler.goindex e51da1a17..0c40ac59c 100644--- a/backend/internal/gitops/handler.go+++ b/backend/internal/gitops/handler.go@@ -280,14 +280,7 @@ func (h *GitOpsSyncHandler) DeleteSync(ctx context.Context, input *DeleteGitOpsSreturn nil, huma.NewError(apiErr.HTTPStatus(), "Failed to delete GitOps sync")}- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{- Message: "Sync deleted successfully",- },- },- }, nil+ return handlerutil.MessageOutput("Sync deleted successfully", ""), nil}// PerformSync manually triggers a sync operation.diff --git a/backend/internal/gitops/helpers.go b/backend/internal/gitops/helpers.goindex ac9878cb2..bbe4bec18 100644--- a/backend/internal/gitops/helpers.go+++ b/backend/internal/gitops/helpers.go@@ -1,143 +1,20 @@package gitopsimport (- "encoding/json""errors"- "fmt""strings"- "time"- "github.com/getarcaneapp/arcane/types/v2/base"- "github.com/getarcaneapp/arcane/types/v2/gitops""go.getarcane.app/kit/pkg"- "gorm.io/gorm"-- "github.com/getarcaneapp/arcane/backend/v2/internal/common"- "github.com/getarcaneapp/arcane/backend/v2/internal/project")-// marshalSyncedFiles converts a list of file paths to JSON for storage-func MarshalSyncedFiles(files []string) *string {- if len(files) == 0 {- return nil- }- data, err := json.Marshal(files)- if err != nil {- return nil- }- return new(string(data))-}--// parseSyncedFiles parses the JSON array of synced file paths from the database-func ParseSyncedFiles(syncedFilesJSON *string) []string {- if syncedFilesJSON == nil || *syncedFilesJSON == "" {- return nil- }- var files []string- if err := json.Unmarshal([]byte(*syncedFilesJSON), &files); err != nil {- return nil- }- return files-}--func isUniqueViolation(err error) bool {- if errors.Is(err, gorm.ErrDuplicatedKey) {- return true- }- message := strings.ToLower(err.Error())- return strings.Contains(message, "unique constraint") || strings.Contains(message, "duplicate key")-}--func isGitOpsSyncOverdue(sync *project.GitOpsSync) bool {- if sync.LastSyncAt == nil {- return true- }- interval := max(sync.SyncInterval, 1)- return time.Now().After(sync.LastSyncAt.Add(time.Duration(interval) * time.Minute))-}--func megabytesToBytes(value int) int64 {- return int64(value) * 1024 * 1024-}-func normalizeSyncLimitSetting(value, defaultValue int) int {return kit.Ternary(value < 0, defaultValue, value)}-func nullableUpdateStringValue(p *string) any {- if p == nil {- return nil- }- trimmed := strings.TrimSpace(*p)- return kit.Ternary[any](trimmed == "", nil, trimmed)-}-+// nullableTrimmedString trims p and maps an empty or missing value to nil, which stores NULL.func nullableTrimmedString(p *string) *string {- if p == nil {- return nil- }- trimmed := strings.TrimSpace(*p)- if trimmed == "" {- return nil- }- return &trimmed-}--// normalizeLifecycleNetworkMode returns the canonical network mode-// string for a user-supplied value. Empty / whitespace / unset all map to-// "none" so the secure-by-default behaviour is restored when the user clears-// the field.-func normalizeLifecycleNetworkMode(p *string) string {- if p == nil {- return "none"- }- trimmed := strings.TrimSpace(*p)- return kit.Ternary(trimmed == "", "none", trimmed)-}--// resolveEffectiveSyncDirectory mirrors the string resolver but for-// the SyncDirectory bool: a nil update keeps the existing value, a non-nil-// update overrides it. On create (current=nil) and no update, defaults to-// false to match the model's create-time default.-func resolveEffectiveSyncDirectory(current *project.GitOpsSync, update *bool) bool {- if update != nil {- return *update- }- if current != nil {- return current.SyncDirectory- }- return false-}--func resolveLifecycleEffectiveTargetType(current *project.GitOpsSync, update *string) string {- if update != nil && strings.TrimSpace(*update) != "" {- return strings.TrimSpace(*update)- }- if current != nil && strings.TrimSpace(current.TargetType) != "" {- return strings.TrimSpace(current.TargetType)- }- return "project"-}--func currentString(sync *project.GitOpsSync, accessor func(*project.GitOpsSync) *string) string {- if sync == nil {- return ""- }- if p := accessor(sync); p != nil {- return *p- }- return ""-}--// resolveLifecycleEffectiveString computes the value a string field-// will take after an update: a nil update leaves the existing value, a-// non-nil update (including empty string, which clears) overrides it. Used-// to validate the post-update state of a sync record.-func resolveLifecycleEffectiveString(existing string, update *string) string {- if update != nil {- return strings.TrimSpace(*update)- }- return strings.TrimSpace(existing)+ trimmed := strings.TrimSpace(kit.FromPtr(p))+ return kit.Ternary(trimmed == "", nil, &trimmed)}func validateSyncLimits(maxFiles *int, maxTotalSize, maxBinarySize *int64) error {@@ -152,14 +29,3 @@ func validateSyncLimits(maxFiles *int, maxTotalSize, maxBinarySize *int64) error}return nil}--func normalizeSyncMode(mode string) (string, error) {- switch strings.TrimSpace(mode) {- case "", gitops.SyncModeDeploy:- return gitops.SyncModeDeploy, nil- case gitops.SyncModeBackup:- return gitops.SyncModeBackup, nil- default:- return "", common.Classify(common.ErrValidation, &base.FieldError{Field: "mode", Err: fmt.Errorf("unsupported sync mode %q", mode)})- }-}diff --git a/backend/internal/gitops/service.go b/backend/internal/gitops/service.goindex a30501164..81fbe4d10 100644--- a/backend/internal/gitops/service.go+++ b/backend/internal/gitops/service.go@@ -19,6 +19,7 @@ import ("github.com/getarcaneapp/arcane/types/v2/lifecycle""github.com/getarcaneapp/arcane/types/v2/scheduler""github.com/getarcaneapp/arcane/types/v2/user"+ "github.com/italypaleale/francis/builtin/workflow""go.getarcane.app/acfs""go.getarcane.app/kit/pkg""go.getarcane.app/kit/pkg/mapping"@@ -37,261 +38,128 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/pagination""github.com/getarcaneapp/arcane/backend/v2/pkg/projects""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/entityjobs"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs")-type GitOpsSyncService struct {- db *database.DB- repoService *gitrepo.GitRepositoryService- projectService *projectpkg.ProjectService- swarmService *swarm.SwarmService- eventService *event.EventService- settingsService *settings.SettingsService-- // jobs carries the scheduler and app lifecycle context, injected- // post-construction via SetScheduler.- jobs *entityjobs.Registry-- backups *backupRuntimeInternal- sync *gitopssync.Service- backup *backup.Service-}-const (defaultGitSyncTimeout = 5 * time.MinutedefaultMaxSyncFiles = 500defaultMaxSyncTotalSizeMB = 50defaultMaxSyncBinarySizeMB = 10- defaultMaxSyncTotalSize = defaultMaxSyncTotalSizeMB * 1024 * 1024- defaultMaxSyncBinarySize = defaultMaxSyncBinarySizeMB * 1024 * 1024- gitOpsSyncAdmissionScopeInternal = "gitops-sync"+ gitOpsSyncAdmissionScope = "gitops-sync")-// lifecycleConfigInputInternal is the slice of CreateSyncRequest /-// UpdateSyncRequest fields relevant to the pre-deploy lifecycle hook. Both-// request types collapse into the same shape so a single validator handles-// both flows.-type lifecycleConfigInputInternal struct {- targetType *string- scriptPath *string- runnerImage *string- env *string- extraMounts *string- timeoutSec *int- networkMode *string- syncDirectory *bool-}--func (s *GitOpsSyncService) validateLifecycleConfigInternal(ctx context.Context, current *projectpkg.GitOpsSync, in lifecycleConfigInputInternal) error {- lifecycleFieldSet := in.scriptPath != nil || in.runnerImage != nil || in.env != nil || in.extraMounts != nil || in.timeoutSec != nil || in.networkMode != nil- syncDirectoryChanging := in.syncDirectory != nil- targetTypeChanging := in.targetType != nil && strings.TrimSpace(*in.targetType) != resolveLifecycleEffectiveTargetType(current, nil)- effectiveScriptPath := resolveLifecycleEffectiveString(currentString(current, func(c *projectpkg.GitOpsSync) *string { return c.PreDeployScriptPath }), in.scriptPath)+type (+ GitOpsSyncService struct {+ db *database.DB+ repoService *gitrepo.GitRepositoryService+ projectService *projectpkg.ProjectService+ swarmService *swarm.SwarmService+ eventService *event.EventService+ settingsService *settings.SettingsService++ // jobs carries the scheduler and app lifecycle context, injected after construction by SetScheduler.+ jobs *entityjobs.Registry+ engine *flow.Engine+ syncWorkflow *flow.Workflow++ backups *backupRuntime+ sync *gitopssync.Service+ backup *backup.Service+ }++ // syncTarget is the gitops-sync workflow input: the sync one scheduled job runs.+ syncTarget struct {+ EnvironmentID string `json:"environmentId"`+ SyncID string `json:"syncId"`+ }++ // lifecycleConfigInput is the pre-deploy hook slice of a create or update request.+ lifecycleConfigInput struct {+ targetType *string+ scriptPath *string+ runnerImage *string+ env *string+ extraMounts *string+ timeoutSec *int+ networkMode *string+ syncDirectory *bool+ }++ // backupRuntime holds the in-memory coordination for backup syncs.+ backupRuntime struct {+ mu sync.Mutex+ branches map[string]*sync.Mutex+ timers map[string]*time.Timer+ debounce time.Duration+ }+)- // If nothing lifecycle-related is being touched and the resulting state- // has no script configured, there's nothing to validate.- if !lifecycleFieldSet && !syncDirectoryChanging && !targetTypeChanging {- return nil+func (s *GitOpsSyncService) validateLifecycleConfig(ctx context.Context, current *projectpkg.GitOpsSync, in lifecycleConfigInput) error {+ var currentScript, currentRunner *string+ currentTargetType, currentSyncDirectory := "", false+ if current != nil {+ currentScript, currentRunner = current.PreDeployScriptPath, current.PreDeployRunnerImage+ currentTargetType, currentSyncDirectory = strings.TrimSpace(current.TargetType), current.SyncDirectory}- if !lifecycleFieldSet && effectiveScriptPath == "" {- return nil+ currentTargetType = cmp.Or(currentTargetType, "project")+ targetType := cmp.Or(strings.TrimSpace(kit.FromPtr(in.targetType)), currentTargetType)+ scriptPath := strings.TrimSpace(kit.FromPtr(cmp.Or(in.scriptPath, currentScript)))+ invalid := func(field, message string) error {+ return common.Classify(common.ErrValidation, &base.FieldError{Field: field, Err: errors.New(message)})}- // Kill switch only blocks updates that actively touch lifecycle fields;- // toggling unrelated fields on an existing config shouldn't be rejected- // just because the global setting was turned off afterwards.+ lifecycleFieldSet := in.scriptPath != nil || in.runnerImage != nil || in.env != nil || in.extraMounts != nil || in.timeoutSec != nil || in.networkMode != nil+ targetTypeChanging := in.targetType != nil && strings.TrimSpace(*in.targetType) != currentTargetType+ // Nothing lifecycle-related changes, or the result has no script to validate.+ if !lifecycleFieldSet && (scriptPath == "" || (in.syncDirectory == nil && !targetTypeChanging)) {+ return nil+ }+ // The kill switch only blocks edits to lifecycle fields, not unrelated edits to a sync that already has a hook.if lifecycleFieldSet && !s.settingsService.GetBoolSetting(ctx, "lifecycleEnabled", false) {- return common.Classify(- common.ErrValidation,- &base.FieldError{- Field: "preDeployScriptPath",- //nolint:staticcheck // Preserve the existing error message.- Err: errors.New(- "Pre-deploy lifecycle hooks are disabled. An admin must enable lifecycleEnabled in settings before they can be configured.",- ),- },- )+ return invalid("preDeployScriptPath", "Pre-deploy lifecycle hooks are disabled. An admin must enable lifecycleEnabled in settings before they can be configured.")}- if effectiveScriptPath != "" {- if err := s.validateLifecycleScriptConfigInternal(ctx, current, in, effectiveScriptPath); err != nil {- return err- }+ // scriptPath is a POSIX repo path, so path.IsAbs behaves the same on Windows.+ cleanedScript := filepath.ToSlash(filepath.Clean(scriptPath))+ switch {+ case scriptPath == "":+ case targetType == "swarm_stack":+ return invalid("preDeployScriptPath", "Pre-deploy lifecycle hooks are only supported for project syncs.")+ case len(scriptPath) > 256:+ return invalid("preDeployScriptPath", "Script path must be 256 characters or fewer.")+ case path.IsAbs(filepath.ToSlash(scriptPath)):+ return invalid("preDeployScriptPath", "Script path must be relative to the project directory.")+ case cleanedScript == ".." || strings.HasPrefix(cleanedScript, "../"):+ return invalid("preDeployScriptPath", "Script path must not escape the project directory.")+ case strings.TrimSpace(kit.FromPtr(cmp.Or(in.runnerImage, currentRunner))) == "" && strings.TrimSpace(s.settingsService.GetStringSetting(ctx, "lifecycleDefaultRunnerImage", "")) == "":+ return invalid("preDeployRunnerImage", "Runner image is required when a script path is set.")+ case !kit.FromPtr(cmp.Or(in.syncDirectory, ¤tSyncDirectory)):+ return invalid("preDeployScriptPath", "Pre-deploy script requires \"Sync entire directory\" so the script is included in the synced files.")}if in.timeoutSec != nil {+ maxTimeoutSec := s.settingsService.GetIntSetting(ctx, "lifecycleMaxTimeoutSec", lifecycle.DefaultMaxTimeoutSec)if *in.timeoutSec < 1 {- return common.Classify(- common.ErrValidation,- &base.FieldError{- Field: "preDeployTimeoutSec",- //nolint:staticcheck // Preserve the existing error message.- Err: errors.New(- "Timeout must be at least 1 second.",- ),- },- )+ return invalid("preDeployTimeoutSec", "Timeout must be at least 1 second.")}- maxTimeoutSec := s.settingsService.GetIntSetting(ctx, "lifecycleMaxTimeoutSec", lifecycle.DefaultMaxTimeoutSec)if maxTimeoutSec > 0 && *in.timeoutSec > maxTimeoutSec {- return common.Classify(- common.ErrValidation,- &base.FieldError{- Field: "preDeployTimeoutSec",- //nolint:staticcheck // Preserve the existing error message.- Err: fmt.Errorf(- "Timeout %ds exceeds the lifecycleMaxTimeoutSec setting (%ds).",- *in.timeoutSec,- maxTimeoutSec,- ),- },- )+ return invalid("preDeployTimeoutSec", fmt.Sprintf("Timeout %ds exceeds the lifecycleMaxTimeoutSec setting (%ds).", *in.timeoutSec, maxTimeoutSec))}}-if _, err := projectpkg.ParseEnvText(in.env); err != nil {- return common.Classify(common.ErrValidation, &base.FieldError{Field: "preDeployEnv", Err: errors.New(err.Error())})+ return invalid("preDeployEnv", err.Error())}if _, err := projectpkg.ParseExtraMountsText(in.extraMounts); err != nil {- return common.Classify(common.ErrValidation, &base.FieldError{Field: "preDeployExtraMounts", Err: errors.New(err.Error())})+ return invalid("preDeployExtraMounts", err.Error())}-return nil}-func (s *GitOpsSyncService) validateLifecycleScriptConfigInternal(ctx context.Context, current *projectpkg.GitOpsSync, in lifecycleConfigInputInternal, scriptPath string) error {- if resolveLifecycleEffectiveTargetType(current, in.targetType) == "swarm_stack" {- return common.Classify(- common.ErrValidation,- &base.FieldError{- Field: "preDeployScriptPath",- //nolint:staticcheck // Preserve the existing error message.- Err: errors.New(- "Pre-deploy lifecycle hooks are only supported for project syncs.",- ),- },- )- }-- if len(scriptPath) > 256 {- return common.Classify(- common.ErrValidation,- &base.FieldError{- Field: "preDeployScriptPath",- //nolint:staticcheck // Preserve the existing error message.- Err: errors.New(- "Script path must be 256 characters or fewer.",- ),- },- )- }-- // scriptPath is a POSIX repo path, not a host path; use path.IsAbs so the- // check behaves the same on Windows-based contributor machines.- if path.IsAbs(filepath.ToSlash(scriptPath)) {- return common.Classify(- common.ErrValidation,- &base.FieldError{- Field: "preDeployScriptPath",- //nolint:staticcheck // Preserve the existing error message.- Err: errors.New(- "Script path must be relative to the project directory.",- ),- },- )- }- cleaned := filepath.ToSlash(filepath.Clean(scriptPath))- if cleaned == ".." || strings.HasPrefix(cleaned, "../") {- return common.Classify(- common.ErrValidation,- &base.FieldError{- Field: "preDeployScriptPath",- //nolint:staticcheck // Preserve the existing error message.- Err: errors.New(- "Script path must not escape the project directory.",- ),- },- )- }-- effectiveRunnerImage := resolveLifecycleEffectiveString(currentString(current, func(c *projectpkg.GitOpsSync) *string { return c.PreDeployRunnerImage }), in.runnerImage)- defaultRunnerImage := strings.TrimSpace(s.settingsService.GetStringSetting(ctx, "lifecycleDefaultRunnerImage", ""))- if effectiveRunnerImage == "" && defaultRunnerImage == "" {- return common.Classify(- common.ErrValidation,- &base.FieldError{- Field: "preDeployRunnerImage",- //nolint:staticcheck // Preserve the existing error message.- Err: errors.New(- "Runner image is required when a script path is set.",- ),- },- )- }-- if !resolveEffectiveSyncDirectory(current, in.syncDirectory) {- return common.Classify(- common.ErrValidation,- &base.FieldError{- Field: "preDeployScriptPath",- //nolint:staticcheck // Preserve the existing error message.- Err: errors.New(- "Pre-deploy script requires \"Sync entire directory\" so the script is included in the synced files.",- ),- },- )- }-- return nil-}--// applyLifecycleFieldsToSyncInternal copies lifecycle config from a Create-// request into a new GitOpsSync row before insert. Strings are trimmed;-// empty strings remain unset (nil pointer) except for fields that have a-// non-null default at the DB level.-func applyLifecycleFieldsToSyncInternal(syncRecord *projectpkg.GitOpsSync, in lifecycleConfigInputInternal) {- syncRecord.PreDeployScriptPath = nullableTrimmedString(in.scriptPath)- syncRecord.PreDeployRunnerImage = nullableTrimmedString(in.runnerImage)- syncRecord.PreDeployEnv = nullableTrimmedString(in.env)- syncRecord.PreDeployExtraMounts = nullableTrimmedString(in.extraMounts)- if in.timeoutSec != nil {- syncRecord.PreDeployTimeoutSec = *in.timeoutSec- } else {- syncRecord.PreDeployTimeoutSec = lifecycle.DefaultTimeoutSec- }- if mode := normalizeLifecycleNetworkMode(in.networkMode); mode != "" {- syncRecord.PreDeployNetworkMode = mode- }-}--// addLifecycleUpdatesInternal appends lifecycle field updates to the GORM-// updates map. nil pointers leave the field unchanged; non-nil empty strings-// clear nullable fields to NULL and reset fixed-default fields to their-// default value.-func addLifecycleUpdatesInternal(updates map[string]any, in lifecycleConfigInputInternal) {- if in.scriptPath != nil {- updates["pre_deploy_script_path"] = nullableUpdateStringValue(in.scriptPath)- }- if in.runnerImage != nil {- updates["pre_deploy_runner_image"] = nullableUpdateStringValue(in.runnerImage)- }- if in.env != nil {- updates["pre_deploy_env"] = nullableUpdateStringValue(in.env)- }- if in.extraMounts != nil {- updates["pre_deploy_extra_mounts"] = nullableUpdateStringValue(in.extraMounts)- }- if in.timeoutSec != nil {- updates["pre_deploy_timeout_sec"] = *in.timeoutSec- }- if in.networkMode != nil {- updates["pre_deploy_network_mode"] = normalizeLifecycleNetworkMode(in.networkMode)- }-}-func NewGitOpsSyncService(db *database.DB,repoService *gitrepo.GitRepositoryService,@@ -307,107 +175,114 @@ func NewGitOpsSyncService(swarmService: swarmService,eventService: eventService,settingsService: settingsService,- jobs: entityjobs.New(entityjobs.GitOpsSyncJobPrefix, gitOpsSyncAdmissionScopeInternal),- backups: newBackupRuntimeInternal(),+ jobs: entityjobs.New(entityjobs.GitOpsSyncJobPrefix, gitOpsSyncAdmissionScope),+ backups: &backupRuntime{branches: map[string]*sync.Mutex{}, timers: map[string]*time.Timer{}, debounce: backup.DefaultBackupSaveDebounce},}- s.sync = gitopssync.New(db, repoService, projectService, swarmService, eventService, s.getEffectiveSyncLimits, s.logSyncError, s.unregisterSyncJobInternal)- s.backup = backup.New(db, repoService, projectService, eventService, s.getEffectiveSyncLimits, s.logSyncError, s.backups.branchLock)+ logError := func(ctx context.Context, syncRecord *projectpkg.GitOpsSync, actor user.Actor, errorMsg string) {+ _, _ = s.eventService.CreateEvent(ctx, event.CreateEventRequest{+ Type: event.EventTypeGitSyncError,+ Severity: event.EventSeverityError,+ Title: "Git sync failed",+ Description: fmt.Sprintf("Failed to sync '%s': %s", syncRecord.Name, errorMsg),+ ResourceType: new("git_sync"),+ ResourceID: new(syncRecord.ID),+ ResourceName: new(syncRecord.Name),+ UserID: new(actor.ID),+ Username: new(actor.Username),+ EnvironmentID: new(syncRecord.EnvironmentID),+ })+ }+ s.sync = gitopssync.New(db, repoService, projectService, swarmService, eventService, s.getEffectiveSyncLimits, logError, s.jobs.Unregister)+ s.backup = backup.New(db, repoService, projectService, eventService, s.getEffectiveSyncLimits, logError, s.backups.branchLock)return s}-// SetScheduler injects the job scheduler and the app lifecycle context. It must be-// called during bootstrap (after the service graph is built) before any per-sync-// jobs are registered. The lifecycle context is used for background sync kicks so-// they outlive the request/bootstrap goroutine that triggered them.-func (s *GitOpsSyncService) SetScheduler(ctx context.Context, jobScheduler scheduler.DynamicScheduler, admissionGate *runs.Admission) error {- return s.jobs.SetScheduler(ctx, jobScheduler, admissionGate)+// RegisterWorkflows defines the gitops-sync workflow each scheduled sync runs. A sync step retries transient+// clone failures; a retry replays the run's pinned revision.+func (s *GitOpsSyncService) RegisterWorkflows(engine *flow.Engine) error {+ s.engine = engine+ syncWorkflow, err := engine.Define(flow.Definition{+ Name: "gitops-sync",+ Version: 1,+ Fingerprint: "f693aa29ca10c373e1b9d39ab25dee6e396ced7cea2bcfef76f41dba571d576a",+ Concurrency: 16,+ Timeout: time.Hour,+ Steps: []workflow.StepSpec{+ workflow.Step("sync", engine.Handler(func(ctx context.Context, t flow.Task) (any, error) {+ var target syncTarget+ if err := t.Payload(&target); err != nil {+ return nil, err+ }+ // The row is re-read on every run, so a deleted sync unregisters itself and a disabled one skips.+ syncRecord, err := s.getSyncByID(ctx, target.EnvironmentID, target.SyncID, false)+ if errors.Is(err, common.ErrNotFound) {+ slog.InfoContext(ctx, "gitops auto-sync job unregistering; sync no longer exists", "syncId", target.SyncID)+ s.jobs.Unregister(ctx, target.SyncID)+ return scheduler.Outcome{Status: scheduler.Skipped}, nil+ }+ if err != nil {+ return nil, common.Classify(common.ErrUnavailable, err)+ }+ if !syncRecord.AutoSync {+ return scheduler.Outcome{Status: scheduler.Skipped}, nil+ }+ if previous, ok := jobcontext.Run(ctx); ok {+ if outcome := jobcontext.ConfirmedTarget(previous, target.SyncID); outcome.Status == scheduler.Succeeded {+ return outcome, nil+ }+ }+ if runningErr := jobcontext.Progress(ctx, scheduler.TargetOutcome{ID: target.SyncID, Status: scheduler.Running}); runningErr != nil {+ return nil, runningErr+ }+ result, err := s.PerformSync(ctx, target.EnvironmentID, target.SyncID, user.SystemUser)+ if err != nil {+ slog.ErrorContext(ctx, "gitops auto-sync run failed", "syncId", target.SyncID, "error", err)+ return nil, err+ }+ if !result.Success {+ return scheduler.Outcome{Status: scheduler.Skipped, Message: result.Message}, nil+ }+ if succeededErr := jobcontext.Progress(ctx, scheduler.TargetOutcome{ID: target.SyncID, Status: scheduler.Succeeded}); succeededErr != nil {+ return nil, succeededErr+ }+ return scheduler.Outcome{Status: scheduler.Succeeded, Message: result.Message}, nil+ }), workflow.WithMaxAttempts(3), workflow.WithRetryBackoff(30*time.Second, 2*time.Minute)),+ },+ })+ s.syncWorkflow = syncWorkflow+ return err}-// runScheduledSyncInternal is the body of a scheduled sync fire. It re-reads the-// sync each run so a row toggled to AutoSync=false self-cancels. If the row is gone-// (e.g. deleted out-of-band via raw SQL), the job unregisters itself instead of-// firing forever; transient load errors are skipped and retried next tick.-func (s *GitOpsSyncService) runScheduledSyncInternal(ctx context.Context, environmentID, syncID string) (scheduler.Outcome, error) {- syncRecord, err := s.getSyncRecordByIDInternal(ctx, environmentID, syncID)- if err != nil {- if errors.Is(err, common.ErrNotFound) {- slog.InfoContext(ctx, "gitops auto-sync job unregistering; sync no longer exists", "syncId", syncID)- s.unregisterSyncJobInternal(ctx, syncID)- return scheduler.Outcome{Status: scheduler.Skipped}, nil- }- slog.DebugContext(ctx, "gitops auto-sync skipped; failed to load sync", "syncId", syncID, "error", err)- return scheduler.Outcome{}, err- }- if !syncRecord.AutoSync {- return scheduler.Outcome{Status: scheduler.Skipped}, nil- }- if previous, ok := jobcontext.Run(ctx); ok {- outcome := jobcontext.ConfirmedTarget(previous, syncID)- if outcome.Status == scheduler.Succeeded {- return outcome, nil- }- }- if progressErr := jobcontext.Progress(ctx, scheduler.TargetOutcome{ID: syncID, Status: scheduler.Running}); progressErr != nil {- return scheduler.Outcome{}, progressErr- }- result, err := s.PerformSync(ctx, environmentID, syncID, user.SystemUser)- if err != nil {- slog.ErrorContext(ctx, "gitops auto-sync run failed", "syncId", syncID, "error", err)- return scheduler.Outcome{}, err- }- if !result.Success {- return scheduler.Outcome{Status: scheduler.Skipped, Message: result.Message}, nil- }- if completeProgressErr := jobcontext.Progress(ctx, scheduler.TargetOutcome{ID: syncID, Status: scheduler.Succeeded}); completeProgressErr != nil {- return scheduler.Outcome{}, completeProgressErr- }- return scheduler.Outcome{Status: scheduler.Succeeded, Message: result.Message}, nil+// SetScheduler injects the scheduler, the admission gate, and the app lifecycle context background syncs run on.+// It must run during bootstrap, before any per-sync job is registered.+func (s *GitOpsSyncService) SetScheduler(ctx context.Context, jobScheduler scheduler.DynamicScheduler, admissionGate *runs.Admission) error {+ return s.jobs.SetScheduler(ctx, jobScheduler, admissionGate)}-// registerSyncJobInternal schedules a single sync on a fixed "@every Nm"-// interval. The run body re-reads the sync each fire, so a row deleted or-// toggled to AutoSync=false self-cancels cleanly, and delegates to PerformSync,-// which owns its own timeout and the per-sync in-flight guard.-func (s *GitOpsSyncService) registerSyncJobInternal(ctx context.Context, syncID, environmentID string, intervalMinutes int) {+// registerSyncJob schedules one sync every intervalMinutes as a gitops-sync workflow run.+func (s *GitOpsSyncService) registerSyncJob(ctx context.Context, syncID, environmentID string, intervalMinutes int) {schedule := fmt.Sprintf("@every %dm", max(intervalMinutes, 1))- s.jobs.Register(ctx, syncID,- func(_ context.Context) string { return schedule },- func(ctx context.Context) (scheduler.Outcome, error) {- return s.runScheduledSyncInternal(ctx, environmentID, syncID)- },- func(ctx context.Context, previous scheduler.Run) (scheduler.Outcome, error) {- outcome := jobcontext.ConfirmedTarget(previous, syncID)- if outcome.Status == scheduler.Succeeded {- return outcome, nil- }- record, err := s.getSyncRecordByIDInternal(ctx, environmentID, syncID)- if err != nil {- return outcome, err- }- return gitopssync.ReconcileRevision(ctx, previous, record, outcome)- },- )-}--func (s *GitOpsSyncService) unregisterSyncJobInternal(ctx context.Context, syncID string) {- s.jobs.Unregister(ctx, syncID)+ s.jobs.Add(ctx, &flow.Job{+ Engine: s.engine,+ Workflow: s.syncWorkflow,+ JobName: s.jobs.JobName(syncID),+ Payload: syncTarget{EnvironmentID: environmentID, SyncID: syncID},+ ScheduleFn: func(context.Context) string { return schedule },+ })}-// kickSyncInternal runs a sync once in the background on the app lifecycle context.-// Used when auto-sync is freshly enabled or when a sync is overdue at startup, so-// the first run does not wait a full interval.-func (s *GitOpsSyncService) kickSyncInternal(ctx context.Context, syncID string) {+// kickSync submits one run of a sync's job now, so it does not wait for its next interval.+func (s *GitOpsSyncService) kickSync(ctx context.Context, syncID, trigger string) {if !s.jobs.Enabled() {return}- if _, err := s.jobs.Scheduler().Submit(ctx, scheduler.Request{JobID: s.jobs.JobName(syncID), EnvironmentID: "0", Trigger: "startup"}); err != nil {- slog.ErrorContext(ctx, "gitops immediate sync admission failed", "syncId", syncID, "error", err)+ if _, err := s.jobs.Scheduler().Submit(ctx, scheduler.Request{JobID: s.jobs.JobName(syncID), EnvironmentID: "0", Trigger: trigger}); err != nil {+ slog.ErrorContext(ctx, "gitops sync admission failed", "syncId", syncID, "trigger", trigger, "error", err)}}-// RegisterAutoSyncJobsOnStartup registers a dynamic job for every auto-sync-enabled-// sync and kicks an immediate run for any that are overdue. This replaces the old-// global polling job so existing syncs keep running after upgrade.+// RegisterAutoSyncJobsOnStartup registers a job for every auto-sync and kicks the overdue ones, and backups+// with pending changes, right away.func (s *GitOpsSyncService) RegisterAutoSyncJobsOnStartup(ctx context.Context) {if !s.jobs.Enabled() {return@@ -419,54 +294,33 @@ func (s *GitOpsSyncService) RegisterAutoSyncJobsOnStartup(ctx context.Context) {slog.ErrorContext(ctx, "Failed to load auto-sync jobs on startup", "error", err)return}- for i := range syncs {- syncRecord := syncs[i]- s.registerSyncJobInternal(ctx, syncRecord.ID, syncRecord.EnvironmentID, syncRecord.SyncInterval)- if isGitOpsSyncOverdue(&syncRecord) || (syncRecord.Mode == gitops.SyncModeBackup && syncRecord.BackupPending) {- s.kickSyncInternal(ctx, syncRecord.ID)+ for _, syncRecord := range syncs {+ s.registerSyncJob(ctx, syncRecord.ID, syncRecord.EnvironmentID, syncRecord.SyncInterval)+ overdue := syncRecord.LastSyncAt == nil || time.Since(*syncRecord.LastSyncAt) > time.Duration(max(syncRecord.SyncInterval, 1))*time.Minute+ if overdue || (syncRecord.Mode == gitops.SyncModeBackup && syncRecord.BackupPending) {+ s.kickSync(ctx, syncRecord.ID, "startup")}}slog.InfoContext(ctx, "Registered gitops auto-sync jobs on startup", "count", len(syncs))}func (s *GitOpsSyncService) getEnvironmentSyncLimits(ctx context.Context) (int, int64, int64) {- if s.settingsService == nil {- return defaultMaxSyncFiles, defaultMaxSyncTotalSize, defaultMaxSyncBinarySize- }-cfg := s.settingsService.GetSettingsOrDefaults(ctx)maxFiles := normalizeSyncLimitSetting(kit.ParseOrDefault(cfg.GitSyncMaxFiles.Value, defaultMaxSyncFiles, strconv.Atoi), defaultMaxSyncFiles)maxTotalSizeMB := normalizeSyncLimitSetting(kit.ParseOrDefault(cfg.GitSyncMaxTotalSizeMb.Value, defaultMaxSyncTotalSizeMB, strconv.Atoi), defaultMaxSyncTotalSizeMB)maxBinarySizeMB := normalizeSyncLimitSetting(kit.ParseOrDefault(cfg.GitSyncMaxBinarySizeMb.Value, defaultMaxSyncBinarySizeMB, strconv.Atoi), defaultMaxSyncBinarySizeMB)-- return maxFiles, megabytesToBytes(maxTotalSizeMB), megabytesToBytes(maxBinarySizeMB)+ return maxFiles, int64(maxTotalSizeMB) << 20, int64(maxBinarySizeMB) << 20}+// getEffectiveSyncLimits returns the sync's own limits, except where an environment override pins the setting.func (s *GitOpsSyncService) getEffectiveSyncLimits(ctx context.Context, syncRecord *projectpkg.GitOpsSync) (int, int64, int64) {- environmentMaxFiles, environmentMaxTotalSize, environmentMaxBinarySize := s.getEnvironmentSyncLimits(ctx)+ maxFiles, maxTotalSize, maxBinarySize := s.getEnvironmentSyncLimits(ctx)if syncRecord == nil {- return environmentMaxFiles, environmentMaxTotalSize, environmentMaxBinarySize+ return maxFiles, maxTotalSize, maxBinarySize}-- maxFiles := syncRecord.MaxSyncFiles- maxTotalSize := syncRecord.MaxSyncTotalSize- maxBinarySize := syncRecord.MaxSyncBinarySize-- if s.gitSyncLimitEnvOverrideActiveInternal("gitSyncMaxFiles") {- maxFiles = environmentMaxFiles- }- if s.gitSyncLimitEnvOverrideActiveInternal("gitSyncMaxTotalSizeMb") {- maxTotalSize = environmentMaxTotalSize- }- if s.gitSyncLimitEnvOverrideActiveInternal("gitSyncMaxBinarySizeMb") {- maxBinarySize = environmentMaxBinarySize- }-- return maxFiles, maxTotalSize, maxBinarySize-}--func (s *GitOpsSyncService) gitSyncLimitEnvOverrideActiveInternal(key string) bool {- return s.settingsService != nil && s.settingsService.IsEnvOverrideActive(key)+ return kit.Ternary(s.settingsService.IsEnvOverrideActive("gitSyncMaxFiles"), maxFiles, syncRecord.MaxSyncFiles),+ kit.Ternary(s.settingsService.IsEnvOverrideActive("gitSyncMaxTotalSizeMb"), maxTotalSize, syncRecord.MaxSyncTotalSize),+ kit.Ternary(s.settingsService.IsEnvOverrideActive("gitSyncMaxBinarySizeMb"), maxBinarySize, syncRecord.MaxSyncBinarySize)}func (s *GitOpsSyncService) GetSyncsPaginated(ctx context.Context, environmentID string, params pagination.QueryParams) ([]gitops.GitOpsSync, pagination.Response, gitops.SyncCounts, error) {@@ -488,10 +342,22 @@ func (s *GitOpsSyncService) GetSyncsPaginated(ctx context.Context, environmentIDq = pagination.ApplyFilter(q, "repository_id", params.Filters["repositoryId"])q = pagination.ApplyFilter(q, "project_id", params.Filters["projectId"])- counts, err := s.getFilteredSyncCounts(q)- if err != nil {+ var total, active, successful, backups int64+ if err := errors.Join(+ q.Session(&gorm.Session{}).Count(&total).Error,+ q.Session(&gorm.Session{}).Where("auto_sync = ?", true).Count(&active).Error,+ q.Session(&gorm.Session{}).Where("last_sync_status = ?", "success").Count(&successful).Error,+ q.Session(&gorm.Session{}).Where("mode = ?", gitops.SyncModeBackup).Count(&backups).Error,+ ); err != nil {return nil, pagination.Response{}, gitops.SyncCounts{}, fmt.Errorf("failed to get sync counts: %w", err)}+ counts := gitops.SyncCounts{+ TotalSyncs: int(total),+ ActiveSyncs: int(active),+ SuccessfulSyncs: int(successful),+ DeploySyncs: int(total - backups),+ BackupSyncs: int(backups),+ }paginationResp, err := pagination.PaginateAndSortDB(params, q.Preload("Repository").Preload("Project"), &syncs)if err != nil {@@ -506,38 +372,8 @@ func (s *GitOpsSyncService) GetSyncsPaginated(ctx context.Context, environmentIDreturn out, paginationResp, counts, nil}-func (s *GitOpsSyncService) getFilteredSyncCounts(query *gorm.DB) (gitops.SyncCounts, error) {- var totalSyncs int64- if err := query.Session(&gorm.Session{}).Count(&totalSyncs).Error; err != nil {- return gitops.SyncCounts{}, err- }-- var activeSyncs int64- if err := query.Session(&gorm.Session{}).Where("auto_sync = ?", true).Count(&activeSyncs).Error; err != nil {- return gitops.SyncCounts{}, err- }-- var successfulSyncs int64- if err := query.Session(&gorm.Session{}).Where("last_sync_status = ?", "success").Count(&successfulSyncs).Error; err != nil {- return gitops.SyncCounts{}, err- }-- var backupSyncs int64- if err := query.Session(&gorm.Session{}).Where("mode = ?", gitops.SyncModeBackup).Count(&backupSyncs).Error; err != nil {- return gitops.SyncCounts{}, err- }-- return gitops.SyncCounts{- TotalSyncs: int(totalSyncs),- ActiveSyncs: int(activeSyncs),- SuccessfulSyncs: int(successfulSyncs),- DeploySyncs: int(totalSyncs - backupSyncs),- BackupSyncs: int(backupSyncs),- }, nil-}-func (s *GitOpsSyncService) GetSyncByID(ctx context.Context, environmentID, id string) (*projectpkg.GitOpsSync, error) {- syncRecord, err := s.getSyncByIDInternal(ctx, environmentID, id, true)+ syncRecord, err := s.getSyncByID(ctx, environmentID, id, true)if err != nil {if errors.Is(err, common.ErrNotFound) {slog.WarnContext(ctx, "GitOps sync not found", "syncId", id, "environmentId", environmentID)@@ -549,7 +385,7 @@ func (s *GitOpsSyncService) GetSyncByID(ctx context.Context, environmentID, id sreturn syncRecord, nil}-func (s *GitOpsSyncService) getSyncByIDInternal(ctx context.Context, environmentID, id string, preloadAssociations bool) (*projectpkg.GitOpsSync, error) {+func (s *GitOpsSyncService) getSyncByID(ctx context.Context, environmentID, id string, preloadAssociations bool) (*projectpkg.GitOpsSync, error) {var syncRecord projectpkg.GitOpsSyncq := s.db.WithContext(ctx).Where("id = ?", id)if preloadAssociations {@@ -567,21 +403,16 @@ func (s *GitOpsSyncService) getSyncByIDInternal(ctx context.Context, environmentreturn &syncRecord, nil}-func (s *GitOpsSyncService) getSyncRecordByIDInternal(ctx context.Context, environmentID, id string) (*projectpkg.GitOpsSync, error) {- return s.getSyncByIDInternal(ctx, environmentID, id, false)-}-func (s *GitOpsSyncService) CreateSync(ctx context.Context, environmentID string, req gitops.CreateSyncRequest, actor user.Actor) (*projectpkg.GitOpsSync, error) {slog.InfoContext(ctx, "Creating GitOps sync", "environmentId", environmentID, "name", req.Name, "repositoryId", req.RepositoryID)- mode, err := normalizeSyncMode(req.Mode)- if err != nil {- return nil, err- }- if mode == gitops.SyncModeDeploy && req.HasBackupOptions() {+ mode := cmp.Or(strings.TrimSpace(req.Mode), gitops.SyncModeDeploy)+ switch {+ case mode != gitops.SyncModeDeploy && mode != gitops.SyncModeBackup:+ return nil, common.Classify(common.ErrValidation, &base.FieldError{Field: "mode", Err: fmt.Errorf("unsupported sync mode %q", req.Mode)})+ case mode == gitops.SyncModeDeploy && req.HasBackupOptions():return nil, common.Classify(common.ErrValidation, &base.FieldError{Field: "mode", Err: errors.New("backup options require mode \"backup\"")})- }- if mode == gitops.SyncModeDeploy && strings.TrimSpace(req.ComposePath) == "" {+ case mode == gitops.SyncModeDeploy && strings.TrimSpace(req.ComposePath) == "":return nil, common.Classify(common.ErrValidation, &base.FieldError{Field: "composePath", Err: errors.New("compose path is required")})}@@ -592,61 +423,10 @@ func (s *GitOpsSyncService) CreateSync(ctx context.Context, environmentID string}slog.InfoContext(ctx, "Found repository for GitOps sync", "repositoryId", req.RepositoryID, "repositoryName", repo.Name)- // Store the project name - use sync name if project name not provided- projectName := cmp.Or(req.ProjectName, req.Name)-- defaultMaxFiles, defaultMaxTotalSize, defaultMaxBinarySize := s.getEnvironmentSyncLimits(ctx)-- syncRecord := projectpkg.GitOpsSync{- Name: req.Name,- EnvironmentID: environmentID,- RepositoryID: req.RepositoryID,- Branch: req.Branch,- ComposePath: req.ComposePath,- TargetType: req.TargetType,- ProjectName: projectName,- ProjectID: nil, // Will be set during first sync- Mode: mode,- AutoSync: false,- SyncInterval: 60,- SyncDirectory: false, // Default to single-file sync- MaxSyncFiles: defaultMaxFiles,- MaxSyncTotalSize: defaultMaxTotalSize,- MaxSyncBinarySize: defaultMaxBinarySize,- BackupOnSave: true,- }-- linkProjectID := strings.TrimSpace(req.ProjectID)-- if req.AutoSync != nil {- syncRecord.AutoSync = *req.AutoSync- }- if req.SyncInterval != nil {- syncRecord.SyncInterval = *req.SyncInterval+ if limitsErr := validateSyncLimits(req.MaxSyncFiles, req.MaxSyncTotalSize, req.MaxSyncBinarySize); limitsErr != nil {+ return nil, limitsErr}- if req.SyncDirectory != nil {- syncRecord.SyncDirectory = *req.SyncDirectory- }- if req.PullImageAfterSync != nil {- syncRecord.PullImageAfterSync = *req.PullImageAfterSync- }- if req.RedeployAfterSync != nil {- syncRecord.RedeployAfterSync = *req.RedeployAfterSync- }- if validateSyncLimitsErr := validateSyncLimits(req.MaxSyncFiles, req.MaxSyncTotalSize, req.MaxSyncBinarySize); validateSyncLimitsErr != nil {- return nil, validateSyncLimitsErr- }- if req.MaxSyncFiles != nil {- syncRecord.MaxSyncFiles = *req.MaxSyncFiles- }- if req.MaxSyncTotalSize != nil {- syncRecord.MaxSyncTotalSize = *req.MaxSyncTotalSize- }- if req.MaxSyncBinarySize != nil {- syncRecord.MaxSyncBinarySize = *req.MaxSyncBinarySize- }-- lifecycleCfg := lifecycleConfigInputInternal{+ lifecycleCfg := lifecycleConfigInput{targetType: &req.TargetType,scriptPath: req.PreDeployScriptPath,runnerImage: req.PreDeployRunnerImage,@@ -656,25 +436,49 @@ func (s *GitOpsSyncService) CreateSync(ctx context.Context, environmentID stringnetworkMode: req.PreDeployNetworkMode,syncDirectory: req.SyncDirectory,}- if validateLifecycleConfigErr := s.validateLifecycleConfigInternal(ctx, nil, lifecycleCfg); validateLifecycleConfigErr != nil {- return nil, validateLifecycleConfigErr+ if lifecycleErr := s.validateLifecycleConfig(ctx, nil, lifecycleCfg); lifecycleErr != nil {+ return nil, lifecycleErr}- applyLifecycleFieldsToSyncInternal(&syncRecord, lifecycleCfg)- adoptedProject, err := s.insertSyncRecordInternal(ctx, &syncRecord, req, mode, linkProjectID)+ defaultMaxFiles, defaultMaxTotalSize, defaultMaxBinarySize := s.getEnvironmentSyncLimits(ctx)+ syncRecord := projectpkg.GitOpsSync{+ Name: req.Name,+ EnvironmentID: environmentID,+ RepositoryID: req.RepositoryID,+ Branch: req.Branch,+ ComposePath: req.ComposePath,+ TargetType: req.TargetType,+ ProjectName: cmp.Or(req.ProjectName, req.Name),+ Mode: mode,+ AutoSync: kit.FromPtr(req.AutoSync),+ SyncInterval: *cmp.Or(req.SyncInterval, new(60)),+ SyncDirectory: kit.FromPtr(req.SyncDirectory),+ PullImageAfterSync: kit.FromPtr(req.PullImageAfterSync),+ RedeployAfterSync: kit.FromPtr(req.RedeployAfterSync),+ MaxSyncFiles: *cmp.Or(req.MaxSyncFiles, &defaultMaxFiles),+ MaxSyncTotalSize: *cmp.Or(req.MaxSyncTotalSize, &defaultMaxTotalSize),+ MaxSyncBinarySize: *cmp.Or(req.MaxSyncBinarySize, &defaultMaxBinarySize),+ BackupOnSave: true,+ PreDeployScriptPath: nullableTrimmedString(req.PreDeployScriptPath),+ PreDeployRunnerImage: nullableTrimmedString(req.PreDeployRunnerImage),+ PreDeployEnv: nullableTrimmedString(req.PreDeployEnv),+ PreDeployExtraMounts: nullableTrimmedString(req.PreDeployExtraMounts),+ PreDeployTimeoutSec: *cmp.Or(req.PreDeployTimeoutSec, new(lifecycle.DefaultTimeoutSec)),+ PreDeployNetworkMode: cmp.Or(strings.TrimSpace(kit.FromPtr(req.PreDeployNetworkMode)), "none"),+ }++ adoptedProject, err := s.insertSyncRecord(ctx, &syncRecord, req, mode)if err != nil {return nil, err}slog.InfoContext(ctx, "GitOps sync created successfully", "syncId", syncRecord.ID, "name", syncRecord.Name)-if adoptedProject != nil {adoptedProject.GitOpsManagedBy = &syncRecord.ID- if ensureGitOpsProjectLinkedErr := s.projectService.EnsureGitOpsProjectLinked(ctx, &syncRecord, adoptedProject); ensureGitOpsProjectLinkedErr != nil {- return nil, fmt.Errorf("failed to link existing project: %w", ensureGitOpsProjectLinkedErr)+ if linkErr := s.projectService.EnsureGitOpsProjectLinked(ctx, &syncRecord, adoptedProject); linkErr != nil {+ return nil, fmt.Errorf("failed to link existing project: %w", linkErr)}}- // Log event_, _ = s.eventService.CreateEvent(ctx, event.CreateEventRequest{Type: event.EventTypeGitSyncCreate,Severity: event.EventSeveritySuccess,@@ -688,60 +492,44 @@ func (s *GitOpsSyncService) CreateSync(ctx context.Context, environmentID stringEnvironmentID: new(syncRecord.EnvironmentID),})- if _, performSyncErr := s.PerformSync(ctx, syncRecord.EnvironmentID, syncRecord.ID, actor); performSyncErr != nil {- slog.ErrorContext(ctx, "Failed to perform initial sync after creation", "syncId", syncRecord.ID, "error", performSyncErr)- // Don't fail the entire creation - the sync config exists and can be retried+ // The sync exists even when its first run fails, so that failure can be retried rather than failing creation.+ if _, syncErr := s.PerformSync(ctx, syncRecord.EnvironmentID, syncRecord.ID, actor); syncErr != nil {+ slog.ErrorContext(ctx, "Failed to perform initial sync after creation", "syncId", syncRecord.ID, "error", syncErr)}-- // Register the recurring job if auto-sync is on. The initial sync above already- // ran once, so no extra kick is needed here.+ // The initial sync above already ran, so the job needs no kick.if syncRecord.AutoSync {- s.registerSyncJobInternal(ctx, syncRecord.ID, syncRecord.EnvironmentID, syncRecord.SyncInterval)+ s.registerSyncJob(ctx, syncRecord.ID, syncRecord.EnvironmentID, syncRecord.SyncInterval)}-return s.GetSyncByID(ctx, "", syncRecord.ID)}-// prepareDeployProjectLinkInternal validates that an existing project can be-// adopted by a deploy sync: it must exist, not be deployed from Git already,-// and not be backed up to Git.-func (s *GitOpsSyncService) prepareDeployProjectLinkInternal(ctx context.Context, tx *gorm.DB, req gitops.CreateSyncRequest) (*projectpkg.Project, error) {- if strings.TrimSpace(req.TargetType) == "swarm_stack" {- return nil, common.Classify(common.ErrValidation, &base.FieldError{Field: "projectId", Err: errors.New("an existing project cannot be linked to a swarm stack sync")})- }- project, err := projectpkg.LockProjectForSync(tx, req.ProjectID)- if err != nil {- return nil, err- }- if project.GitOpsManagedBy != nil && strings.TrimSpace(*project.GitOpsManagedBy) != "" {- return nil, common.Classify(common.ErrConflict, errors.New("project is already deployed from Git"))- }- var backups int64- if countBackupsErr := tx.Model(&projectpkg.GitOpsSync{}).- Where("mode = ? AND project_id = ?", gitops.SyncModeBackup, project.ID).- Count(&backups).Error; countBackupsErr != nil {- return nil, fmt.Errorf("failed to check existing backups: %w", countBackupsErr)- }- if backups > 0 {- return nil, common.Classify(common.ErrConflict, errors.New("project is backed up to Git; disconnect that backup before deploying it from Git"))- }- if ensureProjectPathUnderRootErr := s.projectService.EnsureProjectPathUnderRoot(ctx, project, false); ensureProjectPathUnderRootErr != nil {- return nil, ensureProjectPathUnderRootErr- }- return project, nil-}--// insertSyncRecordInternal validates the project link and inserts the sync in-// one transaction with the project row locked, so a project cannot end up with-// two Git relationships. It returns the adopted project for deploy links.-func (s *GitOpsSyncService) insertSyncRecordInternal(ctx context.Context, syncRecord *projectpkg.GitOpsSync, req gitops.CreateSyncRequest, mode, linkProjectID string) (*projectpkg.Project, error) {+// insertSyncRecord inserts the sync in one transaction with any linked project locked, so a project never+// ends up with two Git relationships. It returns the project a deploy sync adopts.+func (s *GitOpsSyncService) insertSyncRecord(ctx context.Context, syncRecord *projectpkg.GitOpsSync, req gitops.CreateSyncRequest, mode string) (*projectpkg.Project, error) {var adoptedProject *projectpkg.Projecterr := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {- if mode == gitops.SyncModeDeploy && linkProjectID != "" {- project, err := s.prepareDeployProjectLinkInternal(ctx, tx, req)+ // An adopted project must exist, and must not be deployed from or backed up to Git already.+ if mode == gitops.SyncModeDeploy && strings.TrimSpace(req.ProjectID) != "" {+ if strings.TrimSpace(req.TargetType) == "swarm_stack" {+ return common.Classify(common.ErrValidation, &base.FieldError{Field: "projectId", Err: errors.New("an existing project cannot be linked to a swarm stack sync")})+ }+ project, err := projectpkg.LockProjectForSync(tx, req.ProjectID)if err != nil {return err}+ if strings.TrimSpace(kit.FromPtr(project.GitOpsManagedBy)) != "" {+ return common.Classify(common.ErrConflict, errors.New("project is already deployed from Git"))+ }+ var backups int64+ if countErr := tx.Model(&projectpkg.GitOpsSync{}).Where("mode = ? AND project_id = ?", gitops.SyncModeBackup, project.ID).Count(&backups).Error; countErr != nil {+ return fmt.Errorf("failed to check existing backups: %w", countErr)+ }+ if backups > 0 {+ return common.Classify(common.ErrConflict, errors.New("project is backed up to Git; disconnect that backup before deploying it from Git"))+ }+ if rootErr := s.projectService.EnsureProjectPathUnderRoot(ctx, project, false); rootErr != nil {+ return rootErr+ }adoptedProject = projectsyncRecord.ProjectID = &project.IDsyncRecord.ProjectName = project.Name@@ -752,10 +540,10 @@ func (s *GitOpsSyncService) insertSyncRecordInternal(ctx context.Context, syncRe}}- // Select("*") forces explicit zero values (e.g. "0 = unlimited" sync limits and- // unset pre-deploy fields) to persist instead of GORM substituting column defaults.+ // Select("*") persists explicit zero values, such as unlimited sync limits, instead of column defaults.if err := tx.Select("*").Omit("Environment", "Repository", "Project").Create(syncRecord).Error; err != nil { //nolint:unqueryvet // intentional Select("*"); see comment above- if isUniqueViolation(err) {+ message := strings.ToLower(err.Error())+ if errors.Is(err, gorm.ErrDuplicatedKey) || strings.Contains(message, "unique constraint") || strings.Contains(message, "duplicate key") {return common.Classify(common.ErrConflict, errors.New("project already has a Git backup; disconnect it first"))}slog.ErrorContext(ctx, "Failed to create GitOps sync in database", "name", req.Name, "repositoryId", req.RepositoryID, "environmentId", syncRecord.EnvironmentID, "error", err)@@ -775,10 +563,7 @@ func (s *GitOpsSyncService) insertSyncRecordInternal(ctx context.Context, syncRe}return nil})- if err != nil {- return nil, err- }- return adoptedProject, nil+ return adoptedProject, err}func (s *GitOpsSyncService) UpdateSync(ctx context.Context, environmentID, id string, req gitops.UpdateSyncRequest, actor user.Actor) (*projectpkg.GitOpsSync, error) {@@ -787,16 +572,10 @@ func (s *GitOpsSyncService) UpdateSync(ctx context.Context, environmentID, id streturn nil, err}- // Capture state needed to reconcile the dynamic job after the update.+ // Captured before the update so the job can follow the new state.oldAutoSync := syncRecord.AutoSync- newAutoSync := syncRecord.AutoSync- if req.AutoSync != nil {- newAutoSync = *req.AutoSync- }- newInterval := syncRecord.SyncInterval- if req.SyncInterval != nil {- newInterval = *req.SyncInterval- }+ newAutoSync := *cmp.Or(req.AutoSync, &syncRecord.AutoSync)+ newInterval := *cmp.Or(req.SyncInterval, &syncRecord.SyncInterval)updates := make(map[string]any)@@ -804,10 +583,8 @@ func (s *GitOpsSyncService) UpdateSync(ctx context.Context, environmentID, id stupdates["name"] = *req.Name}if req.RepositoryID != nil {- // Validate repository exists- _, getRepositoryByIDErr := s.repoService.GetRepositoryByID(ctx, *req.RepositoryID)- if getRepositoryByIDErr != nil {- return nil, fmt.Errorf("repository not found: %w", getRepositoryByIDErr)+ if _, repoErr := s.repoService.GetRepositoryByID(ctx, *req.RepositoryID); repoErr != nil {+ return nil, fmt.Errorf("repository not found: %w", repoErr)}updates["repository_id"] = *req.RepositoryID}@@ -838,8 +615,8 @@ func (s *GitOpsSyncService) UpdateSync(ctx context.Context, environmentID, id stif req.RedeployAfterSync != nil {updates["redeploy_after_sync"] = *req.RedeployAfterSync}- if validateSyncLimitsErr := validateSyncLimits(req.MaxSyncFiles, req.MaxSyncTotalSize, req.MaxSyncBinarySize); validateSyncLimitsErr != nil {- return nil, validateSyncLimitsErr+ if limitsErr := validateSyncLimits(req.MaxSyncFiles, req.MaxSyncTotalSize, req.MaxSyncBinarySize); limitsErr != nil {+ return nil, limitsErr}if req.MaxSyncFiles != nil {updates["max_sync_files"] = *req.MaxSyncFiles@@ -851,11 +628,11 @@ func (s *GitOpsSyncService) UpdateSync(ctx context.Context, environmentID, id stupdates["max_sync_binary_size"] = *req.MaxSyncBinarySize}- if applyModeUpdatesErr := s.backup.ApplyModeUpdates(ctx, syncRecord, req, updates); applyModeUpdatesErr != nil {- return nil, applyModeUpdatesErr+ if modeErr := s.backup.ApplyModeUpdates(ctx, syncRecord, req, updates); modeErr != nil {+ return nil, modeErr}- lifecycleCfg := lifecycleConfigInputInternal{+ lifecycleCfg := lifecycleConfigInput{targetType: req.TargetType,scriptPath: req.PreDeployScriptPath,runnerImage: req.PreDeployRunnerImage,@@ -865,18 +642,32 @@ func (s *GitOpsSyncService) UpdateSync(ctx context.Context, environmentID, id stnetworkMode: req.PreDeployNetworkMode,syncDirectory: req.SyncDirectory,}- if validateLifecycleConfigErr := s.validateLifecycleConfigInternal(ctx, syncRecord, lifecycleCfg); validateLifecycleConfigErr != nil {- return nil, validateLifecycleConfigErr+ if lifecycleErr := s.validateLifecycleConfig(ctx, syncRecord, lifecycleCfg); lifecycleErr != nil {+ return nil, lifecycleErr+ }+ // A nil field stays unchanged; an empty one clears the column, or resets the network mode to "none".+ for column, value := range map[string]*string{+ "pre_deploy_script_path": lifecycleCfg.scriptPath,+ "pre_deploy_runner_image": lifecycleCfg.runnerImage,+ "pre_deploy_env": lifecycleCfg.env,+ "pre_deploy_extra_mounts": lifecycleCfg.extraMounts,+ } {+ if value != nil {+ updates[column] = nullableTrimmedString(value)+ }+ }+ if lifecycleCfg.timeoutSec != nil {+ updates["pre_deploy_timeout_sec"] = *lifecycleCfg.timeoutSec+ }+ if lifecycleCfg.networkMode != nil {+ updates["pre_deploy_network_mode"] = cmp.Or(strings.TrimSpace(*lifecycleCfg.networkMode), "none")}- addLifecycleUpdatesInternal(updates, lifecycleCfg)if len(updates) > 0 {// Loaded associations must not overwrite explicitly updated foreign keys.- if updateSyncErr := s.db.WithContext(ctx).Model(syncRecord).Omit(clause.Associations).Updates(updates).Error; updateSyncErr != nil {- return nil, fmt.Errorf("failed to update sync: %w", updateSyncErr)+ if updateErr := s.db.WithContext(ctx).Model(syncRecord).Omit(clause.Associations).Updates(updates).Error; updateErr != nil {+ return nil, fmt.Errorf("failed to update sync: %w", updateErr)}-- // Log event_, _ = s.eventService.CreateEvent(ctx, event.CreateEventRequest{Type: event.EventTypeGitSyncUpdate,Severity: event.EventSeveritySuccess,@@ -891,56 +682,42 @@ func (s *GitOpsSyncService) UpdateSync(ctx context.Context, environmentID, id st})}- // Reconcile the dynamic job to match the new state.- switch {- case newAutoSync:- s.registerSyncJobInternal(ctx, syncRecord.ID, syncRecord.EnvironmentID, newInterval)- if !oldAutoSync {- // Freshly enabled — kick a run now so it doesn't wait a full interval.- s.kickSyncInternal(ctx, syncRecord.ID)- }- default:- s.unregisterSyncJobInternal(ctx, syncRecord.ID)+ if !newAutoSync {+ s.jobs.Unregister(ctx, syncRecord.ID)+ return s.GetSyncByID(ctx, environmentID, id)+ }+ s.registerSyncJob(ctx, syncRecord.ID, syncRecord.EnvironmentID, newInterval)+ if !oldAutoSync {+ // A freshly enabled sync runs now instead of waiting a full interval.+ s.kickSync(ctx, syncRecord.ID, "startup")}return s.GetSyncByID(ctx, environmentID, id)}func (s *GitOpsSyncService) DeleteSync(ctx context.Context, environmentID, id string, actor user.Actor) error {- // Stop the recurring job first, unconditionally. Even a sync whose row can no- // longer be loaded (corrupt or environment-mismatched) must stop firing; any- // in-flight run re-reads the row and self-cancels once it is gone.- s.unregisterSyncJobInternal(ctx, id)+ // Even a row that can no longer be loaded must stop firing; an in-flight run re-reads it and skips.+ s.jobs.Unregister(ctx, id)s.backups.cancel(id)- // Best-effort load for the audit-event metadata. A corrupt or env-mismatched row- // must still be deletable, so a load failure falls through to the direct delete- // below instead of aborting.- syncRecord, loadErr := s.getSyncRecordByIDInternal(ctx, environmentID, id)+ // The row is loaded only for the audit event, so a corrupt or environment-mismatched row is still deletable.+ syncRecord, loadErr := s.getSyncByID(ctx, environmentID, id, false)if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {- // Clear gitops_managed_by for any project still pointing at this sync, keyed- // on the sync id so orphaned managed flags are cleared even when the sync row- // (and its ProjectID) could not be loaded.+ // Keyed on the sync ID, so orphaned managed flags clear even when the row could not be loaded.if err := tx.Model(&projectpkg.Project{}).Where("gitops_managed_by = ?", id).Update("gitops_managed_by", nil).Error; err != nil {return fmt.Errorf("failed to clear gitops_managed_by: %w", err)}- if err :=-- // Delete by id only (no environment scoping). The handler already enforced the- // delete permission; env scoping is precisely what made env-mismatched corrupt- // rows undeletable. A zero-row delete is treated as success (idempotent).-- tx.Where("id = ?", id).Delete(&projectpkg.GitOpsSync{}).Error; err != nil {+ // The handler enforced the delete permission, so the delete is not environment-scoped, and deleting nothing succeeds.+ if err := tx.Where("id = ?", id).Delete(&projectpkg.GitOpsSync{}).Error; err != nil {return fmt.Errorf("failed to delete sync: %w", err)}return nil}); err != nil {- // Re-register the recurring job only when we actually loaded an auto-sync row.if loadErr == nil && syncRecord.AutoSync {- s.registerSyncJobInternal(ctx, syncRecord.ID, syncRecord.EnvironmentID, syncRecord.SyncInterval)+ s.registerSyncJob(ctx, syncRecord.ID, syncRecord.EnvironmentID, syncRecord.SyncInterval)}return err}@@ -950,7 +727,6 @@ func (s *GitOpsSyncService) DeleteSync(ctx context.Context, environmentID, id streturn nil}- // Log event_, _ = s.eventService.CreateEvent(ctx, event.CreateEventRequest{Type: event.EventTypeGitSyncDelete,Severity: event.EventSeverityInfo,@@ -968,15 +744,13 @@ func (s *GitOpsSyncService) DeleteSync(ctx context.Context, environmentID, id st}func (s *GitOpsSyncService) PerformSync(ctx context.Context, environmentID, id string, actor user.Actor) (*gitops.SyncResult, error) {- return s.performSyncAdmittedInternal(ctx, environmentID, id, actor, false)+ return s.performSyncAdmitted(ctx, environmentID, id, actor, false)}-// performSyncAdmittedInternal runs one sync under the per-sync admission lease-// and dispatches by mode. backupAdopt makes a backup run replace whatever the-// remote holds in its backup directory.-func (s *GitOpsSyncService) performSyncAdmittedInternal(ctx context.Context, environmentID, id string, actor user.Actor, backupAdopt bool) (*gitops.SyncResult, error) {- // Coalesce overlapping runs for the same sync (scheduled fire, startup/enable- // kick, manual trigger, webhook) so they don't race the clone/redeploy.+// performSyncAdmitted runs one sync under its admission lease and dispatches by mode. backupAdopt makes a+// backup replace whatever the remote holds in its backup directory.+func (s *GitOpsSyncService) performSyncAdmitted(ctx context.Context, environmentID, id string, actor user.Actor, backupAdopt bool) (*gitops.SyncResult, error) {+ // Overlapping runs of one sync (schedule, kick, manual, webhook) must not race the clone and redeploy.lease, admitted, err := s.jobs.TryAcquire(ctx, id)if err != nil {return nil, fmt.Errorf("failed to admit GitOps sync: %w", err)@@ -1035,12 +809,8 @@ func (s *GitOpsSyncService) GetSyncStatus(ctx context.Context, environmentID, idreturn status, nil}-// CleanupLeakedScratchDirsOnStartup removes orphaned GitOps scratch directories-// (.gitops-sync-stage-*, .gitops-backup-*, and the legacy "<name>.gitops-backup-<digits>"-// form) left in the projects directory by a crash or container restart that interrupted-// a sync mid-flight. These are Arcane-internal working dirs; left in place the filesystem-// discovery imports them as phantom projects. It is safe to run at startup because it-// executes before any sync job is registered, so nothing is mid-stage.+// CleanupLeakedScratchDirsOnStartup removes GitOps scratch directories an interrupted sync left in the projects+// directory, which discovery would import as phantom projects. It runs before any sync job is registered.func (s *GitOpsSyncService) CleanupLeakedScratchDirsOnStartup(ctx context.Context) error {projectsDir, err := s.projectService.GetProjectsDirectory(ctx)if err != nil {@@ -1234,38 +1004,7 @@ func (s *GitOpsSyncService) ImportSyncs(ctx context.Context, environmentID strinreturn response, nil}-func (s *GitOpsSyncService) logSyncError(ctx context.Context, syncRecord *projectpkg.GitOpsSync, actor user.Actor, errorMsg string) {- _, _ = s.eventService.CreateEvent(ctx, event.CreateEventRequest{- Type: event.EventTypeGitSyncError,- Severity: event.EventSeverityError,- Title: "Git sync failed",- Description: fmt.Sprintf("Failed to sync '%s': %s", syncRecord.Name, errorMsg),- ResourceType: new("git_sync"),- ResourceID: new(syncRecord.ID),- ResourceName: new(syncRecord.Name),- UserID: new(actor.ID),- Username: new(actor.Username),- EnvironmentID: new(syncRecord.EnvironmentID),- })-}--// backupRuntimeInternal holds the in-memory coordination for backup syncs-type backupRuntimeInternal struct {- mu sync.Mutex- branches map[string]*sync.Mutex- timers map[string]*time.Timer- debounce time.Duration-}--func newBackupRuntimeInternal() *backupRuntimeInternal {- return &backupRuntimeInternal{- branches: make(map[string]*sync.Mutex),- timers: make(map[string]*time.Timer),- debounce: backup.DefaultBackupSaveDebounce,- }-}--func (r *backupRuntimeInternal) branchLock(repositoryID, branch string) *sync.Mutex {+func (r *backupRuntime) branchLock(repositoryID, branch string) *sync.Mutex {r.mu.Lock()defer r.mu.Unlock()key := repositoryID + "\x00" + branch@@ -1277,7 +1016,7 @@ func (r *backupRuntimeInternal) branchLock(repositoryID, branch string) *sync.Mureturn lock}-func (r *backupRuntimeInternal) schedule(syncID string, run func()) {+func (r *backupRuntime) schedule(syncID string, run func()) {r.mu.Lock()defer r.mu.Unlock()if timer, ok := r.timers[syncID]; ok {@@ -1291,7 +1030,7 @@ func (r *backupRuntimeInternal) schedule(syncID string, run func()) {})}-func (r *backupRuntimeInternal) cancel(syncID string) {+func (r *backupRuntime) cancel(syncID string) {r.mu.Lock()defer r.mu.Unlock()if timer, ok := r.timers[syncID]; ok {@@ -1305,11 +1044,11 @@ func (s *GitOpsSyncService) ResolveBackupConflict(ctx context.Context, environmeif req.Strategy != gitops.BackupConflictUseArcane {return nil, common.Classify(common.ErrValidation, &base.FieldError{Field: "strategy", Err: fmt.Errorf("unsupported strategy %q", req.Strategy)})}- syncRecord, err := s.getBackupSyncInternal(ctx, environmentID, id)+ syncRecord, err := s.getBackupSync(ctx, environmentID, id)if err != nil {return nil, err}- return s.performSyncAdmittedInternal(ctx, syncRecord.EnvironmentID, syncRecord.ID, actor, true)+ return s.performSyncAdmitted(ctx, syncRecord.EnvironmentID, syncRecord.ID, actor, true)}// SubscribeProjectFileChanges marks backups pending on save and runs them after a debounce when auto backup is on.@@ -1339,9 +1078,7 @@ func (s *GitOpsSyncService) SubscribeProjectFileChanges(ctx context.Context) {syncID, environmentID := syncRecord.ID, syncRecord.EnvironmentIDs.backups.schedule(syncID, func() {if s.jobs.Enabled() {- if _, err := s.jobs.Scheduler().Submit(runCtx, scheduler.Request{JobID: s.jobs.JobName(syncID), EnvironmentID: "0", Trigger: "save"}); err != nil {- slog.ErrorContext(runCtx, "git backup admission after save failed", "syncId", syncID, "error", err)- }+ s.kickSync(runCtx, syncID, "save")return}go func() {@@ -1359,7 +1096,7 @@ func (s *GitOpsSyncService) PreviewBackup(ctx context.Context, environmentID, idpreviewCtx, cancel := context.WithTimeout(ctx, defaultGitSyncTimeout)defer cancel()- syncRecord, err := s.getBackupSyncInternal(previewCtx, environmentID, id)+ syncRecord, err := s.getBackupSync(previewCtx, environmentID, id)if err != nil {return nil, err}@@ -1371,7 +1108,7 @@ func (s *GitOpsSyncService) GetBackupHistory(ctx context.Context, environmentID,historyCtx, cancel := context.WithTimeout(ctx, defaultGitSyncTimeout)defer cancel()- syncRecord, err := s.getBackupSyncInternal(historyCtx, environmentID, id)+ syncRecord, err := s.getBackupSync(historyCtx, environmentID, id)if err != nil {return nil, err}@@ -1383,7 +1120,7 @@ func (s *GitOpsSyncService) GetBackupRevision(ctx context.Context, environmentIDrevisionCtx, cancel := context.WithTimeout(ctx, defaultGitSyncTimeout)defer cancel()- syncRecord, err := s.getBackupSyncInternal(revisionCtx, environmentID, id)+ syncRecord, err := s.getBackupSync(revisionCtx, environmentID, id)if err != nil {return nil, err}@@ -1409,7 +1146,7 @@ func (s *GitOpsSyncService) ReconcileInterruptedBackupsOnStartup(ctx context.Conreturn nil}-func (s *GitOpsSyncService) getBackupSyncInternal(ctx context.Context, environmentID, id string) (*projectpkg.GitOpsSync, error) {+func (s *GitOpsSyncService) getBackupSync(ctx context.Context, environmentID, id string) (*projectpkg.GitOpsSync, error) {syncRecord, err := s.GetSyncByID(ctx, environmentID, id)if err != nil {return nil, errdiff --git a/backend/internal/gitops/service_test.go b/backend/internal/gitops/service_test.goindex 341ca4e2a..f186283e9 100644--- a/backend/internal/gitops/service_test.go+++ b/backend/internal/gitops/service_test.go@@ -13,7 +13,6 @@ import ("github.com/getarcaneapp/arcane/types/v2/base""github.com/getarcaneapp/arcane/types/v2/gitops"- "github.com/getarcaneapp/arcane/types/v2/lifecycle"schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler""github.com/getarcaneapp/arcane/types/v2/user""github.com/go-git/go-billy/v5/osfs"@@ -26,6 +25,7 @@ import ("go.getarcane.app/kit/pkg""gorm.io/gorm"+ "github.com/getarcaneapp/arcane/backend/v2/internal/activity""github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/database"@@ -37,13 +37,15 @@ import (projectpkg "github.com/getarcaneapp/arcane/backend/v2/internal/project""github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/backend/v2/pkg/gitutil"+ activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity""github.com/getarcaneapp/arcane/backend/v2/pkg/projects""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/entityjobs"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow/flowtest""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing")-func setupGitOpsProjectTestDBInternal(t *testing.T) *database.DB {+func setupGitOpsProjectTestDB(t *testing.T) *database.DB {t.Helper()db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})require.NoError(t, err)@@ -54,7 +56,7 @@ func setupGitOpsProjectTestDBInternal(t *testing.T) *database.DB {return &database.DB{DB: db}}-func newGitOpsSettingsServiceForTestInternal(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {+func newGitOpsSettingsServiceForTest(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {t.Helper()service, err := settings.NewSettingsService(ctx, db)if err == nil {@@ -63,7 +65,7 @@ func newGitOpsSettingsServiceForTestInternal(t testing.TB, ctx context.Context,return service, err}-func newGitOpsAdmissionGateForTestInternal(t testing.TB) *runs.Admission {+func newGitOpsAdmissionGateForTest(t testing.TB) *runs.Admission {t.Helper()runtime := francistest.New(t)gate := runs.NewAdmission(runtime.Service(), t.Name())@@ -76,10 +78,10 @@ func setupGitOpsSyncDirectoryTestService(t *testing.T) (*GitOpsSyncService, *datt.Helper()ctx := t.Context()- db := setupGitOpsProjectTestDBInternal(t)+ db := setupGitOpsProjectTestDB(t)require.NoError(t, db.AutoMigrate(&projectpkg.GitOpsSync{}))- settingsService, err := newGitOpsSettingsServiceForTestInternal(t, ctx, db)+ settingsService, err := newGitOpsSettingsServiceForTest(t, ctx, db)require.NoError(t, err)projectsDir := t.TempDir()@@ -91,15 +93,15 @@ func setupGitOpsSyncDirectoryTestService(t *testing.T) (*GitOpsSyncService, *datreturn NewGitOpsSyncService(db, nil, projectService, nil, eventService, settingsService), db, projectsDir}-func TestGitOpsSyncService_OverlappingSyncPreservesSuccessShapedSkipInternal(t *testing.T) {- gate := newGitOpsAdmissionGateForTestInternal(t)- key := schedulertypes.AdmissionKey{Scope: gitOpsSyncAdmissionScopeInternal, ID: "sync-id"}+func TestGitOpsSyncService_OverlappingSyncPreservesSuccessShapedSkip(t *testing.T) {+ gate := newGitOpsAdmissionGateForTest(t)+ key := schedulertypes.AdmissionKey{Scope: gitOpsSyncAdmissionScope, ID: "sync-id"}lease, admitted, err := gate.TryAcquire(t.Context(), key)require.NoError(t, err)require.True(t, admitted)- service := &GitOpsSyncService{jobs: entityjobs.New(entityjobs.GitOpsSyncJobPrefix, gitOpsSyncAdmissionScopeInternal)}- require.NoError(t, service.SetScheduler(t.Context(), &gitOpsSyncTestSchedulerInternal{}, gate))+ service := &GitOpsSyncService{jobs: entityjobs.New(entityjobs.GitOpsSyncJobPrefix, gitOpsSyncAdmissionScope)}+ require.NoError(t, service.SetScheduler(t.Context(), &gitOpsSyncTestScheduler{}, gate))result, err := service.PerformSync(t.Context(), "0", "sync-id", user.Actor{})require.NoError(t, err)require.False(t, result.Success)@@ -107,36 +109,36 @@ func TestGitOpsSyncService_OverlappingSyncPreservesSuccessShapedSkipInternal(t *lease.Release(t.Context())}-type gitOpsSyncTestSchedulerInternal struct {+type gitOpsSyncTestScheduler struct {submitted []schedulertypes.Requestadded []stringremoved []string}-func (s *gitOpsSyncTestSchedulerInternal) AddJob(_ context.Context, job schedulertypes.Job) error {+func (s *gitOpsSyncTestScheduler) AddJob(_ context.Context, job schedulertypes.Job) error {s.added = append(s.added, job.Name())return nil}-func (s *gitOpsSyncTestSchedulerInternal) RemoveJob(_ context.Context, name string) {+func (s *gitOpsSyncTestScheduler) RemoveJob(_ context.Context, name string) {s.removed = append(s.removed, name)}-func (s *gitOpsSyncTestSchedulerInternal) HasJob(_ string) bool {+func (s *gitOpsSyncTestScheduler) HasJob(_ string) bool {return false}-// setupGitOpsSyncRemoteTestServiceInternal wires a sync service with a real+// setupGitOpsSyncRemoteTestService wires a sync service with a real// scheduler registry over a bare remote "repo-1" whose main branch holds files.-func setupGitOpsSyncRemoteTestServiceInternal(t *testing.T, files map[string]string) (*GitOpsSyncService, *database.DB, string, *gitOpsSyncTestSchedulerInternal) {+func setupGitOpsSyncRemoteTestService(t *testing.T, files map[string]string) (*GitOpsSyncService, *database.DB, string, *gitOpsSyncTestScheduler) {t.Helper()- installBackupTestTransportInternal()+ installBackupTestTransport()ctx := t.Context()- db := setupGitOpsProjectTestDBInternal(t)+ db := setupGitOpsProjectTestDB(t)require.NoError(t, db.AutoMigrate(&projectpkg.GitOpsSync{}, &gitrepo.GitRepository{}))- settingsService, err := newGitOpsSettingsServiceForTestInternal(t, ctx, db)+ settingsService, err := newGitOpsSettingsServiceForTest(t, ctx, db)require.NoError(t, err)projectsDir := t.TempDir()@@ -147,15 +149,15 @@ func setupGitOpsSyncRemoteTestServiceInternal(t *testing.T, files map[string]strrepoService := gitrepo.NewGitRepositoryService(db, t.TempDir(), eventService, settingsService)service := NewGitOpsSyncService(db, repoService, projectService, nil, eventService, settingsService)- testScheduler := &gitOpsSyncTestSchedulerInternal{}- require.NoError(t, service.SetScheduler(ctx, testScheduler, newGitOpsAdmissionGateForTestInternal(t)))+ testScheduler := &gitOpsSyncTestScheduler{}+ require.NoError(t, service.SetScheduler(ctx, testScheduler, newGitOpsAdmissionGateForTest(t)))bare := filepath.Join(t.TempDir(), "sync.git")_, err = gogit.PlainInit(bare, true)require.NoError(t, err)repoURL := "http://localhost" + barerequire.NoError(t, db.Create(&gitrepo.GitRepository{ID: "repo-1", Name: "sync-remote", URL: repoURL, AuthType: "none", Enabled: true}).Error)- pushRemoteCommitInternal(t, git.NewClient(t.TempDir()), repoURL, "seed", files)+ pushRemoteCommit(t, git.NewClient(t.TempDir()), repoURL, "seed", files)return service, db, projectsDir, testScheduler}@@ -165,7 +167,7 @@ func setupGitOpsSyncRemoteTestServiceInternal(t *testing.T, files map[string]str// broken binding, not a "-N" duplicate.func TestGitOpsSyncService_GetOrCreateProject_RefusesDuplicateOnNameCollision(t *testing.T) {ctx := t.Context()- svc, db, projectsDir, _ := setupGitOpsSyncRemoteTestServiceInternal(t, map[string]string{"docker-compose.yaml": "services:\n app:\n image: nginx:alpine\n"})+ svc, db, projectsDir, _ := setupGitOpsSyncRemoteTestService(t, map[string]string{"docker-compose.yaml": "services:\n app:\n image: nginx:alpine\n"})require.NoError(t, os.MkdirAll(filepath.Join(projectsDir, "Dozzle"), 0o755))syncRecord := &projectpkg.GitOpsSync{@@ -194,7 +196,7 @@ func TestGitOpsSyncService_GetOrCreateProject_RefusesDuplicateOnNameCollision(tfunc TestGitOpsSyncService_SyncProjectDirectory_FailsWhenBoundProjectMissing(t *testing.T) {ctx := t.Context()- svc, db, projectsDir, testScheduler := setupGitOpsSyncRemoteTestServiceInternal(t, map[string]string{"apps/demo/docker-compose.yaml": "services:\n app:\n image: nginx:alpine\n"})+ svc, db, projectsDir, testScheduler := setupGitOpsSyncRemoteTestService(t, map[string]string{"apps/demo/docker-compose.yaml": "services:\n app:\n image: nginx:alpine\n"})missingProjectID := "missing-project"syncRecord := &projectpkg.GitOpsSync{@@ -237,7 +239,7 @@ func TestGitOpsSyncService_SyncProjectDirectory_FailsWhenBoundProjectMissing(t *func TestGitOpsSyncService_SyncProjectDirectory_DisablesAutoSyncWhenBoundProjectRecoveryAmbiguous(t *testing.T) {ctx := t.Context()- svc, db, projectsDir, testScheduler := setupGitOpsSyncRemoteTestServiceInternal(t, map[string]string{"apps/media/radarr.yaml": "services:\n app:\n image: lscr.io/linuxserver/radarr:latest\n"})+ svc, db, projectsDir, testScheduler := setupGitOpsSyncRemoteTestService(t, map[string]string{"apps/media/radarr.yaml": "services:\n app:\n image: lscr.io/linuxserver/radarr:latest\n"})for _, dirName := range []string{"Radarr-3", "Radarr-30"} {projectPath := filepath.Join(projectsDir, dirName)@@ -281,9 +283,9 @@ func TestGitOpsSyncService_SyncProjectDirectory_DisablesAutoSyncWhenBoundProjectassert.Contains(t, testScheduler.removed, entityjobs.GitOpsSyncJobPrefix+syncRecord.ID)}-func TestGitOpsSyncService_GetOrCreateProjectInternal_FailsWhenBoundProjectMissing(t *testing.T) {+func TestGitOpsSyncService_GetOrCreateProject_FailsWhenBoundProjectMissing(t *testing.T) {ctx := t.Context()- svc, db, projectsDir, testScheduler := setupGitOpsSyncRemoteTestServiceInternal(t, map[string]string{"apps/demo/docker-compose.yaml": "services:\n app:\n image: nginx:alpine\n"})+ svc, db, projectsDir, testScheduler := setupGitOpsSyncRemoteTestService(t, map[string]string{"apps/demo/docker-compose.yaml": "services:\n app:\n image: nginx:alpine\n"})missingProjectID := "missing-project"syncRecord := &projectpkg.GitOpsSync{@@ -324,23 +326,6 @@ func TestGitOpsSyncService_GetOrCreateProjectInternal_FailsWhenBoundProjectMissiassert.Contains(t, testScheduler.removed, entityjobs.GitOpsSyncJobPrefix+syncRecord.ID)}-func TestApplyLifecycleFieldsToSyncInternal_DefaultsPreDeployTimeout(t *testing.T) {- var syncRecord projectpkg.GitOpsSync-- applyLifecycleFieldsToSyncInternal(&syncRecord, lifecycleConfigInputInternal{})-- require.Equal(t, lifecycle.DefaultTimeoutSec, syncRecord.PreDeployTimeoutSec)-}--func TestApplyLifecycleFieldsToSyncInternal_UsesExplicitPreDeployTimeout(t *testing.T) {- timeoutSec := 90- var syncRecord projectpkg.GitOpsSync-- applyLifecycleFieldsToSyncInternal(&syncRecord, lifecycleConfigInputInternal{timeoutSec: &timeoutSec})-- require.Equal(t, timeoutSec, syncRecord.PreDeployTimeoutSec)-}-func TestGitOpsSyncService_GetSyncByID_ReturnsNotFoundError(t *testing.T) {ctx := t.Context()svc, _, _ := setupGitOpsSyncDirectoryTestService(t)@@ -434,8 +419,8 @@ func TestGitOpsSyncService_RegisterAutoSyncJobsOnStartup_SkipsOrphans(t *testingLastSyncAt: &now,}).Error)- scheduler := &gitOpsSyncTestSchedulerInternal{}- require.NoError(t, svc.SetScheduler(ctx, scheduler, newGitOpsAdmissionGateForTestInternal(t)))+ scheduler := &gitOpsSyncTestScheduler{}+ require.NoError(t, svc.SetScheduler(ctx, scheduler, newGitOpsAdmissionGateForTest(t)))svc.RegisterAutoSyncJobsOnStartup(ctx)require.Equal(t, []string{entityjobs.GitOpsSyncJobPrefix + "sync-live"}, scheduler.added)@@ -472,8 +457,8 @@ func TestGitOpsSyncService_DeleteSync_DeletesStaleProjectReference(t *testing.T)func TestGitOpsSyncService_DeleteSync_SucceedsWhenEnvironmentMismatched(t *testing.T) {ctx := t.Context()svc, db, _ := setupGitOpsSyncDirectoryTestService(t)- scheduler := &gitOpsSyncTestSchedulerInternal{}- require.NoError(t, svc.SetScheduler(ctx, scheduler, newGitOpsAdmissionGateForTestInternal(t)))+ scheduler := &gitOpsSyncTestScheduler{}+ require.NoError(t, svc.SetScheduler(ctx, scheduler, newGitOpsAdmissionGateForTest(t)))syncRecord := &projectpkg.GitOpsSync{ID: "sync-env-mismatch",@@ -523,18 +508,22 @@ func TestGitOpsSyncService_DeleteSync_ClearsOrphanedManagedFlag(t *testing.T) {assert.Nil(t, got.GitOpsManagedBy)}-// TestGitOpsSyncService_RunScheduledSync_UnregistersMissingSync verifies a scheduled-// run whose row no longer exists (e.g. deleted out-of-band via raw SQL) unregisters-// its own job instead of firing forever.-func TestGitOpsSyncService_RunScheduledSync_UnregistersMissingSync(t *testing.T) {+// TestGitOpsSyncWorkflow_UnregistersMissingSync verifies a scheduled run whose row no longer exists+// (e.g. deleted out-of-band via raw SQL) unregisters its own job instead of firing forever.+func TestGitOpsSyncWorkflow_UnregistersMissingSync(t *testing.T) {ctx := t.Context()- svc, _, _ := setupGitOpsSyncDirectoryTestService(t)- scheduler := &gitOpsSyncTestSchedulerInternal{}- require.NoError(t, svc.SetScheduler(ctx, scheduler, newGitOpsAdmissionGateForTestInternal(t)))-- _, scheduledSyncErr := svc.runScheduledSyncInternal(ctx, "0", "ghost-sync")- require.NoError(t, scheduledSyncErr)-+ svc, db, _ := setupGitOpsSyncDirectoryTestService(t)+ require.NoError(t, db.AutoMigrate(&activity.Activity{}, &activity.ActivityMessage{}))+ harness := flowtest.New(t, activity.NewActivityService(db, nil))+ require.NoError(t, svc.RegisterWorkflows(harness.Engine))+ harness.Start(t)+ flowtest.AssertDefinitions(t, harness)+ scheduler := &gitOpsSyncTestScheduler{}+ require.NoError(t, svc.SetScheduler(ctx, scheduler, newGitOpsAdmissionGateForTest(t)))++ outcome, err := harness.Engine.Run(ctx, svc.syncWorkflow, syncTarget{EnvironmentID: "0", SyncID: "ghost-sync"}, activitylib.StartRequest{}, nil)+ require.NoError(t, err)+ assert.Equal(t, schedulertypes.Skipped, outcome.Status)assert.Contains(t, scheduler.removed, entityjobs.GitOpsSyncJobPrefix+"ghost-sync")}@@ -664,8 +653,8 @@ func TestGitOpsSyncService_ReconcileDirectorySyncProjectsOnStartup_SkipsAmbiguoufunc TestGitOpsSyncService_GetEnvironmentSyncLimits(t *testing.T) {ctx := t.Context()- db := setupGitOpsProjectTestDBInternal(t)- settingsSvc, err := newGitOpsSettingsServiceForTestInternal(t, ctx, db)+ db := setupGitOpsProjectTestDB(t)+ settingsSvc, err := newGitOpsSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, settingsSvc.SetIntSetting(ctx, "gitSyncMaxFiles", 123))@@ -687,8 +676,8 @@ func TestGitOpsSyncService_GetEffectiveSyncLimits(t *testing.T) {t.Setenv("GIT_SYNC_MAX_TOTAL_SIZE_MB", "")t.Setenv("GIT_SYNC_MAX_BINARY_SIZE_MB", "")- db := setupGitOpsProjectTestDBInternal(t)- settingsSvc, err := newGitOpsSettingsServiceForTestInternal(t, ctx, db)+ db := setupGitOpsProjectTestDB(t)+ settingsSvc, err := newGitOpsSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, settingsSvc.SetIntSetting(ctx, "gitSyncMaxFiles", 200))@@ -743,7 +732,7 @@ func TestGitOpsSyncService_GetEffectiveSyncLimits(t *testing.T) {t.Setenv("GIT_SYNC_MAX_FILES", "10000")t.Setenv("GIT_SYNC_MAX_TOTAL_SIZE_MB", "1024")t.Setenv("GIT_SYNC_MAX_BINARY_SIZE_MB", "12")- settingsSvcEnv, svcErr := newGitOpsSettingsServiceForTestInternal(t, ctx, db)+ settingsSvcEnv, svcErr := newGitOpsSettingsServiceForTest(t, ctx, db)require.NoError(t, svcErr)svcEnv := &GitOpsSyncService{settingsService: settingsSvcEnv}@@ -764,7 +753,7 @@ func TestGitOpsSyncService_GetEffectiveSyncLimits(t *testing.T) {t.Setenv("GIT_SYNC_MAX_FILES", "0")t.Setenv("GIT_SYNC_MAX_TOTAL_SIZE_MB", "0")t.Setenv("GIT_SYNC_MAX_BINARY_SIZE_MB", "0")- settingsSvcEnv, svcErr := newGitOpsSettingsServiceForTestInternal(t, ctx, db)+ settingsSvcEnv, svcErr := newGitOpsSettingsServiceForTest(t, ctx, db)require.NoError(t, svcErr)svcEnv := &GitOpsSyncService{settingsService: settingsSvcEnv}@@ -795,14 +784,14 @@ func setupLifecycleValidationService(t *testing.T) (*GitOpsSyncService, context.func TestValidateLifecycleConfig_AllNilNoError(t *testing.T) {svc, ctx := setupLifecycleValidationService(t)- require.NoError(t, svc.validateLifecycleConfigInternal(ctx, nil, lifecycleConfigInputInternal{}))+ require.NoError(t, svc.validateLifecycleConfig(ctx, nil, lifecycleConfigInput{}))}func TestValidateLifecycleConfig_RejectsWhenGloballyDisabled(t *testing.T) {svc, _, _ := setupGitOpsSyncDirectoryTestService(t)ctx := t.Context()// lifecycleEnabled defaults to false; do not enable.- err := svc.validateLifecycleConfigInternal(ctx, nil, lifecycleConfigInputInternal{+ err := svc.validateLifecycleConfig(ctx, nil, lifecycleConfigInput{scriptPath: new("scripts/deploy.sh"),runnerImage: new("alpine:latest"),})@@ -812,7 +801,7 @@ func TestValidateLifecycleConfig_RejectsWhenGloballyDisabled(t *testing.T) {func TestValidateLifecycleConfig_RejectsAbsoluteScriptPath(t *testing.T) {svc, ctx := setupLifecycleValidationService(t)- err := svc.validateLifecycleConfigInternal(ctx, nil, lifecycleConfigInputInternal{+ err := svc.validateLifecycleConfig(ctx, nil, lifecycleConfigInput{scriptPath: new("/etc/passwd"),runnerImage: new("alpine:latest"),})@@ -822,7 +811,7 @@ func TestValidateLifecycleConfig_RejectsAbsoluteScriptPath(t *testing.T) {func TestValidateLifecycleConfig_RejectsTraversalScriptPath(t *testing.T) {svc, ctx := setupLifecycleValidationService(t)- err := svc.validateLifecycleConfigInternal(ctx, nil, lifecycleConfigInputInternal{+ err := svc.validateLifecycleConfig(ctx, nil, lifecycleConfigInput{scriptPath: new("../outside.sh"),runnerImage: new("alpine:latest"),})@@ -836,7 +825,7 @@ func TestValidateLifecycleConfig_RejectsOverlongScriptPath(t *testing.T) {for i := range long {long[i] = 'a'}- err := svc.validateLifecycleConfigInternal(ctx, nil, lifecycleConfigInputInternal{+ err := svc.validateLifecycleConfig(ctx, nil, lifecycleConfigInput{scriptPath: new(string(long)),runnerImage: new("alpine:latest"),})@@ -847,7 +836,7 @@ func TestValidateLifecycleConfig_RejectsOverlongScriptPath(t *testing.T) {func TestValidateLifecycleConfig_RejectsScriptWithoutRunnerImageOnCreate(t *testing.T) {svc, ctx := setupLifecycleValidationService(t)require.NoError(t, svc.settingsService.SetStringSetting(ctx, "lifecycleDefaultRunnerImage", " "))- err := svc.validateLifecycleConfigInternal(ctx, nil, lifecycleConfigInputInternal{+ err := svc.validateLifecycleConfig(ctx, nil, lifecycleConfigInput{scriptPath: new("scripts/deploy.sh"),})require.Error(t, err)@@ -857,7 +846,7 @@ func TestValidateLifecycleConfig_RejectsScriptWithoutRunnerImageOnCreate(t *testfunc TestValidateLifecycleConfig_AcceptsScriptWithDefaultRunnerImageOnCreate(t *testing.T) {svc, ctx := setupLifecycleValidationService(t)require.NoError(t, svc.settingsService.SetStringSetting(ctx, "lifecycleDefaultRunnerImage", "alpine:latest"))- require.NoError(t, svc.validateLifecycleConfigInternal(ctx, nil, lifecycleConfigInputInternal{+ require.NoError(t, svc.validateLifecycleConfig(ctx, nil, lifecycleConfigInput{targetType: new("project"),scriptPath: new("scripts/deploy.sh"),syncDirectory: new(true),@@ -868,7 +857,7 @@ func TestValidateLifecycleConfig_AcceptsScriptWithExistingRunnerImageOnUpdate(tsvc, ctx := setupLifecycleValidationService(t)existing := &projectpkg.GitOpsSync{SyncDirectory: true}existing.PreDeployRunnerImage = new("alpine:latest")- require.NoError(t, svc.validateLifecycleConfigInternal(ctx, existing, lifecycleConfigInputInternal{+ require.NoError(t, svc.validateLifecycleConfig(ctx, existing, lifecycleConfigInput{scriptPath: new("scripts/deploy.sh"),}))}@@ -876,7 +865,7 @@ func TestValidateLifecycleConfig_AcceptsScriptWithExistingRunnerImageOnUpdate(tfunc TestValidateLifecycleConfig_RejectsTimeoutZeroOrNegative(t *testing.T) {svc, ctx := setupLifecycleValidationService(t)for _, v := range []int{0, -1, -3600} {- err := svc.validateLifecycleConfigInternal(ctx, nil, lifecycleConfigInputInternal{+ err := svc.validateLifecycleConfig(ctx, nil, lifecycleConfigInput{timeoutSec: new(v),})require.Errorf(t, err, "expected error for timeoutSec=%d", v)@@ -887,7 +876,7 @@ func TestValidateLifecycleConfig_RejectsTimeoutZeroOrNegative(t *testing.T) {func TestValidateLifecycleConfig_RejectsTimeoutAboveSettingCap(t *testing.T) {svc, ctx := setupLifecycleValidationService(t)require.NoError(t, svc.settingsService.SetStringSetting(ctx, "lifecycleMaxTimeoutSec", "120"))- err := svc.validateLifecycleConfigInternal(ctx, nil, lifecycleConfigInputInternal{+ err := svc.validateLifecycleConfig(ctx, nil, lifecycleConfigInput{timeoutSec: new(300),})require.Error(t, err)@@ -896,7 +885,7 @@ func TestValidateLifecycleConfig_RejectsTimeoutAboveSettingCap(t *testing.T) {func TestValidateLifecycleConfig_RejectsInvalidEnvKey(t *testing.T) {svc, ctx := setupLifecycleValidationService(t)- err := svc.validateLifecycleConfigInternal(ctx, nil, lifecycleConfigInputInternal{+ err := svc.validateLifecycleConfig(ctx, nil, lifecycleConfigInput{env: new("FOO-BAR=baz"),})require.Error(t, err)@@ -904,14 +893,14 @@ func TestValidateLifecycleConfig_RejectsInvalidEnvKey(t *testing.T) {func TestValidateLifecycleConfig_AcceptsValidEnv(t *testing.T) {svc, ctx := setupLifecycleValidationService(t)- require.NoError(t, svc.validateLifecycleConfigInternal(ctx, nil, lifecycleConfigInputInternal{+ require.NoError(t, svc.validateLifecycleConfig(ctx, nil, lifecycleConfigInput{env: new("FOO=bar\nBAZ_2=qux"),}))}func TestValidateLifecycleConfig_RejectsRelativeMountSource(t *testing.T) {svc, ctx := setupLifecycleValidationService(t)- err := svc.validateLifecycleConfigInternal(ctx, nil, lifecycleConfigInputInternal{+ err := svc.validateLifecycleConfig(ctx, nil, lifecycleConfigInput{extraMounts: new("relative/path:/in/container"),})require.Error(t, err)@@ -920,7 +909,7 @@ func TestValidateLifecycleConfig_RejectsRelativeMountSource(t *testing.T) {func TestValidateLifecycleConfig_RejectsRelativeMountTarget(t *testing.T) {svc, ctx := setupLifecycleValidationService(t)- err := svc.validateLifecycleConfigInternal(ctx, nil, lifecycleConfigInputInternal{+ err := svc.validateLifecycleConfig(ctx, nil, lifecycleConfigInput{extraMounts: new("/host/path:relative/target"),})require.Error(t, err)@@ -933,14 +922,14 @@ func TestValidateLifecycleConfig_AllowsClearingScriptWithoutImage(t *testing.T)existing.PreDeployScriptPath = new("scripts/old.sh")existing.PreDeployRunnerImage = new("alpine:latest")// User clears the script (empty string in update); image clear is implied not required.- require.NoError(t, svc.validateLifecycleConfigInternal(ctx, existing, lifecycleConfigInputInternal{+ require.NoError(t, svc.validateLifecycleConfig(ctx, existing, lifecycleConfigInput{scriptPath: new(""),}))}func TestValidateLifecycleConfig_RejectsScriptWithoutSyncDirectoryOnCreate(t *testing.T) {svc, ctx := setupLifecycleValidationService(t)- err := svc.validateLifecycleConfigInternal(ctx, nil, lifecycleConfigInputInternal{+ err := svc.validateLifecycleConfig(ctx, nil, lifecycleConfigInput{scriptPath: new("scripts/deploy.sh"),runnerImage: new("alpine:latest"),syncDirectory: new(false),@@ -954,7 +943,7 @@ func TestValidateLifecycleConfig_RejectsScriptWithoutSyncDirectoryOnCreate(t *tefunc TestValidateLifecycleConfig_AcceptsScriptWithSyncDirectoryOnCreate(t *testing.T) {svc, ctx := setupLifecycleValidationService(t)- require.NoError(t, svc.validateLifecycleConfigInternal(ctx, nil, lifecycleConfigInputInternal{+ require.NoError(t, svc.validateLifecycleConfig(ctx, nil, lifecycleConfigInput{targetType: new("project"),scriptPath: new("scripts/deploy.sh"),runnerImage: new("alpine:latest"),@@ -964,7 +953,7 @@ func TestValidateLifecycleConfig_AcceptsScriptWithSyncDirectoryOnCreate(t *testifunc TestValidateLifecycleConfig_RejectsLifecycleHookForSwarmStack(t *testing.T) {svc, ctx := setupLifecycleValidationService(t)- err := svc.validateLifecycleConfigInternal(ctx, nil, lifecycleConfigInputInternal{+ err := svc.validateLifecycleConfig(ctx, nil, lifecycleConfigInput{targetType: new("swarm_stack"),scriptPath: new("scripts/deploy.sh"),runnerImage: new("alpine:latest"),@@ -983,7 +972,7 @@ func TestValidateLifecycleConfig_RejectsSwarmTargetChangeWithExistingLifecycleHoexisting := &projectpkg.GitOpsSync{TargetType: "project", SyncDirectory: true}existing.PreDeployScriptPath = new("scripts/deploy.sh")existing.PreDeployRunnerImage = new("alpine:latest")- err := svc.validateLifecycleConfigInternal(ctx, existing, lifecycleConfigInputInternal{+ err := svc.validateLifecycleConfig(ctx, existing, lifecycleConfigInput{targetType: new("swarm_stack"),})require.Error(t, err)@@ -997,7 +986,7 @@ func TestValidateLifecycleConfig_RejectsSwarmTargetChangeWithExistingLifecycleHofunc TestValidateLifecycleConfig_AcceptsScriptWhenExistingSyncHasSyncDirectory(t *testing.T) {svc, ctx := setupLifecycleValidationService(t)existing := &projectpkg.GitOpsSync{SyncDirectory: true}- require.NoError(t, svc.validateLifecycleConfigInternal(ctx, existing, lifecycleConfigInputInternal{+ require.NoError(t, svc.validateLifecycleConfig(ctx, existing, lifecycleConfigInput{scriptPath: new("scripts/deploy.sh"),runnerImage: new("alpine:latest"),}))@@ -1009,7 +998,7 @@ func TestValidateLifecycleConfig_RejectsSyncDirectoryToggleOffWhileScriptStillSeexisting.PreDeployScriptPath = new("scripts/deploy.sh")existing.PreDeployRunnerImage = new("alpine:latest")// Admin toggles syncDirectory off without clearing the script — should be rejected.- err := svc.validateLifecycleConfigInternal(ctx, existing, lifecycleConfigInputInternal{+ err := svc.validateLifecycleConfig(ctx, existing, lifecycleConfigInput{syncDirectory: new(false),})require.Error(t, err)@@ -1029,27 +1018,27 @@ func TestRedeployAfterSyncFailedError_FormatAndUnwrap(t *testing.T) {require.ErrorIs(t, err, common.ErrRedeployAfterSyncFailed)}-func (s *gitOpsSyncTestSchedulerInternal) Submit(_ context.Context, request schedulertypes.Request) (schedulertypes.Run, error) {+func (s *gitOpsSyncTestScheduler) Submit(_ context.Context, request schedulertypes.Request) (schedulertypes.Run, error) {s.submitted = append(s.submitted, request)return schedulertypes.Run{ID: request.RunID, JobID: request.JobID, EnvironmentID: request.EnvironmentID, Status: schedulertypes.Queued}, nil}-var installBackupTestTransportOnceInternal sync.Once+var installBackupTestTransportOnce sync.Once-// installBackupTestTransportInternal serves bare repositories on disk over the+// installBackupTestTransport serves bare repositories on disk over the// "http" scheme so backups push without a network or the git binary.-func installBackupTestTransportInternal() {- installBackupTestTransportOnceInternal.Do(func() {+func installBackupTestTransport() {+ installBackupTestTransportOnce.Do(func() {client.InstallProtocol("http", server.NewClient(server.NewFilesystemLoader(osfs.New("/"))))})}-// backupTestEnvInternal is one wired-up backup fixture: a bare remote, a git+// backupTestEnv is one wired-up backup fixture: a bare remote, a git// repository row, a project directory and the sync service under test.-type backupTestEnvInternal struct {+type backupTestEnv struct {service *GitOpsSyncServicedb *database.DB- scheduler *gitOpsBackupTestSchedulerInternal+ scheduler *gitOpsBackupTestSchedulerprojectsDir stringprojectPath stringproject *projectpkg.Project@@ -1057,15 +1046,15 @@ type backupTestEnvInternal struct {remote *git.Client}-// gitOpsBackupTestSchedulerInternal runs a submitted job body inline so the+// gitOpsBackupTestScheduler runs a submitted job body inline so the// save-debounce path reaches PerformSync the way the real scheduler does.-type gitOpsBackupTestSchedulerInternal struct {+type gitOpsBackupTestScheduler struct {mu sync.Mutexjobs map[string]schedulertypes.Jobsubmitted []schedulertypes.Request}-func (s *gitOpsBackupTestSchedulerInternal) AddJob(_ context.Context, job schedulertypes.Job) error {+func (s *gitOpsBackupTestScheduler) AddJob(_ context.Context, job schedulertypes.Job) error {s.mu.Lock()defer s.mu.Unlock()if s.jobs == nil {@@ -1075,20 +1064,20 @@ func (s *gitOpsBackupTestSchedulerInternal) AddJob(_ context.Context, job schedureturn nil}-func (s *gitOpsBackupTestSchedulerInternal) RemoveJob(_ context.Context, name string) {+func (s *gitOpsBackupTestScheduler) RemoveJob(_ context.Context, name string) {s.mu.Lock()defer s.mu.Unlock()delete(s.jobs, name)}-func (s *gitOpsBackupTestSchedulerInternal) HasJob(name string) bool {+func (s *gitOpsBackupTestScheduler) HasJob(name string) bool {s.mu.Lock()defer s.mu.Unlock()_, ok := s.jobs[name]return ok}-func (s *gitOpsBackupTestSchedulerInternal) Submit(ctx context.Context, request schedulertypes.Request) (schedulertypes.Run, error) {+func (s *gitOpsBackupTestScheduler) Submit(ctx context.Context, request schedulertypes.Request) (schedulertypes.Run, error) {s.mu.Lock()s.submitted = append(s.submitted, request)job := s.jobs[request.JobID]@@ -1099,21 +1088,21 @@ func (s *gitOpsBackupTestSchedulerInternal) Submit(ctx context.Context, requestreturn schedulertypes.Run{ID: request.RunID, JobID: request.JobID, EnvironmentID: request.EnvironmentID, Status: schedulertypes.Queued}, nil}-func (s *gitOpsBackupTestSchedulerInternal) submitCount() int {+func (s *gitOpsBackupTestScheduler) submitCount() int {s.mu.Lock()defer s.mu.Unlock()return len(s.submitted)}-func setupGitOpsBackupTestServiceInternal(t *testing.T) *backupTestEnvInternal {+func setupGitOpsBackupTestService(t *testing.T) *backupTestEnv {t.Helper()- installBackupTestTransportInternal()+ installBackupTestTransport()ctx := t.Context()- db := setupGitOpsProjectTestDBInternal(t)- require.NoError(t, db.AutoMigrate(&projectpkg.GitOpsSync{}, &projectpkg.ProjectTag{}, &gitrepo.GitRepository{}, &environment.Environment{}))+ db := setupGitOpsProjectTestDB(t)+ require.NoError(t, db.AutoMigrate(&projectpkg.GitOpsSync{}, &projectpkg.ProjectTag{}, &gitrepo.GitRepository{}, &environment.Environment{}, &activity.Activity{}, &activity.ActivityMessage{}))- settingsService, err := newGitOpsSettingsServiceForTestInternal(t, ctx, db)+ settingsService, err := newGitOpsSettingsServiceForTest(t, ctx, db)require.NoError(t, err)projectsDir := t.TempDir()@@ -1124,8 +1113,12 @@ func setupGitOpsBackupTestServiceInternal(t *testing.T) *backupTestEnvInternal {repoService := gitrepo.NewGitRepositoryService(db, t.TempDir(), eventService, settingsService)service := NewGitOpsSyncService(db, repoService, projectService, nil, eventService, settingsService)- scheduler := &gitOpsBackupTestSchedulerInternal{}- require.NoError(t, service.SetScheduler(ctx, scheduler, newGitOpsAdmissionGateForTestInternal(t)))+ // The test scheduler runs submitted jobs inline, which runs the sync workflow.+ harness := flowtest.New(t, activity.NewActivityService(db, nil))+ require.NoError(t, service.RegisterWorkflows(harness.Engine))+ harness.Start(t)+ scheduler := &gitOpsBackupTestScheduler{}+ require.NoError(t, service.SetScheduler(ctx, scheduler, newGitOpsAdmissionGateForTest(t)))bare := filepath.Join(t.TempDir(), "backups.git")_, err = gogit.PlainInit(bare, true)@@ -1142,7 +1135,7 @@ func setupGitOpsBackupTestServiceInternal(t *testing.T) *backupTestEnvInternal {projectPath := filepath.Join(projectsDir, "demo-project")require.NoError(t, os.MkdirAll(projectPath, 0o755))- writeBackupProjectFileInternal(t, projectPath, "compose.yaml", "services:\n app:\n image: nginx:1.27-alpine\n")+ writeBackupProjectFile(t, projectPath, "compose.yaml", "services:\n app:\n image: nginx:1.27-alpine\n")project := &projectpkg.Project{ID: "proj-backup",@@ -1153,7 +1146,7 @@ func setupGitOpsBackupTestServiceInternal(t *testing.T) *backupTestEnvInternal {}require.NoError(t, db.Create(project).Error)- return &backupTestEnvInternal{+ return &backupTestEnv{service: service,db: db,scheduler: scheduler,@@ -1165,14 +1158,14 @@ func setupGitOpsBackupTestServiceInternal(t *testing.T) *backupTestEnvInternal {}}-func writeBackupProjectFileInternal(t *testing.T, root, relative, content string) {+func writeBackupProjectFile(t *testing.T, root, relative, content string) {t.Helper()target := filepath.Join(root, filepath.FromSlash(relative))require.NoError(t, os.MkdirAll(filepath.Dir(target), 0o755))require.NoError(t, os.WriteFile(target, []byte(content), 0o644))}-func (e *backupTestEnvInternal) createBackupInternal(t *testing.T, req gitops.CreateSyncRequest) *projectpkg.GitOpsSync {+func (e *backupTestEnv) createBackup(t *testing.T, req gitops.CreateSyncRequest) *projectpkg.GitOpsSync {t.Helper()req.Name = cmp.Or(req.Name, "demo-backup")req.RepositoryID = cmp.Or(req.RepositoryID, "repo-backup")@@ -1187,15 +1180,15 @@ func (e *backupTestEnvInternal) createBackupInternal(t *testing.T, req gitops.Crreturn syncRecord}-func (e *backupTestEnvInternal) reloadInternal(t *testing.T, id string) *projectpkg.GitOpsSync {+func (e *backupTestEnv) reload(t *testing.T, id string) *projectpkg.GitOpsSync {t.Helper()var syncRecord projectpkg.GitOpsSyncrequire.NoError(t, e.db.Where("id = ?", id).First(&syncRecord).Error)return &syncRecord}-// checkoutRemoteInternal clones the backup branch so the pushed tree can be inspected.-func (e *backupTestEnvInternal) checkoutRemoteInternal(t *testing.T) string {+// checkoutRemote clones the backup branch so the pushed tree can be inspected.+func (e *backupTestEnv) checkoutRemote(t *testing.T) string {t.Helper()repoPath, err := e.remote.Clone(t.Context(), e.repoURL, "main", git.AuthConfig{AuthType: "none"}, 0)require.NoError(t, err)@@ -1203,7 +1196,7 @@ func (e *backupTestEnvInternal) checkoutRemoteInternal(t *testing.T) string {return repoPath}-func (e *backupTestEnvInternal) remoteHeadInternal(t *testing.T) string {+func (e *backupTestEnv) remoteHead(t *testing.T) string {t.Helper()head, exists, err := e.remote.RemoteBranchHead(t.Context(), e.repoURL, "main", git.AuthConfig{AuthType: "none"})require.NoError(t, err)@@ -1211,8 +1204,8 @@ func (e *backupTestEnvInternal) remoteHeadInternal(t *testing.T) string {return head}-// pushRemoteCommitInternal commits directly to the branch as another writer.-func pushRemoteCommitInternal(t *testing.T, remote *git.Client, repoURL, message string, files map[string]string) {+// pushRemoteCommit commits directly to the branch as another writer.+func pushRemoteCommit(t *testing.T, remote *git.Client, repoURL, message string, files map[string]string) {t.Helper()auth := git.AuthConfig{AuthType: "none"}checkout, err := remote.CheckoutForWrite(t.Context(), repoURL, "main", auth)@@ -1228,16 +1221,16 @@ func pushRemoteCommitInternal(t *testing.T, remote *git.Client, repoURL, message}func TestGitOpsBackup_FirstRunPushesSelectedFiles(t *testing.T) {- env := setupGitOpsBackupTestServiceInternal(t)- writeBackupProjectFileInternal(t, env.projectPath, "config/app.conf", "key = value\n")+ env := setupGitOpsBackupTestService(t)+ writeBackupProjectFile(t, env.projectPath, "config/app.conf", "key = value\n")- syncRecord := env.createBackupInternal(t, gitops.CreateSyncRequest{BackupPaths: []string{"compose.yaml", "config"}})+ syncRecord := env.createBackup(t, gitops.CreateSyncRequest{BackupPaths: []string{"compose.yaml", "config"}})require.Equal(t, gitops.SyncModeBackup, syncRecord.Mode)require.Equal(t, "backups/demo", syncRecord.BackupDirectory)require.Equal(t, "backups/demo/compose.yaml", syncRecord.ComposePath)- stored := env.reloadInternal(t, syncRecord.ID)+ stored := env.reload(t, syncRecord.ID)require.NotNil(t, stored.LastSyncStatus)assert.Equal(t, "success", *stored.LastSyncStatus)assert.NotNil(t, stored.LastBackupAt)@@ -1245,10 +1238,10 @@ func TestGitOpsBackup_FirstRunPushesSelectedFiles(t *testing.T) {assert.False(t, stored.BackupConflict)assert.Nil(t, stored.BackupFailureReason)require.NotNil(t, stored.LastSyncCommit)- assert.Equal(t, env.remoteHeadInternal(t), *stored.LastSyncCommit)+ assert.Equal(t, env.remoteHead(t), *stored.LastSyncCommit)require.NotNil(t, stored.LastBackupSnapshot)- repoPath := env.checkoutRemoteInternal(t)+ repoPath := env.checkoutRemote(t)composeBytes, err := os.ReadFile(filepath.Join(repoPath, "backups", "demo", "compose.yaml"))require.NoError(t, err)assert.Contains(t, string(composeBytes), "nginx:1.27-alpine")@@ -1262,32 +1255,32 @@ func TestGitOpsBackup_FirstRunPushesSelectedFiles(t *testing.T) {}func TestGitOpsBackup_UnchangedContentMakesNoNewCommit(t *testing.T) {- env := setupGitOpsBackupTestServiceInternal(t)- syncRecord := env.createBackupInternal(t, gitops.CreateSyncRequest{})- firstHead := env.remoteHeadInternal(t)+ env := setupGitOpsBackupTestService(t)+ syncRecord := env.createBackup(t, gitops.CreateSyncRequest{})+ firstHead := env.remoteHead(t)result, err := env.service.PerformSync(t.Context(), "0", syncRecord.ID, user.SystemUser)require.NoError(t, err)assert.True(t, result.Success)assert.Contains(t, result.Message, "already contains")- assert.Equal(t, firstHead, env.remoteHeadInternal(t))+ assert.Equal(t, firstHead, env.remoteHead(t))- stored := env.reloadInternal(t, syncRecord.ID)+ stored := env.reload(t, syncRecord.ID)require.NotNil(t, stored.LastSyncStatus)assert.Equal(t, "success", *stored.LastSyncStatus)assert.Equal(t, gitops.BackupStateBackedUp, stored.BackupState())}func TestGitOpsBackup_AddsAndRemovesFilesAndKeepsUnrelatedRemoteFiles(t *testing.T) {- env := setupGitOpsBackupTestServiceInternal(t)- writeBackupProjectFileInternal(t, env.projectPath, "config/app.conf", "key = value\n")- writeBackupProjectFileInternal(t, env.projectPath, "config/extra.conf", "extra = 1\n")- syncRecord := env.createBackupInternal(t, gitops.CreateSyncRequest{BackupPaths: []string{"compose.yaml", "config"}})+ env := setupGitOpsBackupTestService(t)+ writeBackupProjectFile(t, env.projectPath, "config/app.conf", "key = value\n")+ writeBackupProjectFile(t, env.projectPath, "config/extra.conf", "extra = 1\n")+ syncRecord := env.createBackup(t, gitops.CreateSyncRequest{BackupPaths: []string{"compose.yaml", "config"}})- pushRemoteCommitInternal(t, env.remote, env.repoURL, "unrelated docs", map[string]string{"docs/readme.md": "docs\n"})+ pushRemoteCommit(t, env.remote, env.repoURL, "unrelated docs", map[string]string{"docs/readme.md": "docs\n"})require.NoError(t, os.Remove(filepath.Join(env.projectPath, "config", "extra.conf")))- writeBackupProjectFileInternal(t, env.projectPath, "scripts/run.sh", "#!/bin/sh\necho hi\n")+ writeBackupProjectFile(t, env.projectPath, "scripts/run.sh", "#!/bin/sh\necho hi\n")_, err := env.service.UpdateSync(t.Context(), "0", syncRecord.ID, gitops.UpdateSyncRequest{BackupPaths: []string{"compose.yaml", "config", "scripts"},@@ -1298,7 +1291,7 @@ func TestGitOpsBackup_AddsAndRemovesFilesAndKeepsUnrelatedRemoteFiles(t *testingrequire.NoError(t, err)require.True(t, result.Success)- repoPath := env.checkoutRemoteInternal(t)+ repoPath := env.checkoutRemote(t)assert.FileExists(t, filepath.Join(repoPath, "backups", "demo", "scripts", "run.sh"))assert.FileExists(t, filepath.Join(repoPath, "backups", "demo", "config", "app.conf"))assert.FileExists(t, filepath.Join(repoPath, "docs", "readme.md"))@@ -1316,15 +1309,15 @@ func TestGitOpsBackup_EnvFilesOnlyIncludedWhenListedExplicitly(t *testing.T) {}for _, test := range tests {t.Run(test.name, func(t *testing.T) {- env := setupGitOpsBackupTestServiceInternal(t)- writeBackupProjectFileInternal(t, env.projectPath, ".env", "TOKEN=secret\n")- writeBackupProjectFileInternal(t, env.projectPath, "config/app.conf", "key = value\n")- writeBackupProjectFileInternal(t, env.projectPath, "config/.env", "TOKEN=nested\n")- writeBackupProjectFileInternal(t, env.projectPath, "config/staging.env", "TOKEN=staging\n")+ env := setupGitOpsBackupTestService(t)+ writeBackupProjectFile(t, env.projectPath, ".env", "TOKEN=secret\n")+ writeBackupProjectFile(t, env.projectPath, "config/app.conf", "key = value\n")+ writeBackupProjectFile(t, env.projectPath, "config/.env", "TOKEN=nested\n")+ writeBackupProjectFile(t, env.projectPath, "config/staging.env", "TOKEN=staging\n")- env.createBackupInternal(t, gitops.CreateSyncRequest{BackupPaths: test.paths})+ env.createBackup(t, gitops.CreateSyncRequest{BackupPaths: test.paths})- repoPath := env.checkoutRemoteInternal(t)+ repoPath := env.checkoutRemote(t)assert.FileExists(t, filepath.Join(repoPath, "backups", "demo", "compose.yaml"))assert.FileExists(t, filepath.Join(repoPath, "backups", "demo", "config", "app.conf"))assert.NoFileExists(t, filepath.Join(repoPath, "backups", "demo", "config", ".env"))@@ -1339,19 +1332,19 @@ func TestGitOpsBackup_EnvFilesOnlyIncludedWhenListedExplicitly(t *testing.T) {}func TestGitOpsBackup_RemoteEditConflictsThenResolvesWithArcane(t *testing.T) {- env := setupGitOpsBackupTestServiceInternal(t)- syncRecord := env.createBackupInternal(t, gitops.CreateSyncRequest{})+ env := setupGitOpsBackupTestService(t)+ syncRecord := env.createBackup(t, gitops.CreateSyncRequest{})- pushRemoteCommitInternal(t, env.remote, env.repoURL, "edited backup outside arcane", map[string]string{+ pushRemoteCommit(t, env.remote, env.repoURL, "edited backup outside arcane", map[string]string{"backups/demo/compose.yaml": "services:\n app:\n image: tampered\n",})- writeBackupProjectFileInternal(t, env.projectPath, "compose.yaml", "services:\n app:\n image: nginx:1.28-alpine\n")+ writeBackupProjectFile(t, env.projectPath, "compose.yaml", "services:\n app:\n image: nginx:1.28-alpine\n")_, err := env.service.PerformSync(t.Context(), "0", syncRecord.ID, user.SystemUser)require.Error(t, err)require.ErrorIs(t, err, common.ErrConflict)- stored := env.reloadInternal(t, syncRecord.ID)+ stored := env.reload(t, syncRecord.ID)assert.True(t, stored.BackupConflict)assert.Equal(t, gitops.BackupStateNeedsAttention, stored.BackupState())require.NotNil(t, stored.BackupFailureReason)@@ -1368,26 +1361,26 @@ func TestGitOpsBackup_RemoteEditConflictsThenResolvesWithArcane(t *testing.T) {require.NoError(t, err)require.True(t, result.Success)- resolved := env.reloadInternal(t, syncRecord.ID)+ resolved := env.reload(t, syncRecord.ID)assert.False(t, resolved.BackupConflict)assert.Nil(t, resolved.BackupFailureReason)assert.Equal(t, gitops.BackupStateBackedUp, resolved.BackupState())- repoPath := env.checkoutRemoteInternal(t)+ repoPath := env.checkoutRemote(t)composeBytes, err := os.ReadFile(filepath.Join(repoPath, "backups", "demo", "compose.yaml"))require.NoError(t, err)assert.Contains(t, string(composeBytes), "nginx:1.28-alpine")}func TestGitOpsBackup_OccupiedDestinationNeedsAttention(t *testing.T) {- env := setupGitOpsBackupTestServiceInternal(t)- pushRemoteCommitInternal(t, env.remote, env.repoURL, "pre-existing files", map[string]string{+ env := setupGitOpsBackupTestService(t)+ pushRemoteCommit(t, env.remote, env.repoURL, "pre-existing files", map[string]string{"backups/demo/unrelated.txt": "not an arcane backup\n",})- syncRecord := env.createBackupInternal(t, gitops.CreateSyncRequest{})+ syncRecord := env.createBackup(t, gitops.CreateSyncRequest{})- stored := env.reloadInternal(t, syncRecord.ID)+ stored := env.reload(t, syncRecord.ID)assert.True(t, stored.BackupConflict)assert.Equal(t, gitops.BackupStateNeedsAttention, stored.BackupState())require.NotNil(t, stored.BackupFailureReason)@@ -1396,9 +1389,9 @@ func TestGitOpsBackup_OccupiedDestinationNeedsAttention(t *testing.T) {}func TestGitOpsBackup_AdoptsMatchingRemoteWithoutSnapshot(t *testing.T) {- env := setupGitOpsBackupTestServiceInternal(t)- syncRecord := env.createBackupInternal(t, gitops.CreateSyncRequest{})- head := env.remoteHeadInternal(t)+ env := setupGitOpsBackupTestService(t)+ syncRecord := env.createBackup(t, gitops.CreateSyncRequest{})+ head := env.remoteHead(t)require.NoError(t, env.db.Model(&projectpkg.GitOpsSync{}).Where("id = ?", syncRecord.ID).Update("last_backup_snapshot", nil).Error)@@ -1406,25 +1399,25 @@ func TestGitOpsBackup_AdoptsMatchingRemoteWithoutSnapshot(t *testing.T) {result, err := env.service.PerformSync(t.Context(), "0", syncRecord.ID, user.SystemUser)require.NoError(t, err)assert.True(t, result.Success)- assert.Equal(t, head, env.remoteHeadInternal(t))+ assert.Equal(t, head, env.remoteHead(t))- stored := env.reloadInternal(t, syncRecord.ID)+ stored := env.reload(t, syncRecord.ID)assert.False(t, stored.BackupConflict)assert.Equal(t, gitops.BackupStateBackedUp, stored.BackupState())}func TestGitOpsBackup_SucceedsOnTopOfAnotherWritersCommit(t *testing.T) {- env := setupGitOpsBackupTestServiceInternal(t)- syncRecord := env.createBackupInternal(t, gitops.CreateSyncRequest{})+ env := setupGitOpsBackupTestService(t)+ syncRecord := env.createBackup(t, gitops.CreateSyncRequest{})- pushRemoteCommitInternal(t, env.remote, env.repoURL, "other writer", map[string]string{"other/notes.txt": "notes\n"})- writeBackupProjectFileInternal(t, env.projectPath, "compose.yaml", "services:\n app:\n image: nginx:1.29-alpine\n")+ pushRemoteCommit(t, env.remote, env.repoURL, "other writer", map[string]string{"other/notes.txt": "notes\n"})+ writeBackupProjectFile(t, env.projectPath, "compose.yaml", "services:\n app:\n image: nginx:1.29-alpine\n")result, err := env.service.PerformSync(t.Context(), "0", syncRecord.ID, user.SystemUser)require.NoError(t, err)require.True(t, result.Success)- repoPath := env.checkoutRemoteInternal(t)+ repoPath := env.checkoutRemote(t)assert.FileExists(t, filepath.Join(repoPath, "other", "notes.txt"))composeBytes, err := os.ReadFile(filepath.Join(repoPath, "backups", "demo", "compose.yaml"))require.NoError(t, err)@@ -1437,13 +1430,13 @@ func TestGitOpsBackup_CreateValidation(t *testing.T) {tests := []struct {name string- mutate func(t *testing.T, env *backupTestEnvInternal)+ mutate func(t *testing.T, env *backupTestEnv)request gitops.CreateSyncRequestwantErr error}{{name: "project deployed from git",- mutate: func(t *testing.T, env *backupTestEnvInternal) {+ mutate: func(t *testing.T, env *backupTestEnv) {t.Helper()require.NoError(t, env.db.Model(&projectpkg.Project{}).Where("id = ?", env.project.ID).Update("gitops_managed_by", "sync-deploy").Error)@@ -1452,16 +1445,16 @@ func TestGitOpsBackup_CreateValidation(t *testing.T) {},{name: "second backup for the same project",- mutate: func(t *testing.T, env *backupTestEnvInternal) {+ mutate: func(t *testing.T, env *backupTestEnv) {t.Helper()- env.createBackupInternal(t, gitops.CreateSyncRequest{})+ env.createBackup(t, gitops.CreateSyncRequest{})},request: gitops.CreateSyncRequest{Name: "second", BackupDirectory: "backups/other"},wantErr: common.ErrConflict,},{name: "overlapping destination on the same branch",- mutate: func(t *testing.T, env *backupTestEnvInternal) {+ mutate: func(t *testing.T, env *backupTestEnv) {t.Helper()other := &projectpkg.Project{ID: "proj-other",@@ -1470,9 +1463,9 @@ func TestGitOpsBackup_CreateValidation(t *testing.T) {Path: filepath.Join(env.projectsDir, "other-project"),Status: projectpkg.ProjectStatusStopped,}- writeBackupProjectFileInternal(t, other.Path, "compose.yaml", "services: {}\n")+ writeBackupProjectFile(t, other.Path, "compose.yaml", "services: {}\n")require.NoError(t, env.db.Create(other).Error)- env.createBackupInternal(t, gitops.CreateSyncRequest{})+ env.createBackup(t, gitops.CreateSyncRequest{})env.project = other},request: gitops.CreateSyncRequest{Name: "nested", BackupDirectory: "backups/demo/nested"},@@ -1492,8 +1485,8 @@ func TestGitOpsBackup_CreateValidation(t *testing.T) {for _, test := range tests {t.Run(test.name, func(t *testing.T) {- env := setupGitOpsBackupTestServiceInternal(t)- writeBackupProjectFileInternal(t, env.projectPath, "config/app.conf", "key = value\n")+ env := setupGitOpsBackupTestService(t)+ writeBackupProjectFile(t, env.projectPath, "config/app.conf", "key = value\n")if test.mutate != nil {test.mutate(t, env)}@@ -1513,7 +1506,7 @@ func TestGitOpsBackup_CreateValidation(t *testing.T) {}func TestGitOpsBackup_CreateWithoutProjectIsRejected(t *testing.T) {- env := setupGitOpsBackupTestServiceInternal(t)+ env := setupGitOpsBackupTestService(t)_, err := env.service.CreateSync(t.Context(), "0", gitops.CreateSyncRequest{Name: "demo-backup",@@ -1528,14 +1521,14 @@ func TestGitOpsBackup_CreateWithoutProjectIsRejected(t *testing.T) {}func TestGitOpsBackup_SaveSignalMarksPendingAndRunsAfterDebounce(t *testing.T) {- env := setupGitOpsBackupTestServiceInternal(t)- syncRecord := env.createBackupInternal(t, gitops.CreateSyncRequest{})- firstHead := env.remoteHeadInternal(t)+ env := setupGitOpsBackupTestService(t)+ syncRecord := env.createBackup(t, gitops.CreateSyncRequest{})+ firstHead := env.remoteHead(t)env.service.backups.debounce = 50 * time.Millisecondenv.service.SubscribeProjectFileChanges(t.Context())- writeBackupProjectFileInternal(t, env.projectPath, "compose.yaml", "services:\n app:\n image: nginx:1.29-alpine\n")+ writeBackupProjectFile(t, env.projectPath, "compose.yaml", "services:\n app:\n image: nginx:1.29-alpine\n")env.service.projectService.FilesChanged.Publish(env.project.ID)require.Eventually(t, func() bool {@@ -1543,23 +1536,23 @@ func TestGitOpsBackup_SaveSignalMarksPendingAndRunsAfterDebounce(t *testing.T) {return err == nil && exists && head != firstHead}, 5*time.Second, 25*time.Millisecond)require.Positive(t, env.scheduler.submitCount())- require.NotEqual(t, firstHead, env.remoteHeadInternal(t))+ require.NotEqual(t, firstHead, env.remoteHead(t))require.Eventually(t, func() bool {- stored := env.reloadInternal(t, syncRecord.ID)+ stored := env.reload(t, syncRecord.ID)return stored.LastSyncStatus != nil && *stored.LastSyncStatus == "success" && !stored.BackupPending}, 5*time.Second, 25*time.Millisecond)- stored := env.reloadInternal(t, syncRecord.ID)+ stored := env.reload(t, syncRecord.ID)require.NotNil(t, stored.LastBackupAt)assert.False(t, stored.BackupPending)assert.Equal(t, gitops.BackupStateBackedUp, stored.BackupState())}func TestGitOpsBackup_SaveSignalOnlyMarksPendingWhenAutoSyncIsOff(t *testing.T) {- env := setupGitOpsBackupTestServiceInternal(t)- syncRecord := env.createBackupInternal(t, gitops.CreateSyncRequest{})- head := env.remoteHeadInternal(t)+ env := setupGitOpsBackupTestService(t)+ syncRecord := env.createBackup(t, gitops.CreateSyncRequest{})+ head := env.remoteHead(t)autoSync := false_, err := env.service.UpdateSync(t.Context(), "0", syncRecord.ID, gitops.UpdateSyncRequest{AutoSync: &autoSync}, user.SystemUser)@@ -1568,31 +1561,31 @@ func TestGitOpsBackup_SaveSignalOnlyMarksPendingWhenAutoSyncIsOff(t *testing.T)env.service.backups.debounce = 50 * time.Millisecondenv.service.SubscribeProjectFileChanges(t.Context())- writeBackupProjectFileInternal(t, env.projectPath, "compose.yaml", "services:\n app:\n image: nginx:1.29-alpine\n")+ writeBackupProjectFile(t, env.projectPath, "compose.yaml", "services:\n app:\n image: nginx:1.29-alpine\n")env.service.projectService.FilesChanged.Publish(env.project.ID)require.Eventually(t, func() bool {- return env.reloadInternal(t, syncRecord.ID).BackupPending+ return env.reload(t, syncRecord.ID).BackupPending}, 5*time.Second, 25*time.Millisecond)time.Sleep(300 * time.Millisecond)- assert.Equal(t, head, env.remoteHeadInternal(t))+ assert.Equal(t, head, env.remoteHead(t))- stored := env.reloadInternal(t, syncRecord.ID)+ stored := env.reload(t, syncRecord.ID)assert.True(t, stored.BackupPending)assert.Equal(t, gitops.BackupStatePaused, stored.BackupState())}func TestGitOpsBackup_ReconcileInterruptedBackupsOnStartup(t *testing.T) {- env := setupGitOpsBackupTestServiceInternal(t)- syncRecord := env.createBackupInternal(t, gitops.CreateSyncRequest{})+ env := setupGitOpsBackupTestService(t)+ syncRecord := env.createBackup(t, gitops.CreateSyncRequest{})require.NoError(t, env.db.Model(&projectpkg.GitOpsSync{}).Where("id = ?", syncRecord.ID).Update("last_sync_status", backup.BackupStatusRunning).Error)require.NoError(t, env.service.ReconcileInterruptedBackupsOnStartup(t.Context()))- stored := env.reloadInternal(t, syncRecord.ID)+ stored := env.reload(t, syncRecord.ID)require.NotNil(t, stored.LastSyncStatus)assert.Equal(t, "failed", *stored.LastSyncStatus)assert.True(t, stored.BackupPending)@@ -1601,8 +1594,8 @@ func TestGitOpsBackup_ReconcileInterruptedBackupsOnStartup(t *testing.T) {}func TestGitOpsBackup_DeletingProjectRemovesBackupSync(t *testing.T) {- env := setupGitOpsBackupTestServiceInternal(t)- syncRecord := env.createBackupInternal(t, gitops.CreateSyncRequest{})+ env := setupGitOpsBackupTestService(t)+ syncRecord := env.createBackup(t, gitops.CreateSyncRequest{})require.NoError(t, env.service.projectService.DestroyProject(t.Context(), env.project.ID, true, false, user.SystemUser))@@ -1612,11 +1605,11 @@ func TestGitOpsBackup_DeletingProjectRemovesBackupSync(t *testing.T) {}func TestGitOpsDeploy_LinksExistingProject(t *testing.T) {- env := setupGitOpsBackupTestServiceInternal(t)+ env := setupGitOpsBackupTestService(t)ctx := t.Context()remoteCompose := "services:\n app:\n image: nginx:1.28-alpine\n"- pushRemoteCommitInternal(t, env.remote, env.repoURL, "seed", map[string]string{"apps/demo/compose.yaml": remoteCompose})- writeBackupProjectFileInternal(t, env.projectPath, ".env", "TOKEN=keep-me\n")+ pushRemoteCommit(t, env.remote, env.repoURL, "seed", map[string]string{"apps/demo/compose.yaml": remoteCompose})+ writeBackupProjectFile(t, env.projectPath, ".env", "TOKEN=keep-me\n")created, err := env.service.CreateSync(ctx, "0", gitops.CreateSyncRequest{Name: "deploy-existing",@@ -1653,7 +1646,7 @@ func TestGitOpsDeploy_LinksExistingProject(t *testing.T) {}func TestGitOpsImport_ForwardsDeployAndLifecycleFields(t *testing.T) {- env := setupGitOpsBackupTestServiceInternal(t)+ env := setupGitOpsBackupTestService(t)ctx := t.Context()require.NoError(t, env.service.settingsService.SetStringSetting(ctx, "lifecycleEnabled", "true"))diff --git a/backend/internal/gitrepo/handler.go b/backend/internal/gitrepo/handler.goindex 515262ab6..38ffad320 100644--- a/backend/internal/gitrepo/handler.go+++ b/backend/internal/gitrepo/handler.go@@ -165,14 +165,7 @@ func (h *GitRepositoryHandler) DeleteRepository(ctx context.Context, input *Delereturn nil, huma.NewError(apiErr.HTTPStatus(), "Failed to delete git repository")}- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{- Message: "Repository deleted successfully",- },- },- }, nil+ return handlerutil.MessageOutput("Repository deleted successfully", ""), nil}// TestRepository tests connectivity and authentication to a git repository.@@ -183,14 +176,7 @@ func (h *GitRepositoryHandler) TestRepository(ctx context.Context, input *TestGireturn nil, huma.Error400BadRequest("Failed to test git repository connection: " + err.Error())}- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{- Message: "Connection successful",- },- },- }, nil+ return handlerutil.MessageOutput("Connection successful", ""), nil}// ListBranches returns all branches from a git repository.@@ -236,12 +222,5 @@ func (h *GitRepositoryHandler) SyncRepositories(ctx context.Context, input *Syncreturn nil, huma.NewError(apiErr.HTTPStatus(), "Failed to sync git repositories: "+err.Error())}- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{- Message: "Repositories synced successfully",- },- },- }, nil+ return handlerutil.MessageOutput("Repositories synced successfully", ""), nil}diff --git a/backend/internal/image/children/patch/service.go b/backend/internal/image/children/patch/service.goindex 767153be2..226449fd7 100644--- a/backend/internal/image/children/patch/service.go+++ b/backend/internal/image/children/patch/service.go@@ -11,6 +11,7 @@ import ("maps""os""path/filepath"+ "slices""strings""time"@@ -46,6 +47,19 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/utils/logging")+// scheduledPatchUser is the actor recorded for scheduled patches and their verification scans.+var (+ scheduledPatchUser = usertypes.Actor{Username: "System"}+ // errPatchInterrupted fails a patch cut off mid-run; its record keeps the message so it never reruns.+ errPatchInterrupted = errors.New("image patch was interrupted before its result was recorded; start a new patch")+)++// Target is one image a scheduled patch run covers.+type Target struct {+ ImageID string `json:"imageId"`+ ImageName string `json:"imageName"`+}+// Service patches image OS packages in place using the Copacetic// library and records each run in the image_patches table.type Service struct {@@ -98,128 +112,186 @@ func NewService(// PatchImage starts a background patch run for the given image and returns the// pending record (carrying the activity ID) immediately.func (s *Service) PatchImage(ctx context.Context, envID, imageID string, opts imagepatch.PatchOptions, user usertypes.Actor) (*imagepatch.PatchRecord, error) {- if strings.TrimSpace(opts.ScanID) != "" {+ if opts.ScanID != "" {if err := s.settingsService.RequireFeature(ctx, features.VulnerabilityManagement); err != nil {return nil, err}}- runCtx := utils.ActivityRuntimeContext(ctx, nil)+ record, run, err := s.startPatch(utils.ActivityRuntimeContext(ctx, nil), envID, imageID, opts, user, "")+ if err != nil {+ return nil, err+ }+ go func() { _ = run() }()+ dto := record.ToDto()+ return &dto, nil+}++// PatchTarget patches one scanned image and waits for the result. recordID keys+// the patch so a redelivered task resumes its own record instead of patching twice.+func (s *Service) PatchTarget(ctx context.Context, envID string, target Target, recordID string) error {+ _, run, err := s.startPatch(ctx, envID, target.ImageID, imagepatch.PatchOptions{ScanID: target.ImageID}, scheduledPatchUser, recordID)+ if err != nil || run == nil {+ return err+ }+ return run()+}++// startPatch validates an image, records its patch as running, and returns the+// function that performs it. A completed record under recordID returns no function.+func (s *Service) startPatch(+ ctx context.Context,+ envID, imageID string,+ opts imagepatch.PatchOptions,+ user usertypes.Actor,+ recordID string,+) (*ImagePatchRecord, func() error, error) {+ var existing ImagePatchRecord+ if recordID != "" {+ if err := s.db.WithContext(ctx).Where("id = ?", recordID).Limit(1).Find(&existing).Error; err != nil {+ return nil, nil, fmt.Errorf("failed to load image patch record: %w", err)+ }+ if existing.Status == string(imagepatch.PatchStatusCompleted) {+ return &existing, nil, nil+ }+ // A record still patching was cut off mid-run, so whether Copacetic already replaced its tag is unknown.+ // Its failure keeps that message, so no later delivery runs the patch again either.+ if existing.Status == string(imagepatch.PatchStatusPatching) || mo.PointerToOption(existing.Error).OrEmpty() == errPatchInterrupted.Error() {+ s.finishPatchRecord(ctx, &existing, imagepatch.PatchStatusFailed, errPatchInterrupted.Error(), nil, 0)+ return nil, nil, errPatchInterrupted+ }+ }dockerClient, err := s.dockerService.GetClient(ctx)if err != nil {- return nil, fmt.Errorf("failed to connect to Docker: %w", err)+ return nil, nil, fmt.Errorf("failed to connect to Docker: %w", err)}- if requireContainerdImageStoreErr := requireContainerdImageStoreInternal(runCtx, dockerClient); requireContainerdImageStoreErr != nil {- return nil, requireContainerdImageStoreErr+ if storeErr := requireContainerdImageStore(ctx, dockerClient); storeErr != nil {+ return nil, nil, storeErr}-- imageInspect, err := dockerClient.ImageInspect(runCtx, imageID)+ imageInspect, err := dockerClient.ImageInspect(ctx, imageID)if err != nil {- return nil, fmt.Errorf("failed to inspect image: %w", err)+ return nil, nil, fmt.Errorf("failed to inspect image: %w", err)}if len(imageInspect.RepoTags) == 0 {- return nil, common.ErrImageUntagged+ return nil, nil, common.ErrImageUntagged}imageRef := imageInspect.RepoTags[0]-+ if existing.ID != "" {+ // A replay patches the source its record names, so the output matches the saved PatchedRef.+ if !slices.Contains(imageInspect.RepoTags, existing.OriginalRef) {+ return nil, nil, fmt.Errorf("image is no longer tagged %s; start a new patch", existing.OriginalRef)+ }+ imageRef = existing.OriginalRef+ }// Never pulled or pushed: BuildKit cannot fetch the image to patch it.if len(imageInspect.RepoDigests) == 0 {- return nil, common.ErrImageLocalOnly+ return nil, nil, common.ErrImageLocalOnly}- originalDigest := imageInspect.RepoDigests[0]- // Resolve the stored scan report when patching from a scan.mode := imagepatch.PatchModeUpdateAllvar reportData []byte- if strings.TrimSpace(opts.ScanID) != "" {- // Scan records are keyed by image ID; refuse a scan belonging to a- // different image than the one selected by the route.+ if opts.ScanID != "" {+ // Scan records are keyed by image ID; refuse a scan of a different image than the route selected.if opts.ScanID != imageInspect.ID {- return nil, common.ErrPatchScanImageMismatch+ return nil, nil, common.ErrPatchScanImageMismatch}var report vulnerability.VulnerabilityReportRecord- if loadScanReportErr := s.db.WithContext(runCtx).First(&report, "image_id = ?", opts.ScanID).Error; loadScanReportErr != nil || report.Data == "" {- return nil, common.ErrPatchScanReportUnavailable+ if reportErr := s.db.WithContext(ctx).First(&report, "image_id = ?", opts.ScanID).Error; reportErr != nil || report.Data == "" {+ return nil, nil, common.ErrPatchScanReportUnavailable}reportData = []byte(report.Data)mode = imagepatch.PatchModeReport}- suffix := strings.TrimSpace(opts.Suffix)- if suffix == "" {- suffix = strings.TrimSpace(s.settingsService.GetSettingsConfig().ImagePatchSuffix.Value)- }- suffix = cmp.Or(suffix, "patched")+ suffix := cmp.Or(strings.TrimSpace(opts.Suffix), strings.TrimSpace(s.settingsService.GetSettingsConfig().ImagePatchSuffix.Value), "patched")patchedRef, err := resolvePatchedRef(imageRef, opts.PatchedTag, suffix)if err != nil {- return nil, err+ return nil, nil, err}- // Copa reads registry credentials from the docker config; refresh it before- // any registry lookups below.- s.writeRegistryAuthConfigInternal(runCtx)+ // Copa reads registry credentials from the docker config; refresh it before any registry lookups.+ s.writeRegistryAuthConfig(ctx)- // Unless the all-platforms setting is enabled, pin the patch target to the- // platform-specific manifest digest matching this image so copa patches a- // single platform and keeps the plain patched tag. Report-mode patches are- // always single-platform in copa, so no pinning is needed there.+ // Unless the all-platforms setting is on, pin update-all patches to this image's platform manifest so copa+ // patches one platform and keeps the plain patched tag. Report-mode patches are always single-platform.copaImageRef := imageRefif mode == imagepatch.PatchModeUpdateAll && !s.settingsService.GetSettingsConfig().ImagePatchAllPlatforms.IsTrue() {- if pinned := platformPinnedRefInternal(runCtx, imageRef, v1.Platform{+ copaImageRef = cmp.Or(platformPinnedRef(ctx, imageRef, v1.Platform{OS: imageInspect.Os,Architecture: imageInspect.Architecture,Variant: imageInspect.Variant,- }); pinned != "" {- copaImageRef = pinned- }- }-- activityID := s.startPatchActivityInternal(runCtx, envID, imageID, imageRef, &user)- runCtx = s.activityService.Track(runCtx, activityID)-- record := &ImagePatchRecord{- EnvironmentID: envID,- OriginalImageID: imageInspect.ID,- OriginalRef: imageRef,- OriginalDigest: originalDigest,- PatchedRef: patchedRef,- Mode: string(mode),- Status: string(imagepatch.PatchStatusPatching),- ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer(),+ }), imageRef)+ }++ activityID := ""+ if s.activityService != nil {+ // Each attempt gets its own activity, since a finished activity never reopens.+ started, startErr := s.activityService.StartActivity(ctx, activity.StartActivityRequest{+ EnvironmentID: envID,+ Type: activitytypes.TypeImagePatch,+ ResourceType: new("image"),+ ResourceID: &imageID,+ ResourceName: &imageRef,+ StartedBy: &user,+ Progress: new(0),+ LatestMessage: "Image patch queued",+ })+ if startErr != nil {+ slog.WarnContext(ctx, "failed to create image patch activity", "error", startErr, "imageRef", imageRef)+ } else {+ activityID = started.ID+ }+ }+ runCtx := s.activityService.Track(ctx, activityID)++ record := &existing+ var saveErr error+ if record.ID == "" {+ record = &ImagePatchRecord{+ EnvironmentID: envID,+ OriginalImageID: imageInspect.ID,+ OriginalRef: imageRef,+ OriginalDigest: imageInspect.RepoDigests[0],+ PatchedRef: patchedRef,+ Mode: string(mode),+ Status: string(imagepatch.PatchStatusPatching),+ ActivityID: mo.EmptyableToOption(activityID).ToPointer(),+ }+ record.ID = recordID+ saveErr = s.db.WithContext(ctx).Create(record).Error+ } else {+ // A retried record runs again under this attempt's activity, without the earlier attempt's error.+ record.Status, record.Error, record.ActivityID = string(imagepatch.PatchStatusPatching), nil, mo.EmptyableToOption(activityID).ToPointer()+ saveErr = s.db.WithContext(ctx).Model(record).Select("status", "error", "activity_id").Updates(record).Error}- if createPatchRecordErr := s.db.WithContext(runCtx).Create(record).Error; createPatchRecordErr != nil {- return nil, fmt.Errorf("failed to create image patch record: %w", createPatchRecordErr)+ if saveErr != nil {+ // No worker will run, so settle the activity and release its cancel registration here.+ saveErr = fmt.Errorf("failed to save image patch record: %w", saveErr)+ s.completePatchActivity(runCtx, activityID, false, saveErr.Error())+ return nil, nil, saveErr}- slog.InfoContext(runCtx, "image patch queued",- "environmentId", envID,- "imageRef", imageRef,- "patchedRef", patchedRef,- "mode", mode,- "patchTarget", copaImageRef,- )-- go s.patchInBackgroundInternal(runCtx, record, opts, reportData, copaImageRef, activityID)-- dto := record.ToDto()- return &dto, nil+ slog.InfoContext(ctx, "image patch queued", "environmentId", envID, "imageRef", imageRef, "patchedRef", patchedRef, "mode", mode, "patchTarget", copaImageRef)+ return record, func() error { return s.runPatch(runCtx, record, opts, reportData, copaImageRef, activityID) }, nil}-func (s *Service) patchInBackgroundInternal(ctx context.Context, record *ImagePatchRecord, opts imagepatch.PatchOptions, reportData []byte, copaImageRef, activityID string) {+// runPatch executes one recorded patch with Copacetic and settles its record and activity.+func (s *Service) runPatch(ctx context.Context, record *ImagePatchRecord, opts imagepatch.PatchOptions, reportData []byte, copaImageRef, activityID string) error {+ fail := func(err error, durationMs int64) error {+ s.finishPatchRecord(ctx, record, imagepatch.PatchStatusFailed, err.Error(), nil, durationMs)+ s.completePatchActivity(ctx, activityID, false, err.Error())+ return err+ }select {case s.patchSlot <- struct{}{}:defer func() { <-s.patchSlot }()case <-ctx.Done():- s.finishPatchRecordInternal(ctx, record, imagepatch.PatchStatusFailed, ctx.Err().Error(), nil, 0)- s.completePatchActivityInternal(ctx, activityID, false, ctx.Err().Error())- return+ return fail(ctx.Err(), 0)}if record.Mode == string(imagepatch.PatchModeReport) {if err := s.settingsService.RequireFeature(ctx, features.VulnerabilityManagement); err != nil {- s.finishPatchRecordInternal(ctx, record, imagepatch.PatchStatusFailed, err.Error(), nil, 0)- s.completePatchActivityInternal(ctx, activityID, false, err.Error())- return+ return fail(err, 0)}}@@ -238,9 +310,7 @@ func (s *Service) patchInBackgroundInternal(ctx context.Context, record *ImagePa}()}if err != nil {- s.finishPatchRecordInternal(ctx, record, imagepatch.PatchStatusFailed, err.Error(), nil, 0)- s.completePatchActivityInternal(ctx, activityID, false, err.Error())- return+ return fail(err, 0)}}@@ -286,7 +356,7 @@ func (s *Service) patchInBackgroundInternal(ctx context.Context, record *ImagePa// Mirror copa's log output into the activity while the patch runs; BuildKit// step progress stays quiet until copa exposes a progress writer upstream.- s.appendPatchActivityInternal(ctx, activityID, 30, "Patching image packages via BuildKit")+ s.appendPatchActivity(ctx, activityID, 30, "Patching image packages via BuildKit")patchOut := activitylib.NewWriter(ctx, s.activityService, activityID, nil, "Patching image")removeMirror := logging.AddLogrusMirror(patchOut)patchErr := patch.Patch(ctx, copaOpts)@@ -301,15 +371,12 @@ func (s *Service) patchInBackgroundInternal(ctx context.Context, record *ImagePaif strings.Contains(patchErr.Error(), `exporter "docker" could not be found`) {patchErr = common.ErrPatchRequiresContainerdImageStore}- s.finishPatchRecordInternal(ctx, record, imagepatch.PatchStatusFailed, patchErr.Error(), nil, durationMs)- slog.WarnContext(ctx, "image patch failed",- "environmentId", record.EnvironmentID,- "imageRef", record.OriginalRef,- "durationMs", durationMs,- "error", patchErr,- )- s.completePatchActivityInternal(ctx, activityID, false, patchErr.Error())- return+ // Interrupted mid-run, Copacetic may already have replaced the tag, so the record must never rerun.+ if ctx.Err() != nil {+ patchErr = errPatchInterrupted+ }+ slog.WarnContext(ctx, "image patch failed", "environmentId", record.EnvironmentID, "imageRef", record.OriginalRef, "durationMs", durationMs, "error", patchErr)+ return fail(patchErr, durationMs)}// Count VEX statements as the number of patched packages, when available.@@ -326,42 +393,37 @@ func (s *Service) patchInBackgroundInternal(ctx context.Context, record *ImagePa}}- s.verifyPatchedImageInternal(ctx, record, activityID)+ s.verifyPatchedImage(ctx, record, activityID)- s.finishPatchRecordInternal(ctx, record, imagepatch.PatchStatusCompleted, "", packagesUpdated, durationMs)- slog.InfoContext(ctx, "image patch completed",- "environmentId", record.EnvironmentID,- "imageRef", record.OriginalRef,- "patchedRef", record.PatchedRef,- "durationMs", durationMs,- )- s.completePatchActivityInternal(ctx, activityID, true, "")+ s.finishPatchRecord(ctx, record, imagepatch.PatchStatusCompleted, "", packagesUpdated, durationMs)+ slog.InfoContext(ctx, "image patch completed", "environmentId", record.EnvironmentID, "imageRef", record.OriginalRef, "patchedRef", record.PatchedRef, "durationMs", durationMs)+ s.completePatchActivity(ctx, activityID, true, "")+ return nil}-func (s *Service) verifyPatchedImageInternal(ctx context.Context, record *ImagePatchRecord, activityID string) {- // Warn when the expected patched tag is missing from the daemon (e.g. copa- // took the multi-platform path and arch-suffixed the tag). When it exists,- // re-scan it so the security page can show whether the patch worked.+// verifyPatchedImage warns when the expected tag is missing (copa may arch-suffix+// multi-platform tags) and otherwise re-scans it to show whether the patch worked.+func (s *Service) verifyPatchedImage(ctx context.Context, record *ImagePatchRecord, activityID string) {if dockerClient, err := s.dockerService.GetClient(ctx); err == nil {- if patchedInspect, imageInspectErr := dockerClient.ImageInspect(ctx, record.PatchedRef); imageInspectErr != nil {- slog.WarnContext(ctx, "patched image tag not found after patching", "patchedRef", record.PatchedRef, "error", imageInspectErr)- s.appendPatchActivityInternal(ctx, activityID, 95, "Patched image was created but the expected tag "+record.PatchedRef+" was not found; check the image list")+ if patchedInspect, inspectErr := dockerClient.ImageInspect(ctx, record.PatchedRef); inspectErr != nil {+ slog.WarnContext(ctx, "patched image tag not found after patching", "patchedRef", record.PatchedRef, "error", inspectErr)+ s.appendPatchActivity(ctx, activityID, 95, "Patched image was created but the expected tag "+record.PatchedRef+" was not found; check the image list")} else if s.vulnerabilityService != nil && s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) {- s.appendPatchActivityInternal(ctx, activityID, 95, "Re-scanning patched image to verify the patch")- if _, scanImageErr := s.vulnerabilityService.ScanImage(context.WithoutCancel(ctx), record.EnvironmentID, patchedInspect.ID, usertypes.Actor{Username: "System"}); scanImageErr != nil {- slog.WarnContext(ctx, "failed to start verification scan of patched image", "patchedRef", record.PatchedRef, "error", scanImageErr)+ s.appendPatchActivity(ctx, activityID, 95, "Re-scanning patched image to verify the patch")+ if _, scanErr := s.vulnerabilityService.ScanImage(context.WithoutCancel(ctx), record.EnvironmentID, patchedInspect.ID, scheduledPatchUser); scanErr != nil {+ slog.WarnContext(ctx, "failed to start verification scan of patched image", "patchedRef", record.PatchedRef, "error", scanErr)}}}}-// platformPinnedRefInternal resolves a tag reference to the manifest digest of+// platformPinnedRef resolves a tag reference to the manifest digest of// the given platform when the registry serves a multi-platform index. This// makes copa patch a single platform (keeping the plain patched tag) instead// of every platform in the index. Returns "" when the reference is not a// multi-platform index or cannot be resolved, in which case the plain tag is// used and copa's own discovery decides.-func platformPinnedRefInternal(ctx context.Context, imageRef string, target v1.Platform) string {+func platformPinnedRef(ctx context.Context, imageRef string, target v1.Platform) string {ref, err := name.ParseReference(imageRef)if err != nil {return ""@@ -411,11 +473,11 @@ func resolvePatchedRef(imageRef, patchedTag, suffix string) (string, error) {return localName + ":" + tag, nil}-// writeRegistryAuthConfigInternal merges Arcane's registry credentials into the+// writeRegistryAuthConfig merges Arcane's registry credentials into the// docker config file copa reads for BuildKit registry auth. It only touches the// config when DOCKER_CONFIG is set, i.e. the deployment (normally Arcane's own// startup) owns that directory — a user's ~/.docker/config.json is never modified.-func (s *Service) writeRegistryAuthConfigInternal(ctx context.Context) {+func (s *Service) writeRegistryAuthConfig(ctx context.Context) {configDir := strings.TrimSpace(os.Getenv("DOCKER_CONFIG"))if configDir == "" || s.registryService == nil {return@@ -473,7 +535,7 @@ func (s *Service) writeRegistryAuthConfigInternal(ctx context.Context) {}}-func (s *Service) finishPatchRecordInternal(ctx context.Context, record *ImagePatchRecord, status imagepatch.PatchStatus, errMessage string, packagesUpdated *int, durationMs int64) {+func (s *Service) finishPatchRecord(ctx context.Context, record *ImagePatchRecord, status imagepatch.PatchStatus, errMessage string, packagesUpdated *int, durationMs int64) {updates := map[string]any{"status": string(status),"duration_ms": durationMs,@@ -517,8 +579,8 @@ func (s *Service) ListPatches(ctx context.Context, envID string, params paginatireturn dtos, paginationResp, nil}-// PatchedRefs returns the set of patched image references recorded for an-// environment, used to keep patch outputs from being patched again.+// PatchedRefs returns every name a completed patch output can be scanned under+// (normalized and familiar), used to keep patch outputs from being patched again.func (s *Service) PatchedRefs(ctx context.Context, envID string) (map[string]struct{}, error) {var refs []stringif err := s.db.WithContext(ctx).@@ -528,9 +590,12 @@ func (s *Service) PatchedRefs(ctx context.Context, envID string) (map[string]strPluck("patched_ref", &refs).Error; err != nil {return nil, fmt.Errorf("failed to list patched image refs: %w", err)}- set := make(map[string]struct{}, len(refs))+ set := make(map[string]struct{}, len(refs)*2)for _, ref := range refs {set[ref] = struct{}{}+ if named, err := reference.ParseNormalizedNamed(ref); err == nil {+ set[reference.FamiliarString(named)] = struct{}{}+ }}return set, nil}@@ -547,13 +612,7 @@ func (s *Service) ListPatchTargets(ctx context.Context, envID string, params pagif err != nil {return nil, pagination.Response{}, err}- excludedNames := make([]string, 0, len(patchedRefs)*2)- for ref := range patchedRefs {- excludedNames = append(excludedNames, ref)- if named, parseNormalizedNamedErr := reference.ParseNormalizedNamed(ref); parseNormalizedNamedErr == nil {- excludedNames = append(excludedNames, reference.FamiliarString(named))- }- }+ excludedNames := slices.Collect(maps.Keys(patchedRefs))// Only list images that are actionable (fixable vulnerabilities) or carry// patch history.@@ -599,7 +658,7 @@ func (s *Service) ListPatchTargets(ctx context.Context, envID string, params pagfor i := range scans {imageIDs = append(imageIDs, scans[i].ID)}- lastPatchByImageID, lastPatchScanByImageID, err := s.latestPatchesByImageInternal(ctx, envID, imageIDs)+ lastPatchByImageID, lastPatchScanByImageID, err := s.latestPatchesByImage(ctx, envID, imageIDs)if err != nil {return nil, pagination.Response{}, err}@@ -636,10 +695,10 @@ func (s *Service) ListPatchTargets(ctx context.Context, envID string, params pagreturn targets, paginationResp, nil}-// latestPatchesByImageInternal loads the most recent patch run per original+// latestPatchesByImage loads the most recent patch run per original// image in two batched queries, plus the scan of each completed patch's output// image so the original's row can show whether the patch worked.-func (s *Service) latestPatchesByImageInternal(ctx context.Context, envID string, imageIDs []string) (map[string]*ImagePatchRecord, map[string]*imagepatch.PatchScanSummary, error) {+func (s *Service) latestPatchesByImage(ctx context.Context, envID string, imageIDs []string) (map[string]*ImagePatchRecord, map[string]*imagepatch.PatchScanSummary, error) {var patchRows []ImagePatchRecordif err := s.db.WithContext(ctx).Where("environment_id = ? AND original_image_id IN ?", envID, imageIDs).@@ -704,75 +763,54 @@ func (s *Service) latestPatchesByImageInternal(ctx context.Context, envID stringreturn lastPatchByImageID, lastPatchScanByImageID, nil}-// PatchFlaggedImages patches every image whose latest completed vulnerability-// scan found fixable vulnerabilities and stored a raw report. Images that are-// themselves patch outputs are skipped (never patch a patched tag), as are-// images already patched since their latest scan. Used by the scheduled-// auto-patch job.-func (s *Service) PatchFlaggedImages(ctx context.Context, envID string, user usertypes.Actor) (patched, skipped int, err error) {- if requireFeatureErr := s.settingsService.RequireFeature(ctx, features.VulnerabilityManagement); requireFeatureErr != nil {- return 0, 0, requireFeatureErr+// Targets lists images whose latest completed scan found fixable vulnerabilities+// and kept a report, skipping patch outputs and images patched since that scan.+func (s *Service) Targets(ctx context.Context, envID string) ([]Target, error) {+ if err := s.settingsService.RequireFeature(ctx, features.VulnerabilityManagement); err != nil {+ return nil, err}dockerClient, err := s.dockerService.GetClient(ctx)if err != nil {- return 0, 0, fmt.Errorf("failed to connect to Docker: %w", err)+ return nil, fmt.Errorf("failed to connect to Docker: %w", err)}- if requireContainerdImageStoreErr := requireContainerdImageStoreInternal(ctx, dockerClient); requireContainerdImageStoreErr != nil {- return 0, 0, requireContainerdImageStoreErr+ if storeErr := requireContainerdImageStore(ctx, dockerClient); storeErr != nil {+ return nil, storeErr}var scans []vulnerability.VulnerabilityScanRecord- if loadFixableScansErr := s.db.WithContext(ctx).+ if scansErr := s.db.WithContext(ctx).Where("status = ? AND fixable_count > 0", vulnerability.ScanStatusCompleted).Where("id IN (?)", s.db.Model(&vulnerability.VulnerabilityReportRecord{}).Select("image_id")).Where("image_name NOT LIKE 'sha256:%' AND image_name NOT LIKE '%<none>%' AND image_name <> id").- Find(&scans).Error; loadFixableScansErr != nil {- return 0, 0, fmt.Errorf("failed to list vulnerability scans: %w", loadFixableScansErr)+ Find(&scans).Error; scansErr != nil {+ return nil, fmt.Errorf("failed to list vulnerability scans: %w", scansErr)}-patchedRefs, err := s.PatchedRefs(ctx, envID)if err != nil {- return 0, 0, err+ return nil, err}+ targets := make([]Target, 0, len(scans))for i := range scans {- if !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) {- return patched, skipped + len(scans) - i, fmt.Errorf("remaining patches stopped because feature %s was disabled: %w", features.VulnerabilityManagement, common.ErrFeatureDisabled)- }scan := &scans[i]-if _, isPatchOutput := patchedRefs[scan.ImageName]; isPatchOutput {- skipped++continue}-- // Skip images already patched (or being patched) since their latest scan.+ // Skip images already patched, or being patched, since their latest scan.var recent int64- if countRecentPatchesErr := s.db.WithContext(ctx).+ if countErr := s.db.WithContext(ctx).Model(&ImagePatchRecord{}).Where("environment_id = ? AND original_image_id = ? AND status IN ? AND created_at >= ?",envID, scan.ID, []string{string(imagepatch.PatchStatusCompleted), string(imagepatch.PatchStatusPatching)}, scan.ScanTime).- Count(&recent).Error; countRecentPatchesErr == nil && recent > 0 {- skipped++- continue- }-- if _, patchImageErr := s.PatchImage(ctx, envID, scan.ID, imagepatch.PatchOptions{ScanID: scan.ID}, user); patchImageErr != nil {- slog.WarnContext(ctx, "scheduled image patch failed to start",- "imageId", scan.ID,- "imageName", scan.ImageName,- "error", patchImageErr,- )- skipped+++ Count(&recent).Error; countErr == nil && recent > 0 {continue}- patched+++ targets = append(targets, Target{ImageID: scan.ID, ImageName: scan.ImageName})}-- return patched, skipped, nil+ return targets, nil}// Copa needs BuildKit's docker exporter, which dockerd only offers with the containerd image store.-func requireContainerdImageStoreInternal(ctx context.Context, dockerClient *client.Client) error {+func requireContainerdImageStore(ctx context.Context, dockerClient *client.Client) error {info, err := dockerClient.Info(ctx, client.InfoOptions{})if err != nil {return fmt.Errorf("failed to inspect Docker: %w", err)@@ -785,28 +823,7 @@ func requireContainerdImageStoreInternal(ctx context.Context, dockerClient *cliereturn common.ErrPatchRequiresContainerdImageStore}-func (s *Service) startPatchActivityInternal(ctx context.Context, envID, imageID, imageRef string, user *usertypes.Actor) string {- if s.activityService == nil {- return ""- }- started, err := s.activityService.StartActivity(ctx, activity.StartActivityRequest{- EnvironmentID: envID,- Type: activitytypes.TypeImagePatch,- ResourceType: new("image"),- ResourceID: &imageID,- ResourceName: &imageRef,- StartedBy: user,- Progress: new(0),- LatestMessage: "Image patch queued",- })- if err != nil {- slog.WarnContext(ctx, "failed to create image patch activity", "error", err, "imageRef", imageRef)- return ""- }- return started.ID-}--func (s *Service) appendPatchActivityInternal(ctx context.Context, activityID string, progress int, message string) {+func (s *Service) appendPatchActivity(ctx context.Context, activityID string, progress int, message string) {if s.activityService == nil || activityID == "" {return}@@ -819,7 +836,7 @@ func (s *Service) appendPatchActivityInternal(ctx context.Context, activityID st}}-func (s *Service) completePatchActivityInternal(ctx context.Context, activityID string, success bool, errMessage string) {+func (s *Service) completePatchActivity(ctx context.Context, activityID string, success bool, errMessage string) {if s.activityService == nil || activityID == "" {return}@@ -827,18 +844,13 @@ func (s *Service) completePatchActivityInternal(ctx context.Context, activityIDstatus := activitytypes.StatusSuccessmessage := "Image patch completed"var errorPtr *string- if !success {- if activitylib.CancelledByContext(ctx) {- status = activitytypes.StatusCancelled- message = "Image patch cancelled"- } else {- status = activitytypes.StatusFailed- message = "Image patch failed"- if strings.TrimSpace(errMessage) != "" {- errorPtr = &errMessage- message = errMessage- }- }+ switch {+ case success:+ case activitylib.CancelledByContext(ctx):+ status, message = activitytypes.StatusCancelled, "Image patch cancelled"+ default:+ status, message = activitytypes.StatusFailed, cmp.Or(errMessage, "Image patch failed")+ errorPtr = mo.EmptyableToOption(errMessage).ToPointer()}if _, err := s.activityService.CompleteActivity(utils.ActivityRuntimeContext(ctx, nil), activityID, status, message, errorPtr); err != nil {diff --git a/backend/internal/image/children/patch/service_test.go b/backend/internal/image/children/patch/service_test.goindex 55b6c2a28..c22a378a4 100644--- a/backend/internal/image/children/patch/service_test.go+++ b/backend/internal/image/children/patch/service_test.go@@ -95,7 +95,7 @@ func TestListPatchTargets_ExcludesUntaggedImages(t *testing.T) {require.Equal(t, "nginx:latest", targets[0].ImageRef)}-func patchFeatureSettingsInternal(t *testing.T) (*settings.SettingsService, *database.DB) {+func patchFeatureSettings(t *testing.T) (*settings.SettingsService, *database.DB) {t.Helper()gdb, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})require.NoError(t, err)@@ -110,7 +110,7 @@ func patchFeatureSettingsInternal(t *testing.T) (*settings.SettingsService, *dat}func TestVulnerabilityFeatureBlocksReportPatchesOnly(t *testing.T) {- settingsSvc, db := patchFeatureSettingsInternal(t)+ settingsSvc, db := patchFeatureSettings(t)require.NoError(t, settingsSvc.SetBoolSetting(t.Context(), features.VulnerabilityManagementSettingKey, false))svc := &Service{settingsService: settingsSvc,@@ -121,7 +121,7 @@ func TestVulnerabilityFeatureBlocksReportPatchesOnly(t *testing.T) {require.ErrorIs(t, err, common.ErrFeatureDisabled)_, _, err = svc.ListPatchTargets(t.Context(), "0", pagination.QueryParams{})require.ErrorIs(t, err, common.ErrFeatureDisabled)- _, _, err = svc.PatchFlaggedImages(t.Context(), "0", user.Actor{})+ _, err = svc.Targets(t.Context(), "0")require.ErrorIs(t, err, common.ErrFeatureDisabled)// Standalone patching still reaches Docker; this test deliberately has no daemon._, err = svc.PatchImage(t.Context(), "0", "test-image", imagepatch.PatchOptions{}, user.Actor{})@@ -131,7 +131,7 @@ func TestVulnerabilityFeatureBlocksReportPatchesOnly(t *testing.T) {}func TestQueuedReportPatchStopsWhenFeatureDisabled(t *testing.T) {- settingsSvc, db := patchFeatureSettingsInternal(t)+ settingsSvc, db := patchFeatureSettings(t)svc := &Service{settingsService: settingsSvc, db: db, patchSlot: make(chan struct{}, 1)}record := &ImagePatchRecord{EnvironmentID: "0", OriginalImageID: "test-image", Mode: string(imagepatch.PatchModeReport), Status: string(imagepatch.PatchStatusPatching)}require.NoError(t, db.Create(record).Error)@@ -139,7 +139,7 @@ func TestQueuedReportPatchStopsWhenFeatureDisabled(t *testing.T) {done := make(chan struct{})go func() {defer close(done)- svc.patchInBackgroundInternal(t.Context(), record, imagepatch.PatchOptions{ScanID: "test-image"}, nil, "", "")+ _ = svc.runPatch(t.Context(), record, imagepatch.PatchOptions{ScanID: "test-image"}, nil, "", "")}()require.NoError(t, settingsSvc.SetBoolSetting(t.Context(), features.VulnerabilityManagementSettingKey, false))// The active patch owns its slot until it finishes normally.@@ -159,7 +159,7 @@ func TestQueuedReportPatchStopsWhenFeatureDisabled(t *testing.T) {}func TestDisabledFeatureSkipsPatchVerificationScan(t *testing.T) {- settingsSvc, _ := patchFeatureSettingsInternal(t)+ settingsSvc, _ := patchFeatureSettings(t)require.NoError(t, settingsSvc.SetBoolSetting(t.Context(), features.VulnerabilityManagementSettingKey, false))var inspected atomic.Int32server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {@@ -178,7 +178,7 @@ func TestDisabledFeatureSkipsPatchVerificationScan(t *testing.T) {vulnerabilityService: &vulnerability.VulnerabilityService{},}require.NotPanics(t, func() {- svc.verifyPatchedImageInternal(t.Context(), &ImagePatchRecord{EnvironmentID: "0", PatchedRef: "test:patched"}, "")+ svc.verifyPatchedImage(t.Context(), &ImagePatchRecord{EnvironmentID: "0", PatchedRef: "test:patched"}, "")})require.Equal(t, int32(1), inspected.Load())}diff --git a/backend/internal/image/handler.go b/backend/internal/image/handler.goindex a20faacda..571115ca5 100644--- a/backend/internal/image/handler.go+++ b/backend/internal/image/handler.go@@ -16,7 +16,7 @@ import ("github.com/getarcaneapp/arcane/types/v2/system"uploadtypes "github.com/getarcaneapp/arcane/types/v2/upload""go.getarcane.app/builds/types"- "go.getarcane.app/kit/pkg"+ kit "go.getarcane.app/kit/pkg""gorm.io/gorm""github.com/getarcaneapp/arcane/backend/v2/internal/activity"@@ -203,12 +203,7 @@ func (h *ImageHandler) TagImage(ctx context.Context, input *TagImageInput) (*hanreturn nil, huma.Error500InternalServerError(fmt.Sprintf("failed to tag image: %v", tagImageErr))}- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{Message: "Image tagged successfully"},- },- }, nil+ return handlerutil.MessageOutput("Image tagged successfully", ""), nil}// GetImageHistory returns Docker image layer history.@@ -304,14 +299,7 @@ func (h *ImageHandler) RemoveImage(ctx context.Context, input *RemoveImageInput)return nil, huma.Error500InternalServerError("Failed to remove image: " + removeImageErr.Error())}- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{- Message: "Image removed successfully",- },- },- }, nil+ return handlerutil.MessageOutput("Image removed successfully", ""), nil}// PullImage pulls a Docker image with streaming progress.diff --git a/backend/internal/image/service.go b/backend/internal/image/service.goindex 142967fcc..edddafcf2 100644--- a/backend/internal/image/service.go+++ b/backend/internal/image/service.go@@ -13,11 +13,16 @@ import ("time""github.com/distribution/reference"+ "github.com/getarcaneapp/arcane/types/v2"+ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity""github.com/getarcaneapp/arcane/types/v2/containerregistry"+ "github.com/getarcaneapp/arcane/types/v2/features"imagetypes "github.com/getarcaneapp/arcane/types/v2/image"+ "github.com/getarcaneapp/arcane/types/v2/scheduler""github.com/getarcaneapp/arcane/types/v2/system"usertypes "github.com/getarcaneapp/arcane/types/v2/user"vulnerabilitytypes "github.com/getarcaneapp/arcane/types/v2/vulnerability"+ "github.com/italypaleale/francis/builtin/workflow""github.com/moby/moby/api/types/container""github.com/moby/moby/api/types/image""github.com/moby/moby/client"@@ -30,6 +35,7 @@ import ("golang.org/x/sync/errgroup""github.com/getarcaneapp/arcane/backend/v2/internal/activity"+ "github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/internal/database"dockerInternal "github.com/getarcaneapp/arcane/backend/v2/internal/docker""github.com/getarcaneapp/arcane/backend/v2/internal/event"@@ -39,12 +45,19 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/registry""github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability"+ activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/registryauth""github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow""github.com/getarcaneapp/arcane/backend/v2/pkg/utils""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/imageref")+// patchSlot is one patch task's output; failures travel as task errors.+type patchSlot struct {+ Skipped bool `json:"skipped,omitempty"`+}+type ImageService struct {db *database.DBdockerService *dockerInternal.DockerClientService@@ -53,8 +66,10 @@ type ImageService struct {vulnerabilityService *vulnerability.VulnerabilityServiceeventService *event.EventService- patch *patch.Service- projectIDCache *hot.HotCache[struct{}, map[string]string]+ settingsService *settings.SettingsService+ patch *patch.Service+ patchWorkflow *flow.Workflow+ projectIDCache *hot.HotCache[struct{}, map[string]string]}func NewImageService(@@ -74,6 +89,7 @@ func NewImageService(imageUpdateService: imageUpdateService,vulnerabilityService: vulnerabilityService,eventService: eventService,+ settingsService: settingsService,patch: patch.NewService(db, dockerService, settingsService, activityService, registryService, vulnerabilityService),projectIDCache: hot.NewHotCache[struct{}, map[string]string](hot.LRU, 1).WithTTL(projectIDCacheTTL).@@ -81,11 +97,101 @@ func NewImageService(}}-// PatchFlaggedImages patches eligible images for the scheduled patch job.-func (s *ImageService) PatchFlaggedImages(ctx context.Context, environmentID string, user usertypes.Actor) (patched, skipped int, err error) {- return s.patch.PatchFlaggedImages(ctx, environmentID, user)+// RegisterWorkflows defines the scheduled auto-patch: find images with fixable+// findings, patch each one in turn, then report. Call it before the host starts.+func (s *ImageService) RegisterWorkflows(engine *flow.Engine) error {+ patchWorkflow, err := engine.Define(flow.Definition{+ Name: "auto-patch",+ Version: 1,+ Fingerprint: "e399c9c07ea15a7d6a39f086529110040a4d2cc95daf72c1464363c7f5bb5efc",+ // Copacetic runs one BuildKit patch at a time and mirrors a process-wide logger.+ Concurrency: 1,+ Timeout: 12 * time.Hour,+ Activity: activitylib.StartRequest{+ Type: activitytypes.TypeImagePatch,+ ResourceType: new("images"),+ ResourceName: new("Scheduled image patch"),+ Step: "Finding patchable images",+ LatestMessage: "Scheduled image patch started",+ },+ Labels: map[string]string{+ "discover": "Finding patchable images",+ "patch": "Patching images",+ "finalize": "Finishing image patch",+ },+ Steps: []workflow.StepSpec{+ workflow.Step("discover", engine.Handler(func(ctx context.Context, _ flow.Task) (any, error) {+ if !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) {+ return []patch.Target{}, nil+ }+ targets, err := s.patch.Targets(ctx, types.LocalDockerEnvironmentID)+ if errors.Is(err, common.ErrPatchRequiresContainerdImageStore) {+ return nil, err+ }+ if err != nil {+ return nil, common.Classify(common.ErrUnavailable, err)+ }+ return targets, nil+ })),+ workflow.ForEach("patch", engine.Handler(func(ctx context.Context, t flow.Task) (any, error) {+ var target patch.Target+ if err := t.DecodeItem(&target); err != nil {+ return nil, err+ }+ if !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) {+ return patchSlot{Skipped: true}, nil+ }+ // The record ID is stable per task, so a redelivery resumes its own patch.+ return nil, s.patch.PatchTarget(ctx, types.LocalDockerEnvironmentID, target, fmt.Sprintf("%s-%d", t.ActivityID(), t.Index()))+ }),+ workflow.WithItemsFrom("discover"),+ workflow.WithMaxParallel(1),+ workflow.WithFailurePolicy(workflow.TolerateFailures),+ workflow.WithMaxAttempts(2),+ workflow.WithRetryBackoff(30*time.Second, 2*time.Minute)),+ workflow.Step("finalize", engine.Handler(func(ctx context.Context, t flow.Task) (any, error) {+ var targets []patch.Target+ if err := t.DecodeOutput("discover", &targets); err != nil {+ return nil, err+ }+ slots, err := flow.Results[patchSlot](t, "patch")+ if err != nil {+ return nil, err+ }+ outcome := scheduler.Outcome{Status: scheduler.Succeeded}+ patched, skipped := 0, 0+ for index, slot := range slots {+ switch {+ case slot.Err != "":+ outcome.Targets = append(outcome.Targets, scheduler.TargetOutcome{ResourceType: "image", ID: targets[index].ImageName, Status: scheduler.Failed, Message: slot.Err})+ case slot.Value.Skipped:+ skipped+++ default:+ patched+++ }+ }+ outcome.Message = fmt.Sprintf("Image patch completed: %d patched, %d failed", patched, len(outcome.Targets))+ switch {+ case skipped > 0 || (len(targets) == 0 && !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement)):+ outcome.Status = kit.Ternary(patched > 0, scheduler.Partial, scheduler.Skipped)+ outcome.Message = fmt.Sprintf("Remaining patches stopped because feature %s was disabled", features.VulnerabilityManagement)+ case len(outcome.Targets) > 0:+ outcome.Status = scheduler.Partial+ }+ return outcome, nil+ }), workflow.WithInputFrom("discover")),+ },+ })+ if err != nil {+ return err+ }+ s.patchWorkflow = patchWorkflow+ return nil}+// PatchWorkflow is the scheduled auto-patch workflow.+func (s *ImageService) PatchWorkflow() *flow.Workflow { return s.patchWorkflow }+// newAttestationsServiceInternal builds the attestation reader from the image// service's Docker client and registry credentials.func (s *ImageService) newAttestationsServiceInternal() *attestations.Service {diff --git a/backend/internal/image/service_test.go b/backend/internal/image/service_test.goindex cb2fa6cc8..d2423a797 100644--- a/backend/internal/image/service_test.go+++ b/backend/internal/image/service_test.go@@ -35,6 +35,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/registry""github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow/flowtest""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/imageref")@@ -742,3 +743,10 @@ func TestImageServiceContainerTagUpdatesStayScoped(t *testing.T) {require.NotContains(t, checks, "moving-no-id", "a missing runtime image ID matches nothing")require.NotSame(t, checks["moving"], checks["moving-twin"], "each container receives its own copy")}++func TestPatchWorkflowDefinitions(t *testing.T) {+ harness := flowtest.New(t, nil)+ require.NoError(t, (&ImageService{}).RegisterWorkflows(harness.Engine))+ harness.Start(t)+ flowtest.AssertDefinitions(t, harness)+}diff --git a/backend/internal/imageupdate/handler.go b/backend/internal/imageupdate/handler.goindex e28802e3d..e02d2773c 100644--- a/backend/internal/imageupdate/handler.go+++ b/backend/internal/imageupdate/handler.go@@ -99,7 +99,7 @@ func (h *ImageUpdateHandler) CheckMultipleImages(ctx context.Context, input *Che}runtimeCtx := utils.ActivityRuntimeContext(ctx, h.appCtx)- results, err := h.imageUpdateService.CheckMultipleImages(runtimeCtx, input.Body.ImageRefs, input.Body.Credentials)+ results, err := h.imageUpdateService.CheckImages(runtimeCtx, imageupdate.CheckRequest{ImageRefs: input.Body.ImageRefs}, input.Body.Credentials)if err != nil {return nil, huma.Error500InternalServerError("Failed to check image updates: " + err.Error())}@@ -114,7 +114,7 @@ func (h *ImageUpdateHandler) CheckMultipleImages(ctx context.Context, input *Chefunc (h *ImageUpdateHandler) CheckAllImages(ctx context.Context, input *CheckAllImagesInput) (*handlerutil.Out[imageupdate.BatchResponse], error) {runtimeCtx := utils.ActivityRuntimeContext(ctx, h.appCtx)- results, err := h.imageUpdateService.CheckAllImages(runtimeCtx, 0, input.Body.Credentials)+ results, err := h.imageUpdateService.CheckImages(runtimeCtx, imageupdate.CheckRequest{All: true}, input.Body.Credentials)if err != nil {return nil, huma.Error500InternalServerError("Failed to check all images: " + err.Error())}diff --git a/backend/internal/imageupdate/service.go b/backend/internal/imageupdate/service.goindex 044bf709c..b6aaa91e6 100644--- a/backend/internal/imageupdate/service.go+++ b/backend/internal/imageupdate/service.go@@ -11,25 +11,29 @@ import ("strings""sync""time"+ "uuid""github.com/containerd/errdefs""github.com/distribution/reference"activitytypes "github.com/getarcaneapp/arcane/types/v2/activity""github.com/getarcaneapp/arcane/types/v2/containerregistry""github.com/getarcaneapp/arcane/types/v2/imageupdate"+ "github.com/getarcaneapp/arcane/types/v2/scheduler""github.com/getarcaneapp/arcane/types/v2/user"+ "github.com/italypaleale/francis/builtin/workflow""github.com/moby/moby/api/types/container""github.com/moby/moby/api/types/image""github.com/moby/moby/client"+ "github.com/samber/hot""github.com/samber/mo"- "go.getarcane.app/kit/pkg"+ kit "go.getarcane.app/kit/pkg""go.getarcane.app/sys/crypto""go.getarcane.app/updater/digest""go.getarcane.app/updater/refs"- "golang.org/x/sync/errgroup""gorm.io/gorm""github.com/getarcaneapp/arcane/backend/v2/internal/activity"+ "github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/docker""github.com/getarcaneapp/arcane/backend/v2/internal/event"@@ -42,11 +46,24 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/registryauth""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts""github.com/getarcaneapp/arcane/backend/v2/pkg/projects"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow""github.com/getarcaneapp/arcane/backend/v2/pkg/utils""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/imageref""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/notifications")+const (+ imageCheckConcurrency = 10+ imageCheckAttempts = 3+)++var (+ // errCheckInterrupted fails the images of a manual check that a restart interrupted.+ errCheckInterrupted = errors.New("image check was interrupted by a restart; run it again")+ imageCheckRetryBackoff = 30 * time.Second+ imageCheckRetryBackoffMax = 2 * time.Minute+)+type ImageUpdateService struct {db *database.DBsettingsService *settings.SettingsService@@ -57,7 +74,13 @@ type ImageUpdateService struct {registryLimiter *ratelimit.RegistryRateLimitertags *tags.ServiceactivityService *activity.ActivityService- notifyMu sync.Mutex+ engine *flow.Engine+ checkWorkflow *flow.Workflow+ credentials *hot.HotCache[string, []containerregistry.Credential]+ // requestCredentials holds credentials a caller supplied for one check, by its input's CredentialsKey,+ // so they never reach the workflow journal. They expire with the check's timeout if nothing deletes them.+ requestCredentials *hot.HotCache[string, []containerregistry.Credential]+ notifyMu sync.Mutex}type ImageParts struct {@@ -66,6 +89,36 @@ type ImageParts struct {Tag string}+// imageCheckResultInternal is a check task's output; failures travel as errors.+type imageCheckResultInternal struct {+ Local bool `json:"local,omitempty"`+ // Result is the ref's own result, kept when the caller reports results; tags sharing an image share its stored row.+ Result *imageupdate.Response `json:"result,omitempty"`+ // Failure is a reported check's error, which keeps its Result instead of failing the slot.+ Failure string `json:"failure,omitempty"`+}++// imageCheckInput is an image-check payload; Report keeps every ref's result in the output for the caller.+type imageCheckInput struct {+ imageupdate.CheckRequest+ Report bool `json:"report,omitempty"`+ CredentialsKey string `json:"credentialsKey,omitempty"`+}++// imageCheckOutput is the image-check outcome plus, when reported, each ref's result and the current+// container tag results, failed checks included.+type imageCheckOutput struct {+ scheduler.Outcome+ Results imageupdate.BatchResponse `json:"results,omitempty"`+ ContainerUpdates map[string]*imageupdate.Response `json:"containerUpdates,omitempty"`+}++// tagCheckOutput is the tags step's output: failed container checks and, when reported, every container's result.+type tagCheckOutput struct {+ Failures []scheduler.TargetOutcome `json:"failures"`+ Updates map[string]*imageupdate.Response `json:"updates,omitempty"`+}+type localImageSnapshot struct {ImageID stringRepository string@@ -96,6 +149,10 @@ func NewImageUpdateService(registryLimiter: registryLimiter,tags: tags.NewService(registryService, dockerService, settingsService, registryLimiter),activityService: activityService,+ credentials: hot.NewHotCache[string, []containerregistry.Credential](hot.LRU, 8).WithTTL(time.Minute).Build(),+ // They outlive the longest check: its slot wait plus its workflow timeout.+ requestCredentials: hot.NewHotCache[string, []containerregistry.Credential](hot.LRU, 64).+ WithTTL(timeouts.DefaultActivitySlotWait + timeouts.DefaultImageUpdateScan).Build(),}}@@ -177,21 +234,20 @@ func (s *ImageUpdateService) composeBuildImageRefsInternal(ctx context.Context)return buildRefs, nil}-func (s *ImageUpdateService) startImageUpdateActivityInternal(ctx context.Context, resourceName string, count int) string {+func (s *ImageUpdateService) startImageUpdateActivityInternal(ctx context.Context, imageRef string) string {if s.activityService == nil {return ""}- resourceType := kit.Ternary(count > 1, "images", "image")localActivity, err := s.activityService.StartActivity(ctx, activity.StartActivityRequest{EnvironmentID: "0",Type: activitytypes.TypeImageUpdateCheck,Queue: true,- ResourceType: &resourceType,- ResourceName: mo.EmptyableToOption(strings.TrimSpace(resourceName)).ToPointer(),+ ResourceType: new("image"),+ ResourceName: mo.EmptyableToOption(strings.TrimSpace(imageRef)).ToPointer(),Step: "Checking image updates",LatestMessage: "Image update check started",Metadata: database.JSON{- "imageCount": count,+ "imageCount": 1,},})if err != nil {@@ -201,7 +257,9 @@ func (s *ImageUpdateService) startImageUpdateActivityInternal(ctx context.Contexreturn localActivity.ID}-func (s *ImageUpdateService) appendImageUpdateActivityMessageInternal(ctx context.Context, activityID string, level activitytypes.MessageLevel, message string, progress int, step string) {+// appendImageUpdateActivityMessageInternal appends a check message; a nil+// progress leaves progress to the activity's owner, such as the workflow engine.+func (s *ImageUpdateService) appendImageUpdateActivityMessageInternal(ctx context.Context, activityID string, level activitytypes.MessageLevel, message string, progress *int, step string) {if s.activityService == nil || activityID == "" || strings.TrimSpace(message) == "" {return}@@ -209,7 +267,7 @@ func (s *ImageUpdateService) appendImageUpdateActivityMessageInternal(ctx contexif _, err := s.activityService.AppendMessage(ctx, activityID, activity.AppendActivityMessageRequest{Level: level,Message: message,- Progress: &progress,+ Progress: progress,Step: step,}); err != nil {slog.DebugContext(ctx, "failed to append image update activity message", "activityId", activityID, "error", err)@@ -242,14 +300,14 @@ func (s *ImageUpdateService) completeImageUpdateActivityInternal(ctx context.Confunc (s *ImageUpdateService) CheckImageUpdate(ctx context.Context, imageRef string) (*imageupdate.Response, error) {startTime := time.Now()- activityID := s.startImageUpdateActivityInternal(ctx, imageRef, 1)+ activityID := s.startImageUpdateActivityInternal(ctx, imageRef)ctx = s.activityService.Track(ctx, activityID)activitylib.AwaitHandlerActivitySlot(ctx, s.activityService, activityID, "0")if result, ok := digestPinnedImageUpdateResultInternal(imageRef).Get(); ok {result.ResponseTimeMs = int(time.Since(startTime).Milliseconds())result.ActivityID = mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()- s.recordDigestPinnedSkipInternal(ctx, activityID, imageRef, result, 100)+ s.recordDigestPinnedSkipInternal(ctx, activityID, imageRef, result, new(100))if s.eventService != nil {metadata := database.JSON{"action": "check_update",@@ -269,7 +327,7 @@ func (s *ImageUpdateService) CheckImageUpdate(ctx context.Context, imageRef strireturn result, nil}- s.appendImageUpdateActivityMessageInternal(ctx, activityID, activitytypes.MessageLevelInfo, "Checking "+imageRef, 20, "Checking remote digest")+ s.appendImageUpdateActivityMessageInternal(ctx, activityID, activitytypes.MessageLevelInfo, "Checking "+imageRef, new(20), "Checking remote digest")parts := s.parseImageReference(imageRef)if parts == nil {@@ -333,7 +391,7 @@ func (s *ImageUpdateService) CheckImageUpdate(ctx context.Context, imageRef stridigestResult.ResponseTimeMs = int(time.Since(startTime).Milliseconds())digestResult.ActivityID = mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()if digestResult.UpdateType == UpdateTypeLocal {- s.appendImageUpdateActivityMessageInternal(ctx, activityID, activitytypes.MessageLevelInfo, imageRef+" — local build, registry check skipped", 100, "Skipping image update check")+ s.appendImageUpdateActivityMessageInternal(ctx, activityID, activitytypes.MessageLevelInfo, imageRef+" — local build, registry check skipped", new(100), "Skipping image update check")}metadata := database.JSON{"action": "check_update",@@ -985,29 +1043,6 @@ func buildImageUpdateRepositoryInternal(parts *ImageParts) string {return fmt.Sprintf("%s/%s", strings.TrimSpace(parts.Registry), repository)}-func countBatchResultOutcomesInternal(imageRefs []string, results map[string]*imageupdate.Response) (int, int) {- successCount := 0- errorCount := 0-- for _, imageRef := range imageRefs {- result := results[imageRef]- hasError := result == nil- if result != nil {- hasError = strings.TrimSpace(result.Error) != ""- for _, containerUpdate := range result.ContainerUpdates {- hasError = hasError || containerUpdate.Error != ""- }- }- if !hasError {- successCount++- continue- }- errorCount++- }-- return successCount, errorCount-}-// imageCheckResultMessageInternal derives an activity message level and text from// a per-image update check result: errors become ERROR, available updates become// SUCCESS, and up-to-date images stay INFO.@@ -1367,70 +1402,27 @@ func (s *ImageUpdateService) MarkUpdatesAsNotified(ctx context.Context, imageIDsUpdate("notification_sent", true).Error}-type batchImage struct {- refs []string- canonicalRef string- parts *ImageParts-}--type batchImageProgressRecorder struct {- mu sync.Mutex- results map[string]*imageupdate.Response- completed int- total int-}--func (r *batchImageProgressRecorder) recordInternal(localRefs []string, res *imageupdate.Response) int {- r.mu.Lock()- defer r.mu.Unlock()-- r.completed++- progress := 10 + int(float64(r.completed)/float64(r.total)*80)- for _, imageRef := range localRefs {- r.results[imageRef] = res- }- return progress-}--func (s *ImageUpdateService) parseAndGroupImagesInternal(imageRefs []string) (map[string]map[string]struct{}, map[string]*imageupdate.Response, []batchImage) {- regRepos := make(map[string]map[string]struct{})- results := make(map[string]*imageupdate.Response)- var images []batchImage+// groupImageRefsInternal dedupes refs naming the same registry/repo:tag into one+// check; the first ref of a group is canonical. Digest-pinned and invalid refs+// each form their own group.+func (s *ImageUpdateService) groupImageRefsInternal(imageRefs []string) [][]string {+ var groups [][]stringindexByNormalizedRef := make(map[string]int)-for _, imageRef := range imageRefs {- if result, ok := digestPinnedImageUpdateResultInternal(imageRef).Get(); ok {- results[imageRef] = result- continue- }-parts := s.parseImageReference(imageRef)- if parts == nil {- results[imageRef] = &imageupdate.Response{- Error: "Invalid image reference format",- CheckTime: time.Now(),- ResponseTimeMs: 0,- }+ if parts == nil || digestPinnedImageUpdateResultInternal(imageRef).IsPresent() {+ groups = append(groups, []string{imageRef})continue}- if _, ok := regRepos[parts.Registry]; !ok {- regRepos[parts.Registry] = make(map[string]struct{})- }- regRepos[parts.Registry][parts.Repository] = struct{}{}normalizedRef := strings.ToLower(fmt.Sprintf("%s/%s:%s", parts.Registry, parts.Repository, parts.Tag))- if idx, exists := indexByNormalizedRef[normalizedRef]; exists {- images[idx].refs = append(images[idx].refs, imageRef)+ if index, exists := indexByNormalizedRef[normalizedRef]; exists {+ groups[index] = append(groups[index], imageRef)continue}-- indexByNormalizedRef[normalizedRef] = len(images)- images = append(images, batchImage{- refs: []string{imageRef},- canonicalRef: imageRef,- parts: parts,- })+ indexByNormalizedRef[normalizedRef] = len(groups)+ groups = append(groups, []string{imageRef})}- return regRepos, results, images+ return groups}func (@@ -1443,23 +1435,21 @@ func () (*imageupdate.Response,*localImageSnapshot,+ error,) {if s.registryService == nil {- return &imageupdate.Response{- Error: "registry service unavailable",- CheckTime: time.Now(),- ResponseTimeMs: 0,- }, nil+ err := errors.New("registry service unavailable")+ return &imageupdate.Response{Error: err.Error(), CheckTime: time.Now()}, nil, err}start := time.Now()imageRef := fmt.Sprintf("%s/%s:%s", parts.Registry, parts.Repository, parts.Tag)snapshot, ldErr := s.inspectLocalImageSnapshotInternal(ctx, imageRef, composeBuildRefs)if ldErr == nil && snapshot.IsLocalBuild {- return localBuildImageUpdateResultInternal(snapshot, int(time.Since(start).Milliseconds())), snapshot+ return localBuildImageUpdateResultInternal(snapshot, int(time.Since(start).Milliseconds())), snapshot, nil}if ldErr != nil && errdefs.IsNotFound(ldErr) && isLocalBuildImageRefInternal(imageRef, composeBuildRefs) {- return missingLocalBuildImageUpdateResultInternal(parts.Tag, int(time.Since(start).Milliseconds())), nil+ return missingLocalBuildImageUpdateResultInternal(parts.Tag, int(time.Since(start).Milliseconds())), nil, nil}registryCtx, registryCancel := s.registryContextInternal(ctx)@@ -1477,7 +1467,7 @@ func (resp.AuthRegistry = digestResult.AuthRegistryresp.UsedCredential = digestResult.UsedCredential}- return resp, nil+ return resp, nil, digestErr}if ldErr != nil {@@ -1497,7 +1487,7 @@ func (AuthUsername: digestResult.AuthUsername,AuthRegistry: digestResult.AuthRegistry,UsedCredential: digestResult.UsedCredential,- }, nil+ }, nil, nil}return &imageupdate.Response{Error: ldErr.Error(),@@ -1507,7 +1497,7 @@ func (AuthUsername: digestResult.AuthUsername,AuthRegistry: digestResult.AuthRegistry,UsedCredential: digestResult.UsedCredential,- }, nil+ }, nil, ldErr}}@@ -1532,30 +1522,40 @@ func (AuthUsername: digestResult.AuthUsername,AuthRegistry: digestResult.AuthRegistry,UsedCredential: digestResult.UsedCredential,- }, snapshot+ }, snapshot, nil}-func (s *ImageUpdateService) resolveBatchCredentialsInternal(ctx context.Context, externalCreds []containerregistry.Credential) []containerregistry.Credential {- if len(externalCreds) > 0 {- filtered := make([]containerregistry.Credential, 0, len(externalCreds))- for _, cred := range externalCreds {- if !cred.Enabled || strings.TrimSpace(cred.URL) == "" || strings.TrimSpace(cred.Username) == "" || strings.TrimSpace(cred.Token) == "" {- continue- }- filtered = append(filtered, cred)- }- return filtered+// resolveBatchCredentialsInternal returns the caller's credentials for the check, or else the enabled stored+// registry credentials, cached per scan so its tasks decrypt them once while each new scan sees registry edits.+// Failures are not cached. Caller credentials live only in memory, so a check resumed after a restart uses the stored ones.+func (s *ImageUpdateService) resolveBatchCredentialsInternal(ctx context.Context, t flow.Task, input imageCheckInput) []containerregistry.Credential {+ if supplied, ok, _ := s.requestCredentials.Get(input.CredentialsKey); ok && input.CredentialsKey != "" {+ return supplied}-if s.registryService == nil {return nil}-- registries, err := s.registryService.GetEnabledRegistries(ctx)+ scanID := t.ActivityID()+ credentials, _, err := s.credentials.GetWithLoaders(scanID, func([]string) (map[string][]containerregistry.Credential, error) {+ loaded, loadErr := s.loadStoredCredentialsInternal(ctx)+ if loadErr != nil {+ return nil, loadErr+ }+ return map[string][]containerregistry.Credential{scanID: loaded}, nil+ })if err != nil {slog.DebugContext(ctx, "failed to load enabled registries for batch check", "error", err.Error())return nil}+ return credentials+}++// loadStoredCredentialsInternal decrypts the enabled stored registry credentials.+func (s *ImageUpdateService) loadStoredCredentialsInternal(ctx context.Context) ([]containerregistry.Credential, error) {+ registries, err := s.registryService.GetEnabledRegistries(ctx)+ if err != nil {+ return nil, err+ }credentials := make([]containerregistry.Credential, 0, len(registries))for _, reg := range registries {@@ -1581,215 +1581,351 @@ func (s *ImageUpdateService) resolveBatchCredentialsInternal(ctx context.Context})}- return credentials+ return credentials, nil}-func (- s *ImageUpdateService,-) checkBatchImageInternal(- ctx context.Context,- activityID string,- resolvedCreds []containerregistry.Credential,- img batchImage,- composeBuildRefs map[string]struct{},- recorder *batchImageProgressRecorder,-) error {- registryRecord := img.parts.Registry-- if err := s.registryLimiter.Acquire(ctx, registryRecord); err != nil {- slog.DebugContext(ctx, "skipping image check: registry limiter acquire failed",- "imageRef", img.canonicalRef,- "registry", registryRecord,- "error", err.Error())- if ctx.Err() != nil {- return ctx.Err()- }-- res := &imageupdate.Response{- Error: err.Error(),- CheckTime: time.Now(),- ResponseTimeMs: 0,- ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer(),- }- s.recordBatchImageCheckInternal(ctx, activityID, img, res, nil, recorder)- return nil+// recordImageCheckInternal saves a check result and reports it on the activity.+func (s *ImageUpdateService) recordImageCheckInternal(ctx context.Context, activityID, imageRef string, res *imageupdate.Response, snapshot *localImageSnapshot) {+ level, message := imageCheckResultMessageInternal(imageRef, res)+ s.appendImageUpdateActivityMessageInternal(ctx, activityID, level, message, nil, "")+ if err := s.saveUpdateResultWithSnapshotInternal(ctx, imageRef, res, snapshot); err != nil {+ slog.WarnContext(ctx, "Failed to save update result", "imageRef", imageRef, "error", err.Error())}- defer s.registryLimiter.Release(registryRecord)+}- res, snapshot := s.checkSingleImageInBatchInternal(ctx, resolvedCreds, img.parts, composeBuildRefs)- if ctx.Err() != nil {- return ctx.Err()+// recordDigestPinnedSkipInternal notes a digest-pinned ref as skipped on the+// activity stream and persists its unchanged result.+func (s *ImageUpdateService) recordDigestPinnedSkipInternal(ctx context.Context, activityID, imageRef string, result *imageupdate.Response, progress *int) {+ step := ""+ if progress != nil {+ step = "Skipping image update check"}- if res != nil {- res.ActivityID = mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()+ s.appendImageUpdateActivityMessageInternal(ctx, activityID, activitytypes.MessageLevelInfo, imageRef+" — digest pinned, skipped", progress, step)+ if err := s.saveUpdateResultWithSnapshotInternal(ctx, imageRef, result, nil); err != nil {+ slog.WarnContext(ctx, "Failed to save digest-pinned update result", "imageRef", imageRef, "error", err.Error())}+}- s.recordBatchImageCheckInternal(ctx, activityID, img, res, snapshot, recorder)+// RegisterWorkflows defines the image-check workflow; call it before the host starts.+func (s *ImageUpdateService) RegisterWorkflows(engine *flow.Engine) error {+ checkWorkflow, err := engine.Define(flow.Definition{+ Name: "image-check",+ Version: 3,+ Fingerprint: "e508fae9a6d269cc6aaac6375643f761c459e18571aab0efb61d5c0efb94b16d",+ Concurrency: imageCheckConcurrency,+ Timeout: timeouts.DefaultImageUpdateScan,+ Activity: activity.StartActivityRequest{+ Type: activitytypes.TypeImageUpdateCheck,+ Queue: true,+ ResourceType: new("images"),+ Step: "Checking image updates",+ LatestMessage: "Image update check started",+ },+ Labels: map[string]string{+ "prepare": "Preparing image update check",+ "discover": "Discovering images",+ "check": "Checking images",+ "tags": "Checking tag policies",+ "finalize": "Finishing image update check",+ },+ Steps: []workflow.StepSpec{+ workflow.Step("prepare", engine.Handler(s.prepareImageCheckInternal)),+ workflow.Step("discover", engine.Handler(s.discoverImagesInternal)),+ workflow.ForEach("check", engine.Handler(s.checkImageInternal),+ workflow.WithItemsFrom("discover"),+ workflow.WithInputFrom("prepare"),+ workflow.WithMaxParallel(imageCheckConcurrency),+ workflow.WithFailurePolicy(workflow.TolerateFailures),+ workflow.WithMaxAttempts(imageCheckAttempts),+ workflow.WithRetryBackoff(imageCheckRetryBackoff, imageCheckRetryBackoffMax),+ workflow.WithAttemptTimeout(2*time.Minute)),+ workflow.Step("tags", engine.Handler(s.checkTagPoliciesInternal), workflow.WithInputFrom("discover")),+ workflow.Step("finalize", engine.Handler(s.finalizeImageCheckInternal), workflow.WithInputFrom("discover", "check")),+ },+ // A reported check returns every ref's result, which outgrows the default output limit on large hosts.+ Options: []workflow.Option{workflow.WithMaxOutputSize(4 << 20), workflow.WithMaxJournalSize(16 << 20)},+ })+ if err != nil {+ return err+ }+ s.engine = engine+ s.checkWorkflow = checkWorkflowreturn nil}-func (- s *ImageUpdateService,-) recordBatchImageCheckInternal(- ctx context.Context,- activityID string,- img batchImage,- res *imageupdate.Response,- snapshot *localImageSnapshot,- recorder *batchImageProgressRecorder,-) {- progress := recorder.recordInternal(img.refs, res)-- level, message := imageCheckResultMessageInternal(img.canonicalRef, res)- s.appendImageUpdateActivityMessageInternal(ctx, activityID, level, message, progress, "Checking image")+// CheckWorkflow is the image-check workflow, for use as a child step.+func (s *ImageUpdateService) CheckWorkflow() *flow.Workflow { return s.checkWorkflow }- if err := s.saveUpdateResultWithSnapshotInternal(ctx, img.canonicalRef, res, snapshot); err != nil {- slog.WarnContext(ctx, "Failed to save update result", "imageRef", img.canonicalRef, "error", err.Error())- }+// RunImageCheck checks the requested images and returns the check outcome;+// failed images are reported as failed targets.+func (s *ImageUpdateService) RunImageCheck(ctx context.Context, request imageupdate.CheckRequest) (scheduler.Outcome, error) {+ return s.engine.Run(ctx, s.checkWorkflow, request, activitylib.StartRequest{}, nil)}-// recordDigestPinnedSkipInternal notes a digest-pinned ref as skipped on the-// activity stream and persists its unchanged result.-func (s *ImageUpdateService) recordDigestPinnedSkipInternal(ctx context.Context, activityID, imageRef string, result *imageupdate.Response, progress int) {- s.appendImageUpdateActivityMessageInternal(ctx, activityID, activitytypes.MessageLevelInfo, imageRef+" — digest pinned, skipped", progress, "Skipping image update check")- if err := s.saveUpdateResultWithSnapshotInternal(ctx, imageRef, result, nil); err != nil {- slog.WarnContext(ctx, "Failed to save digest-pinned update result", "imageRef", imageRef, "error", err.Error())+// CheckImages checks the requested images and returns each ref's result. Usable credentials replace the stored+// registries for this check, as they always have.+func (s *ImageUpdateService) CheckImages(ctx context.Context, request imageupdate.CheckRequest, credentials []containerregistry.Credential) (imageupdate.BatchResponse, error) {+ if !request.All && len(request.ImageRefs) == 0 {+ return imageupdate.BatchResponse{}, nil}-}--// recordInitialBatchResultsInternal stamps the activity ID on the parse-stage-// results and persists digest-pinned refs as skipped, since they never reach-// the registry check stage.-func (s *ImageUpdateService) recordInitialBatchResultsInternal(ctx context.Context, activityID string, initialResults map[string]*imageupdate.Response) {- for imageRef, result := range initialResults {- if result == nil {+ input := imageCheckInput{CheckRequest: request, Report: true}+ credentials = slices.DeleteFunc(slices.Clone(credentials), func(credential containerregistry.Credential) bool {+ return !credential.Enabled || strings.TrimSpace(credential.URL) == "" || strings.TrimSpace(credential.Username) == "" || strings.TrimSpace(credential.Token) == ""+ })+ if len(credentials) > 0 {+ input.CredentialsKey = uuid.NewV7().String()+ s.requestCredentials.Set(input.CredentialsKey, credentials)+ }+ var output imageCheckOutput+ outcome, err := s.engine.Run(ctx, s.checkWorkflow, input, activitylib.StartRequest{}, &output)+ if err != nil {+ // The workflow outlives a caller that stopped waiting, so finalize releases the credentials.+ return nil, err+ }+ s.requestCredentials.Delete(input.CredentialsKey)+ if outcome.Status == scheduler.Failed || outcome.Status == scheduler.Canceled {+ return nil, errors.New(cmp.Or(outcome.Message, "image update check failed"))+ }+ results := output.Results+ if results == nil {+ results = imageupdate.BatchResponse{}+ }+ for _, target := range outcome.Targets {+ if target.ResourceType != "image" {continue}- result.ActivityID = mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()- if strings.TrimSpace(result.Error) != "" {- continue+ if results[target.ID] == nil {+ results[target.ID] = &imageupdate.Response{CheckTime: time.Now()}}- if _, ok := digest.FromReferenceSuffix(imageRef); !ok {- continue- }-- s.recordDigestPinnedSkipInternal(ctx, activityID, imageRef, result, 5)+ results[target.ID].Error = target.Message}+ // Failed images get their entries first, so their containers' tag results still attach.+ tags.AttachContainerUpdates(results, output.ContainerUpdates)+ for _, result := range results {+ result.ActivityID = mo.EmptyableToOption(outcome.ActivityID).ToPointer()+ }+ return results, nil}-func (s *ImageUpdateService) CheckMultipleImages(ctx context.Context, imageRefs []string, externalCreds []containerregistry.Credential) (results map[string]*imageupdate.Response, err error) {- startBatch := time.Now()- results = make(map[string]*imageupdate.Response, len(imageRefs))- if len(imageRefs) == 0 {- return results, nil+// MonitoredImageRefs keeps the refs whose containers still permit update checks.+func (s *ImageUpdateService) MonitoredImageRefs(ctx context.Context, imageRefs []string) ([]string, error) {+ dockerClient, err := s.dockerClientInternal(ctx)+ if err != nil {+ return nil, err}+ apiCtx, cancel := s.dockerAPIContextInternal(ctx)+ listed, err := dockerClient.ContainerList(apiCtx, client.ContainerListOptions{All: true})+ cancel()+ if err != nil {+ return nil, fmt.Errorf("list containers for update-check eligibility: %w", err)+ }+ eligibility := newMonitoringEligibilityInternal(listed.Items)+ return slices.DeleteFunc(slices.Clone(imageRefs), func(imageRef string) bool {+ return !eligibility.imageEligible("", refs.NormalizeImageUpdateRef(imageRef))+ }), nil+}- activityID := s.startImageUpdateActivityInternal(ctx, fmt.Sprintf("%d images", len(imageRefs)), len(imageRefs))- ctx = s.activityService.Track(ctx, activityID)+// prepareImageCheckInternal loads the compose build refs every check task needs, once per scan.+func (s *ImageUpdateService) prepareImageCheckInternal(ctx context.Context, _ flow.Task) (any, error) {+ buildRefs, err := s.composeBuildImageRefsInternal(ctx)+ if err != nil {+ return nil, common.Classify(common.ErrUnavailable, err)+ }+ return slices.Sorted(maps.Keys(buildRefs)), nil+}- // A single deferred finalizer owns the terminal activity write so it runs- // on success, error, and panic alike — any early return that skipped- // completion would strand the row in running forever. It reads the Track- // ctx so a user cancellation still records a cancelled status.- defer func() {- if panicErr := utils.PanicToError(recover()); panicErr != nil {- // Don't re-panic: the caller is the long-lived watcher goroutine.- err = fmt.Errorf("image update check panicked: %w", panicErr)- slog.ErrorContext(ctx, "image update check panicked", "activityId", activityID, "error", err)- }+func (s *ImageUpdateService) discoverImagesInternal(ctx context.Context, t flow.Task) (any, error) {+ var request imageCheckInput+ if err := t.Payload(&request); err != nil {+ return nil, err+ }+ imageRefs := request.ImageRefs+ if request.All {+ discovered, err := s.getAllImageRefsInternal(ctx, 0)if err != nil {- s.completeImageUpdateActivityInternal(ctx, activityID, false, "Image update check failed: "+err.Error())- return- }- successCount, errorCount := countBatchResultOutcomesInternal(imageRefs, results)- finalMessage := fmt.Sprintf("Image update check completed: %d checked, %d errors", successCount, errorCount)- s.completeImageUpdateActivityInternal(ctx, activityID, errorCount == 0, finalMessage)- }()-- if activityID != "" {- // Bounded slot wait: update-all runs share these slots and can hold- // them for a long time; parking here forever would wedge every future- // scan behind the watcher's single-flight gate. On timeout the- // finalizer above flips the queued row to failed.- if slotErr := s.activityService.AwaitActivitySlotBounded(ctx, activityID, "0"); slotErr != nil {- return results, slotErr- }- }- s.appendImageUpdateActivityMessageInternal(ctx, activityID, activitytypes.MessageLevelInfo, fmt.Sprintf("Checking %d image references", len(imageRefs)), 5, "Preparing image update check")- slog.DebugContext(ctx, "Starting batch image update check", "imageCount", len(imageRefs), "externalCredCount", len(externalCreds))-- regRepos, initialResults, images := s.parseAndGroupImagesInternal(imageRefs)- maps.Copy(results, initialResults)- s.recordInitialBatchResultsInternal(ctx, activityID, initialResults)-- // The aggregate scan gets its own deadline: individual registry RPCs are- // bounded, but the batch as a whole (including registry limiter waits)- // was not, and a wedged scan holds its activity slot indefinitely.- scanCtx, cancelScan := context.WithTimeout(ctx, timeouts.DefaultImageUpdateScan)- defer cancelScan()-- composeBuildRefs, composeErr := s.composeBuildImageRefsInternal(scanCtx)- if composeErr != nil {- err = fmt.Errorf("prepare compose build image references: %w", composeErr)- return results, err+ return nil, common.Classify(common.ErrUnavailable, fmt.Errorf("list image references: %w", err))+ }+ imageRefs = discovered}+ s.appendImageUpdateActivityMessageInternal(ctx, t.ActivityID(), activitytypes.MessageLevelInfo, fmt.Sprintf("Checking %d image references", len(imageRefs)), nil, "")+ return s.groupImageRefsInternal(imageRefs), nil+}- resolvedCreds := s.resolveBatchCredentialsInternal(scanCtx, externalCreds)-- slog.DebugContext(ctx, "Resolved batch registry credentials", "credentialCount", len(resolvedCreds), "registryCount", len(regRepos))-- recorder := &batchImageProgressRecorder{- results: results,- total: len(images),+// checkImageInternal checks one group of refs. Retryable failures are recorded+// only on the last attempt so earlier attempts never replace a good result.+func (s *ImageUpdateService) checkImageInternal(ctx context.Context, t flow.Task) (any, error) {+ var group []string+ if err := t.DecodeItem(&group); err != nil || len(group) == 0 {+ return nil, fmt.Errorf("decode image group: %w", cmp.Or(err, errors.New("empty group")))}- g, groupCtx := errgroup.WithContext(scanCtx)- g.SetLimit(10) // Limit concurrency-- for _, img := range images {- g.Go(func() (checkErr error) {- // x/sync's errgroup does not recover goroutine panics (they crash- // the process), so convert them to errors here; the deferred- // finalizer then records the failed terminal status.- defer func() {- if panicErr := utils.PanicToError(recover()); panicErr != nil {- checkErr = fmt.Errorf("image update check panicked: %w", panicErr)- slog.ErrorContext(groupCtx, "image update check worker panicked", "activityId", activityID, "imageRef", img.canonicalRef, "error", checkErr)- }- }()- return s.checkBatchImageInternal(groupCtx, activityID, resolvedCreds, img, composeBuildRefs, recorder)- })+ var input imageCheckInput+ if err := t.Payload(&input); err != nil {+ return nil, err}-- if err = g.Wait(); err != nil {- if ctx.Err() == nil && errors.Is(scanCtx.Err(), context.DeadlineExceeded) {- err = fmt.Errorf("image update check timed out after %s: %w", timeouts.DefaultImageUpdateScan, err)- }- slog.ErrorContext(ctx, "Batch check error", "error", err)- return results, err+ // A manual check answers its caller, who is gone after a restart; its requester was not rechecked here either.+ if input.Report && t.Recovered() {+ return nil, errCheckInterrupted}+ imageRef := group[0]+ activityID := t.ActivityID()+ if result, ok := digestPinnedImageUpdateResultInternal(imageRef).Get(); ok {+ result.ActivityID = mo.EmptyableToOption(activityID).ToPointer()+ s.recordDigestPinnedSkipInternal(ctx, activityID, imageRef, result, nil)+ return imageCheckResultInternal{Result: kit.Ternary(input.Report, result, nil)}, nil+ }+ parts := s.parseImageReference(imageRef)+ if parts == nil {+ err := common.Classify(common.ErrValidation, errors.New("invalid image reference format"))+ s.appendImageUpdateActivityMessageInternal(ctx, activityID, activitytypes.MessageLevelError, imageRef+": "+err.Error(), nil, "")+ return nil, err+ }+ if err := s.registryLimiter.Acquire(ctx, parts.Registry); err != nil {+ return nil, err+ }+ defer s.registryLimiter.Release(parts.Registry)+ var buildRefs []string+ if err := t.DecodeOutput("prepare", &buildRefs); err != nil {+ return nil, err+ }+ composeBuildRefs := make(map[string]struct{}, len(buildRefs))+ for _, buildRef := range buildRefs {+ composeBuildRefs[buildRef] = struct{}{}+ }+ res, snapshot, checkErr := s.checkSingleImageInBatchInternal(ctx, s.resolveBatchCredentialsInternal(ctx, t, input), parts, composeBuildRefs)+ // Shutdown hands the task back for redelivery rather than recording it as a failed check.+ if ctx.Err() != nil {+ return nil, ctx.Err()+ }+ // A reported check answers a waiting caller, so it gets one attempt as the batch endpoint always has.+ if checkErr != nil && flow.Retryable(checkErr) && !input.Report && t.Attempt() < imageCheckAttempts {+ return nil, checkErr+ }+ res.ActivityID = mo.EmptyableToOption(activityID).ToPointer()+ s.recordImageCheckInternal(ctx, activityID, imageRef, res, snapshot)+ if checkErr != nil && input.Report {+ // The waiting caller gets the failed response, auth details included, alongside its error.+ return imageCheckResultInternal{Result: res, Failure: checkErr.Error()}, nil+ }+ if checkErr != nil {+ return nil, checkErr+ }+ return imageCheckResultInternal{Local: res.UpdateType == UpdateTypeLocal, Result: kit.Ternary(input.Report, res, nil)}, nil+}+// checkTagPoliciesInternal checks container tag policies for the non-local refs.+func (s *ImageUpdateService) checkTagPoliciesInternal(ctx context.Context, t flow.Task) (any, error) {+ groups, results, err := imageCheckResultsInternal(t)+ if err != nil {+ return nil, err+ }var tagRefs []string- for _, imageRef := range imageRefs {- if res := results[imageRef]; res != nil && res.UpdateType != UpdateTypeLocal {- tagRefs = append(tagRefs, imageRef)+ for index, group := range groups {+ if !results[index].Value.Local {+ tagRefs = append(tagRefs, group...)}}- containerUpdates, tagErr := s.checkContainerTagUpdatesInternal(scanCtx, tagRefs, resolvedCreds)- if tagErr != nil {- return results, tagErr+ var input imageCheckInput+ if payloadErr := t.Payload(&input); payloadErr != nil {+ return nil, payloadErr+ }+ if input.Report && t.Recovered() {+ return tagCheckOutput{Failures: []scheduler.TargetOutcome{}}, nil+ }+ containerUpdates, err := s.checkContainerTagUpdatesInternal(ctx, tagRefs, s.resolveBatchCredentialsInternal(ctx, t, input))+ if err != nil {+ return nil, common.Classify(common.ErrUnavailable, err)+ }+ output := tagCheckOutput{Failures: []scheduler.TargetOutcome{}}+ for containerID, update := range containerUpdates {+ if update.Error != "" {+ output.Failures = append(output.Failures, scheduler.TargetOutcome{ResourceType: "container", ID: containerID, Status: scheduler.Failed, Message: update.Error})+ }}- tags.AttachContainerUpdates(results, containerUpdates)- successCount, errorCount := countBatchResultOutcomesInternal(imageRefs, results)- slog.InfoContext(ctx, "Batch image update check completed",- "totalImages", len(imageRefs),- "successCount", successCount,- "errorCount", errorCount,- "duration", time.Since(startBatch))+ if input.Report {+ output.Updates = containerUpdates+ }+ return output, nil+}+func (s *ImageUpdateService) finalizeImageCheckInternal(ctx context.Context, t flow.Task) (any, error) {+ var request imageCheckInput+ if err := t.Payload(&request); err != nil {+ return nil, err+ }+ // Every check task has run, so the caller's credentials are no longer needed.+ s.requestCredentials.Delete(request.CredentialsKey)+ groups, results, err := imageCheckResultsInternal(t)+ if err != nil {+ return nil, err+ }+ var tagged tagCheckOutput+ if decodeErr := t.DecodeOutput("tags", &tagged); decodeErr != nil {+ return nil, decodeErr+ }+ tagFailures := tagged.Failures+ checked, failed := 0, 0+ var targets []scheduler.TargetOutcome+ for index, group := range groups {+ checked += len(group)+ if results[index].Err == "" {+ continue+ }+ failed += len(group)+ for _, imageRef := range group {+ targets = append(targets, scheduler.TargetOutcome{ResourceType: "image", ID: imageRef, Status: scheduler.Failed, Message: results[index].Err})+ }+ }+ targets = append(targets, tagFailures...)// A failed flush is already logged; the records stay pending for the next one._ = s.SendBatchUpdateNotifications(ctx)+ if request.All && (!request.Report || !t.Recovered()) {+ if cleanupErr := s.CleanupOrphanedRecords(ctx); cleanupErr != nil {+ slog.WarnContext(ctx, "failed to cleanup orphaned image update records after check-all", "error", cleanupErr.Error())+ }+ }+ outcome := scheduler.Outcome{+ Status: scheduler.Succeeded,+ Message: fmt.Sprintf("Image update check completed: %d checked, %d errors", checked, failed),+ Targets: targets,+ }+ if len(tagFailures) > 0 {+ outcome.Message += fmt.Sprintf(", %d tag policy errors", len(tagFailures))+ }+ if len(targets) > 0 {+ outcome.Status = scheduler.Partial+ }+ slog.InfoContext(ctx, "Image update check completed", "totalImages", checked, "failedImages", failed, "failedContainers", len(tagFailures))+ if !request.Report {+ return outcome, nil+ }+ output := imageCheckOutput{Outcome: outcome, Results: imageupdate.BatchResponse{}, ContainerUpdates: tagged.Updates}+ for index, group := range groups {+ if result := results[index].Value.Result; result != nil {+ for _, imageRef := range group {+ output.Results[imageRef] = new(*result)+ }+ }+ }+ return output, nil+}- return results, nil+// imageCheckResultsInternal reads the discovered groups and their check slots.+func imageCheckResultsInternal(t flow.Task) ([][]string, []flow.Result[imageCheckResultInternal], error) {+ var groups [][]string+ if err := t.DecodeOutput("discover", &groups); err != nil {+ return nil, nil, err+ }+ results, err := flow.Results[imageCheckResultInternal](t, "check")+ if err != nil {+ return nil, nil, err+ }+ if len(results) != len(groups) {+ return nil, nil, fmt.Errorf("image check produced %d results for %d groups", len(results), len(groups))+ }+ for index := range results {+ results[index].Err = cmp.Or(results[index].Err, results[index].Value.Failure)+ }+ return groups, results, nil}// SendBatchUpdateNotifications delivers pending update notifications. It@@ -1837,21 +1973,7 @@ func (s *ImageUpdateService) SendBatchUpdateNotifications(ctx context.Context) efor imageID, record := range unnotifiedUpdates {imageRef := containerUpdateNotificationKeyInternal(record)- updatesToNotify[imageRef] = &imageupdate.Response{- HasUpdate: record.HasUpdate,- UpdateType: record.UpdateType,- CurrentVersion: record.CurrentVersion,- LatestVersion: mo.PointerToOption(record.LatestVersion).OrEmpty(),- CurrentDigest: mo.PointerToOption(record.CurrentDigest).OrEmpty(),- LatestDigest: mo.PointerToOption(record.LatestDigest).OrEmpty(),- CheckTime: record.CheckTime,- ResponseTimeMs: record.ResponseTimeMs,- Error: mo.PointerToOption(record.LastError).OrEmpty(),- AuthMethod: mo.PointerToOption(record.AuthMethod).OrEmpty(),- AuthUsername: mo.PointerToOption(record.AuthUsername).OrEmpty(),- AuthRegistry: mo.PointerToOption(record.AuthRegistry).OrEmpty(),- UsedCredential: record.UsedCredential,- }+ updatesToNotify[imageRef] = responseFromRecordInternal(record)imageIDsToMark = append(imageIDsToMark, imageID)}@@ -1877,28 +1999,6 @@ func (s *ImageUpdateService) SendBatchUpdateNotifications(ctx context.Context) ereturn nil}-func (s *ImageUpdateService) CheckAllImages(ctx context.Context, limit int, externalCreds []containerregistry.Credential) (map[string]*imageupdate.Response, error) {- imageRefs, err := s.getAllImageRefsInternal(ctx, limit)- if err != nil {- return nil, fmt.Errorf("failed to get image references: %w", err)- }-- if len(imageRefs) == 0 {- return make(map[string]*imageupdate.Response), nil- }-- results, err := s.CheckMultipleImages(ctx, imageRefs, externalCreds)- if err != nil {- return nil, err- }-- if cleanupOrphanedRecordsErr := s.CleanupOrphanedRecords(ctx); cleanupOrphanedRecordsErr != nil {- slog.WarnContext(ctx, "failed to cleanup orphaned image update records after check-all", "error", cleanupOrphanedRecordsErr.Error())- }-- return results, nil-}-// DeleteRecordsForImages removes image-level rows (keyed by id) and// container/project rows (keyed by image_id) for the given images.func (s *ImageUpdateService) DeleteRecordsForImages(ctx context.Context, imageIDs []string) error {@@ -2077,6 +2177,24 @@ func (s *ImageUpdateService) saveContainerTagResultInternal(ctx context.Context,})}+func responseFromRecordInternal(record *ImageUpdateRecord) *imageupdate.Response {+ return &imageupdate.Response{+ HasUpdate: record.HasUpdate,+ UpdateType: record.UpdateType,+ CurrentVersion: record.CurrentVersion,+ LatestVersion: mo.PointerToOption(record.LatestVersion).OrEmpty(),+ CurrentDigest: mo.PointerToOption(record.CurrentDigest).OrEmpty(),+ LatestDigest: mo.PointerToOption(record.LatestDigest).OrEmpty(),+ CheckTime: record.CheckTime,+ ResponseTimeMs: record.ResponseTimeMs,+ Error: mo.PointerToOption(record.LastError).OrEmpty(),+ AuthMethod: mo.PointerToOption(record.AuthMethod).OrEmpty(),+ AuthUsername: mo.PointerToOption(record.AuthUsername).OrEmpty(),+ AuthRegistry: mo.PointerToOption(record.AuthRegistry).OrEmpty(),+ UsedCredential: record.UsedCredential,+ }+}+func containerUpdateNotificationKeyInternal(record *ImageUpdateRecord) string {imageRef := fmt.Sprintf("%s:%s", record.Repository, record.Tag)if record.ContainerID != "" {diff --git a/backend/internal/imageupdate/service_test.go b/backend/internal/imageupdate/service_test.goindex 4cafb6bef..cc3b13d9c 100644--- a/backend/internal/imageupdate/service_test.go+++ b/backend/internal/imageupdate/service_test.go@@ -15,8 +15,8 @@ import ("github.com/distribution/reference"activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"- "github.com/getarcaneapp/arcane/types/v2/containerregistry""github.com/getarcaneapp/arcane/types/v2/imageupdate"+ "github.com/getarcaneapp/arcane/types/v2/scheduler""github.com/libtnb/sqlite""github.com/moby/moby/api/pkg/authconfig""github.com/moby/moby/api/types/container"@@ -41,6 +41,9 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/notification""github.com/getarcaneapp/arcane/backend/v2/internal/registry""github.com/getarcaneapp/arcane/backend/v2/internal/settings"+ activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow/flowtest""github.com/getarcaneapp/arcane/backend/v2/pkg/utils""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/imageref""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/notifications"@@ -406,16 +409,10 @@ func TestImageUpdateService_CheckImageUpdate_ComposeBuildSkipsRegistryWithRepoDiassert.Nil(t, saved.LastError)}-func TestImageUpdateService_CheckMultipleImages_ComposeBuildSkipsRegistryWithRepoDigests(t *testing.T) {+func TestImageUpdateService_CheckImages_ComposeBuildSkipsRegistryWithRepoDigests(t *testing.T) {svc, registryCalls := newComposeBuildImageUpdateServiceInternal(t)- credentials := []containerregistry.Credential{{- URL: "https://index.docker.io/v1/",- Username: "unused",- Token: "unused",- Enabled: true,- }}- results, err := svc.CheckMultipleImages(t.Context(), []string{"test2:latest"}, credentials)+ results, err := checkImagesInternal(t, t.Context(), svc, "test2:latest")require.NoError(t, err)result := results["test2:latest"]require.NotNil(t, result)@@ -425,7 +422,7 @@ func TestImageUpdateService_CheckMultipleImages_ComposeBuildSkipsRegistryWithRepassert.Zero(t, registryCalls.Load())}-func TestImageUpdateService_CheckMultipleImages_ComposeBuildMissingLocallySkipsRegistry(t *testing.T) {+func TestImageUpdateService_CheckImages_ComposeBuildMissingLocallySkipsRegistry(t *testing.T) {db := setupImageUpdateTestDB(t)buildRefsJSON := `["test2:latest"]`require.NoError(t, db.Create(&testProjectRow{@@ -461,7 +458,7 @@ func TestImageUpdateService_CheckMultipleImages_ComposeBuildMissingLocallySkipsReventService := event.NewEventService(db, nil, nil)svc := NewImageUpdateService(db, nil, registryService, dockerService, eventService, nil, nil)- results, err := svc.CheckMultipleImages(t.Context(), []string{"test2:latest"}, nil)+ results, err := checkImagesInternal(t, t.Context(), svc, "test2:latest")require.NoError(t, err)result := results["test2:latest"]require.NotNil(t, result)@@ -540,10 +537,10 @@ func TestImageUpdateService_CheckImageUpdate_ArcaneLocalHostSkipsRegistry(t *tesassert.Nil(t, saved.LastError)}-func TestImageUpdateService_CheckMultipleImages_ArcaneLocalMissingImageSkipsRegistry(t *testing.T) {+func TestImageUpdateService_CheckImages_ArcaneLocalMissingImageSkipsRegistry(t *testing.T) {svc, registryCalls := newArcaneLocalImageUpdateServiceInternal(t, false)- results, err := svc.CheckMultipleImages(t.Context(), []string{"arcane.local/demo-2ab41b29/worker:latest"}, nil)+ results, err := checkImagesInternal(t, t.Context(), svc, "arcane.local/demo-2ab41b29/worker:latest")require.NoError(t, err)result := results["arcane.local/demo-2ab41b29/worker:latest"]require.NotNil(t, result)@@ -553,10 +550,10 @@ func TestImageUpdateService_CheckMultipleImages_ArcaneLocalMissingImageSkipsRegiassert.Zero(t, registryCalls.Load())}-func TestImageUpdateService_CheckMultipleImages_OtherDottedRegistryStillChecked(t *testing.T) {+func TestImageUpdateService_CheckImages_OtherDottedRegistryStillChecked(t *testing.T) {svc, registryCalls := newArcaneLocalImageUpdateServiceInternal(t, false)- results, err := svc.CheckMultipleImages(t.Context(), []string{"registry.local/team/app:latest"}, nil)+ results, err := checkImagesInternal(t, t.Context(), svc, "registry.local/team/app:latest")require.NoError(t, err)result := results["registry.local/team/app:latest"]require.NotNil(t, result)@@ -783,20 +780,15 @@ func TestImageUpdateService_CheckImageUpdate_SkipsDigestPinnedReferenceInternal(assert.Empty(t, result.Error)}-func TestImageUpdateService_CheckMultipleImages_SkipsDigestPinnedReferenceInternal(t *testing.T) {+func TestImageUpdateService_CheckImages_SkipsDigestPinnedReferenceInternal(t *testing.T) {db := setupImageUpdateTestDB(t)pinnedDigest := digest.FromString("batch-pinned-newt").String()imageRef := "ghcr.io/fosrl/newt@" + pinnedDigestsvc := NewImageUpdateService(db, nil, nil, nil, nil, nil, nil)- regRepos, initialResults, grouped := svc.parseAndGroupImagesInternal([]string{imageRef})- require.Empty(t, regRepos)- require.Empty(t, grouped)- require.Contains(t, initialResults, imageRef)- require.NotNil(t, initialResults[imageRef])- assert.Empty(t, initialResults[imageRef].Error)+ require.Equal(t, [][]string{{imageRef}}, svc.groupImageRefsInternal([]string{imageRef}))- results, err := svc.CheckMultipleImages(t.Context(), []string{imageRef}, nil)+ results, err := checkImagesInternal(t, t.Context(), svc, imageRef)require.NoError(t, err)require.Contains(t, results, imageRef)require.NotNil(t, results[imageRef])@@ -807,7 +799,7 @@ func TestImageUpdateService_CheckMultipleImages_SkipsDigestPinnedReferenceInternassert.Empty(t, results[imageRef].Error)}-func TestImageUpdateService_CheckMultipleImages_SkippedDigestPinnedReferenceClearsStaleErrorInternal(t *testing.T) {+func TestImageUpdateService_CheckImages_SkippedDigestPinnedReferenceClearsStaleErrorInternal(t *testing.T) {db := setupImageUpdateTestDB(t)pinnedDigest := digest.FromString("stale-pinned-newt").String()imageRef := "ghcr.io/fosrl/newt@" + pinnedDigest@@ -825,7 +817,7 @@ func TestImageUpdateService_CheckMultipleImages_SkippedDigestPinnedReferenceClea}).Error)svc := NewImageUpdateService(db, nil, nil, nil, nil, nil, nil)- results, err := svc.CheckMultipleImages(t.Context(), []string{imageRef}, nil)+ results, err := checkImagesInternal(t, t.Context(), svc, imageRef)require.NoError(t, err)require.Contains(t, results, imageRef)assert.Empty(t, results[imageRef].Error)@@ -838,7 +830,7 @@ func TestImageUpdateService_CheckMultipleImages_SkippedDigestPinnedReferenceCleaassert.Equal(t, pinnedDigest, mo.PointerToOption(saved.LatestDigest).OrEmpty())}-func TestImageUpdateService_CheckMultipleImages_DigestPinnedTagPreservedWhenLocalImageHasNoRepoTags(t *testing.T) {+func TestImageUpdateService_CheckImages_DigestPinnedTagPreservedWhenLocalImageHasNoRepoTags(t *testing.T) {db := setupImageUpdateTestDB(t)pinnedDigest := digest.FromString("valkey-pinned").String()@@ -853,7 +845,7 @@ func TestImageUpdateService_CheckMultipleImages_DigestPinnedTagPreservedWhenLocadockerService := &docker.DockerClientService{Client: newImageUpdateTestDockerClientInternal(t, server)}svc := NewImageUpdateService(db, nil, nil, dockerService, nil, nil, nil)- results, err := svc.CheckMultipleImages(t.Context(), []string{imageRef}, nil)+ results, err := checkImagesInternal(t, t.Context(), svc, imageRef)require.NoError(t, err)require.Contains(t, results, imageRef)assert.False(t, results[imageRef].HasUpdate)@@ -930,7 +922,7 @@ func TestImageUpdateService_CheckImageUpdate_UsesRegistryFallback(t *testing.T)assert.Equal(t, remoteDigest, mo.PointerToOption(saved.LatestDigest).OrEmpty())}-func TestImageUpdateService_CheckMultipleImages_UsesRegistryFallback(t *testing.T) {+func TestImageUpdateService_CheckImages_UsesRegistryFallback(t *testing.T) {db := setupImageUpdateTestDB(t)localDigest := digest.FromString("batchlocal").String()remoteDigest := digest.FromString("batchremote").String()@@ -955,7 +947,7 @@ func TestImageUpdateService_CheckMultipleImages_UsesRegistryFallback(t *testing.eventService := event.NewEventService(db, nil, nil)svc := NewImageUpdateService(db, nil, registryService, dockerService, eventService, nil, nil)- results, err := svc.CheckMultipleImages(t.Context(), []string{imageRef}, nil)+ results, err := checkImagesInternal(t, t.Context(), svc, imageRef)require.NoError(t, err)require.Contains(t, results, imageRef)@@ -972,7 +964,7 @@ func TestImageUpdateService_CheckMultipleImages_UsesRegistryFallback(t *testing.assert.Equal(t, remoteDigest, mo.PointerToOption(saved.LatestDigest).OrEmpty())}-func TestImageUpdateService_CheckMultipleImagesCompletesActivityWhenRequestContextCanceledInternal(t *testing.T) {+func TestImageUpdateService_CheckImagesCompletesActivityAfterRequestContextCanceledInternal(t *testing.T) {db := setupImageUpdateTestDB(t)require.NoError(t, db.AutoMigrate(&activity.Activity{}, &activity.ActivityMessage{}))@@ -982,44 +974,43 @@ func TestImageUpdateService_CheckMultipleImagesCompletesActivityWhenRequestContefor range 5 {require.NoError(t, svc.registryLimiter.Acquire(t.Context(), "docker.io"))}- defer func() {- for range 5 {- svc.registryLimiter.Release("docker.io")- }- }()+ startCheckWorkflowInternal(t, svc)ctx, cancel := context.WithCancel(t.Context())errCh := make(chan error, 1)go func() {- _, err := svc.CheckMultipleImages(ctx, []string{"nginx:latest"}, nil)+ _, err := svc.CheckImages(ctx, imageupdate.CheckRequest{ImageRefs: []string{"nginx:latest"}}, nil)errCh <- err}()var localActivity activity.Activityrequire.Eventually(t, func() bool {return db.Where("type = ?", activitytypes.TypeImageUpdateCheck).First(&localActivity).Error == nil- }, time.Second, 10*time.Millisecond)+ }, 5*time.Second, 10*time.Millisecond)cancel()select {case err := <-errCh:require.ErrorIs(t, err, context.Canceled)- case <-time.After(time.Second):+ case <-time.After(5 * time.Second):require.FailNow(t, "timed out waiting for image update check to return")}+ // The durable check outlives the request and still finalizes its activity.+ for range 5 {+ svc.registryLimiter.Release("docker.io")+ }require.Eventually(t, func() bool {if err := db.First(&localActivity, "id = ?", localActivity.ID).Error; err != nil {return false}return localActivity.Status == activitytypes.StatusFailed- }, time.Second, 10*time.Millisecond)- assert.Equal(t, "Image update check complete", localActivity.Step)- assert.Contains(t, localActivity.LatestMessage, "Image update check failed")+ }, 15*time.Second, 10*time.Millisecond)+ assert.NotNil(t, localActivity.EndedAt)}-func TestImageUpdateService_CheckMultipleImagesTimesOutStalledRegistryCheckInternal(t *testing.T) {+func TestImageUpdateService_CheckImagesRetriesStalledRegistryCheckInternal(t *testing.T) {db := setupImageUpdateTestDB(t)require.NoError(t, db.AutoMigrate(&activity.Activity{}, &activity.ActivityMessage{}))@@ -1036,7 +1027,8 @@ func TestImageUpdateService_CheckMultipleImagesTimesOutStalledRegistryCheckInter}, nil}, nil, nil)- parentCtx, cancel := context.WithTimeout(t.Context(), 2500*time.Millisecond)+ shortenImageCheckRetryBackoffInternal(t)+ parentCtx, cancel := context.WithTimeout(t.Context(), 10*time.Second)defer cancel()svc := NewImageUpdateService(@@ -1053,25 +1045,26 @@ func TestImageUpdateService_CheckMultipleImagesTimesOutStalledRegistryCheckInter)start := time.Now()- results, err := svc.CheckMultipleImages(parentCtx, []string{"registry.example.com/team/app:1.2.3"}, nil)+ results, err := checkImagesInternal(t, parentCtx, svc, "registry.example.com/team/app:1.2.3")elapsed := time.Since(start)require.NoError(t, err)- require.Less(t, elapsed, 2*time.Second)+ // Each of the three attempts is bounded by the one-second registry timeout.+ require.Less(t, elapsed, 8*time.Second)require.Contains(t, results, "registry.example.com/team/app:1.2.3")require.NotNil(t, results["registry.example.com/team/app:1.2.3"])require.Contains(t, results["registry.example.com/team/app:1.2.3"].Error, context.DeadlineExceeded.Error())var localActivity activity.Activity- require.NoError(t, db.Where("type = ?", activitytypes.TypeImageUpdateCheck).First(&localActivity).Error)+ require.Eventually(t, func() bool {+ return db.Where("type = ?", activitytypes.TypeImageUpdateCheck).First(&localActivity).Error == nil && localActivity.EndedAt != nil+ }, 5*time.Second, 10*time.Millisecond)require.Equal(t, activitytypes.StatusFailed, localActivity.Status)- require.NotNil(t, localActivity.EndedAt)require.NotNil(t, localActivity.DurationMs)- require.Equal(t, "Image update check complete", localActivity.Step)- require.Contains(t, localActivity.LatestMessage, "0 checked, 1 errors")+ require.Contains(t, localActivity.LatestMessage, "1 checked, 1 errors")}-func TestImageUpdateService_CheckMultipleImagesPanicMarksActivityFailedInternal(t *testing.T) {+func TestImageUpdateService_CheckImagesPanicFailsOnlyThatImageInternal(t *testing.T) {db := setupImageUpdateTestDB(t)require.NoError(t, db.AutoMigrate(&activity.Activity{}, &activity.ActivityMessage{}))@@ -1100,16 +1093,16 @@ func TestImageUpdateService_CheckMultipleImagesPanicMarksActivityFailedInternal(activityService,)- _, err := svc.CheckMultipleImages(t.Context(), []string{"registry.example.com/team/app:1.2.3"}, nil)- require.Error(t, err)- require.Contains(t, err.Error(), "image update check panicked")- require.Contains(t, err.Error(), "registry check exploded")+ results, err := checkImagesInternal(t, t.Context(), svc, "registry.example.com/team/app:1.2.3")+ require.NoError(t, err)+ require.Contains(t, results["registry.example.com/team/app:1.2.3"].Error, "task panicked: registry check exploded")var localActivity activity.Activity- require.NoError(t, db.Where("type = ?", activitytypes.TypeImageUpdateCheck).First(&localActivity).Error)+ require.Eventually(t, func() bool {+ return db.Where("type = ?", activitytypes.TypeImageUpdateCheck).First(&localActivity).Error == nil && localActivity.EndedAt != nil+ }, 5*time.Second, 10*time.Millisecond)require.Equal(t, activitytypes.StatusFailed, localActivity.Status)- require.NotNil(t, localActivity.EndedAt)- require.Contains(t, localActivity.LatestMessage, "Image update check failed")+ require.Contains(t, localActivity.LatestMessage, "1 errors")}func TestImageUpdateService_GetAllImageRefsUsesDockerAPITimeoutInternal(t *testing.T) {@@ -1148,7 +1141,7 @@ func TestImageUpdateService_InspectLocalImageSnapshotUsesDockerAPITimeoutInternarequire.Contains(t, err.Error(), context.DeadlineExceeded.Error())}-func TestImageUpdateService_CheckMultipleImages_UsesDockerHubCredentialsOnFirstAttempt(t *testing.T) {+func TestImageUpdateService_CheckImages_UsesDockerHubCredentialsOnFirstAttempt(t *testing.T) {db := setupImageUpdateRegistryTestDBInternal(t)require.NoError(t, db.AutoMigrate(&ImageUpdateRecord{}, &event.Event{}, &testProjectRow{}))createImageUpdateTestPullRegistryInternal(t, db, "https://index.docker.io/v1/", "docker-user", "docker-token")@@ -1184,7 +1177,7 @@ func TestImageUpdateService_CheckMultipleImages_UsesDockerHubCredentialsOnFirstAeventService := event.NewEventService(db, nil, nil)svc := NewImageUpdateService(db, nil, registryService, dockerService, eventService, nil, nil)- results, err := svc.CheckMultipleImages(t.Context(), []string{"docker.io/library/registry:3"}, nil)+ results, err := checkImagesInternal(t, t.Context(), svc, "docker.io/library/registry:3")require.NoError(t, err)require.Contains(t, results, "docker.io/library/registry:3")@@ -1200,7 +1193,7 @@ func TestImageUpdateService_CheckMultipleImages_UsesDockerHubCredentialsOnFirstAassert.True(t, result.UsedCredential)}-func TestImageUpdateService_CheckMultipleImages_ReportsNotPulledWhenLocalImageMissing(t *testing.T) {+func TestImageUpdateService_CheckImages_ReportsNotPulledWhenLocalImageMissing(t *testing.T) {db := setupImageUpdateTestDB(t)remoteDigest := digest.FromString("registry-only-remote").String()@@ -1223,7 +1216,7 @@ func TestImageUpdateService_CheckMultipleImages_ReportsNotPulledWhenLocalImageMieventService := event.NewEventService(db, nil, nil)svc := NewImageUpdateService(db, nil, registryService, dockerService, eventService, nil, nil)- results, err := svc.CheckMultipleImages(t.Context(), []string{imageRef}, nil)+ results, err := checkImagesInternal(t, t.Context(), svc, imageRef)require.NoError(t, err)require.Contains(t, results, imageRef)require.NotNil(t, results[imageRef])@@ -2122,26 +2115,10 @@ func TestImageUpdateService_ParseAndGroupImages_DedupesNormalizedRefs(t *testing"docker.io/library/redis:7",}- regRepos, initialResults, grouped := svc.parseAndGroupImagesInternal(refs)- require.Empty(t, initialResults)+ grouped := svc.groupImageRefsInternal(refs)require.Len(t, grouped, 2)- require.Contains(t, regRepos, "docker.io")- require.Len(t, regRepos["docker.io"], 2)-- firstRefSet := map[string]struct{}{}- for _, imageRef := range grouped[0].refs {- firstRefSet[imageRef] = struct{}{}- }- secondRefSet := map[string]struct{}{}- for _, imageRef := range grouped[1].refs {- secondRefSet[imageRef] = struct{}{}- }-- // Each normalized image should only be checked once, while retaining all aliases.- assert.True(t, (containsAll(firstRefSet, "nginx:latest", "docker.io/library/nginx:latest") &&- containsAll(secondRefSet, "redis:7", "docker.io/library/redis:7")) ||- (containsAll(secondRefSet, "nginx:latest", "docker.io/library/nginx:latest") &&- containsAll(firstRefSet, "redis:7", "docker.io/library/redis:7")))+ assert.ElementsMatch(t, []string{"nginx:latest", "docker.io/library/nginx:latest"}, grouped[0])+ assert.ElementsMatch(t, []string{"redis:7", "docker.io/library/redis:7"}, grouped[1])}func newImageUpdateTestSettingsServiceInternal(t *testing.T, registryTimeout, dockerAPITimeout string) *settings.SettingsService {@@ -2183,15 +2160,6 @@ func newBlockedDockerAPIServerInternal(t *testing.T, pathContains string) *httptreturn server}-func containsAll(set map[string]struct{}, refs ...string) bool {- for _, imageRef := range refs {- if _, ok := set[imageRef]; !ok {- return false- }- }- return true-}-func newImageUpdateDiscoveryServerInternal(t *testing.T,images []image.Summary,@@ -3017,3 +2985,163 @@ func TestContainerTagChecksPersistResultsFinishedBeforeScanDeadlineInternal(t *trequire.Equal(t, "1.1.0", mo.PointerToOption(records[0].LatestVersion).OrEmpty())require.Empty(t, mo.PointerToOption(records[0].LastError).OrEmpty(), "a stored result must not be overwritten by the cancellation")}++func TestImageUpdateService_RunImageCheckExhaustedRateLimitKeepsPreviousResult(t *testing.T) {+ // Serial: it shortens the package-wide retry backoff.+ const imageRef = "registry.example.com/team/app:1"+ var calls atomic.Int32+ svc, db := newWorkflowCheckServiceInternal(t, map[string]string{imageRef: "sha256:app"}, func(string) (client.DistributionInspectResult, error) {+ calls.Add(1)+ return client.DistributionInspectResult{}, errors.New("toomanyrequests: rate limit exceeded")+ })+ shortenImageCheckRetryBackoffInternal(t)+ previousDigest := digest.FromString("previous-remote").String()+ require.NoError(t, db.Create(&ImageUpdateRecord{+ ID: "sha256:app", Repository: "registry.example.com/team/app", Tag: "1", HasUpdate: true,+ UpdateType: UpdateTypeDigest, CurrentVersion: "1", LatestDigest: &previousDigest, CheckTime: time.Now().Add(-time.Hour),+ }).Error)++ // A scheduled check retries across workflow attempts; a reported one answers its caller after a single attempt.+ if svc.engine == nil {+ startCheckWorkflowInternal(t, svc)+ }+ outcome, err := svc.RunImageCheck(t.Context(), imageupdate.CheckRequest{ImageRefs: []string{imageRef}})+ require.NoError(t, err)+ require.Len(t, outcome.Targets, 1)+ require.Contains(t, outcome.Targets[0].Message, "toomanyrequests")+ require.Equal(t, int32(5*imageCheckAttempts), calls.Load(), "each workflow attempt spends the registry's own retry budget")++ var saved ImageUpdateRecord+ require.NoError(t, db.First(&saved, "id = ?", "sha256:app").Error)+ assert.True(t, saved.HasUpdate)+ assert.Nil(t, saved.LastError)+ assert.Equal(t, previousDigest, mo.PointerToOption(saved.LatestDigest).OrEmpty())+}++func TestImageUpdateService_RunImageCheckReportsPartialFailure(t *testing.T) {+ const okRef = "registry.example.com/team/app:1"+ const deniedRef = "registry.example.com/team/private:1"+ remoteDigest := digest.FromString("remote").String()+ svc, _ := newWorkflowCheckServiceInternal(t, map[string]string{okRef: "sha256:app"}, func(imageRef string) (client.DistributionInspectResult, error) {+ if strings.Contains(imageRef, "private") {+ return client.DistributionInspectResult{}, errors.New("unauthorized: authentication required")+ }+ return client.DistributionInspectResult{Descriptor: v1.Descriptor{Digest: digest.Digest(remoteDigest)}}, nil+ })+ harness := startCheckWorkflowInternal(t, svc)+ flowtest.AssertDefinitions(t, harness)++ request := imageupdate.CheckRequest{ImageRefs: []string{okRef, deniedRef}}+ outcome, err := svc.RunImageCheck(t.Context(), request)+ require.NoError(t, err)+ require.Equal(t, scheduler.Partial, outcome.Status)+ require.Len(t, outcome.Targets, 1)+ assert.Equal(t, deniedRef, outcome.Targets[0].ID)+ assert.Equal(t, scheduler.Failed, outcome.Targets[0].Status)+ require.Len(t, outcome.Steps, 5)++ results, err := svc.CheckImages(t.Context(), request, nil)+ require.NoError(t, err)+ assert.True(t, results[okRef].HasUpdate)+ assert.Empty(t, results[okRef].Error)+ assert.Equal(t, remoteDigest, results[okRef].LatestDigest)+ assert.Contains(t, results[deniedRef].Error, "unauthorized")+}++// startCheckWorkflowInternal registers the image-check workflow on an isolated host.+func startCheckWorkflowInternal(t *testing.T, svc *ImageUpdateService) *flowtest.Harness {+ t.Helper()+ var activities flow.Activities = noopActivitiesInternal{}+ if svc.activityService != nil {+ activities = svc.activityService+ }+ harness := flowtest.New(t, activities)+ require.NoError(t, svc.RegisterWorkflows(harness.Engine))+ harness.Start(t)+ return harness+}++// checkImagesInternal runs refs through the image-check workflow.+func checkImagesInternal(t *testing.T, ctx context.Context, svc *ImageUpdateService, imageRefs ...string) (imageupdate.BatchResponse, error) {+ t.Helper()+ if svc.engine == nil {+ startCheckWorkflowInternal(t, svc)+ }+ return svc.CheckImages(ctx, imageupdate.CheckRequest{ImageRefs: imageRefs}, nil)+}++// newWorkflowCheckServiceInternal serves one local image per repository named in+// localImages and answers registry lookups through inspect.+func newWorkflowCheckServiceInternal(+ t *testing.T,+ localImages map[string]string,+ inspect func(imageRef string) (client.DistributionInspectResult, error),+) (*ImageUpdateService, *database.DB) {+ t.Helper()+ db := setupImageUpdateTestDB(t)+ server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ if strings.HasSuffix(r.URL.Path, "/containers/json") {+ assert.NoError(t, json.NewEncoder(w).Encode([]container.Summary{}))+ return+ }+ for imageRef, imageID := range localImages {+ if strings.Contains(r.URL.Path, "/images/"+imageRef+"/json") {+ repository := imageRef[:strings.LastIndex(imageRef, ":")]+ assert.NoError(t, json.NewEncoder(w).Encode(image.InspectResponse{+ ID: imageID,+ RepoTags: []string{imageRef},+ RepoDigests: []string{repository + "@" + digest.FromString(imageID).String()},+ }))+ return+ }+ }+ http.Error(w, "No such image", http.StatusNotFound)+ }))+ t.Cleanup(server.Close)+ registryService := registry.NewContainerRegistryService(db, func(context.Context) (registry.RegistryDaemonClient, error) {+ return &fakeRegistryDaemonClient{+ distributionInspectFn: func(_ context.Context, imageRef string, _ client.DistributionInspectOptions) (client.DistributionInspectResult, error) {+ return inspect(imageRef)+ },+ }, nil+ }, nil, nil)+ dockerService := &docker.DockerClientService{Client: newImageUpdateTestDockerClientInternal(t, server)}+ return NewImageUpdateService(db, nil, registryService, dockerService, event.NewEventService(db, nil, nil), nil, nil), db+}++func shortenImageCheckRetryBackoffInternal(t *testing.T) {+ t.Helper()+ initial, maximum := imageCheckRetryBackoff, imageCheckRetryBackoffMax+ imageCheckRetryBackoff, imageCheckRetryBackoffMax = 10*time.Millisecond, 20*time.Millisecond+ t.Cleanup(func() { imageCheckRetryBackoff, imageCheckRetryBackoffMax = initial, maximum })+}++// noopActivitiesInternal stands in for the activity service in tests that do not inspect activities.+type noopActivitiesInternal struct{}++func (noopActivitiesInternal) StartActivity(_ context.Context, req activitylib.StartRequest) (*activitytypes.Activity, error) {+ return &activitytypes.Activity{ID: req.ID}, nil+}++func (noopActivitiesInternal) CompleteActivity(_ context.Context, activityID string, status activitytypes.Status, _ string, _ *string, _ ...string) (*activitytypes.Activity, error) {+ return &activitytypes.Activity{ID: activityID, Status: status}, nil+}++func (noopActivitiesInternal) AppendMessage(context.Context, string, activitylib.AppendMessageRequest) (*activitytypes.Message, error) {+ return &activitytypes.Message{}, nil+}++func (noopActivitiesInternal) AppendMessages(context.Context, string, []activitylib.AppendMessageRequest) ([]activitytypes.Message, error) {+ return nil, nil+}++func (noopActivitiesInternal) Track(ctx context.Context, _ string) context.Context { return ctx }++func (noopActivitiesInternal) UpdateActivity(_ context.Context, activityID string, _ activitylib.UpdateRequest) (*activitytypes.Activity, error) {+ return &activitytypes.Activity{ID: activityID}, nil+}++func (noopActivitiesInternal) AwaitActivitySlotBounded(context.Context, string, string) error {+ return nil+}diff --git a/backend/internal/job/service.go b/backend/internal/job/service.goindex 376f476b3..c3620bde7 100644--- a/backend/internal/job/service.go+++ b/backend/internal/job/service.go@@ -33,10 +33,8 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/backend/v2/pkg/authz""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"- "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"scheduleutil "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/schedule"- "github.com/getarcaneapp/arcane/backend/v2/pkg/utils")// JobService manages configuration for background job schedules.@@ -624,7 +622,7 @@ func (s *JobService) executeRunInternal(ctx context.Context, run scheduler.Run)return scheduler.Outcome{Status: scheduler.Waiting, Message: "Waiting for Docker"}, err}}- ctx = s.runContextInternal(ctx, run)+ ctx = s.runs.ExecutionContext(ctx, run, "")if run.JobID == "environment-health" && s.RunEnvironmentHealthNow != nil {err := s.RunEnvironmentHealthNow(ctx)return classifyOutcomeInternal(run.JobID, scheduler.Outcome{}, err)@@ -697,7 +695,7 @@ func (s *JobService) reconcileRunInternal(ctx context.Context, run scheduler.Runif s.scheduler != nil {if job, ok := s.scheduler.GetJob(run.JobID); ok {if reconciler, localOk := job.(scheduler.Reconciler); localOk {- return reconciler.Reconcile(s.runContextInternal(ctx, run), run)+ return reconciler.Reconcile(s.runs.ExecutionContext(ctx, run, ""), run)}}}@@ -723,7 +721,7 @@ func (s *JobService) reconcileRunInternal(ctx context.Context, run scheduler.Run}if job, ok := s.scheduler.GetJob(run.JobID); ok {if reconciler, localOk2 := job.(scheduler.Reconciler); localOk2 {- outcome, err := reconciler.Reconcile(s.runContextInternal(ctx, run), run)+ outcome, err := reconciler.Reconcile(s.runs.ExecutionContext(ctx, run, ""), run)if err != nil {slog.WarnContext(ctx, "Job reconciliation requires attention", "runId", run.ID, "error", err)}@@ -733,29 +731,6 @@ func (s *JobService) reconcileRunInternal(ctx context.Context, run scheduler.Runreturn scheduler.Outcome{Status: scheduler.Failed, Message: "Interrupted operation has no confirmed completion", Targets: run.Outcome.Targets}, nil}-func (s *JobService) runContextInternal(ctx context.Context, run scheduler.Run) context.Context {- ctx = utils.WithActivityBatchID(ctx, run.ID)- ctx = jobcontext.WithExecution(ctx, run, func(target scheduler.TargetOutcome) error {- return s.runs.UpdateRun(ctx, run, func(current *scheduler.Run) error {- if current.Status != scheduler.Running || current.Owner != run.Owner {- return runs.ErrRunConflict- }- for index := range current.Outcome.Targets {- if current.Outcome.Targets[index].ID == target.ID {- if len(target.RecoveryData) == 0 {- target.RecoveryData = current.Outcome.Targets[index].RecoveryData- }- current.Outcome.Targets[index] = target- return nil- }- }- current.Outcome.Targets = append(current.Outcome.Targets, target)- return nil- })- })- return ctx-}-func requiresDockerInternal(jobID string) bool {switch jobID {case "auto-update", "auto-patch", "auto-heal", "scheduled-prune", "vulnerability-scan", "image-polling":diff --git a/backend/internal/job/service_test.go b/backend/internal/job/service_test.goindex f24d9163b..3f042d277 100644--- a/backend/internal/job/service_test.go+++ b/backend/internal/job/service_test.go@@ -642,7 +642,7 @@ func TestJobActivityVisibilityAndGrouping(t *testing.T) {require.NoError(t, getActivityDetailErr)require.Equal(t, test.environment, detail.Activity.Metadata["environmentId"])}- child, err := activities.StartActivity(svc.runContextInternal(t.Context(), run), activity.StartActivityRequest{Type: activitytypes.TypeImagePull})+ child, err := activities.StartActivity(svc.runs.ExecutionContext(t.Context(), run, ""), activity.StartActivityRequest{Type: activitytypes.TypeImagePull})require.NoError(t, err)require.Equal(t, run.ID, *child.BatchID)})diff --git a/backend/internal/network/handler.go b/backend/internal/network/handler.goindex 5ff31c368..6954d5eae 100644--- a/backend/internal/network/handler.go+++ b/backend/internal/network/handler.go@@ -14,7 +14,7 @@ import ("github.com/moby/moby/api/types/network""github.com/moby/moby/client""github.com/samber/mo"- "go.getarcane.app/kit/pkg"+ kit "go.getarcane.app/kit/pkg""go.getarcane.app/kit/pkg/mapping""github.com/getarcaneapp/arcane/backend/v2/internal/activity"@@ -285,12 +285,7 @@ func (h *NetworkHandler) DeleteNetwork(ctx context.Context, input *DeleteNetworkreturn nil, huma.Error500InternalServerError("Failed to remove network: " + err.Error())}- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{Message: "Network removed successfully", ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()},- },- }, nil+ return handlerutil.MessageOutput("Network removed successfully", activityID), nil}func (h *NetworkHandler) ConnectContainer(ctx context.Context, input *ConnectContainerInput) (*handlerutil.Out[base.MessageResponse], error) {@@ -324,12 +319,7 @@ func (h *NetworkHandler) ConnectContainer(ctx context.Context, input *ConnectConreturn nil, huma.Error500InternalServerError("Failed to connect container to network: " + err.Error())}- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{Message: "Container connected successfully", ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()},- },- }, nil+ return handlerutil.MessageOutput("Container connected successfully", activityID), nil}func (h *NetworkHandler) DisconnectContainer(ctx context.Context, input *DisconnectContainerInput) (*handlerutil.Out[base.MessageResponse], error) {@@ -361,12 +351,7 @@ func (h *NetworkHandler) DisconnectContainer(ctx context.Context, input *Disconnreturn nil, huma.Error500InternalServerError("Failed to disconnect container from network: " + err.Error())}- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{Message: "Container disconnected successfully", ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()},- },- }, nil+ return handlerutil.MessageOutput("Container disconnected successfully", activityID), nil}func (h *NetworkHandler) PruneNetworks(ctx context.Context, input *PruneNetworksInput) (*handlerutil.Out[networktypes.PruneReport], error) {diff --git a/backend/internal/notification/handler.go b/backend/internal/notification/handler.goindex d0fac7c41..0ffacc4b2 100644--- a/backend/internal/notification/handler.go+++ b/backend/internal/notification/handler.go@@ -9,7 +9,7 @@ import ("github.com/danielgtaylor/huma/v2""github.com/getarcaneapp/arcane/types/v2/base""github.com/getarcaneapp/arcane/types/v2/notification"- "go.getarcane.app/kit/pkg"+ kit "go.getarcane.app/kit/pkg""github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/internal/config"@@ -168,12 +168,7 @@ func (h *NotificationHandler) DeleteNotificationSettings(ctx context.Context, inreturn nil, huma.Error500InternalServerError("Failed to delete notification settings: " + err.Error())}- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{Message: "Settings deleted successfully"},- },- }, nil+ return handlerutil.MessageOutput("Settings deleted successfully", ""), nil}func (h *NotificationHandler) TestNotification(ctx context.Context, input *TestNotificationInput) (*handlerutil.Out[notification.TestResponse], error) {diff --git a/backend/internal/passkey/handler.go b/backend/internal/passkey/handler.goindex 767494a62..964bdf1e9 100644--- a/backend/internal/passkey/handler.go+++ b/backend/internal/passkey/handler.go@@ -369,7 +369,7 @@ func (h *PasskeyHandler) DeletePasskey(ctx context.Context, input *DeletePasskeyif deletePasskeyErr := h.passkeyService.DeletePasskey(ctx, userModel.ID, input.ID, sessionID, input.StepUpToken, oidcEnabled); deletePasskeyErr != nil {return nil, passkeyHTTPErrorInternal(deletePasskeyErr)}- return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Passkey deleted"}}}, nil+ return handlerutil.MessageOutput("Passkey deleted", ""), nil}func (h *PasskeyHandler) BeginStepUp(ctx context.Context, _ *BeginStepUpInput) (*handlerutil.Out[passkeyBeginResponse], error) {@@ -455,7 +455,7 @@ func (h *PasskeyHandler) DisableMFA(ctx context.Context, input *MFASettingsInputreturn nil, passkeyHTTPErrorInternal(disableMFAErr)}h.authService.InvalidateUserTokenCache(userModel.ID)- return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Passkey MFA disabled"}}}, nil+ return handlerutil.MessageOutput("Passkey MFA disabled", ""), nil}func (h *PasskeyHandler) RegenerateRecoveryCodes(ctx context.Context, input *MFASettingsInput) (*handlerutil.Out[RecoveryCodesResponse], error) {diff --git a/backend/internal/project/handler.go b/backend/internal/project/handler.goindex 8b47f38f2..ac7ef2401 100644--- a/backend/internal/project/handler.go+++ b/backend/internal/project/handler.go@@ -422,15 +422,7 @@ func (h *ProjectHandler) DownProject(ctx context.Context, input *DownProjectInpuactivitylib.FlushWriter(activityWriter)activitylib.CompleteHandlerActivity(runtimeCtx, h.activityService, activityID, "Project stopped", nil)- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{- Message: "Project brought down successfully",- ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer(),- },- },- }, nil+ return handlerutil.MessageOutput("Project brought down successfully", activityID), nil}func projectUpdateHTTPErrorInternal(err error) error {@@ -687,15 +679,7 @@ func (h *ProjectHandler) DestroyProject(ctx context.Context, input *DestroyProjeactivitylib.FlushWriter(activityWriter)activitylib.CompleteHandlerActivity(runtimeCtx, h.activityService, activityID, "Project destroyed", nil)- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{- Message: "Project destroyed successfully",- ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer(),- },- },- }, nil+ return handlerutil.MessageOutput("Project destroyed successfully", activityID), nil}// UpdateProject updates a Docker Compose project.@@ -943,12 +927,7 @@ func (h *ProjectHandler) ArchiveProject(ctx context.Context, input *ArchiveProjereturn nil, huma.Error500InternalServerError("Failed to archive project: " + archiveProjectErr.Error())}- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{Message: "Project archived successfully"},- },- }, nil+ return handlerutil.MessageOutput("Project archived successfully", ""), nil}func (h *ProjectHandler) UnarchiveProject(ctx context.Context, input *UnarchiveProjectInput) (*handlerutil.Out[base.MessageResponse], error) {@@ -965,12 +944,7 @@ func (h *ProjectHandler) UnarchiveProject(ctx context.Context, input *UnarchivePreturn nil, huma.Error500InternalServerError("Failed to unarchive project: " + unarchiveProjectErr.Error())}- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{Message: "Project unarchived successfully"},- },- }, nil+ return handlerutil.MessageOutput("Project unarchived successfully", ""), nil}// PullProjectImages pulls all images for a project with streaming progress.diff --git a/backend/internal/registry/handler.go b/backend/internal/registry/handler.goindex 8c0cbf86f..96a61aa93 100644--- a/backend/internal/registry/handler.go+++ b/backend/internal/registry/handler.go@@ -8,7 +8,7 @@ import ("github.com/danielgtaylor/huma/v2""github.com/getarcaneapp/arcane/types/v2/base""github.com/getarcaneapp/arcane/types/v2/containerregistry"- "go.getarcane.app/kit/pkg"+ kit "go.getarcane.app/kit/pkg""go.getarcane.app/sys/crypto""github.com/getarcaneapp/arcane/backend/v2/internal/common"@@ -161,14 +161,7 @@ func (h *ContainerRegistryHandler) DeleteRegistry(ctx context.Context, input *Deh.triggerRemoteRegistrySync(ctx, "registry deletion")- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{- Message: "Container registry deleted successfully",- },- },- }, nil+ return handlerutil.MessageOutput("Container registry deleted successfully", ""), nil}// TestRegistry tests connectivity to a container registry.@@ -184,14 +177,7 @@ func (h *ContainerRegistryHandler) TestRegistry(ctx context.Context, input *Testif testECRRegistryErr := h.registryService.TestECRRegistry(ctx, reg); testECRRegistryErr != nil {return nil, huma.Error400BadRequest("Registry test failed: " + testECRRegistryErr.Error())}- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{- Message: "ECR authentication succeeded",- },- },- }, nil+ return handlerutil.MessageOutput("ECR authentication succeeded", ""), nil}decryptedToken, err := crypto.Decrypt(reg.Token)@@ -206,14 +192,7 @@ func (h *ContainerRegistryHandler) TestRegistry(ctx context.Context, input *TestnoCredentials := strings.TrimSpace(reg.Username) == "" && strings.TrimSpace(decryptedToken) == ""msg := kit.Ternary(noCredentials, "Registry saved (no credentials to test)", "Authentication succeeded")- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{- Message: msg,- },- },- }, nil+ return handlerutil.MessageOutput(msg, ""), nil}// SyncRegistries syncs container registries from a remote source.@@ -223,14 +202,7 @@ func (h *ContainerRegistryHandler) SyncRegistries(ctx context.Context, input *Syreturn nil, huma.NewError(apiErr.HTTPStatus(), "Failed to sync registries: "+err.Error())}- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{- Message: "Registries synced successfully",- },- },- }, nil+ return handlerutil.MessageOutput("Registries synced successfully", ""), nil}func (h *ContainerRegistryHandler) triggerRemoteRegistrySync(ctx context.Context, reason string) {diff --git a/backend/internal/registry/service.go b/backend/internal/registry/service.goindex 86dc199e8..65bfef964 100644--- a/backend/internal/registry/service.go+++ b/backend/internal/registry/service.go@@ -7,6 +7,7 @@ import ("errors""fmt""log/slog"+ "net""net/http""runtime""strings"@@ -16,6 +17,7 @@ import ("github.com/aws/aws-sdk-go-v2/credentials""github.com/aws/aws-sdk-go-v2/service/ecr""github.com/cenkalti/backoff/v5"+ "github.com/containerd/errdefs""github.com/distribution/reference""github.com/getarcaneapp/arcane/types/v2/base""github.com/getarcaneapp/arcane/types/v2/containerregistry"@@ -23,6 +25,7 @@ import ("github.com/google/go-containerregistry/pkg/name""github.com/google/go-containerregistry/pkg/v1""github.com/google/go-containerregistry/pkg/v1/remote"+ "github.com/google/go-containerregistry/pkg/v1/remote/transport"dockerregistry "github.com/moby/moby/api/types/registry""github.com/moby/moby/client""github.com/samber/hot"@@ -870,7 +873,7 @@ func (s *ContainerRegistryService) ImageDigest(ctx context.Context, imageRef strfunc (s *ContainerRegistryService) InspectImageDigest(ctx context.Context, imageRef string, externalCreds []containerregistry.Credential) (*containerregistry.DigestResult, error) {parts, err := refs.NormalizeReference(imageRef)if err != nil {- return nil, err+ return nil, common.Classify(common.ErrValidation, err)}var lastResult *containerregistry.DigestResult@@ -933,15 +936,46 @@ func (s *ContainerRegistryService) InspectImageDigest(ctx context.Context, image}, backoff.WithBackOff(bo), backoff.WithMaxTries(5))if retryErr != nil {- if errors.Is(retryErr, context.Canceled) || errors.Is(retryErr, context.DeadlineExceeded) {+ if errors.Is(retryErr, context.Canceled) {return lastResult, retryErr}- return lastResult, lastErr+ if errors.Is(retryErr, context.DeadlineExceeded) {+ return lastResult, common.Classify(common.ErrTimeout, retryErr)+ }+ return lastResult, classifyDigestErrorInternal(lastErr)}return lastResult, nil}+// classifyDigestErrorInternal tags digest lookup failures so callers can retry+// transient ones. Retryable kinds are checked first because fallback errors are joined.+func classifyDigestErrorInternal(err error) error {+ if err == nil {+ return nil+ }+ errLower := strings.ToLower(err.Error())+ transportErr, isTransport := errors.AsType[*transport.Error](err)+ var netErr net.Error+ var dnsErr *net.DNSError+ switch {+ case errors.As(err, &dnsErr) && dnsErr.IsNotFound:+ // A registry host that does not resolve is a bad reference, not an outage worth retrying.+ return err+ case isRateLimitErrorInternal(err), errdefs.IsUnavailable(err), errors.As(err, &netErr),+ strings.Contains(errLower, "connection refused"), isTransport && transportErr.StatusCode >= http.StatusInternalServerError:+ return common.Classify(common.ErrUnavailable, err)+ case errors.Is(err, context.DeadlineExceeded), errdefs.IsDeadlineExceeded(err):+ return common.Classify(common.ErrTimeout, err)+ case browse.IsUnauthorizedRegistryError(err):+ return common.Classify(common.ErrUnauthorized, err)+ case errdefs.IsNotFound(err), isTransport && transportErr.StatusCode == http.StatusNotFound, strings.Contains(errLower, "manifest unknown"):+ return common.Classify(common.ErrNotFound, err)+ default:+ return err+ }+}+func (s *ContainerRegistryService,) inspectImageDigestViaDaemonInternal(diff --git a/backend/internal/registry/service_test.go b/backend/internal/registry/service_test.goindex 8e014e4c0..793e1c9e7 100644--- a/backend/internal/registry/service_test.go+++ b/backend/internal/registry/service_test.go@@ -7,6 +7,7 @@ import ("errors""io""log"+ "net""net/http""net/http/httptest""net/url"@@ -22,6 +23,7 @@ import ("github.com/google/go-containerregistry/pkg/v1/mutate""github.com/google/go-containerregistry/pkg/v1/random""github.com/google/go-containerregistry/pkg/v1/remote"+ "github.com/google/go-containerregistry/pkg/v1/remote/transport""github.com/libtnb/sqlite""github.com/moby/moby/api/pkg/authconfig"dockerregistry "github.com/moby/moby/api/types/registry"@@ -1593,3 +1595,28 @@ func TestContainerRegistryService_ImageVersionLabelErrorNotCachedInternal(t *tesrequire.NoError(t, err)assert.Equal(t, "v2.8.0-next.67", label)}++func TestClassifyDigestError(t *testing.T) {+ cases := map[string]struct {+ err error+ kind error+ }{+ "rate limit": {errors.New("toomanyrequests: rate limit exceeded"), common.ErrUnavailable},+ "server error": {&transport.Error{StatusCode: http.StatusBadGateway}, common.ErrUnavailable},+ "connection refused": {errors.New("dial tcp 10.0.0.1:443: connect: connection refused"), common.ErrUnavailable},+ "unauthorized": {&transport.Error{StatusCode: http.StatusUnauthorized}, common.ErrUnauthorized},+ "not found": {&transport.Error{StatusCode: http.StatusNotFound}, common.ErrNotFound},+ "manifest unknown": {errors.New("MANIFEST_UNKNOWN: manifest unknown"), common.ErrNotFound},+ }+ for name, tc := range cases {+ require.ErrorIs(t, classifyDigestErrorInternal(tc.err), tc.kind, name)+ }++ plain := errors.New("x509: certificate signed by unknown authority")+ classified := classifyDigestErrorInternal(plain)+ require.Equal(t, plain, classified)+ require.NotErrorIs(t, classified, common.ErrUnavailable)+ // A host that does not resolve never comes back, so it must not be retried like an outage.+ unknownHost := &url.Error{Op: "Get", URL: "https://invalid-registry.example.com/v2/", Err: &net.DNSError{Err: "no such host", Name: "invalid-registry.example.com", IsNotFound: true}}+ require.NotErrorIs(t, classifyDigestErrorInternal(unknownHost), common.ErrUnavailable)+}diff --git a/backend/internal/role/handler.go b/backend/internal/role/handler.goindex ec0da38ee..48ee426c6 100644--- a/backend/internal/role/handler.go+++ b/backend/internal/role/handler.go@@ -152,9 +152,7 @@ func (h *RoleHandler) DeleteRole(ctx context.Context, input *DeleteRoleInput) (*}return nil, huma.Error500InternalServerError("failed to delete role: " + err.Error())}- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "role deleted"}},- }, nil+ return handlerutil.MessageOutput("role deleted", ""), nil}func (h *RoleHandler) GetPermissionsManifest(_ context.Context, _ *struct{}) (*handlerutil.Out[roletypes.PermissionsManifest], error) {diff --git a/backend/internal/s3/handler.go b/backend/internal/s3/handler.goindex 5e4ebeac3..25be76263 100644--- a/backend/internal/s3/handler.go+++ b/backend/internal/s3/handler.go@@ -158,7 +158,7 @@ func (h *s3DestinationHandlerInternal) deleteInternal(ctx context.Context, inputreturn nil, huma.Error500InternalServerError(err.Error())}h.triggerRemoteSyncInternal(ctx, "S3 destination deletion")- return &handlerutil.Out[base.MessageResponse]{Body: successMessageResponseInternal("S3 destination deleted successfully")}, nil+ return handlerutil.MessageOutput("S3 destination deleted successfully", ""), nil}func (h *s3DestinationHandlerInternal) testInternal(ctx context.Context, input *testS3DestinationInputInternal) (*handlerutil.Out[base.MessageResponse], error) {@@ -168,11 +168,7 @@ func (h *s3DestinationHandlerInternal) testInternal(ctx context.Context, input *}return nil, huma.Error400BadRequest(err.Error())}- return &handlerutil.Out[base.MessageResponse]{Body: successMessageResponseInternal(s3ConnectionTestSuccessInternal)}, nil-}--func successMessageResponseInternal(message string) base.ApiResponse[base.MessageResponse] {- return base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: message}}+ return handlerutil.MessageOutput(s3ConnectionTestSuccessInternal, ""), nil}func (h *s3DestinationHandlerInternal) triggerRemoteSyncInternal(ctx context.Context, reason string) {@@ -191,12 +187,12 @@ func (h *s3DestinationHandlerInternal) testConfigurationInternal(ctx context.Conif err := h.service.TestS3DestinationConfiguration(ctx, input.Body); err != nil {return nil, huma.Error400BadRequest(err.Error())}- return &handlerutil.Out[base.MessageResponse]{Body: successMessageResponseInternal(s3ConnectionTestSuccessInternal)}, nil+ return handlerutil.MessageOutput(s3ConnectionTestSuccessInternal, ""), nil}func (h *s3DestinationHandlerInternal) syncInternal(ctx context.Context, input *syncS3DestinationsInputInternal) (*handlerutil.Out[base.MessageResponse], error) {if err := h.service.SyncS3Destinations(ctx, input.Body.Destinations); err != nil {return nil, huma.Error400BadRequest(err.Error())}- return &handlerutil.Out[base.MessageResponse]{Body: successMessageResponseInternal("S3 destinations synced successfully")}, nil+ return handlerutil.MessageOutput("S3 destinations synced successfully", ""), nil}diff --git a/backend/internal/s3/service.go b/backend/internal/s3/service.goindex d0854e983..9812cbf27 100644--- a/backend/internal/s3/service.go+++ b/backend/internal/s3/service.go@@ -4,6 +4,7 @@ import ("context""errors""fmt"+ "log/slog""strings""github.com/getarcaneapp/arcane/types/v2/backup"@@ -83,17 +84,22 @@ func (s *S3DestinationService) ListAllS3Destinations(ctx context.Context) ([]bacreturn s3DestinationsToDTOsInternal(destinations), nil}-// ListS3DestinationsByID indexes destination DTOs for backup metadata lookups.-func (s *S3DestinationService) ListS3DestinationsByID(ctx context.Context) (map[string]backup.S3Destination, error) {+// DestinationsByID indexes destinations to decorate backup records, such as with their names. Decoration never fails+// its caller, so a missing service or failed listing yields an empty index.+func (s *S3DestinationService) DestinationsByID(ctx context.Context) map[string]backup.S3Destination {+ if s == nil {+ return nil+ }destinations, err := s.ListAllS3Destinations(ctx)if err != nil {- return nil, err+ slog.WarnContext(ctx, "could not list S3 destinations", "error", err)+ return nil}indexed := make(map[string]backup.S3Destination, len(destinations))for _, destination := range destinations {indexed[destination.ID] = destination}- return indexed, nil+ return indexed}func (s *S3DestinationService) getS3DestinationModelInternal(ctx context.Context, id string) (*S3Destination, error) {diff --git a/backend/internal/s3/service_test.go b/backend/internal/s3/service_test.goindex 27b494570..e4b6dc405 100644--- a/backend/internal/s3/service_test.go+++ b/backend/internal/s3/service_test.go@@ -343,22 +343,19 @@ func TestS3DestinationService_TestS3DestinationRoundTrip(t *testing.T) {}}-func TestListS3DestinationsByIDInternal(t *testing.T) {+func TestDestinationsByID(t *testing.T) {service, db := setupS3DestinationServiceTestInternal(t)- indexed, err := service.ListS3DestinationsByID(t.Context())- require.NoError(t, err)- require.Empty(t, indexed)+ require.Empty(t, service.DestinationsByID(t.Context()))destination := S3Destination{Name: "Offsite", Bucket: "backups", SecretAccessKey: "encrypted"}require.NoError(t, db.Create(&destination).Error)- indexed, err = service.ListS3DestinationsByID(t.Context())- require.NoError(t, err)+ indexed := service.DestinationsByID(t.Context())require.Len(t, indexed, 1)require.Equal(t, "Offsite", indexed[destination.ID].Name)require.Equal(t, "backups", indexed[destination.ID].Bucket)require.True(t, indexed[destination.ID].SecretConfigured)require.Empty(t, indexed["missing"].Name)+ // Listing failures and a missing service leave records undecorated rather than failing.require.NoError(t, db.Migrator().DropTable(&S3Destination{}))- indexed, err = service.ListS3DestinationsByID(t.Context())- require.ErrorContains(t, err, "failed to list S3 destinations")- require.Nil(t, indexed)+ require.Nil(t, service.DestinationsByID(t.Context()))+ require.Nil(t, (*S3DestinationService)(nil).DestinationsByID(t.Context()))}diff --git a/backend/internal/swarm/children/configs/handler.go b/backend/internal/swarm/children/configs/handler.goindex 782b7cc62..40866a4e7 100644--- a/backend/internal/swarm/children/configs/handler.go+++ b/backend/internal/swarm/children/configs/handler.go@@ -133,5 +133,5 @@ func (h *Handler) DeleteConfig(ctx context.Context, input *DeleteSwarmConfigInpuh.audit(ctx, input.EnvironmentID, "config.delete", "swarm_config", input.ConfigID, "", map[string]any{"configId": input.ConfigID})- return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Swarm config removed successfully"}}}, nil+ return handlerutil.MessageOutput("Swarm config removed successfully", ""), nil}diff --git a/backend/internal/swarm/children/nodes/handler.go b/backend/internal/swarm/children/nodes/handler.goindex 8cf1d6ac1..ad8079f83 100644--- a/backend/internal/swarm/children/nodes/handler.go+++ b/backend/internal/swarm/children/nodes/handler.go@@ -295,7 +295,7 @@ func (h *Handler) DeleteNodeAgentBinding(ctx context.Context, input *DeleteSwarmif err := h.environmentService.DetachSwarmNodeEnvironment(ctx, input.EnvironmentID, input.NodeID); err != nil {return nil, huma.Error500InternalServerError(err.Error())}- return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Swarm node environment detached"}}}, nil+ return handlerutil.MessageOutput("Swarm node environment detached", ""), nil}// DeleteNodeAgentDeployment removes a dedicated hidden node-agent registration.@@ -313,7 +313,7 @@ func (h *Handler) DeleteNodeAgentDeployment(ctx context.Context, input *DeleteSw); deleteSwarmNodeAgentDeploymentErr != nil {return nil, huma.Error500InternalServerError(deleteSwarmNodeAgentDeploymentErr.Error())}- return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Dedicated swarm node agent registration removed"}}}, nil+ return handlerutil.MessageOutput("Dedicated swarm node agent registration removed", ""), nil}// GetNodeIdentity returns the swarm identity of the node serving the current request.@@ -359,7 +359,7 @@ func (h *Handler) UpdateNode(ctx context.Context, input *UpdateSwarmNodeInput) (h.audit(ctx, input.EnvironmentID, "node.update", "swarm_node", input.NodeID, "", map[string]any{"nodeId": input.NodeID})- return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Swarm node updated successfully"}}}, nil+ return handlerutil.MessageOutput("Swarm node updated successfully", ""), nil}// DeleteNode removes a swarm node from the cluster.@@ -380,7 +380,7 @@ func (h *Handler) DeleteNode(ctx context.Context, input *DeleteSwarmNodeInput) (h.audit(ctx, input.EnvironmentID, "node.delete", "swarm_node", input.NodeID, "", map[string]any{"nodeId": input.NodeID, "force": input.Force})- return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Swarm node removed successfully"}}}, nil+ return handlerutil.MessageOutput("Swarm node removed successfully", ""), nil}// PromoteNode promotes a swarm worker to manager.@@ -401,7 +401,7 @@ func (h *Handler) PromoteNode(ctx context.Context, input *PromoteSwarmNodeInput)h.audit(ctx, input.EnvironmentID, "node.promote", "swarm_node", input.NodeID, "", map[string]any{"nodeId": input.NodeID})- return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Swarm node promoted successfully"}}}, nil+ return handlerutil.MessageOutput("Swarm node promoted successfully", ""), nil}// DemoteNode demotes a swarm manager to worker.@@ -422,7 +422,7 @@ func (h *Handler) DemoteNode(ctx context.Context, input *DemoteSwarmNodeInput) (h.audit(ctx, input.EnvironmentID, "node.demote", "swarm_node", input.NodeID, "", map[string]any{"nodeId": input.NodeID})- return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Swarm node demoted successfully"}}}, nil+ return handlerutil.MessageOutput("Swarm node demoted successfully", ""), nil}// ListNodeTasks lists tasks currently associated with a swarm node.diff --git a/backend/internal/swarm/children/secrets/handler.go b/backend/internal/swarm/children/secrets/handler.goindex e50c080af..3d0c4fdad 100644--- a/backend/internal/swarm/children/secrets/handler.go+++ b/backend/internal/swarm/children/secrets/handler.go@@ -133,5 +133,5 @@ func (h *Handler) DeleteSecret(ctx context.Context, input *DeleteSwarmSecretInpuh.audit(ctx, input.EnvironmentID, "secret.delete", "swarm_secret", input.SecretID, "", map[string]any{"secretId": input.SecretID})- return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Swarm secret removed successfully"}}}, nil+ return handlerutil.MessageOutput("Swarm secret removed successfully", ""), nil}diff --git a/backend/internal/swarm/children/services/handler.go b/backend/internal/swarm/children/services/handler.goindex 079302c04..19ea0f36d 100644--- a/backend/internal/swarm/children/services/handler.go+++ b/backend/internal/swarm/children/services/handler.go@@ -191,7 +191,7 @@ func (h *Handler) DeleteService(ctx context.Context, input *DeleteSwarmServiceInh.audit(ctx, input.EnvironmentID, "service.delete", "swarm_service", input.ServiceID, "", map[string]any{"serviceId": input.ServiceID})- return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Swarm service removed successfully"}}}, nil+ return handlerutil.MessageOutput("Swarm service removed successfully", ""), nil}// ListServiceTasks lists tasks belonging to a specific swarm service.diff --git a/backend/internal/swarm/children/stacks/handler.go b/backend/internal/swarm/children/stacks/handler.goindex bb7f72bdc..651a3e7e4 100644--- a/backend/internal/swarm/children/stacks/handler.go+++ b/backend/internal/swarm/children/stacks/handler.go@@ -217,7 +217,7 @@ func (h *Handler) DeleteStack(ctx context.Context, input *DeleteSwarmStackInput)h.audit(ctx, input.EnvironmentID, "stack.delete", "swarm_stack", input.Name, input.Name, map[string]any{"stack": input.Name})- return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Swarm stack removed successfully"}}}, nil+ return handlerutil.MessageOutput("Swarm stack removed successfully", ""), nil}// ListStackServices lists services belonging to a swarm stack.diff --git a/backend/internal/swarm/handler.go b/backend/internal/swarm/handler.goindex 161d7f0cf..997445b9d 100644--- a/backend/internal/swarm/handler.go+++ b/backend/internal/swarm/handler.go@@ -198,7 +198,7 @@ func (h *SwarmHandler) JoinSwarm(ctx context.Context, input *JoinSwarmInput) (*hh.auditSwarmMutation(ctx, input.EnvironmentID, "lifecycle.join", "swarm", "cluster", "cluster", map[string]any{"remoteAddrs": input.Body.RemoteAddrs})- return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Joined swarm successfully"}}}, nil+ return handlerutil.MessageOutput("Joined swarm successfully", ""), nil}// LeaveSwarm removes the target engine from its current swarm cluster.@@ -219,7 +219,7 @@ func (h *SwarmHandler) LeaveSwarm(ctx context.Context, input *LeaveSwarmInput) (h.auditSwarmMutation(ctx, input.EnvironmentID, "lifecycle.leave", "swarm", "cluster", "cluster", map[string]any{"force": input.Body.Force})- return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Left swarm successfully"}}}, nil+ return handlerutil.MessageOutput("Left swarm successfully", ""), nil}// UnlockSwarm unlocks a swarm manager using the supplied unlock key.@@ -240,7 +240,7 @@ func (h *SwarmHandler) UnlockSwarm(ctx context.Context, input *UnlockSwarmInput)h.auditSwarmMutation(ctx, input.EnvironmentID, "lifecycle.unlock", "swarm", "cluster", "cluster", map[string]any{})- return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Swarm unlocked successfully"}}}, nil+ return handlerutil.MessageOutput("Swarm unlocked successfully", ""), nil}// GetUnlockKey returns the current swarm manager unlock key.@@ -310,7 +310,7 @@ func (h *SwarmHandler) RotateJoinTokens(ctx context.Context, input *RotateSwarmJ},)- return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Swarm join tokens rotated successfully"}}}, nil+ return handlerutil.MessageOutput("Swarm join tokens rotated successfully", ""), nil}// UpdateSwarmSpec updates the swarm cluster specification.@@ -331,7 +331,7 @@ func (h *SwarmHandler) UpdateSwarmSpec(ctx context.Context, input *UpdateSwarmSph.auditSwarmMutation(ctx, input.EnvironmentID, "lifecycle.update_spec", "swarm", "cluster", "cluster", map[string]any{})- return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Swarm spec updated successfully"}}}, nil+ return handlerutil.MessageOutput("Swarm spec updated successfully", ""), nil}// auditSwarmMutation writes an informational event for a completed swarm mutation.diff --git a/backend/internal/system/children/backup/children/volumes/service.go b/backend/internal/system/children/backup/children/volumes/service.goindex 91db944de..ba9047407 100644--- a/backend/internal/system/children/backup/children/volumes/service.go+++ b/backend/internal/system/children/backup/children/volumes/service.go@@ -6,76 +6,147 @@ import ("errors""fmt""log/slog"+ "path""slices""strings"+ "time""uuid"activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"backuptypes "github.com/getarcaneapp/arcane/types/v2/backup""github.com/getarcaneapp/arcane/types/v2/scheduler"usertypes "github.com/getarcaneapp/arcane/types/v2/user"- "go.getarcane.app/kit/pkg"+ "github.com/italypaleale/francis/builtin/workflow"+ kit "go.getarcane.app/kit/pkg""gorm.io/gorm""github.com/getarcaneapp/arcane/backend/v2/internal/activity""github.com/getarcaneapp/arcane/backend/v2/internal/backup""github.com/getarcaneapp/arcane/backend/v2/internal/database"- "github.com/getarcaneapp/arcane/backend/v2/internal/middleware""github.com/getarcaneapp/arcane/backend/v2/internal/s3""github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/backend/v2/internal/volume""github.com/getarcaneapp/arcane/backend/v2/pkg/authz"activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/entityjobs"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs""github.com/getarcaneapp/arcane/backend/v2/pkg/utils")-// Dependencies are the services and system admission hooks volume backups run on.+const (+ systemVolumeBackupConfigKey = "systemVolumeBackupConfig"+ systemVolumeBackupJobPrefix = "volumes:"+ // systemVolumePlanTarget records a scheduled run's frozen selection.+ systemVolumePlanTarget = "system-volume-plan"+ defaultSystemVolumeSchedule = "0 0 2 * * *"+)++// Dependencies are the services and system admission volume backups run on.type Dependencies struct {- DB *database.DB- Engine *backup.Engine- Volumes *volume.VolumeService- S3Destinations *s3.S3DestinationService- Activity *activity.ActivityService- Settings *settings.SettingsService- Jobs *entityjobs.Registry- AcquireRun func(ctx context.Context) (*runs.Lease, error)- AcquireDurableRun func(ctx context.Context, runID string) (*runs.Lease, error)- AlreadyRunning error+ DB *database.DB+ Engine *backup.Engine+ Volumes *volume.VolumeService+ S3Destinations *s3.S3DestinationService+ Activity *activity.ActivityService+ Settings *settings.SettingsService+ Jobs *entityjobs.Registry+ AlreadyRunning error}// Service owns system-managed volume backup policies and their runs.type Service struct {- db *database.DB- engine *backup.Engine- volumes *volume.VolumeService- s3Destinations *s3.S3DestinationService- activity *activity.ActivityService- settings *settings.SettingsService- jobs *entityjobs.Registry- acquireRun func(ctx context.Context) (*runs.Lease, error)- acquireDurableRun func(ctx context.Context, runID string) (*runs.Lease, error)- alreadyRunning error+ db *database.DB+ engine *backup.Engine+ volumes *volume.VolumeService+ s3Destinations *s3.S3DestinationService+ activity *activity.ActivityService+ settings *settings.SettingsService+ jobs *entityjobs.Registry+ alreadyRunning error+ flow *flow.Engine+ // backupWorkflow runs the selections Start accepted; policyWorkflow runs scheduled policies.+ backupWorkflow *flow.Workflow+ policyWorkflow *flow.Workflow+}++// selectionInput freezes the policy and volumes one run backs up; a manual run also carries its requester.+type selectionInput struct {+ Policy backuptypes.SystemVolumeBackupPolicy `json:"policy"`+ ManualPolicy bool `json:"manualPolicy"`+ Candidates []backuptypes.SystemVolumeBackupOption `json:"candidates"`+ Requester *backuptypes.Requester `json:"requester,omitempty"`+ // Claim names the manual run that parked the system admission for its own task.+ Claim string `json:"claim,omitempty"`}func NewService(deps Dependencies) *Service {return &Service{- db: deps.DB,- engine: deps.Engine,- volumes: deps.Volumes,- s3Destinations: deps.S3Destinations,- activity: deps.Activity,- settings: deps.Settings,- jobs: deps.Jobs,- acquireRun: deps.AcquireRun,- acquireDurableRun: deps.AcquireDurableRun,- alreadyRunning: deps.AlreadyRunning,+ db: deps.DB,+ engine: deps.Engine,+ volumes: deps.Volumes,+ s3Destinations: deps.S3Destinations,+ activity: deps.Activity,+ settings: deps.Settings,+ jobs: deps.Jobs,+ alreadyRunning: deps.AlreadyRunning,+ }+}++// RegisterWorkflows defines the manual and scheduled volume backup workflows while the host is still unstarted.+func (s *Service) RegisterWorkflows(engine *flow.Engine) error {+ var err error+ s.flow = engine+ template := activitylib.StartRequest{Type: activitytypes.TypeResourceAction, EnvironmentID: "0", ResourceType: new("system_backup"), ResourceID: new("volumes"), ResourceName: new("Volumes")}+ s.backupWorkflow, err = engine.Define(flow.Definition{+ Name: "system-volume-backup",+ Version: 1,+ Fingerprint: "9b4a5e46a984446bc08df39a71e563e031548db3af1825a2413e8c96515db6ab",+ Concurrency: 1,+ Timeout: 24 * time.Hour,+ Activity: template,+ Labels: map[string]string{"backup": "Backing up volumes"},+ Steps: []workflow.StepSpec{workflow.Step("backup", engine.Handler(func(ctx context.Context, t flow.Task) (any, error) {+ var input selectionInput+ if decodeErr := t.Payload(&input); decodeErr != nil {+ return nil, decodeErr+ }+ result, backupErr := s.backUpSelection(ctx, input, volume.VolumeBackupTriggerManual, t.ActivityID())+ if backupErr != nil {+ return nil, backupErr+ }+ return selectionOutcome(result, false), nil+ }), workflow.WithMaxAttempts(1))},+ })+ if err != nil {+ return err}+ template.StartedBy, template.Step, template.LatestMessage = &usertypes.SystemUser, "Backing up volumes", "Creating scheduled system-managed volume backups"+ s.policyWorkflow, err = engine.Define(flow.Definition{+ Name: "system-volume-backup-policy",+ Version: 1,+ Fingerprint: "94a13d1bac530e6eddcb73c27990a07d69915dc947ce5b559166eb3fb85e39ef",+ Concurrency: 1,+ Timeout: 24 * time.Hour,+ Activity: template,+ Labels: map[string]string{"backup": "Backing up volumes"},+ Steps: []workflow.StepSpec{workflow.Step("backup", engine.Handler(func(ctx context.Context, t flow.Task) (any, error) {+ previous, _ := jobcontext.Run(ctx)+ if slices.ContainsFunc(previous.Outcome.Targets, func(target scheduler.TargetOutcome) bool { return target.ID == systemVolumePlanTarget }) {+ return s.resumeSelection(ctx, previous, t.ActivityID())+ }+ var policyID string+ if decodeErr := t.Payload(&policyID); decodeErr != nil {+ return nil, decodeErr+ }+ return s.runScheduledSystemVolumeBackup(ctx, policyID, t.ActivityID())+ }), workflow.WithMaxAttempts(1))},+ })+ return err}-func (s *Service) loadSystemVolumeBackupPoliciesInternal() (*backuptypes.SystemVolumeBackupPolicyCollection, error) {+func (s *Service) loadSystemVolumeBackupPolicies() (*backuptypes.SystemVolumeBackupPolicyCollection, error) {collection := &backuptypes.SystemVolumeBackupPolicyCollection{Policies: []backuptypes.SystemVolumeBackupPolicy{}}if s.settings == nil {return collection, nil@@ -93,31 +164,24 @@ func (s *Service) loadSystemVolumeBackupPoliciesInternal() (*backuptypes.SystemVreturn collection, nil}-func (s *Service) systemVolumeBackupPolicyInternal(policyID string) (*backuptypes.SystemVolumeBackupPolicy, error) {- collection, err := s.loadSystemVolumeBackupPoliciesInternal()+func (s *Service) systemVolumeBackupPolicy(policyID string) (*backuptypes.SystemVolumeBackupPolicy, error) {+ collection, err := s.loadSystemVolumeBackupPolicies()if err != nil {return nil, err}- for i := range collection.Policies {- if collection.Policies[i].ID == policyID {- policy := collection.Policies[i]- return &policy, nil- }+ index := slices.IndexFunc(collection.Policies, func(policy backuptypes.SystemVolumeBackupPolicy) bool { return policy.ID == policyID })+ if index < 0 {+ return nil, nil}- return nil, nil+ return &collection.Policies[index], nil}func (s *Service) GetConfig(ctx context.Context) (*backuptypes.SystemVolumeBackupPolicyCollection, error) {- collection, err := s.loadSystemVolumeBackupPoliciesInternal()+ collection, err := s.loadSystemVolumeBackupPolicies()if err != nil {return nil, err}- destinations := make(map[string]backuptypes.S3Destination)- if s.s3Destinations != nil {- if available, listErr := s.s3Destinations.ListS3DestinationsByID(ctx); listErr == nil {- destinations = available- }- }+ destinations := s.s3Destinations.DestinationsByID(ctx)for i := range collection.Policies {policy := &collection.Policies[i]policy.S3DestinationName = destinations[policy.S3DestinationID].Name@@ -143,9 +207,10 @@ func (s *Service) GetConfig(ctx context.Context) (*backuptypes.SystemVolumeBackureturn collection, nil}-func (s *Service) normalizeSystemVolumePolicyUpdateInternal(ctx context.Context, input backuptypes.UpdateSystemVolumeBackupPolicy) (backuptypes.SystemVolumeBackupPolicy, error) {- if err := validateSystemVolumeSelectionInternal(input.SelectionMode); err != nil {- return backuptypes.SystemVolumeBackupPolicy{}, err+func (s *Service) normalizeSystemVolumePolicyUpdate(ctx context.Context, input backuptypes.UpdateSystemVolumeBackupPolicy) (backuptypes.SystemVolumeBackupPolicy, error) {+ mode := input.SelectionMode+ if mode != backuptypes.SystemVolumeSelectionAll && mode != backuptypes.SystemVolumeSelectionAllowlist && mode != backuptypes.SystemVolumeSelectionBlocklist {+ return backuptypes.SystemVolumeBackupPolicy{}, errors.New("selectionMode must be all, allowlist, or blocklist")}update, err := backup.ValidatePolicyUpdate(ctx, "system-managed volume", input.UpdateBackupPolicy, s.s3Destinations)if err != nil {@@ -153,13 +218,13 @@ func (s *Service) normalizeSystemVolumePolicyUpdateInternal(ctx context.Context,}names := kit.Unique(kit.TrimNonEmpty(input.VolumeNames))slices.Sort(names)- if input.SelectionMode == backuptypes.SystemVolumeSelectionAll || names == nil {+ if mode == backuptypes.SystemVolumeSelectionAll || names == nil {names = []string{}}return backuptypes.SystemVolumeBackupPolicy{ID: input.ID, Enabled: update.Enabled, Schedule: update.Schedule, RetentionCount: update.RetentionCount,StopContainers: update.StopContainers, LocalEnabled: update.LocalEnabled, S3Enabled: update.S3Enabled,- S3DestinationID: update.S3DestinationID, SelectionMode: input.SelectionMode,+ S3DestinationID: update.S3DestinationID, SelectionMode: mode,VolumeNames: names, IgnoreAnonymous: input.IgnoreAnonymous,}, nil}@@ -168,7 +233,7 @@ func (s *Service) UpdateConfig(ctx context.Context, updates []backuptypes.Updateif s.settings == nil {return nil, errors.New("settings service is unavailable")}- existing, err := s.loadSystemVolumeBackupPoliciesInternal()+ existing, err := s.loadSystemVolumeBackupPolicies()if err != nil {return nil, err}@@ -181,7 +246,7 @@ func (s *Service) UpdateConfig(ctx context.Context, updates []backuptypes.Updatereturn backuptypes.SystemVolumeBackupPolicy{ID: uuid.New().String()}},Build: func(ctx context.Context, policy *backuptypes.SystemVolumeBackupPolicy, update backuptypes.UpdateSystemVolumeBackupPolicy) error {- normalized, normalizeErr := s.normalizeSystemVolumePolicyUpdateInternal(ctx, update)+ normalized, normalizeErr := s.normalizeSystemVolumePolicyUpdate(ctx, update)if normalizeErr != nil {return normalizeErr}@@ -189,11 +254,17 @@ func (s *Service) UpdateConfig(ctx context.Context, updates []backuptypes.Update*policy = normalizedreturn nil},- Persist: s.saveSystemVolumeBackupPoliciesInternal,+ Persist: func(ctx context.Context, policies []backuptypes.SystemVolumeBackupPolicy) error {+ encoded, encodeErr := json.Marshal(backuptypes.SystemVolumeBackupPolicyCollection{Policies: policies})+ if encodeErr != nil {+ return fmt.Errorf("encode policies: %w", encodeErr)+ }+ return s.settings.UpdateSetting(ctx, systemVolumeBackupConfigKey, string(encoded))+ },Unregister: func(ctx context.Context, policyID string) {s.jobs.Unregister(ctx, systemVolumeBackupJobPrefix+policyID)},- Reschedule: s.rescheduleSystemVolumeBackupInternal,+ Reschedule: s.rescheduleSystemVolumeBackup,}if runErr := reconcile.Run(ctx, updates); runErr != nil {return nil, runErr@@ -210,7 +281,7 @@ func (s *Service) ListOptions(ctx context.Context) ([]backuptypes.SystemVolumeBaif err != nil {return nil, err}- collection, err := s.loadSystemVolumeBackupPoliciesInternal()+ collection, err := s.loadSystemVolumeBackupPolicies()if err != nil {return nil, err}@@ -232,152 +303,132 @@ func (s *Service) ListOptions(ctx context.Context) ([]backuptypes.SystemVolumeBareturn options, nil}-func (s *Service) resolveSystemVolumeRunPolicyInternal(ctx context.Context, request backuptypes.RunSystemVolumeBackupsRequest) (backuptypes.SystemVolumeBackupPolicy, bool, error) {- if request.PolicyID != "" {- if request.Custom != nil {- return backuptypes.SystemVolumeBackupPolicy{}, false, errors.New("select a saved policy or custom configuration, not both")- }- policy, err := s.systemVolumeBackupPolicyInternal(request.PolicyID)+// prepareSystemVolumeBackups resolves the requested policy, takes the system admission, and selects the live volumes.+func (s *Service) prepareSystemVolumeBackups(ctx context.Context, request backuptypes.RunSystemVolumeBackupsRequest) (selectionInput, *runs.Lease, error) {+ if s.volumes == nil {+ return selectionInput{}, nil, errors.New("volume service is unavailable")+ }+ var input selectionInput+ switch {+ case request.PolicyID != "" && request.Custom != nil:+ return selectionInput{}, nil, errors.New("select a saved policy or custom configuration, not both")+ case request.PolicyID != "":+ policy, err := s.systemVolumeBackupPolicy(request.PolicyID)if err != nil {- return backuptypes.SystemVolumeBackupPolicy{}, false, err+ return selectionInput{}, nil, err}if policy == nil {- return backuptypes.SystemVolumeBackupPolicy{}, false, errors.New("system-managed volume backup policy not found")+ return selectionInput{}, nil, errors.New("system-managed volume backup policy not found")}- return *policy, false, nil- }- custom := request.Custom- if custom != nil &&- custom.Destination != backuptypes.SystemBackupDestinationLocal &&- custom.Destination != backuptypes.SystemBackupDestinationS3 &&- custom.Destination != backuptypes.SystemBackupDestinationLocalS3 {- return backuptypes.SystemVolumeBackupPolicy{}, false, errors.New("destination must be local, s3, or local_s3")- }- policy, err := s.normalizeSystemVolumePolicyUpdateInternal(ctx, customSystemVolumePolicyInternal(custom))- return policy, true, err-}--type preparedSystemVolumeBackupInternal struct {- policy backuptypes.SystemVolumeBackupPolicy- manualPolicy bool- candidates []backuptypes.SystemVolumeBackupOption- lease *runs.Lease-}--func (- s *Service,-) runSystemVolumeBackupsInternal(- ctx context.Context,- request backuptypes.RunSystemVolumeBackupsRequest,- trigger volume.VolumeBackupTrigger,-) (- *backuptypes.SystemVolumeBackupRunResult,- error,-) {- prepared, err := s.prepareSystemVolumeBackupsInternal(ctx, request)- if err != nil {- return nil, err+ input.Policy = *policy+ default:+ custom := backuptypes.UpdateSystemVolumeBackupPolicy{+ Enabled: true, Schedule: defaultSystemVolumeSchedule, LocalEnabled: true,+ SelectionMode: backuptypes.SystemVolumeSelectionAll, VolumeNames: []string{}, IgnoreAnonymous: true,+ }+ if run := request.Custom; run != nil {+ localEnabled := run.Destination == backuptypes.SystemBackupDestinationLocal || run.Destination == backuptypes.SystemBackupDestinationLocalS3+ s3Enabled := run.Destination == backuptypes.SystemBackupDestinationS3 || run.Destination == backuptypes.SystemBackupDestinationLocalS3+ if !localEnabled && !s3Enabled {+ return selectionInput{}, nil, errors.New("destination must be local, s3, or local_s3")+ }+ custom = backuptypes.UpdateSystemVolumeBackupPolicy{+ Enabled: true, Schedule: defaultSystemVolumeSchedule, StopContainers: run.StopContainers,+ LocalEnabled: localEnabled, S3Enabled: s3Enabled, S3DestinationID: run.S3DestinationID,+ SelectionMode: run.SelectionMode, VolumeNames: run.VolumeNames, IgnoreAnonymous: run.IgnoreAnonymous,+ }+ }+ policy, err := s.normalizeSystemVolumePolicyUpdate(ctx, custom)+ if err != nil {+ return selectionInput{}, nil, err+ }+ input.Policy, input.ManualPolicy = policy, true}- defer prepared.lease.Release(ctx)- frozen, err := json.Marshal(manualSystemVolumesInternal{Policy: prepared.policy, ManualPolicy: prepared.manualPolicy, Candidates: prepared.candidates})+ lease, admitted, err := s.engine.TryAcquireRun(ctx, backup.SystemAdmissionScope, backup.SystemAdmissionID)if err != nil {- return nil, err- }- if progressErr := jobcontext.Progress(- ctx,- scheduler.TargetOutcome{- ResourceType: "backup_plan",- ID: "system-volume-plan",- Status: scheduler.Succeeded,- RecoveryData: frozen,- },- ); progressErr != nil {- return nil, progressErr- }- return s.executeSystemVolumeBackupsInternal(ctx, prepared.policy, prepared.manualPolicy, prepared.candidates, trigger, "")-}--func (s *Service) prepareSystemVolumeBackupsInternal(ctx context.Context, request backuptypes.RunSystemVolumeBackupsRequest) (*preparedSystemVolumeBackupInternal, error) {- if s.volumes == nil {- return nil, errors.New("volume service is unavailable")+ return selectionInput{}, nil, err}- policyConfig, manualPolicy, err := s.resolveSystemVolumeRunPolicyInternal(ctx, request)- if err != nil {- return nil, err- }- lease, err := s.acquireRun(ctx)- if err != nil {- return nil, err+ if !admitted {+ return selectionInput{}, nil, s.alreadyRunning}options, err := s.volumes.ListBackupVolumeOptions(ctx)if err != nil {lease.Release(ctx)- return nil, err+ return selectionInput{}, nil, err}- candidates := selectSystemVolumeBackupCandidatesInternal(policyConfig, options)- return &preparedSystemVolumeBackupInternal{policy: policyConfig, manualPolicy: manualPolicy, candidates: candidates, lease: lease}, nil+ input.Candidates = selectSystemVolumeBackupCandidates(input.Policy, options)+ return input, lease, nil}-func (- s *Service,-) executeSystemVolumeBackupsInternal(- ctx context.Context,- policyConfig backuptypes.SystemVolumeBackupPolicy,- manualPolicy bool,- candidates []backuptypes.SystemVolumeBackupOption,- trigger volume.VolumeBackupTrigger,- activityID string,-) (- result *backuptypes.SystemVolumeBackupRunResult,- err error,-) {- result = &backuptypes.SystemVolumeBackupRunResult{- Matched: len(candidates), Failures: make([]backuptypes.SystemVolumeBackupFailure, 0),- }- defer s.updateSystemVolumeProgressInternal(context.WithoutCancel(ctx), activityID, policyConfig.ID, candidates, result)+func (s *Service) executeSystemVolumeBackups(+ ctx context.Context, input selectionInput, trigger volume.VolumeBackupTrigger, activityID string,+) (result *backuptypes.SystemVolumeBackupRunResult, err error) {+ result = &backuptypes.SystemVolumeBackupRunResult{Matched: len(input.Candidates), Failures: make([]backuptypes.SystemVolumeBackupFailure, 0)}+ reportProgress := func(ctx context.Context) {+ if activityID == "" {+ return+ }+ names := make([]string, len(input.Candidates))+ for i, candidate := range input.Candidates {+ names[i] = candidate.Name+ }+ progress := 100+ if result.Matched > 0 {+ progress = 100 * (result.Succeeded + result.Failed + result.Skipped) / result.Matched+ }+ _, updateErr := s.activity.UpdateActivity(ctx, activityID, activitylib.UpdateRequest{Progress: &progress, Metadata: database.JSON{+ "action": "run_system_volume_backups", "policyId": input.Policy.ID, "volumeNames": names,+ "matched": result.Matched, "succeeded": result.Succeeded, "failed": result.Failed, "skipped": result.Skipped, "failures": result.Failures,+ }})+ if updateErr != nil {+ slog.WarnContext(ctx, "Failed to report system-managed volume backup progress", "activityId", activityID, "policyId", input.Policy.ID, "error", updateErr)+ }+ }+ defer reportProgress(context.WithoutCancel(ctx))defer utils.RecoverToError(&err, "system-managed volume backup")policy := backuptypes.UpdateBackupPolicy{- Enabled: true, Schedule: policyConfig.Schedule, RetentionCount: policyConfig.RetentionCount,- StopContainers: policyConfig.StopContainers, LocalEnabled: policyConfig.LocalEnabled, S3Enabled: policyConfig.S3Enabled,- S3DestinationID: policyConfig.S3DestinationID,+ Enabled: true, Schedule: input.Policy.Schedule, RetentionCount: input.Policy.RetentionCount,+ StopContainers: input.Policy.StopContainers, LocalEnabled: input.Policy.LocalEnabled, S3Enabled: input.Policy.S3Enabled,+ S3DestinationID: input.Policy.S3DestinationID,}- for _, candidate := range candidates {- status := completedVolumeBackupStatusInternal(ctx, candidate.Name)- if status == scheduler.Succeeded {+ previous, _ := jobcontext.Run(ctx)+ for _, candidate := range input.Candidates {+ done := slices.IndexFunc(previous.Outcome.Targets, func(target scheduler.TargetOutcome) bool {+ return target.ID == candidate.Name && (target.Status == scheduler.Succeeded || target.Status == scheduler.Skipped)+ })+ if done >= 0 && previous.Outcome.Targets[done].Status == scheduler.Succeeded {result.Succeeded++continue}- if status == scheduler.Skipped {+ if done >= 0 {result.Skipped++continue}if cancellationErr := ctx.Err(); cancellationErr != nil {return result, cancellationErr}- s.updateSystemVolumeProgressInternal(ctx, activityID, policyConfig.ID, candidates, result)- overridden, policyErr := s.volumes.HasEnabledBackupPolicy(ctx, candidate.Name)- if policyErr != nil {- result.Failed++- result.Failures = append(result.Failures, backuptypes.SystemVolumeBackupFailure{VolumeName: candidate.Name, Error: policyErr.Error()})- continue- }- if overridden {+ reportProgress(ctx)+ overridden, backupErr := s.volumes.HasEnabledBackupPolicy(ctx, candidate.Name)+ if backupErr == nil && overridden {if skipProgressErr := jobcontext.Progress(ctx, scheduler.TargetOutcome{ID: candidate.Name, Status: scheduler.Skipped}); skipProgressErr != nil {return result, skipProgressErr}result.Skipped++continue}- seriesID := systemVolumePolicyIDInternal(policyConfig.ID, candidate.Name)- if manualPolicy {- seriesID = systemVolumeManualPolicyIDInternal(candidate.Name)+ if backupErr == nil {+ seriesID := backuptypes.SystemVolumePolicyPrefix + kit.Ternary(input.ManualPolicy, "manual", input.Policy.ID) + ":" + kit.SHA256Hex(candidate.Name)[:16]+ _, backupErr = s.volumes.CreateSystemManagedBackup(ctx, candidate.Name, usertypes.SystemUser, trigger, seriesID, policy)}- _, backupErr := s.volumes.CreateSystemManagedBackup(ctx, candidate.Name, usertypes.SystemUser, trigger, seriesID, policy)if errors.Is(backupErr, volume.ErrVolumeBackupAlreadyRunning) {result.Skipped++continue}+ if backupErr != nil && ctx.Err() != nil {+ // Shutdown interrupted this volume; hand the task back instead of counting it failed.+ return result, ctx.Err()+ }if backupErr != nil {result.Failed++result.Failures = append(result.Failures, backuptypes.SystemVolumeBackupFailure{VolumeName: candidate.Name, Error: backupErr.Error()})@@ -391,279 +442,124 @@ func (return result, nil}-func (s *Service) runScheduledSystemVolumeBackupInternal(ctx context.Context, policyID string) (scheduler.Outcome, error) {- policy, err := s.systemVolumeBackupPolicyInternal(policyID)+func (s *Service) runScheduledSystemVolumeBackup(ctx context.Context, policyID, activityID string) (scheduler.Outcome, error) {+ policy, err := s.systemVolumeBackupPolicy(policyID)if err != nil {return scheduler.Outcome{}, err}if policy == nil || !policy.Enabled {return scheduler.Outcome{Status: scheduler.Skipped}, nil}- remoteDisabled, checkErr := s.disableMissingVolumeS3Internal(ctx, policy)- if checkErr != nil {- return scheduler.Outcome{}, checkErr+ var remoteErr error+ if policy.S3Enabled {+ root := path.Join(backup.VolumeRoot, s.settings.GetSettingsConfig().InstanceID.Value)+ remoteErr = backup.CheckScheduledRemote(ctx, s.db, s.s3Destinations, "volume_backups", policy.S3DestinationID, root)+ }+ remoteDisabled, err := backup.DisableMissingRemote(remoteErr, policy.LocalEnabled, &policy.S3Enabled, &policy.Enabled, func(field string) (bool, error) {+ collection, loadErr := s.loadSystemVolumeBackupPolicies()+ if loadErr != nil {+ return false, loadErr+ }+ index := slices.IndexFunc(collection.Policies, func(current backuptypes.SystemVolumeBackupPolicy) bool {+ return current.ID == policy.ID && current.S3DestinationID == policy.S3DestinationID && current.S3Enabled == policy.S3Enabled &&+ current.Enabled == policy.Enabled && current.LocalEnabled == policy.LocalEnabled+ })+ if index < 0 {+ return false, nil+ }+ current := &collection.Policies[index]+ *kit.Ternary(field == "enabled", ¤t.Enabled, ¤t.S3Enabled) = false+ encoded, encodeErr := json.Marshal(collection)+ if encodeErr != nil {+ return false, fmt.Errorf("encode policies: %w", encodeErr)+ }+ return true, s.settings.UpdateSetting(ctx, systemVolumeBackupConfigKey, string(encoded))+ })+ if err != nil {+ return scheduler.Outcome{}, err+ }+ if remoteDisabled {+ s.rescheduleSystemVolumeBackup(ctx, policy)}if remoteDisabled && !policy.LocalEnabled {return scheduler.Outcome{Status: scheduler.NeedsAttention, Message: backup.RemoteDisabledMessage}, nil}- result, err := s.runSystemVolumeBackupsInternal(ctx, backuptypes.RunSystemVolumeBackupsRequest{PolicyID: policyID}, volume.VolumeBackupTriggerScheduled)+ input, lease, err := s.prepareSystemVolumeBackups(ctx, backuptypes.RunSystemVolumeBackupsRequest{PolicyID: policyID})if errors.Is(err, s.alreadyRunning) {slog.InfoContext(ctx, "Scheduled system-managed volume backups skipped; a system backup is running", "policyId", policyID)- return scheduler.Outcome{Status: scheduler.Skipped}, nil+ return scheduler.Outcome{Status: scheduler.Skipped, Message: "Skipped: a system backup is running"}, nil+ }+ var result *backuptypes.SystemVolumeBackupRunResult+ if err == nil {+ defer lease.Release(ctx)+ var frozen []byte+ if frozen, err = json.Marshal(input); err == nil {+ err = jobcontext.Progress(ctx, scheduler.TargetOutcome{ResourceType: "backup_plan", ID: systemVolumePlanTarget, Status: scheduler.Succeeded, RecoveryData: frozen})+ }+ if err == nil {+ result, err = s.executeSystemVolumeBackups(ctx, input, volume.VolumeBackupTriggerScheduled, activityID)+ }}if err != nil {slog.ErrorContext(ctx, "Scheduled system-managed volume backups failed", "policyId", policyID, "error", err)return scheduler.Outcome{}, err}- slog.InfoContext(- ctx,- "Scheduled system-managed volume backups completed",- "policyId",- policyID,- "matched",- result.Matched,- "succeeded",- result.Succeeded,- "failed",- result.Failed,- "skipped",- result.Skipped,- )- outcome := scheduler.Outcome{Status: scheduler.Succeeded}- if remoteDisabled {- outcome.Message = backup.RemoteDisabledMessage- }- if result.Failed > 0 || remoteDisabled {- outcome.Status = scheduler.Partial- }- for _, failure := range result.Failures {- outcome.Targets = append(outcome.Targets, scheduler.TargetOutcome{ID: failure.VolumeName, Status: scheduler.Failed, Message: failure.Error})- }- return outcome, nil+ slog.InfoContext(ctx, "Scheduled system-managed volume backups completed", "policyId", policyID,+ "matched", result.Matched, "succeeded", result.Succeeded, "failed", result.Failed, "skipped", result.Skipped)+ return selectionOutcome(result, remoteDisabled), nil}-func (s *Service) rescheduleSystemVolumeBackupInternal(ctx context.Context, policy *backuptypes.SystemVolumeBackupPolicy) {- if policy == nil {- return- }- jobID := systemVolumeBackupJobPrefix + policy.ID- if !policy.Enabled {- s.jobs.Unregister(ctx, jobID)- return- }- policyID := policy.ID- s.jobs.Register(ctx, jobID, func(context.Context) string {- current, err := s.systemVolumeBackupPolicyInternal(policyID)- if err != nil || current == nil {- return defaultSystemVolumeSchedule- }- return current.Schedule- }, func(ctx context.Context) (scheduler.Outcome, error) {- return s.runScheduledSystemVolumeBackupInternal(ctx, policyID)- }, func(ctx context.Context, previous scheduler.Run) (scheduler.Outcome, error) {- for _, target := range previous.Outcome.Targets {- if target.ID != "system-volume-plan" || len(target.RecoveryData) == 0 {- continue- }- if err := s.ExecuteDurable(ctx, previous.ID, target.RecoveryData, true); err != nil {- return scheduler.Outcome{Status: scheduler.NeedsAttention, Message: err.Error()}, err- }- return scheduler.Outcome{Status: scheduler.Succeeded}, nil- }- return scheduler.Outcome{Status: scheduler.NeedsAttention, Message: "The interrupted volume backup has no frozen selection"}, nil+// resumeSelection finishes the frozen selection an interrupted scheduled run recorded.+func (s *Service) resumeSelection(ctx context.Context, previous scheduler.Run, activityID string) (scheduler.Outcome, error) {+ index := slices.IndexFunc(previous.Outcome.Targets, func(target scheduler.TargetOutcome) bool {+ return target.ID == systemVolumePlanTarget && len(target.RecoveryData) > 0})-}--func (s *Service) saveSystemVolumeBackupPoliciesInternal(ctx context.Context, policies []backuptypes.SystemVolumeBackupPolicy) error {- encoded, err := json.Marshal(backuptypes.SystemVolumeBackupPolicyCollection{Policies: policies})- if err != nil {- return fmt.Errorf("encode policies: %w", err)- }- return s.settings.UpdateSetting(ctx, systemVolumeBackupConfigKey, string(encoded))-}--func (s *Service) disableMissingVolumeS3Internal(ctx context.Context, policy *backuptypes.SystemVolumeBackupPolicy) (bool, error) {- if !policy.S3Enabled {- return false, nil+ if index < 0 {+ return scheduler.Outcome{Status: scheduler.NeedsAttention, Message: "The interrupted volume backup has no frozen selection"}, nil}- root := "arcane-volume-backups/" + s.settings.GetSettingsConfig().InstanceID.Value- err := backup.CheckScheduledRemote(ctx, s.db, s.s3Destinations, "volume_backups", policy.S3DestinationID, root)- if !errors.Is(err, backup.ErrRemoteRepositoryMissing) {- return false, nil+ var input selectionInput+ if err := json.Unmarshal(previous.Outcome.Targets[index].RecoveryData, &input); err != nil {+ return scheduler.Outcome{}, err}- collection, err := s.loadSystemVolumeBackupPoliciesInternal()+ result, err := s.backUpSelection(ctx, input, volume.VolumeBackupTriggerScheduled, activityID)if err != nil {- return false, err+ return scheduler.Outcome{Status: scheduler.NeedsAttention, Message: err.Error()}, err}- for i := range collection.Policies {- current := &collection.Policies[i]- if current.ID != policy.ID {- continue- }- if current.S3DestinationID != policy.S3DestinationID || current.S3Enabled != policy.S3Enabled || current.Enabled != policy.Enabled || current.LocalEnabled != policy.LocalEnabled {- return false, nil- }- if current.LocalEnabled {- current.S3Enabled = false- } else {- current.Enabled = false- }- if saveSystemVolumeBackupPoliciesErr := s.saveSystemVolumeBackupPoliciesInternal(ctx, collection.Policies); saveSystemVolumeBackupPoliciesErr != nil {- return false, saveSystemVolumeBackupPoliciesErr- }- *policy = *current- s.rescheduleSystemVolumeBackupInternal(ctx, policy)- return true, nil- }- return false, nil+ return selectionOutcome(result, false), nil}-// Start freezes the selected policy and volumes before returning.-func (s *Service) Start(ctx context.Context, user usertypes.Actor, request backuptypes.RunSystemVolumeBackupsRequest) (*backuptypes.BackupRunAccepted, error) {- prepared, err := s.prepareSystemVolumeBackupsInternal(ctx, request)+// backUpSelection takes over the system admission, resumes the selection's interrupted volumes, and backs up the rest.+func (s *Service) backUpSelection(ctx context.Context, input selectionInput, trigger volume.VolumeBackupTrigger, activityID string) (*backuptypes.SystemVolumeBackupRunResult, error) {+ lease, admitted, err := s.engine.AcquireRun(ctx, backup.SystemAdmissionScope, backup.SystemAdmissionID, input.Claim)if err != nil {return nil, err}- policy, manualPolicy, candidates, lease := prepared.policy, prepared.manualPolicy, prepared.candidates, prepared.lease-- names := make([]string, len(candidates))- for i, candidate := range candidates {- names[i] = candidate.Name- }- activityID, workCtx := activitylib.StartHandlerActivity(ctx, s.activity, "0", activitytypes.TypeResourceAction, "system_backup", "volumes", "Volumes", &user,- "Backing up volumes", "Creating system-managed volume backups", database.JSON{- "action": "run_system_volume_backups",- "policyId": policy.ID,- "volumeNames": names,- "matched": len(- candidates,- ),- "succeeded": 0,- "failed": 0,- "skipped": 0,- "failures": []backuptypes.SystemVolumeBackupFailure{},- }, false)- if activityID == "" {- lease.Release(ctx)- return nil, errors.New("failed to create system-managed volume backup activity")- }- finish := func(runErr error) {- defer lease.Release(ctx)- activitylib.CompleteHandlerActivity(workCtx, s.activity, activityID, "System-managed volume backups completed", runErr)- }- keyID, _ := ctx.Value(middleware.ContextKeyApiKeyID).(string)- payload, err := json.Marshal(manualSystemVolumesInternal{Policy: policy, ManualPolicy: manualPolicy, Candidates: candidates, ActivityID: activityID, UserID: user.ID})- if err == nil {- err = s.engine.SubmitDurableRun(- workCtx,- backuptypes.DurableRunCommand{- Kind: "system-volumes",- RunID: activityID,- ActivityID: activityID,- Payload: payload,- UserID: user.ID,- EnvironmentID: "0",- Permission: authz.PermSystemBackupsManage,- RequestedWithKey: keyID,- },- lease,- )- }-- if err != nil {- finish(err)- return nil, err- }- return &backuptypes.BackupRunAccepted{ActivityID: activityID, Status: "running"}, nil-}--func (- s *Service,-) updateSystemVolumeProgressInternal(- ctx context.Context,- activityID, policyID string,- candidates []backuptypes.SystemVolumeBackupOption,- result *backuptypes.SystemVolumeBackupRunResult,-) {- if activityID == "" {- return- }- names := make([]string, len(candidates))- for i, candidate := range candidates {- names[i] = candidate.Name- }- progress := 100- if result.Matched > 0 {- progress = 100 * (result.Succeeded + result.Failed + result.Skipped) / result.Matched- }- _, err := s.activity.UpdateActivity(ctx, activityID, activitylib.UpdateRequest{Progress: &progress, Metadata: database.JSON{- "action": "run_system_volume_backups", "policyId": policyID, "volumeNames": names,- "matched": result.Matched, "succeeded": result.Succeeded, "failed": result.Failed, "skipped": result.Skipped, "failures": result.Failures,- }})- if err != nil {- slog.WarnContext(ctx, "Failed to report system-managed volume backup progress", "activityId", activityID, "policyId", policyID, "error", err)- }-}--type manualSystemVolumesInternal struct {- Policy backuptypes.SystemVolumeBackupPolicy `json:"policy"`- ManualPolicy bool `json:"manualPolicy"`- Candidates []backuptypes.SystemVolumeBackupOption `json:"candidates"`- ActivityID string `json:"activityId"`- UserID string `json:"userId"`-}--// ExecuteDurable runs a submitted or interrupted system-volumes run.-func (s *Service) ExecuteDurable(ctx context.Context, runID string, payload []byte, interrupted bool) (err error) {- var command manualSystemVolumesInternal- if decodeErr := json.Unmarshal(payload, &command); decodeErr != nil {- return decodeErr- }- defer func() {- if ctx.Err() == nil {- activitylib.CompleteHandlerActivity(ctx, s.activity, command.ActivityID, "System-managed volume backups completed", err)- }- }()- lease, err := s.acquireDurableRun(ctx, runID)- if err != nil {- return err+ if !admitted {+ return nil, s.alreadyRunning}defer lease.Release(ctx)- if interrupted {- ctx, err = s.reconcileVolumeCandidatesInternal(ctx, command.Candidates)- if err != nil {- return err+ if input.Requester != nil {+ if authorizeErr := s.engine.Authorize(ctx, *input.Requester); authorizeErr != nil {+ return nil, authorizeErr}}- result, err := s.executeSystemVolumeBackupsInternal(ctx, command.Policy, command.ManualPolicy, command.Candidates, volume.VolumeBackupTriggerManual, command.ActivityID)- if err != nil {- return err- }- if result.Failed > 0 {- return fmt.Errorf("%d volume backups failed", result.Failed)- }- return nil-}--func (s *Service) reconcileVolumeCandidatesInternal(ctx context.Context, candidates []backuptypes.SystemVolumeBackupOption) (context.Context, error) {previous, _ := jobcontext.Run(ctx)- for _, candidate := range candidates {- pending := false- for _, target := range previous.Outcome.Targets {- if target.ID == candidate.Name && target.Status != scheduler.Succeeded && target.Status != scheduler.Skipped {- pending = true- break- }- }- if !pending {+ for _, candidate := range input.Candidates {+ if !slices.ContainsFunc(previous.Outcome.Targets, func(target scheduler.TargetOutcome) bool {+ return target.ID == candidate.Name && target.Status != scheduler.Succeeded && target.Status != scheduler.Skipped+ }) {continue}outcome, recoveryErr := s.volumes.ReconcileBackup(ctx, previous, candidate.Name)+ if errors.Is(recoveryErr, backup.ErrBackupSettled) {+ // That volume's backup already failed, so the selection backs it up again below.+ continue+ }if recoveryErr != nil {- return ctx, recoveryErr+ return nil, recoveryErr}if outcome.Status != scheduler.Succeeded {- return ctx, errors.New(outcome.Message)+ return nil, errors.New(outcome.Message)}for i := range previous.Outcome.Targets {if previous.Outcome.Targets[i].ID == candidate.Name {@@ -673,86 +569,109 @@ func (s *Service) reconcileVolumeCandidatesInternal(ctx context.Context, candida}progressCtx := ctxctx = jobcontext.WithExecution(ctx, previous, func(target scheduler.TargetOutcome) error { return jobcontext.Progress(progressCtx, target) })- return ctx, nil+ return s.executeSystemVolumeBackups(ctx, input, trigger, activityID)}-// FailDurable completes the activity of a system-volumes run that cannot continue.-func (s *Service) FailDurable(ctx context.Context, _ string, payload []byte, runErr error) error {- var command manualSystemVolumesInternal- if err := json.Unmarshal(payload, &command); err != nil {- return err+// selectionOutcome reports a run's result; failed volumes or disabled remote storage make it partial.+func selectionOutcome(result *backuptypes.SystemVolumeBackupRunResult, remoteDisabled bool) scheduler.Outcome {+ outcome := scheduler.Outcome{Status: scheduler.Succeeded, Message: "System-managed volume backups completed"}+ if result.Failed > 0 {+ outcome.Status, outcome.Message = scheduler.Partial, fmt.Sprintf("%d volume backups failed", result.Failed)}- activitylib.CompleteHandlerActivity(ctx, s.activity, command.ActivityID, "System-managed volume backups completed", runErr)- return nil-}--// RegisterJobsOnStartup schedules every saved policy and returns how many were loaded.-func (s *Service) RegisterJobsOnStartup(ctx context.Context) int {- policies, err := s.loadSystemVolumeBackupPoliciesInternal()- if err != nil {- slog.ErrorContext(ctx, "Failed to load system-managed volume backup policies", "error", err)- return 0+ if remoteDisabled {+ outcome.Status, outcome.Message = scheduler.Partial, backup.RemoteDisabledMessage}- for i := range policies.Policies {- s.rescheduleSystemVolumeBackupInternal(ctx, &policies.Policies[i])+ for _, failure := range result.Failures {+ outcome.Targets = append(outcome.Targets, scheduler.TargetOutcome{ID: failure.VolumeName, Status: scheduler.Failed, Message: failure.Error})}- return len(policies.Policies)+ return outcome}-func completedVolumeBackupStatusInternal(ctx context.Context, volumeName string) scheduler.RunStatus {- previous, ok := jobcontext.Run(ctx)- if !ok {- return ""+func (s *Service) rescheduleSystemVolumeBackup(ctx context.Context, policy *backuptypes.SystemVolumeBackupPolicy) {+ if policy == nil {+ return}- for _, target := range previous.Outcome.Targets {- if target.ID == volumeName && (target.Status == scheduler.Succeeded || target.Status == scheduler.Skipped) {- return target.Status- }+ jobID := systemVolumeBackupJobPrefix + policy.ID+ if !policy.Enabled {+ s.jobs.Unregister(ctx, jobID)+ return}- return ""+ policyID := policy.ID+ s.jobs.Add(ctx, &flow.Job{+ Engine: s.flow,+ Workflow: s.policyWorkflow,+ JobName: s.jobs.JobName(jobID),+ Payload: policyID,+ Activity: activitylib.StartRequest{Metadata: database.JSON{"action": "scheduled_system_volume_backups", "policyId": policyID, "schedule": policy.Schedule}},+ ScheduleFn: func(context.Context) string {+ current, err := s.systemVolumeBackupPolicy(policyID)+ if err != nil || current == nil {+ return defaultSystemVolumeSchedule+ }+ return current.Schedule+ },+ FallbackFn: func(ctx context.Context, previous scheduler.Run) (scheduler.Outcome, error) {+ return s.resumeSelection(ctx, previous, "")+ },+ })}-func customSystemVolumePolicyInternal(custom *backuptypes.SystemVolumeBackupCustomRun) backuptypes.UpdateSystemVolumeBackupPolicy {- if custom == nil {- return backuptypes.UpdateSystemVolumeBackupPolicy{- Enabled: true, Schedule: defaultSystemVolumeSchedule, LocalEnabled: true,- SelectionMode: backuptypes.SystemVolumeSelectionAll, VolumeNames: []string{}, IgnoreAnonymous: true,- }+// Start freezes the selected policy and volumes, then submits their workflow.+func (s *Service) Start(ctx context.Context, user usertypes.Actor, request backuptypes.RunSystemVolumeBackupsRequest) (*backuptypes.BackupRunAccepted, error) {+ requester := backup.NewRequester(ctx, user, "0", authz.PermSystemBackupsManage)+ if err := s.engine.Authorize(ctx, requester); err != nil {+ return nil, err}- localEnabled := custom.Destination == backuptypes.SystemBackupDestinationLocal || custom.Destination == backuptypes.SystemBackupDestinationLocalS3- s3Enabled := custom.Destination == backuptypes.SystemBackupDestinationS3 || custom.Destination == backuptypes.SystemBackupDestinationLocalS3- return backuptypes.UpdateSystemVolumeBackupPolicy{- Enabled: true, Schedule: defaultSystemVolumeSchedule, RetentionCount: 0, StopContainers: custom.StopContainers,- LocalEnabled: localEnabled, S3Enabled: s3Enabled, S3DestinationID: custom.S3DestinationID,- SelectionMode: custom.SelectionMode, VolumeNames: custom.VolumeNames, IgnoreAnonymous: custom.IgnoreAnonymous,+ input, lease, err := s.prepareSystemVolumeBackups(ctx, request)+ if err != nil {+ return nil, err}+ input.Requester = &requester+ names := make([]string, len(input.Candidates))+ for i, candidate := range input.Candidates {+ names[i] = candidate.Name+ }+ input.Claim = uuid.NewV7().String()+ release := s.engine.Hold(backup.SystemAdmissionScope, backup.SystemAdmissionID, input.Claim, lease)+ activityID, err := s.flow.Submit(ctx, s.backupWorkflow, input, activitylib.StartRequest{+ StartedBy: &user, Step: "Backing up volumes", LatestMessage: "Creating system-managed volume backups",+ Metadata: database.JSON{+ "action": "run_system_volume_backups", "policyId": input.Policy.ID, "volumeNames": names, "matched": len(names),+ "succeeded": 0, "failed": 0, "skipped": 0, "failures": []backuptypes.SystemVolumeBackupFailure{},+ },+ })+ if err != nil {+ release(ctx)+ return nil, err+ }+ // A run canceled before its task starts never takes the parked lease.+ go func() {+ _, _ = s.flow.Wait(context.WithoutCancel(ctx), s.backupWorkflow, activityID)+ release(context.WithoutCancel(ctx))+ }()+ return &backuptypes.BackupRunAccepted{ActivityID: activityID, Status: "running"}, nil}-const (- systemVolumeBackupConfigKey = "systemVolumeBackupConfig"- systemVolumeBackupJobPrefix = "volumes:"- defaultSystemVolumeSchedule = "0 0 2 * * *"-)--func systemVolumeManualPolicyIDInternal(volumeName string) string {- return backuptypes.SystemVolumePolicyPrefix + "manual:" + kit.SHA256Hex(volumeName)[:16]-}--func systemVolumePolicyIDInternal(policyID, volumeName string) string {- return backuptypes.SystemVolumePolicyPrefix + policyID + ":" + kit.SHA256Hex(volumeName)[:16]+// RegisterJobsOnStartup schedules every saved policy and returns how many were loaded.+func (s *Service) RegisterJobsOnStartup(ctx context.Context) int {+ policies, err := s.loadSystemVolumeBackupPolicies()+ if err != nil {+ slog.ErrorContext(ctx, "Failed to load system-managed volume backup policies", "error", err)+ return 0+ }+ for i := range policies.Policies {+ s.rescheduleSystemVolumeBackup(ctx, &policies.Policies[i])+ }+ return len(policies.Policies)}-func selectSystemVolumeBackupCandidatesInternal(policy backuptypes.SystemVolumeBackupPolicy, options []backuptypes.SystemVolumeBackupOption) []backuptypes.SystemVolumeBackupOption {- configured := make(map[string]struct{}, len(policy.VolumeNames))- for _, name := range policy.VolumeNames {- configured[name] = struct{}{}- }+func selectSystemVolumeBackupCandidates(policy backuptypes.SystemVolumeBackupPolicy, options []backuptypes.SystemVolumeBackupOption) []backuptypes.SystemVolumeBackupOption {result := make([]backuptypes.SystemVolumeBackupOption, 0, len(options))for _, option := range options {if !option.Available {continue}- _, selected := configured[option.Name]+ selected := slices.Contains(policy.VolumeNames, option.Name)matches := policy.SelectionMode == backuptypes.SystemVolumeSelectionAll ||(policy.SelectionMode == backuptypes.SystemVolumeSelectionAllowlist && selected) ||(policy.SelectionMode == backuptypes.SystemVolumeSelectionBlocklist && !selected)@@ -762,10 +681,3 @@ func selectSystemVolumeBackupCandidatesInternal(policy backuptypes.SystemVolumeB}return result}--func validateSystemVolumeSelectionInternal(mode backuptypes.SystemVolumeSelectionMode) error {- if mode != backuptypes.SystemVolumeSelectionAll && mode != backuptypes.SystemVolumeSelectionAllowlist && mode != backuptypes.SystemVolumeSelectionBlocklist {- return errors.New("selectionMode must be all, allowlist, or blocklist")- }- return nil-}diff --git a/backend/internal/system/children/backup/children/volumes/service_test.go b/backend/internal/system/children/backup/children/volumes/service_test.goindex e844504b5..7770e30cd 100644--- a/backend/internal/system/children/backup/children/volumes/service_test.go+++ b/backend/internal/system/children/backup/children/volumes/service_test.go@@ -7,7 +7,7 @@ import ("github.com/stretchr/testify/require")-func TestSystemVolumeBackupSelectionInternal(t *testing.T) {+func TestSystemVolumeBackupSelection(t *testing.T) {options := []backup.SystemVolumeBackupOption{{Name: "app", Available: true},{Name: "cache", Available: true},@@ -37,7 +37,7 @@ func TestSystemVolumeBackupSelectionInternal(t *testing.T) {}for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {- selected := selectSystemVolumeBackupCandidatesInternal(tt.config, options)+ selected := selectSystemVolumeBackupCandidates(tt.config, options)names := make([]string, len(selected))for i := range selected {names[i] = selected[i].Namediff --git a/backend/internal/system/children/backup/handler.go b/backend/internal/system/children/backup/handler.goindex 87f25f56e..c98d526f1 100644--- a/backend/internal/system/children/backup/handler.go+++ b/backend/internal/system/children/backup/handler.go@@ -3,13 +3,11 @@ package backupimport ("context""errors"- "strings""github.com/danielgtaylor/huma/v2"activitytypes "github.com/getarcaneapp/arcane/types/v2/activity""github.com/getarcaneapp/arcane/types/v2/backup""github.com/getarcaneapp/arcane/types/v2/base"- "github.com/samber/mo""github.com/getarcaneapp/arcane/backend/v2/internal/activity""github.com/getarcaneapp/arcane/backend/v2/internal/common"@@ -248,7 +246,7 @@ func (h *Handler) Restore(ctx context.Context, input *RestoreSystemBackupInput)if err != nil {return nil, huma.Error500InternalServerError(err.Error())}- return messageOutputInternal("Arcane system restore started", activityID), nil+ return handlerutil.MessageOutput("Arcane system restore started", activityID), nil}// BrowseFiles returns one lazy-loaded project tree page.@@ -285,7 +283,7 @@ func (h *Handler) RestoreFiles(ctx context.Context, input *RestoreSystemBackupFiif err != nil {return nil, huma.Error500InternalServerError(err.Error())}- return messageOutputInternal("Arcane project files restored successfully", activityID), nil+ return handlerutil.MessageOutput("Arcane project files restored successfully", activityID), nil}func (h *Handler) Upload(ctx context.Context, input *UploadSystemBackupInput) (*SystemBackupOutput, error) {@@ -324,21 +322,5 @@ func (h *Handler) Delete(ctx context.Context, input *DeleteSystemBackupInput) (*if err != nil {return nil, huma.Error500InternalServerError(err.Error())}- return messageOutputInternal("Arcane system backup deleted successfully", activityID), nil-}--func messageOutputInternal(message, activityID string) *handlerutil.Out[base.MessageResponse] {- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{- Message: message,- ActivityID: mo.EmptyableToOption(- strings.TrimSpace(- activityID,- ),- ).ToPointer(),- },- },- }+ return handlerutil.MessageOutput("Arcane system backup deleted successfully", activityID), nil}diff --git a/backend/internal/system/children/backup/service.go b/backend/internal/system/children/backup/service.goindex 46047cefb..797219455 100644--- a/backend/internal/system/children/backup/service.go+++ b/backend/internal/system/children/backup/service.go@@ -23,14 +23,14 @@ import ("github.com/getarcaneapp/arcane/types/v2/recovery""github.com/getarcaneapp/arcane/types/v2/scheduler"usertypes "github.com/getarcaneapp/arcane/types/v2/user"+ "github.com/italypaleale/francis/builtin/workflow""github.com/moby/moby/api/types/container""github.com/moby/moby/api/types/mount""github.com/moby/moby/client""go.getarcane.app/docker""go.getarcane.app/docker/compat"- "go.getarcane.app/kit/pkg"+ kit "go.getarcane.app/kit/pkg""go.getarcane.app/sys/cgroup"- "go.getarcane.app/sys/crypto""gorm.io/gorm""github.com/getarcaneapp/arcane/backend/v2/internal/activity"@@ -39,7 +39,6 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/database"dockerInternal "github.com/getarcaneapp/arcane/backend/v2/internal/docker"- "github.com/getarcaneapp/arcane/backend/v2/internal/middleware""github.com/getarcaneapp/arcane/backend/v2/internal/s3""github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/backend/v2/internal/system/children/backup/children/snapshots"@@ -50,6 +49,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/pagination""github.com/getarcaneapp/arcane/backend/v2/pkg/projects""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/entityjobs"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs""github.com/getarcaneapp/arcane/backend/v2/pkg/utils"@@ -59,7 +59,6 @@ import (const (defaultSystemBackupSchedule = "0 0 3 * * *"systemRecoveryHelperPath = "/app/arcane-recovery-helper"- systemAdmissionID = "system")var ErrSystemBackupAlreadyRunning = errors.New("an Arcane system backup is already running")@@ -88,7 +87,7 @@ type Store struct {unregister func(context.Context, string),reschedule func(context.Context, *backuptypes.SystemBackupPolicy),) error- DisablePolicyRemote func(ctx context.Context, policy *backuptypes.SystemBackupPolicy) (bool, error)+ DisablePolicyRemote func(ctx context.Context, policy *backuptypes.SystemBackupPolicy, field string) (bool, error)CheckScheduledRemote func(ctx context.Context, destinationID string) errorListHistory func(ctx context.Context, params pagination.QueryParams) ([]backuptypes.HistoryEntry, pagination.Response, error)}@@ -136,6 +135,29 @@ type Service struct {jobs *entityjobs.Registrysnapshots *snapshots.Servicevolumes *volumes.Service+ flow *flow.Engine+ // backupWorkflow runs backups StartBackup accepted; policyWorkflow runs scheduled policies.+ backupWorkflow *flow.Workflow+ policyWorkflow *flow.Workflow+}++// manualBackupInput is a manual backup's workflow payload; the run row is created before it starts.+type manualBackupInput struct {+ Checkpoint systemBackupRecovery `json:"checkpoint"`+ Requester backuptypes.Requester `json:"requester"`+}++// systemBackupRecovery is the checkpoint a run records so an interrupted backup can resume.+type systemBackupRecovery struct {+ BackupID string `json:"backupId"`+ LocalEnabled bool `json:"localEnabled"`+ S3Enabled bool `json:"s3Enabled"`+}++type preparedSystemBackup struct {+ run *backuptypes.SystemBackupRun+ recoveryKey string+ checkpoint systemBackupRecovery}func NewService(deps Dependencies) *Service {@@ -149,78 +171,29 @@ func NewService(deps Dependencies) *Service {deps.Engine,deps.SQLDB,deps.ActorStore,- service.localRepositoryInternal,- service.remoteRepositoryInternal,- service.databaseFileInternal,- service.projectsDirectoryInternal,- service.recoveryEnvironmentInternal,+ service.localRepository,+ service.remoteRepository,+ service.databaseFile,+ func(ctx context.Context) string {+ return projects.ResolveConfiguredContainerDirectory(service.projectsSetting(ctx), "/app/data/projects")+ },+ service.recoveryEnvironment,)service.volumes = volumes.NewService(volumes.Dependencies{- DB: deps.DB,- Engine: deps.Engine,- Volumes: deps.Volumes,- S3Destinations: deps.S3Destinations,- Activity: deps.Activity,- Settings: deps.Settings,- Jobs: service.jobs,- AcquireRun: service.acquireRunInternal,- AcquireDurableRun: service.acquireDurableRunInternal,- AlreadyRunning: ErrSystemBackupAlreadyRunning,+ DB: deps.DB,+ Engine: deps.Engine,+ Volumes: deps.Volumes,+ S3Destinations: deps.S3Destinations,+ Activity: deps.Activity,+ Settings: deps.Settings,+ Jobs: service.jobs,+ AlreadyRunning: ErrSystemBackupAlreadyRunning,})- if deps.Engine != nil {- deps.Engine.RegisterRunKind("system", service.executeDurableBackupInternal, service.failDurableBackupInternal)- deps.Engine.RegisterRunKind("system-volumes", service.volumes.ExecuteDurable, service.volumes.FailDurable)- }return service}-// acquireRunInternal takes the shared system backup admission lease.-func (s *Service) acquireRunInternal(ctx context.Context) (*runs.Lease, error) {- lease, admitted, err := s.engine.TryAcquireRun(ctx, backup.SystemAdmissionScope, systemAdmissionID)- if err != nil {- return nil, err- }- if !admitted {- return nil, ErrSystemBackupAlreadyRunning- }- return lease, nil-}--// acquireDurableRunInternal takes the shared system backup admission lease for a durable run.-func (s *Service) acquireDurableRunInternal(ctx context.Context, runID string) (*runs.Lease, error) {- lease, admitted, err := s.engine.AcquireDurableRun(ctx, runID, backup.SystemAdmissionScope, systemAdmissionID)- if err != nil {- return nil, err- }- if !admitted {- return nil, ErrSystemBackupAlreadyRunning- }- return lease, nil-}--// SupportedDatabaseProvider reports whether system recovery works on the-// configured database. Recovery is currently SQLite-only.-func (s *Service) SupportedDatabaseProvider() bool {- return strings.HasPrefix(s.config.DatabaseURL, "file:")-}--func recoveryHelperExecutableInternal(mounts []container.MountPoint, executablePath string) (string, *mount.Mount, error) {- executablePath = strings.TrimSpace(executablePath)- if executablePath == "" {- return "", nil, errors.New("arcane executable path is empty")- }- executableMount := docker.MountForSubpath(mounts, executablePath, systemRecoveryHelperPath)- if executableMount == nil {- return executablePath, nil, nil- }- if executableMount.Type != mount.TypeBind {- return "", nil, errors.New("the Arcane executable must come from the image or a bind mount")- }- executableMount.ReadOnly = true- return systemRecoveryHelperPath, executableMount, nil-}--func (s *Service) recoveryEnvironmentInternal(ctx context.Context) map[string]string {+// recoveryEnvironment is the runtime configuration, secrets included, that a restored instance needs.+func (s *Service) recoveryEnvironment(ctx context.Context) map[string]string {result := make(map[string]string)var visit func(reflect.Value)visit = func(value reflect.Value) {@@ -233,10 +206,8 @@ func (s *Service) recoveryEnvironmentInternal(ctx context.Context) map[string]stif value.Kind() != reflect.Struct {return}- typeOf := value.Type()for i := range value.NumField() {- fieldType := typeOf.Field(i)- field := value.Field(i)+ fieldType, field := value.Type().Field(i), value.Field(i)if fieldType.Anonymous {visit(field)continue@@ -245,26 +216,88 @@ func (s *Service) recoveryEnvironmentInternal(ctx context.Context) map[string]stif name == "" || !field.CanInterface() {continue}- if field.Type() == reflect.TypeFor[os.FileMode]() {- mode, ok := reflect.TypeAssert[os.FileMode](field)- if ok {- result[name] = fmt.Sprintf("%#o", uint32(mode))- }+ if mode, ok := reflect.TypeAssert[os.FileMode](field); ok {+ result[name] = fmt.Sprintf("%#o", uint32(mode))} else {result[name] = fmt.Sprint(field.Interface())}}}visit(reflect.ValueOf(s.config))- if value := s.projectsSettingInternal(ctx); value != "" {+ if value := s.projectsSetting(ctx); value != "" {result["PROJECTS_DIRECTORY"] = value}return result}-// databaseFileInternal resolves the SQLite database file: under /app/data in+// acquireRun takes the shared system backup admission lease.+func (s *Service) acquireRun(ctx context.Context) (*runs.Lease, error) {+ lease, admitted, err := s.engine.TryAcquireRun(ctx, backup.SystemAdmissionScope, backup.SystemAdmissionID)+ if err != nil {+ return nil, err+ }+ if !admitted {+ return nil, ErrSystemBackupAlreadyRunning+ }+ return lease, nil+}++// RegisterWorkflows defines the manual and scheduled backup workflows while the host is still unstarted.+func (s *Service) RegisterWorkflows(engine *flow.Engine) error {+ var err error+ s.flow = engine+ template := activitylib.StartRequest{Type: activitytypes.TypeResourceAction, EnvironmentID: "0", ResourceType: new("system_backup"), ResourceName: new("Arcane")}+ s.backupWorkflow, err = engine.Define(flow.Definition{+ Name: "system-backup",+ Version: 1,+ Fingerprint: "9b44e4c38b7fcf8951468cc43a51b66c644d1104f1a56fa49bf8c065dc51c046",+ Concurrency: 1,+ Timeout: 24 * time.Hour,+ Activity: template,+ Labels: map[string]string{"backup": "Creating system backup"},+ Steps: []workflow.StepSpec{workflow.Step("backup", engine.Handler(s.runManualBackup), workflow.WithMaxAttempts(1))},+ })+ if err != nil {+ return err+ }+ template.StartedBy, template.Step, template.LatestMessage = &usertypes.SystemUser, "Creating scheduled system backup", "Creating scheduled Arcane system backup"+ s.policyWorkflow, err = engine.Define(flow.Definition{+ Name: "system-backup-policy",+ Version: 1,+ Fingerprint: "42fa9de20fefac192148a5f6d68132e34ae87948160b5bb0cc5876d506843bcf",+ Concurrency: 1,+ Timeout: 24 * time.Hour,+ Activity: template,+ Labels: map[string]string{"backup": "Creating scheduled system backup"},+ Steps: []workflow.StepSpec{workflow.Step("backup", engine.Handler(func(ctx context.Context, t flow.Task) (any, error) {+ var policyID string+ if decodeErr := t.Payload(&policyID); decodeErr != nil {+ return nil, decodeErr+ }+ previous, _ := jobcontext.Run(ctx)+ if slices.ContainsFunc(previous.Outcome.Targets, func(target scheduler.TargetOutcome) bool {+ return target.ID == policyID && len(target.RecoveryData) > 0+ }) {+ return s.resumeScheduledBackup(ctx, previous, policyID)+ }+ return s.runScheduledBackup(ctx, policyID)+ }), workflow.WithMaxAttempts(1))},+ })+ if err != nil {+ return err+ }+ return s.volumes.RegisterWorkflows(engine)+}++// SupportedDatabaseProvider reports whether system recovery works on the+// configured database. Recovery is currently SQLite-only.+func (s *Service) SupportedDatabaseProvider() bool {+ return strings.HasPrefix(s.config.DatabaseURL, "file:")+}++// databaseFile resolves the SQLite database file: under /app/data in// the container, the local data directory in host development.-func (s *Service) databaseFileInternal() (string, error) {+func (s *Service) databaseFile() (string, error) {databasePath, err := kit.SQLitePathFromDSN(s.config.DatabaseURL)if err != nil {return "", fmt.Errorf("failed to parse database URL: %w", err)@@ -279,17 +312,7 @@ func (s *Service) databaseFileInternal() (string, error) {return absolute, nil}-// dataDirectoryInternal resolves the directory holding the SQLite database.-// The recovery manifest must live inside it so every snapshot includes it.-func (s *Service) dataDirectoryInternal() (string, error) {- databaseFile, err := s.databaseFileInternal()- if err != nil {- return "", err- }- return filepath.Dir(databaseFile), nil-}--func (s *Service) localRepositoryInternal(ctx context.Context, dockerClient *client.Client, readOnly bool) (backup.Repository, error) {+func (s *Service) localRepository(ctx context.Context, dockerClient *client.Client, readOnly bool) (backup.Repository, error) {storage, err := s.volumeService.BackupStorageMount(ctx, dockerClient, "/repository", readOnly)if err != nil {return backup.Repository{}, err@@ -301,7 +324,7 @@ func (s *Service) localRepositoryInternal(ctx context.Context, dockerClient *cli}, nil}-func (s *Service) remoteRepositoryInternal(ctx context.Context, destinationID string) (backup.Repository, error) {+func (s *Service) remoteRepository(ctx context.Context, destinationID string) (backup.Repository, error) {configuration, err := s.s3Destinations.Configuration(ctx, destinationID)if err != nil {return backup.Repository{}, errors.New("the selected S3 backup destination is not configured")@@ -312,7 +335,7 @@ func (s *Service) remoteRepositoryInternal(ctx context.Context, destinationID st}, nil}-func (s *Service) recoveryKeyInternal(ctx context.Context, supplied string) (string, error) {+func (s *Service) recoveryKey(ctx context.Context, supplied string) (string, error) {if strings.TrimSpace(supplied) != "" {if err := backup.ValidateRecoveryKey(supplied); err != nil {return "", err@@ -329,299 +352,172 @@ func (s *Service) recoveryKeyInternal(ctx context.Context, supplied string) (strreturn key, nil}-// resolveBackupPlanInternal resolves which destinations a run targets, from-// either the policy or the request's destination enum.-func (- s *Service,-) resolveBackupPlanInternal(- ctx context.Context,- request backuptypes.CreateSystemBackupRequest,-) (- localEnabled, s3Enabled bool,- destinationID string,- destination backuptypes.SystemBackupDestination,- err error,-) {- destinationID = strings.TrimSpace(request.S3DestinationID)+func (s *Service) CreateBackup(ctx context.Context, user usertypes.Actor, trigger string, request backuptypes.CreateSystemBackupRequest) (*backuptypes.SystemBackupRun, error) {+ lease, err := s.acquireRun(ctx)+ if err != nil {+ return nil, err+ }+ defer lease.Release(ctx)+ prepared, err := s.prepareBackup(ctx, trigger, request)+ if err != nil {+ return nil, err+ }+ return s.executeBackup(ctx, prepared)+}++// prepareBackup resolves the run's destinations from its policy or destination enum, creates the run row, and records its checkpoint.+func (s *Service) prepareBackup(ctx context.Context, trigger string, request backuptypes.CreateSystemBackupRequest) (*preparedSystemBackup, error) {+ if !s.SupportedDatabaseProvider() {+ return nil, errors.New("arcane system recovery currently requires the SQLite database provider")+ }+ recoveryKey, err := s.recoveryKey(ctx, request.RecoveryKey)+ if err != nil {+ return nil, err+ }+ var checkpoint systemBackupRecovery+ destinationID := strings.TrimSpace(request.S3DestinationID)if request.PolicyID != "" {policy, policyErr := s.store.Policy(ctx, request.PolicyID)if policyErr != nil {- return false, false, "", "", policyErr+ return nil, policyErr}if policy == nil {- return false, false, "", "", errors.New("system backup policy not found")+ return nil, errors.New("system backup policy not found")}- localEnabled, s3Enabled, destinationID = policy.LocalEnabled, policy.S3Enabled, policy.S3DestinationID+ checkpoint.LocalEnabled, checkpoint.S3Enabled, destinationID = policy.LocalEnabled, policy.S3Enabled, policy.S3DestinationID} else {switch request.Destination {case backuptypes.SystemBackupDestinationLocal, "":- localEnabled = true+ checkpoint.LocalEnabled = truecase backuptypes.SystemBackupDestinationS3:- s3Enabled = true+ checkpoint.S3Enabled = truecase backuptypes.SystemBackupDestinationLocalS3:- localEnabled, s3Enabled = true, true+ checkpoint.LocalEnabled, checkpoint.S3Enabled = true, truedefault:- return false, false, "", "", errors.New("unknown system backup destination")+ return nil, errors.New("unknown system backup destination")}}- if !localEnabled && !s3Enabled {- return false, false, "", "", errors.New("select at least one system backup destination")+ if !checkpoint.LocalEnabled && !checkpoint.S3Enabled {+ return nil, errors.New("select at least one system backup destination")}- if s3Enabled && destinationID == "" {- return false, false, "", "", errors.New("select an S3 destination for the system backup")+ if checkpoint.S3Enabled && destinationID == "" {+ return nil, errors.New("select an S3 destination for the system backup")}- destination = backuptypes.SystemBackupDestinationLocal- if localEnabled && s3Enabled {+ destination := backuptypes.SystemBackupDestinationS3+ switch {+ case checkpoint.LocalEnabled && checkpoint.S3Enabled:destination = backuptypes.SystemBackupDestinationLocalS3- } else if s3Enabled {- destination = backuptypes.SystemBackupDestinationS3- }- return localEnabled, s3Enabled, destinationID, destination, nil-}--func (s *Service) CreateBackup(ctx context.Context, user usertypes.Actor, trigger string, request backuptypes.CreateSystemBackupRequest) (*backuptypes.SystemBackupRun, error) {- lease, err := s.acquireRunInternal(ctx)- if err != nil {- return nil, err- }- defer lease.Release(ctx)- return s.createBackupInternal(ctx, trigger, request)-}--func (- s *Service,-) createStagedSystemRecoverySnapshotInternal(- ctx context.Context,- dockerClient *client.Client,- recoveryKey, backupID, destinationID string,- localEnabled, s3Enabled bool,-) (- backup.Snapshot,- backup.Repository,- bool,- error,-) {- localRepository, err := s.localRepositoryInternal(ctx, dockerClient, false)- var snapshot backup.Snapshot- if err == nil {- snapshot, err = s.snapshots.Create(ctx, dockerClient, localRepository, recoveryKey, backupID)- }- if err == nil {- return snapshot, localRepository, false, nil- }- localStagingErr := fmt.Errorf("failed to create local system recovery staging snapshot: %w", err)- if localEnabled || !s3Enabled {- return backup.Snapshot{}, backup.Repository{}, false, localStagingErr- }- // S3-only backups normally use local repository staging. If local storage is unusable, write directly to- // S3 so remote backups and restore safety snapshots remain available.- remoteRepository, repoErr := s.remoteRepositoryInternal(ctx, destinationID)- if repoErr != nil {- return backup.Snapshot{}, backup.Repository{}, false, errors.Join(localStagingErr, repoErr)- }- remoteSnapshot, snapshotErr := s.snapshots.Create(ctx, dockerClient, remoteRepository, recoveryKey, backupID)- if snapshotErr != nil {- return backup.Snapshot{}, backup.Repository{}, false, errors.Join(localStagingErr, fmt.Errorf("failed to create S3 system recovery snapshot: %w", snapshotErr))- }- return remoteSnapshot, remoteRepository, true, nil-}--type systemBackupRecoveryInternal struct {- BackupID string `json:"backupId"`- LocalEnabled bool `json:"localEnabled"`- S3Enabled bool `json:"s3Enabled"`-}--type preparedSystemBackupInternal struct {- run *backuptypes.SystemBackupRun- recoveryKey string- localEnabled bool- s3Enabled bool-}--func (s *Service) createBackupInternal(ctx context.Context, trigger string, request backuptypes.CreateSystemBackupRequest) (*backuptypes.SystemBackupRun, error) {- prepared, err := s.prepareBackupInternal(ctx, trigger, request)- if err != nil {- return nil, err- }- return s.executeBackupInternal(ctx, prepared)-}--func (s *Service) prepareBackupInternal(ctx context.Context, trigger string, request backuptypes.CreateSystemBackupRequest) (*preparedSystemBackupInternal, error) {- if !s.SupportedDatabaseProvider() {- return nil, errors.New("arcane system recovery currently requires the SQLite database provider")- }- recoveryKey, err := s.recoveryKeyInternal(ctx, request.RecoveryKey)- if err != nil {- return nil, err- }- localEnabled, s3Enabled, destinationID, destination, err := s.resolveBackupPlanInternal(ctx, request)- if err != nil {- return nil, err+ case checkpoint.LocalEnabled:+ destination = backuptypes.SystemBackupDestinationLocal}run := &backuptypes.SystemBackupRun{- CreatedAt: time.Now().UTC(), Status: backuptypes.SystemBackupStatusRunning,+ ID: "system-" + uuid.New().String(), CreatedAt: time.Now().UTC(), Status: backuptypes.SystemBackupStatusRunning,Trigger: trigger, Destination: destination, S3DestinationID: destinationID, PolicyID: request.PolicyID,}- run.ID = "system-" + uuid.New().String()- if createBackupRunErr := s.store.CreateRun(ctx, run); createBackupRunErr != nil {- return nil, createBackupRunErr+ if createErr := s.store.CreateRun(ctx, run); createErr != nil {+ return nil, createErr}- checkpoint, err := json.Marshal(systemBackupRecoveryInternal{BackupID: run.ID, LocalEnabled: localEnabled, S3Enabled: s3Enabled})+ checkpoint.BackupID = run.ID+ recoveryData, err := json.Marshal(checkpoint)if err == nil {- err = jobcontext.Progress(ctx, scheduler.TargetOutcome{ResourceType: "system_backup", ID: cmp.Or(run.PolicyID, run.ID), Status: scheduler.Running, RecoveryData: checkpoint})+ err = jobcontext.Progress(ctx, scheduler.TargetOutcome{ResourceType: "system_backup", ID: cmp.Or(run.PolicyID, run.ID), Status: scheduler.Running, RecoveryData: recoveryData})}if err != nil {return nil, err}- return &preparedSystemBackupInternal{run: run, recoveryKey: recoveryKey, localEnabled: localEnabled, s3Enabled: s3Enabled}, nil+ return &preparedSystemBackup{run: run, recoveryKey: recoveryKey, checkpoint: checkpoint}, nil}-func (s *Service) executeBackupInternal(ctx context.Context, prepared *preparedSystemBackupInternal) (_ *backuptypes.SystemBackupRun, err error) {- run, recoveryKey := prepared.run, prepared.recoveryKey- localEnabled, s3Enabled, destinationID := prepared.localEnabled, prepared.s3Enabled, run.S3DestinationID-- defer func() { err = s.completeBackupInternal(ctx, run, err) }()+// executeBackup stages one snapshot, keeps it locally and/or replicates it to S3, then records the run's result.+func (s *Service) executeBackup(ctx context.Context, prepared *preparedSystemBackup) (_ *backuptypes.SystemBackupRun, err error) {+ run, key, checkpoint := prepared.run, prepared.recoveryKey, prepared.checkpoint+ defer func() {+ if err = cmp.Or(err, ctx.Err()); err != nil {+ run.Status, run.Error = backuptypes.SystemBackupStatusFailed, err.Error()+ } else {+ run.Status, run.Error = backuptypes.SystemBackupStatusSucceeded, ""+ }+ if saveErr := s.store.SaveRun(context.WithoutCancel(ctx), run); saveErr != nil {+ err = errors.Join(err, fmt.Errorf("failed to save system backup result: %w", saveErr))+ }+ if err == nil {+ err = jobcontext.Progress(ctx, scheduler.TargetOutcome{ResourceType: "system_backup", ID: cmp.Or(run.PolicyID, run.ID), Status: scheduler.Succeeded})+ }+ }()defer utils.RecoverToError(&err, "system backup")+ progress := func(stage string, status scheduler.RunStatus, message string) error {+ return jobcontext.Progress(ctx, scheduler.TargetOutcome{ResourceType: "backup_destination", ID: run.ID + ":" + stage, Status: status, Message: message})+ }dockerClient, err := s.dockerService.GetClient(ctx)if err != nil {return run, err}- if progressErr := jobcontext.Progress(ctx, scheduler.TargetOutcome{ResourceType: "backup_destination", ID: run.ID + ":stage", Status: scheduler.Running}); progressErr != nil {+ if progressErr := progress("stage", scheduler.Running, ""); progressErr != nil {return run, progressErr}- stagedSnapshot, stagingRepository, directRemote, err := s.createStagedSystemRecoverySnapshotInternal(- ctx, dockerClient, recoveryKey, run.ID, destinationID, localEnabled, s3Enabled,- )- if err != nil {- return run, err- }- if startDestinationProgressErr := jobcontext.Progress(- ctx,- scheduler.TargetOutcome{- ResourceType: "backup_destination",- ID: run.ID + ":stage",- Status: scheduler.Succeeded,- Message: stagedSnapshot.ID,- },- ); startDestinationProgressErr != nil {- return run, startDestinationProgressErr+ local, err := s.localRepository(ctx, dockerClient, false)+ var staged backup.Snapshot+ if err == nil {+ staged, err = s.snapshots.Create(ctx, dockerClient, local, key, run.ID)}+ directRemote := err != nilif directRemote {- run.RemoteSnapshotID, run.Size = stagedSnapshot.ID, stagedSnapshot.Size- if saveDisabledBackupErr := s.store.SaveRun(ctx, run); saveDisabledBackupErr != nil {- return run, saveDisabledBackupErr- }- return run, nil- }- localRepository := stagingRepository- // Both destinations use the same staged database copy.- // The staged snapshot is kept only for local-enabled runs. Forgetting it in- // a defer covers the failure paths too: a failed replication otherwise- // orphans one unreferenced snapshot in the staging repository per run.- defer func() {- if localEnabled {- return+ stagingErr := fmt.Errorf("failed to create local system recovery staging snapshot: %w", err)+ if checkpoint.LocalEnabled || !checkpoint.S3Enabled {+ return run, stagingErr}- cleanupCtx := context.WithoutCancel(ctx)- if forgetErr := s.engine.ForgetSnapshots(cleanupCtx, dockerClient, localRepository, recoveryKey, []string{stagedSnapshot.ID}); forgetErr != nil {- slog.WarnContext(cleanupCtx, "failed to remove staged system recovery snapshot", "snapshotId", stagedSnapshot.ID, "error", forgetErr)+ // S3-only backups write straight to S3 when local staging storage is unusable.+ direct, repoErr := s.remoteRepository(ctx, run.S3DestinationID)+ if repoErr != nil {+ return run, errors.Join(stagingErr, repoErr)}- }()- if localEnabled {- run.LocalSnapshotID, run.Size = stagedSnapshot.ID, stagedSnapshot.Size- if saveLocalBackupErr := s.store.SaveRun(ctx, run); saveLocalBackupErr != nil {- return run, saveLocalBackupErr- }- if localDestinationProgressErr := jobcontext.Progress(- ctx,- scheduler.TargetOutcome{- ResourceType: "backup_destination",- ID: run.ID + ":local",- Status: scheduler.Succeeded,- Message: stagedSnapshot.ID,- },- ); localDestinationProgressErr != nil {- return run, localDestinationProgressErr+ if staged, err = s.snapshots.Create(ctx, dockerClient, direct, key, run.ID); err != nil {+ return run, errors.Join(stagingErr, fmt.Errorf("failed to create S3 system recovery snapshot: %w", err))}}- if s3Enabled {- if replicateBackupErr := s.replicateBackupInternal(ctx, dockerClient, prepared, localRepository, stagedSnapshot); replicateBackupErr != nil {- return run, replicateBackupErr- }+ if progressErr := progress("stage", scheduler.Succeeded, staged.ID); progressErr != nil {+ return run, progressErr}- return run, nil-}--func (s *Service) completeBackupInternal(ctx context.Context, run *backuptypes.SystemBackupRun, err error) error {- if err == nil {- err = ctx.Err()+ if directRemote {+ run.RemoteSnapshotID, run.Size = staged.ID, staged.Size+ return run, s.store.SaveRun(ctx, run)}- if err != nil {- run.Status, run.Error = backuptypes.SystemBackupStatusFailed, err.Error()+ if checkpoint.LocalEnabled {+ run.LocalSnapshotID, run.Size = staged.ID, staged.Size+ if saveErr := s.store.SaveRun(ctx, run); saveErr != nil {+ return run, saveErr+ }+ if progressErr := progress("local", scheduler.Succeeded, staged.ID); progressErr != nil {+ return run, progressErr+ }} else {- run.Status, run.Error = backuptypes.SystemBackupStatusSucceeded, ""- }- if saveErr := s.store.SaveRun(context.WithoutCancel(ctx), run); saveErr != nil {- err = errors.Join(err, fmt.Errorf("failed to save system backup result: %w", saveErr))- }- if err == nil {- err = jobcontext.Progress(ctx, scheduler.TargetOutcome{ResourceType: "system_backup", ID: cmp.Or(run.PolicyID, run.ID), Status: scheduler.Succeeded})- }- return err-}--func (- s *Service,-) replicateBackupInternal(- ctx context.Context,- dockerClient *client.Client,- prepared *preparedSystemBackupInternal,- localRepository backup.Repository,- stagedSnapshot backup.Snapshot,-) error {- run := prepared.run- remoteRepository, repoErr := s.remoteRepositoryInternal(ctx, run.S3DestinationID)- if repoErr != nil {- return repoErr+ // S3-only runs forget the staged snapshot even when replication fails.+ defer func() {+ cleanupCtx := context.WithoutCancel(ctx)+ if forgetErr := s.engine.ForgetSnapshots(cleanupCtx, dockerClient, local, key, []string{staged.ID}); forgetErr != nil {+ slog.WarnContext(cleanupCtx, "failed to remove staged system recovery snapshot", "snapshotId", staged.ID, "error", forgetErr)+ }+ }()}- if err := jobcontext.Progress(ctx, scheduler.TargetOutcome{ResourceType: "backup_destination", ID: run.ID + ":remote", Status: scheduler.Running}); err != nil {- return err+ if !checkpoint.S3Enabled {+ return run, nil}- remoteSnapshot, replicateErr := s.engine.Replicate(- ctx,- dockerClient,- localRepository,- stagedSnapshot.ID,- remoteRepository,- prepared.recoveryKey,- "arcane-system-recovery",- backup.RunSnapshotTag(- run.ID,- ),- )- if replicateErr != nil {- return fmt.Errorf("failed to create S3 system recovery snapshot: %w", replicateErr)+ remote, err := s.remoteRepository(ctx, run.S3DestinationID)+ if err != nil {+ return run, err}- run.RemoteSnapshotID = remoteSnapshot.ID- if run.Size == 0 {- run.Size = remoteSnapshot.Size+ if progressErr := progress("remote", scheduler.Running, ""); progressErr != nil {+ return run, progressErr}- if err := s.store.SaveRun(ctx, run); err != nil {- return err+ snapshot, err := s.engine.Replicate(ctx, dockerClient, local, staged.ID, remote, key, "arcane-system-recovery", backup.RunSnapshotTag(run.ID))+ if err != nil {+ return run, fmt.Errorf("failed to create S3 system recovery snapshot: %w", err)}- if err := jobcontext.Progress(- ctx,- scheduler.TargetOutcome{- ResourceType: "backup_destination",- ID: run.ID + ":remote",- Status: scheduler.Succeeded,- Message: remoteSnapshot.ID,- },- ); err != nil {- return err+ run.RemoteSnapshotID, run.Size = snapshot.ID, cmp.Or(run.Size, snapshot.Size)+ if saveErr := s.store.SaveRun(ctx, run); saveErr != nil {+ return run, saveErr}- return nil+ return run, progress("remote", scheduler.Succeeded, snapshot.ID)}func (s *Service) ListBackups(ctx context.Context, params pagination.QueryParams) ([]backuptypes.SystemBackupRun, pagination.Response, error) {@@ -629,16 +525,11 @@ func (s *Service) ListBackups(ctx context.Context, params pagination.QueryParamsif err != nil {return nil, pagination.Response{}, fmt.Errorf("failed to list system backups: %w", err)}- names := make(map[string]backuptypes.S3Destination)- if s.s3Destinations != nil {- if available, listErr := s.s3Destinations.ListS3DestinationsByID(ctx); listErr == nil {- names = available- }- }- remoteAvailable := backup.RemoteSnapshotChecker(ctx, s.s3Destinations, "arcane-system-recovery")+ names := s.s3Destinations.DestinationsByID(ctx)+ remoteAvailable := backup.RemoteSnapshotChecker(ctx, s.s3Destinations)for i := range result {result[i].S3DestinationName = names[result[i].S3DestinationID].Name- result[i].RemoteAvailable = remoteAvailable(result[i].S3DestinationID, result[i].RemoteSnapshotID)+ result[i].RemoteAvailable = remoteAvailable(result[i].S3DestinationID, "arcane-system-recovery", result[i].RemoteSnapshotID)}return result, response, nil}@@ -656,7 +547,7 @@ func (s *Service) BrowseBackupFiles(ctx context.Context, id, recoveryKey, requesif run.Status != backuptypes.SystemBackupStatusSucceeded {return nil, pagination.Response{}, errors.New("only successful system backups can be opened")}- key, err := s.recoveryKeyInternal(ctx, recoveryKey)+ key, err := s.recoveryKey(ctx, recoveryKey)if err != nil {return nil, pagination.Response{}, err}@@ -670,7 +561,7 @@ func (s *Service) BrowseBackupFiles(ctx context.Context, id, recoveryKey, reques// RestoreBackupFiles restores selected project files into the current projects// directory after creating a safety snapshot.func (s *Service) RestoreBackupFiles(ctx context.Context, id string, request backuptypes.RestoreSystemBackupFilesRequest, user usertypes.Actor) error {- lease, err := s.acquireRunInternal(ctx)+ lease, err := s.acquireRun(ctx)if err != nil {return err}@@ -682,7 +573,7 @@ func (s *Service) RestoreBackupFiles(ctx context.Context, id string, request bacif run.Status != backuptypes.SystemBackupStatusSucceeded {return errors.New("only successful system backups can be restored")}- key, err := s.recoveryKeyInternal(ctx, request.RecoveryKey)+ key, err := s.recoveryKey(ctx, request.RecoveryKey)if err != nil {return err}@@ -690,17 +581,20 @@ func (s *Service) RestoreBackupFiles(ctx context.Context, id string, request bacif err != nil {return err}- return s.snapshots.RestoreFiles(ctx, dockerClient, key, *run, request.RestoreSelection, func(- ctx context.Context, safetyRequest backuptypes.CreateSystemBackupRequest,- ) (*backuptypes.SystemBackupRun, error) {- return s.createBackupInternal(ctx, backuptypes.SystemBackupTriggerSafety, safetyRequest)- })+ safetyBackup := func(ctx context.Context, safetyRequest backuptypes.CreateSystemBackupRequest) (*backuptypes.SystemBackupRun, error) {+ prepared, prepareErr := s.prepareBackup(ctx, backuptypes.SystemBackupTriggerSafety, safetyRequest)+ if prepareErr != nil {+ return nil, prepareErr+ }+ return s.executeBackup(ctx, prepared)+ }+ return s.snapshots.RestoreFiles(ctx, dockerClient, key, *run, request.RestoreSelection, safetyBackup)}func (s *Service) DeleteBackup(ctx context.Context, id, recoveryKey string) error {// Deletes contend with create/restore/upload on the system run lease so a// slow operation cannot resurrect or double-forget snapshots.- lease, err := s.acquireRunInternal(ctx)+ lease, err := s.acquireRun(ctx)if err != nil {return err}@@ -709,27 +603,27 @@ func (s *Service) DeleteBackup(ctx context.Context, id, recoveryKey string) erroif err != nil {return err}- return s.deleteRunsInternal(ctx, []*backuptypes.SystemBackupRun{run}, recoveryKey, true)+ return s.deleteRuns(ctx, []*backuptypes.SystemBackupRun{run}, recoveryKey, true)}-// deleteRunsInternal forgets snapshots grouped by repository under the caller's run lease.-func (s *Service) deleteRunsInternal(ctx context.Context, localRuns2 []*backuptypes.SystemBackupRun, recoveryKey string, includeRemote bool) error {+// deleteRuns forgets snapshots grouped by repository under the caller's run lease.+func (s *Service) deleteRuns(ctx context.Context, backupRuns []*backuptypes.SystemBackupRun, recoveryKey string, includeRemote bool) error {var localRuns []*backuptypes.SystemBackupRun+ var localIDs []stringremoteGroups := make(map[string][]*backuptypes.SystemBackupRun)- for _, run := range localRuns2 {+ for _, run := range backupRuns {if run.LocalSnapshotID != "" {- localRuns = append(localRuns, run)+ localRuns, localIDs = append(localRuns, run), append(localIDs, run.LocalSnapshotID)}if includeRemote && run.RemoteSnapshotID != "" {remoteGroups[run.S3DestinationID] = append(remoteGroups[run.S3DestinationID], run)}}- // A run without snapshots (a failed attempt) is just a row; deleting it- // needs neither the key nor Rustic.+ // A run without snapshots (a failed attempt) is just a row; deleting it needs neither the key nor Rustic.key := ""if len(localRuns) > 0 || len(remoteGroups) > 0 {var err error- if key, err = s.recoveryKeyInternal(ctx, recoveryKey); err != nil {+ if key, err = s.recoveryKey(ctx, recoveryKey); err != nil {return err}}@@ -737,14 +631,27 @@ func (s *Service) deleteRunsInternal(ctx context.Context, localRuns2 []*backuptyif err != nil {return err}- deleteErr := s.forgetLocalSnapshotsInternal(ctx, dockerClient, key, localRuns)+ var deleteErr error+ if len(localRuns) > 0 {+ repository, repoErr := s.localRepository(ctx, dockerClient, false)+ if repoErr == nil {+ repoErr = s.engine.ForgetSnapshots(ctx, dockerClient, repository, key, localIDs)+ }+ if repoErr != nil {+ deleteErr = fmt.Errorf("failed to delete local snapshots: %w", repoErr)+ } else {+ for _, run := range localRuns {+ run.LocalSnapshotID = ""+ }+ }+ }for destinationID, group := range remoteGroups {- deleteErr = errors.Join(deleteErr, s.forgetRemoteSnapshotsInternal(ctx, dockerClient, key, destinationID, group))+ deleteErr = errors.Join(deleteErr, s.forgetRemoteSnapshots(ctx, dockerClient, key, destinationID, group))}- for _, run := range localRuns2 {+ for _, run := range backupRuns {if run.LocalSnapshotID == "" && run.RemoteSnapshotID == "" {- if deleteBackupRunErr := s.store.DeleteRun(ctx, run.ID); deleteBackupRunErr != nil {- deleteErr = errors.Join(deleteErr, fmt.Errorf("failed to delete system backup record: %w", deleteBackupRunErr))+ if deleteRunErr := s.store.DeleteRun(ctx, run.ID); deleteRunErr != nil {+ deleteErr = errors.Join(deleteErr, fmt.Errorf("failed to delete system backup record: %w", deleteRunErr))}continue}@@ -763,6 +670,34 @@ func (s *Service) deleteRunsInternal(ctx context.Context, localRuns2 []*backuptyreturn deleteErr}+// forgetRemoteSnapshots forgets one S3 destination's snapshots and drops the runs' references to them.+func (s *Service) forgetRemoteSnapshots(ctx context.Context, dockerClient *client.Client, key, destinationID string, group []*backuptypes.SystemBackupRun) error {+ snapshotIDs := make([]string, len(group))+ for index, run := range group {+ snapshotIDs[index] = run.RemoteSnapshotID+ }+ repository, repoErr := s.remoteRepository(ctx, destinationID)+ var observation backuptypes.RepositoryObservation+ if repoErr == nil {+ observation, repoErr = backup.CheckRemoteRepository(ctx, s.s3Destinations, destinationID, "arcane-system-recovery")+ }+ switch {+ case repoErr != nil:+ case !observation.Available:+ // Nothing remains to forget; drop the references so the run can be removed.+ slog.WarnContext(ctx, "S3 repository is missing; dropping remote snapshot references", "destinationId", destinationID, "snapshots", len(snapshotIDs))+ default:+ repoErr = s.engine.ForgetSnapshots(ctx, dockerClient, repository, key, snapshotIDs)+ }+ if repoErr != nil {+ return fmt.Errorf("failed to delete S3 snapshots: %w", repoErr)+ }+ for _, run := range group {+ run.RemoteSnapshotID, run.S3DestinationID = "", ""+ }+ return nil+}+// PruneLocalRepository frees the data of deleted local system backups once Rustic's keep-delete window passes.func (s *Service) PruneLocalRepository(ctx context.Context) error {key, err := s.recoveryKeys.Get(ctx)@@ -776,67 +711,19 @@ func (s *Service) PruneLocalRepository(ctx context.Context) error {if err != nil {return err}- repository, err := s.localRepositoryInternal(ctx, dockerClient, false)+ repository, err := s.localRepository(ctx, dockerClient, false)if err != nil {return err}return s.engine.ForgetSnapshots(ctx, dockerClient, repository, key, nil)}-func (s *Service) forgetLocalSnapshotsInternal(ctx context.Context, dockerClient *client.Client, key string, localRuns []*backuptypes.SystemBackupRun) error {- if len(localRuns) == 0 {- return nil- }- snapshotIDs := make([]string, len(localRuns))- for index, run := range localRuns {- snapshotIDs[index] = run.LocalSnapshotID- }- repository, repoErr := s.localRepositoryInternal(ctx, dockerClient, false)- if repoErr == nil {- repoErr = s.engine.ForgetSnapshots(ctx, dockerClient, repository, key, snapshotIDs)- }- if repoErr != nil {- return fmt.Errorf("failed to delete local snapshots: %w", repoErr)- }- for _, run := range localRuns {- run.LocalSnapshotID = ""- }- return nil-}--func (s *Service) forgetRemoteSnapshotsInternal(ctx context.Context, dockerClient *client.Client, key, destinationID string, localRuns []*backuptypes.SystemBackupRun) error {- snapshotIDs := make([]string, len(localRuns))- for index, run := range localRuns {- snapshotIDs[index] = run.RemoteSnapshotID- }- repository, repoErr := s.remoteRepositoryInternal(ctx, destinationID)- if repoErr == nil {- var observation backuptypes.RepositoryObservation- observation, repoErr = backup.CheckRemoteRepository(ctx, s.s3Destinations, destinationID, "arcane-system-recovery")- switch {- case repoErr != nil:- case !observation.Available:- // Nothing remains to forget; drop the references so the run can be removed.- slog.WarnContext(ctx, "S3 repository is missing; dropping remote snapshot references", "destinationId", destinationID, "snapshots", len(snapshotIDs))- default:- repoErr = s.engine.ForgetSnapshots(ctx, dockerClient, repository, key, snapshotIDs)- }- }- if repoErr != nil {- return fmt.Errorf("failed to delete S3 snapshots: %w", repoErr)- }- for _, run := range localRuns {- run.RemoteSnapshotID, run.S3DestinationID = "", ""- }- return nil-}-func (s *Service) DiscoverRemoteBackups(ctx context.Context, request backuptypes.DiscoverSystemBackupsRequest) (int, error) {- recoveryKey, err := s.recoveryKeyInternal(ctx, request.RecoveryKey)+ recoveryKey, err := s.recoveryKey(ctx, request.RecoveryKey)if err != nil {return 0, err}- repository, err := s.remoteRepositoryInternal(ctx, request.S3DestinationID)+ repository, err := s.remoteRepository(ctx, request.S3DestinationID)if err != nil {return 0, err}@@ -852,28 +739,17 @@ func (s *Service) DiscoverRemoteBackups(ctx context.Context, request backuptypesif err != nil {return 0, err}- known := make(map[string]struct{}, len(knownIDs))- for _, id := range knownIDs {- known[id] = struct{}{}- }created := 0for _, snapshot := range listedSnapshots {- if strings.TrimSpace(snapshot.ID) == "" {- continue- }- if _, exists := known[snapshot.ID]; exists {+ if strings.TrimSpace(snapshot.ID) == "" || slices.Contains(knownIDs, snapshot.ID) {continue}- createdAt := snapshot.Time- if createdAt.IsZero() {- createdAt = time.Now().UTC()- }run := &backuptypes.SystemBackupRun{- Size: snapshot.Summary.TotalBytesProcessed, CreatedAt: createdAt, Status: backuptypes.SystemBackupStatusSucceeded,+ ID: fmt.Sprintf("remote-%s-%s", request.S3DestinationID, snapshot.ID), Size: snapshot.Summary.TotalBytesProcessed,+ CreatedAt: kit.Ternary(snapshot.Time.IsZero(), time.Now().UTC(), snapshot.Time), Status: backuptypes.SystemBackupStatusSucceeded,Trigger: backuptypes.SystemBackupTriggerManual, Destination: backuptypes.SystemBackupDestinationS3,RemoteSnapshotID: snapshot.ID, S3DestinationID: request.S3DestinationID,}- run.ID = fmt.Sprintf("remote-%s-%s", request.S3DestinationID, snapshot.ID)if createDiscoveredBackupErr := s.store.CreateRun(ctx, run); createDiscoveredBackupErr != nil {return created, fmt.Errorf("failed to save discovered system backup: %w", createDiscoveredBackupErr)}@@ -883,7 +759,7 @@ func (s *Service) DiscoverRemoteBackups(ctx context.Context, request backuptypes}func (s *Service) UploadBackup(ctx context.Context, id string, request backuptypes.UploadSystemBackupRequest) (*backuptypes.SystemBackupRun, error) {- lease, err := s.acquireRunInternal(ctx)+ lease, err := s.acquireRun(ctx)if err != nil {return nil, err}@@ -898,7 +774,7 @@ func (s *Service) UploadBackup(ctx context.Context, id string, request backuptypif run.RemoteSnapshotID != "" {return nil, errors.New("system backup has already been uploaded")}- key, err := s.recoveryKeyInternal(ctx, request.RecoveryKey)+ key, err := s.recoveryKey(ctx, request.RecoveryKey)if err != nil {return nil, err}@@ -909,11 +785,11 @@ func (s *Service) UploadBackup(ctx context.Context, id string, request backuptypif err != nil {return nil, err}- localRepository, err := s.localRepositoryInternal(ctx, dockerClient, true)+ localRepository, err := s.localRepository(ctx, dockerClient, true)if err != nil {return nil, err}- remoteRepository, err := s.remoteRepositoryInternal(ctx, request.S3DestinationID)+ remoteRepository, err := s.remoteRepository(ctx, request.S3DestinationID)if err != nil {return nil, err}@@ -937,7 +813,7 @@ func (s *Service) RestoreBackup(ctx context.Context, id, recoveryKey string, useif run.Status != backuptypes.SystemBackupStatusSucceeded {return errors.New("only successful system backups can be restored")}- key, err := s.recoveryKeyInternal(ctx, recoveryKey)+ key, err := s.recoveryKey(ctx, recoveryKey)if err != nil {return err}@@ -959,17 +835,17 @@ func (s *Service) RestoreBackup(ctx context.Context, id, recoveryKey string, useif appDataMount == nil {return errors.New("arcane system restore requires /app/data to be mounted")}- dataDirectory, err := s.dataDirectoryInternal()+ databaseFile, err := s.databaseFile()if err != nil {return err}- projectsDirectory := s.projectsDirectoryInternal(ctx)+ dataDirectory := filepath.Dir(databaseFile)+ projectsDirectory := projects.ResolveConfiguredContainerDirectory(s.projectsSetting(ctx), "/app/data/projects")plan, err := s.snapshots.PlanRestore(ctx, dockerClient, key, *run, current.Mounts, dataDirectory, projectsDirectory)if err != nil {return err}- // A local safety snapshot is created before the detached helper is allowed to- // stop Arcane and replace its data.+ // A local safety snapshot is created before the detached helper is allowed to stop Arcane and replace its data.safetyBackup, err := s.CreateBackup(ctx, user, backuptypes.SystemBackupTriggerSafety, backuptypes.CreateSystemBackupRequest{Destination: backuptypes.SystemBackupDestinationLocal,RecoveryKey: key,@@ -981,15 +857,23 @@ func (s *Service) RestoreBackup(ctx context.Context, id, recoveryKey string, useif err != nil {return err}+ // Rustic needs a named network to reach network-local S3 once Arcane stops; bridge and none carry no DNS.+ rusticNetwork := ""+ if current.NetworkSettings != nil {+ names := slices.Sorted(maps.Keys(current.NetworkSettings.Networks))+ if index := slices.IndexFunc(names, func(name string) bool { return name != "bridge" && name != "none" }); index >= 0 {+ rusticNetwork = names[index]+ }+ }request := recovery.RestoreRequest{BackupID: run.ID, ContainerID: current.ID, ContainerImage: current.Config.Image, SnapshotID: plan.SnapshotID, RecoveryKey: key,LocalSnapshotID: run.LocalSnapshotID, RemoteSnapshotID: run.RemoteSnapshotID, S3DestinationID: run.S3DestinationID,- Size: run.Size, NetworkMode: rusticRestoreNetworkModeInternal(¤t),+ Size: run.Size, NetworkMode: rusticNetwork,SafetyBackup: &recovery.SafetyBackup{ID: safetyBackup.ID, LocalSnapshotID: safetyBackup.LocalSnapshotID,Size: safetyBackup.Size, CreatedAt: safetyBackup.CreatedAt,},- ProjectsSetting: s.projectsSettingInternal(ctx), ProjectsIncluded: plan.ProjectsIncluded,+ ProjectsSetting: s.projectsSetting(ctx), ProjectsIncluded: plan.ProjectsIncluded,Stages: plan.Stages, RollbackStages: rollbackStages,}requestData, err := json.Marshal(request)@@ -1025,14 +909,13 @@ func (s *Service) RestoreBackup(ctx context.Context, id, recoveryKey string, useif err != nil {return fmt.Errorf("resolve Arcane executable: %w", err)}- helperExecutable, executableMount, err := recoveryHelperExecutableInternal(current.Mounts, executablePath)- if err != nil {- return fmt.Errorf("resolve recovery helper executable: %w", err)- }- mounts := append([]mount.Mount{}, runtimeMounts...)- mounts = append(mounts, *appDataMount)- if executableMount != nil {- mounts = append(mounts, *executableMount)+ helperExecutable, mounts := executablePath, append(slices.Clone(runtimeMounts), *appDataMount)+ if executableMount := docker.MountForSubpath(current.Mounts, executablePath, systemRecoveryHelperPath); executableMount != nil {+ if executableMount.Type != mount.TypeBind {+ return errors.New("resolve recovery helper executable: the Arcane executable must come from the image or a bind mount")+ }+ executableMount.ReadOnly = true+ helperExecutable, mounts = systemRecoveryHelperPath, append(mounts, *executableMount)}hostConfig := &container.HostConfig{AutoRemove: true, Mounts: mounts, NetworkMode: networkMode}if current.HostConfig != nil {@@ -1059,22 +942,6 @@ func (s *Service) RestoreBackup(ctx context.Context, id, recoveryKey string, usereturn nil}-// rusticRestoreNetworkModeInternal picks a named network Arcane is attached to-// so the detached restore's Rustic container can still resolve network-local-// S3 endpoints once Arcane is stopped. The default bridge carries no DNS, so-// it (and none) fall back to the default network.-func rusticRestoreNetworkModeInternal(current *container.InspectResponse) string {- if current == nil || current.NetworkSettings == nil {- return ""- }- for _, name := range slices.Sorted(maps.Keys(current.NetworkSettings.Networks)) {- if name != "bridge" && name != "none" {- return name- }- }- return ""-}-func (s *Service) GenerateRecoveryKey() (*backuptypes.SystemBackupRecoveryKey, error) {recoveryKey, err := backup.GenerateRecoveryKey()if err != nil {@@ -1087,15 +954,13 @@ func (s *Service) SetRecoveryKey(ctx context.Context, recoveryKey string) (*backif err := backup.ValidateRecoveryKey(recoveryKey); err != nil {return nil, err}- // The repositories are encrypted with the key that initialized them and a- // different key cannot open them, so a rotation would silently break every- // scheduled backup and keyless restore. Refuse it while backups exist.+ // Repositories only open with the key that initialized them, so refuse a rotation while backups exist.configured, err := s.recoveryKeys.Configured(ctx)if err != nil {return nil, err}if configured {- if current, keyErr := s.recoveryKeyInternal(ctx, ""); keyErr == nil && current != recoveryKey {+ if current, keyErr := s.recoveryKeys.Get(ctx); keyErr == nil && current != recoveryKey {localRuns, countLocalBackupsErr := s.store.CountRuns(ctx)if countLocalBackupsErr != nil {return nil, countLocalBackupsErr@@ -1135,12 +1000,7 @@ func (s *Service) GetPolicies(ctx context.Context) (*backuptypes.SystemBackupPolresult := &backuptypes.SystemBackupPolicyCollection{Policies: make([]backuptypes.SystemBackupPolicy, 0, len(policies)), RecoveryKeyStored: configured,}- destinations := make(map[string]backuptypes.S3Destination)- if s.s3Destinations != nil {- if available, listErr := s.s3Destinations.ListS3DestinationsByID(ctx); listErr == nil {- destinations = available- }- }+ destinations := s.s3Destinations.DestinationsByID(ctx)for i := range policies {lastRun, runErr := s.store.LatestPolicyRun(ctx, policies[i].ID)if runErr != nil {@@ -1178,7 +1038,7 @@ func (s *Service) UpdatePolicies(ctx context.Context, updates []backuptypes.Updapolicy.LocalEnabled, policy.S3Enabled, policy.S3DestinationID = update.LocalEnabled, update.S3Enabled, update.S3DestinationIDreturn nil}- if runErr := s.store.ReconcilePolicies(ctx, updates, build, s.jobs.Unregister, s.rescheduleSystemBackupPolicyInternal); runErr != nil {+ if runErr := s.store.ReconcilePolicies(ctx, updates, build, s.jobs.Unregister, s.rescheduleSystemBackupPolicy); runErr != nil {return nil, runErr}return s.GetPolicies(ctx)@@ -1190,7 +1050,7 @@ func (s *Service) SetScheduler(ctx context.Context, dynamicScheduler scheduler.Dreturn s.jobs.SetScheduler(ctx, dynamicScheduler, admissionGate)}-func (s *Service) runScheduledBackupInternal(ctx context.Context, policyID string) (scheduler.Outcome, error) {+func (s *Service) runScheduledBackup(ctx context.Context, policyID string) (scheduler.Outcome, error) {policy, loadErr := s.store.Policy(ctx, policyID)if loadErr != nil {return scheduler.Outcome{}, loadErr@@ -1198,64 +1058,64 @@ func (s *Service) runScheduledBackupInternal(ctx context.Context, policyID strinif policy == nil || !policy.Enabled {return scheduler.Outcome{Status: scheduler.Skipped}, nil}- if previous, ok := jobcontext.Run(ctx); ok {- outcome := jobcontext.ConfirmedTarget(previous, policy.ID)- if outcome.Status == scheduler.Succeeded {- if policy.RetentionCount > 0 {- if err := s.applyRetentionInternal(ctx, policy.ID, policy.RetentionCount, policy.S3Enabled); err != nil {- outcome.Status = scheduler.Partial- return outcome, err- }- }- return outcome, nil- }+ var remoteErr error+ if policy.S3Enabled {+ remoteErr = s.store.CheckScheduledRemote(ctx, policy.S3DestinationID)}- remoteDisabled, checkErr := s.disableMissingS3Internal(ctx, policy)- if checkErr != nil {- return scheduler.Outcome{}, checkErr+ remoteDisabled, disableErr := backup.DisableMissingRemote(remoteErr, policy.LocalEnabled, &policy.S3Enabled, &policy.Enabled, func(field string) (bool, error) {+ return s.store.DisablePolicyRemote(ctx, policy, field)+ })+ if disableErr != nil {+ return scheduler.Outcome{}, disableErr+ }+ if remoteDisabled {+ s.rescheduleSystemBackupPolicy(ctx, policy)}if remoteDisabled && !policy.LocalEnabled {return scheduler.Outcome{Status: scheduler.NeedsAttention, Message: backup.RemoteDisabledMessage}, nil}- var run *backuptypes.SystemBackupRun- activityID, runErr := activitylib.RunHandlerActivity(ctx, s.activityService, activitylib.HandlerOptions{- EnvironmentID: "0", Type: activitytypes.TypeResourceAction, ResourceType: "system_backup",- ResourceID: policy.ID, ResourceName: "Arcane", User: &usertypes.SystemUser,- Step: "Creating scheduled system backup", Message: "Creating scheduled Arcane system backup",- SuccessMessage: "Scheduled Arcane system backup created successfully",- Metadata: database.JSON{- "action": "scheduled_system_backup", "policyId": policy.ID, "schedule": policy.Schedule,- "retentionCount": policy.RetentionCount, "localEnabled": policy.LocalEnabled,- "s3Enabled": policy.S3Enabled, "s3DestinationId": policy.S3DestinationID,- },- }, func(activityCtx context.Context) error {- var backupErr error- run, backupErr = s.CreateBackup(activityCtx, usertypes.SystemUser, backuptypes.SystemBackupTriggerScheduled,- backuptypes.CreateSystemBackupRequest{PolicyID: policy.ID})- return backupErr- })+ run, runErr := s.CreateBackup(ctx, usertypes.SystemUser, backuptypes.SystemBackupTriggerScheduled, backuptypes.CreateSystemBackupRequest{PolicyID: policy.ID})if errors.Is(runErr, ErrSystemBackupAlreadyRunning) {slog.InfoContext(ctx, "Scheduled Arcane system backup skipped; another backup is running", "policyId", policy.ID)- return scheduler.Outcome{Status: scheduler.Skipped}, nil+ return scheduler.Outcome{Status: scheduler.Skipped, Message: "Skipped: another system backup is running"}, nil}if runErr != nil {slog.ErrorContext(ctx, "Scheduled Arcane system backup failed", "policyId", policy.ID, "error", runErr)- return scheduler.Outcome{ActivityID: activityID}, runErr+ return scheduler.Outcome{}, runErr}if policy.RetentionCount > 0 {- if retentionErr := s.applyRetentionInternal(ctx, policy.ID, policy.RetentionCount, policy.S3Enabled); retentionErr != nil {+ if retentionErr := s.applyRetention(ctx, policy.ID, policy.RetentionCount, policy.S3Enabled); retentionErr != nil {slog.ErrorContext(ctx, "System backup retention failed", "policyId", policy.ID, "error", retentionErr)- return scheduler.Outcome{Status: scheduler.Partial, ActivityID: activityID, Message: "Backup completed but retention failed"}, retentionErr+ return scheduler.Outcome{Status: scheduler.Partial, Message: "Backup completed but retention failed"}, retentionErr}}slog.InfoContext(ctx, "Scheduled Arcane system backup completed", "backupId", run.ID, "policyId", policy.ID)if remoteDisabled {- return scheduler.Outcome{Status: scheduler.Partial, ActivityID: activityID, Message: backup.RemoteDisabledMessage}, nil+ return scheduler.Outcome{Status: scheduler.Partial, Message: backup.RemoteDisabledMessage}, nil+ }+ return scheduler.Outcome{Status: scheduler.Succeeded, Message: "Scheduled Arcane system backup created successfully"}, nil+}++// resumeScheduledBackup finishes the backup an interrupted scheduled run recorded, then applies retention.+func (s *Service) resumeScheduledBackup(ctx context.Context, previous scheduler.Run, policyID string) (scheduler.Outcome, error) {+ outcome, err := s.reconcileBackup(ctx, previous, policyID, "")+ if err != nil || outcome.Status != scheduler.Succeeded {+ return outcome, err+ }+ current, err := s.store.Policy(ctx, policyID)+ if err != nil {+ return outcome, err+ }+ if current != nil && current.RetentionCount > 0 {+ if applyRetentionErr := s.applyRetention(ctx, current.ID, current.RetentionCount, current.S3Enabled); applyRetentionErr != nil {+ outcome.Status = scheduler.Partial+ return outcome, applyRetentionErr+ }}- return scheduler.Outcome{Status: scheduler.Succeeded, ActivityID: activityID}, nil+ return outcome, nil}-func (s *Service) rescheduleSystemBackupPolicyInternal(ctx context.Context, policy *backuptypes.SystemBackupPolicy) {+func (s *Service) rescheduleSystemBackupPolicy(ctx context.Context, policy *backuptypes.SystemBackupPolicy) {if policy == nil {return}@@ -1264,35 +1124,30 @@ func (s *Service) rescheduleSystemBackupPolicyInternal(ctx context.Context, polireturn}policyID := policy.ID- s.jobs.Register(ctx, policyID,- func(ctx context.Context) string {+ s.jobs.Add(ctx, &flow.Job{+ Engine: s.flow,+ Workflow: s.policyWorkflow,+ JobName: s.jobs.JobName(policyID),+ Payload: policyID,+ Activity: activitylib.StartRequest{+ ResourceID: new(policyID),+ Metadata: database.JSON{+ "action": "scheduled_system_backup", "policyId": policyID, "schedule": policy.Schedule,+ "retentionCount": policy.RetentionCount, "localEnabled": policy.LocalEnabled,+ "s3Enabled": policy.S3Enabled, "s3DestinationId": policy.S3DestinationID,+ },+ },+ ScheduleFn: func(ctx context.Context) string {current, err := s.store.Policy(ctx, policyID)if err != nil || current == nil {return defaultSystemBackupSchedule}return current.Schedule},- func(ctx context.Context) (scheduler.Outcome, error) {- return s.runScheduledBackupInternal(ctx, policyID)+ FallbackFn: func(ctx context.Context, previous scheduler.Run) (scheduler.Outcome, error) {+ return s.resumeScheduledBackup(ctx, previous, policyID)},- func(ctx context.Context, previous scheduler.Run) (scheduler.Outcome, error) {- outcome, err := s.reconcileBackupInternal(ctx, previous, policyID)- if err != nil || outcome.Status != scheduler.Succeeded {- return outcome, err- }- current, err := s.store.Policy(ctx, policyID)- if err != nil {- return outcome, err- }- if current != nil && current.RetentionCount > 0 {- if applyRetentionErr := s.applyRetentionInternal(ctx, current.ID, current.RetentionCount, current.S3Enabled); applyRetentionErr != nil {- outcome.Status = scheduler.Partial- return outcome, applyRetentionErr- }- }- return outcome, nil- },- )+ })}func (s *Service) RegisterBackupJobOnStartup(ctx context.Context) {@@ -1302,52 +1157,31 @@ func (s *Service) RegisterBackupJobOnStartup(ctx context.Context) {return}for i := range policies {- s.rescheduleSystemBackupPolicyInternal(ctx, &policies[i])+ s.rescheduleSystemBackupPolicy(ctx, &policies[i])}volumePolicyCount := s.volumes.RegisterJobsOnStartup(ctx)slog.InfoContext(ctx, "Registered backup schedules", "systemPolicies", len(policies), "volumePolicies", volumePolicyCount)}-func (s *Service) applyRetentionInternal(ctx context.Context, policyID string, keep int, includeRemote bool) error {+func (s *Service) applyRetention(ctx context.Context, policyID string, keep int, includeRemote bool) error {expired, err := s.store.ExpiredRunIDs(ctx, policyID, keep)if err != nil || len(expired) == 0 {return err}- lease, err := s.acquireRunInternal(ctx)+ lease, err := s.acquireRun(ctx)if err != nil {return err}defer lease.Release(ctx)- localRuns, err := s.store.RunsByID(ctx, expired)+ expiredRuns, err := s.store.RunsByID(ctx, expired)if err != nil {return err}- return s.deleteRunsInternal(ctx, localRuns, "", includeRemote)+ return s.deleteRuns(ctx, expiredRuns, "", includeRemote)}-func (s *Service) disableMissingS3Internal(ctx context.Context, policy *backuptypes.SystemBackupPolicy) (bool, error) {- if !policy.S3Enabled {- return false, nil- }- err := s.store.CheckScheduledRemote(ctx, policy.S3DestinationID)- if !errors.Is(err, backup.ErrRemoteRepositoryMissing) {- return false, nil- }- disabled, err := s.store.DisablePolicyRemote(ctx, policy)- if err != nil || !disabled {- return false, err- }- if policy.LocalEnabled {- policy.S3Enabled = false- } else {- policy.Enabled = false- }- s.rescheduleSystemBackupPolicyInternal(ctx, policy)- return true, nil-}--// projectsSettingInternal returns the effective projectsDirectory setting.-func (s *Service) projectsSettingInternal(ctx context.Context) string {+// projectsSetting returns the effective projectsDirectory setting.+func (s *Service) projectsSetting(ctx context.Context) string {fallback := ""if s.config != nil {fallback = s.config.ProjectsDirectory@@ -1358,178 +1192,181 @@ func (s *Service) projectsSettingInternal(ctx context.Context) string {return s.settingsService.GetStringSetting(ctx, "projectsDirectory", fallback)}-// projectsDirectoryInternal resolves the configured projects directory to the-// container path Arcane reads, dropping any host-path mapping suffix.-func (s *Service) projectsDirectoryInternal(ctx context.Context) string {- return projects.ResolveConfiguredContainerDirectory(s.projectsSettingInternal(ctx), "/app/data/projects")-}--// StartBackup prepares a manual backup before submitting its snapshot work.+// StartBackup prepares a manual backup, then submits its workflow.func (s *Service) StartBackup(ctx context.Context, user usertypes.Actor, request backuptypes.CreateSystemBackupRequest) (*backuptypes.SystemBackupRun, error) {- lease, err := s.acquireRunInternal(ctx)+ requester := backup.NewRequester(ctx, user, "0", authz.PermSystemBackupsManage)+ if err := s.engine.Authorize(ctx, requester); err != nil {+ return nil, err+ }+ lease, err := s.acquireRun(ctx)if err != nil {return nil, err}- prepared, err := s.prepareBackupInternal(ctx, backuptypes.SystemBackupTriggerManual, request)+ prepared, err := s.prepareBackup(ctx, backuptypes.SystemBackupTriggerManual, request)if err != nil {lease.Release(ctx)return nil, err}- activityID, workCtx := activitylib.StartHandlerActivity(ctx, s.activityService, "0", activitytypes.TypeResourceAction, "system_backup", "arcane", "Arcane", &user,- "Creating system backup", "Creating Arcane system backup", database.JSON{- "action": "create_system_backup",- "backupId": prepared.run.ID,- "destination": prepared.run.Destination,- "s3DestinationId": prepared.run.S3DestinationID,- "policyId": prepared.run.PolicyID,- }, false)- finish := func(runErr error) {- defer lease.Release(ctx)- if runErr != nil {- if saveErr := s.store.FailRun(context.WithoutCancel(workCtx), prepared.run.ID, runErr.Error()); saveErr != nil {- runErr = errors.Join(runErr, fmt.Errorf("save backup failure: %w", saveErr))- }+ run := prepared.run+ release := s.engine.Hold(backup.SystemAdmissionScope, backup.SystemAdmissionID, run.ID, lease)+ // An entered key is held encrypted, outside the workflow payload, so a restart can still resume the backup.+ if request.RecoveryKey != "" {+ if holdErr := s.recoveryKeys.HoldRunKey(ctx, run.ID, prepared.recoveryKey); holdErr != nil {+ release(ctx)+ return nil, errors.Join(holdErr, s.store.FailRun(context.WithoutCancel(ctx), run.ID, holdErr.Error()))}- activitylib.CompleteHandlerActivity(workCtx, s.activityService, activityID, "Arcane system backup created successfully", runErr)}- if activityID == "" {- err = errors.New("failed to create system backup activity")- finish(err)- return nil, err+ input := manualBackupInput{Checkpoint: prepared.checkpoint, Requester: requester}+ activityID, err := s.flow.Submit(ctx, s.backupWorkflow, input, activitylib.StartRequest{+ ResourceID: new("arcane"), StartedBy: &user, Step: "Creating system backup", LatestMessage: "Creating Arcane system backup",+ Metadata: database.JSON{"action": "create_system_backup", "backupId": run.ID, "destination": run.Destination, "s3DestinationId": run.S3DestinationID, "policyId": run.PolicyID},+ }, backup.AcceptedTarget(run.ID))+ if err != nil {+ release(ctx)+ return nil, errors.Join(err, s.recoveryKeys.ReleaseRunKey(context.WithoutCancel(ctx), run.ID), s.store.FailRun(context.WithoutCancel(ctx), run.ID, err.Error()))}- dto := *prepared.run+ // A backup canceled before its task runs never takes the parked lease or uses the held key.+ go func() {+ background := context.WithoutCancel(ctx)+ s.watchBackup(background, run.ID, activityID)+ release(background)+ }()+ dto := *rundto.ActivityID = activityID- keyID, _ := ctx.Value(middleware.ContextKeyApiKeyID).(string)- encryptedKey, keyErr := crypto.Encrypt(prepared.recoveryKey)- if keyErr != nil {- finish(keyErr)- return nil, keyErr- }- payload, err := json.Marshal(- manualSystemBackupInternal{- Checkpoint: systemBackupRecoveryInternal{- BackupID: prepared.run.ID,- LocalEnabled: prepared.localEnabled,- S3Enabled: prepared.s3Enabled,- },- EncryptedKey: encryptedKey,- ActivityID: activityID,- UserID: user.ID,- },- )- if err == nil {- err = s.engine.SubmitDurableRun(- workCtx,- backuptypes.DurableRunCommand{- Kind: "system",- RunID: prepared.run.ID,- ActivityID: activityID,- Payload: payload,- UserID: user.ID,- EnvironmentID: "0",- Permission: authz.PermSystemBackupsManage,- RequestedWithKey: keyID,- },- lease,- )- }+ return &dto, nil+}++// WatchAcceptedBackups restores, at startup, the watch over manual backups whose saved workflows resume.+func (s *Service) WatchAcceptedBackups(ctx context.Context) error {+ standalone, err := s.flow.Standalone(ctx)if err != nil {- finish(err)- return nil, err+ return err}-- return &dto, nil+ for _, run := range standalone {+ if backupID := backup.AcceptedBackupID(run); backupID != "" && run.JobID == s.backupWorkflow.Francis().Name() {+ go s.watchBackup(context.WithoutCancel(ctx), backupID, run.ID)+ }+ }+ return nil}-func (s *Service) reconcileBackupInternal(ctx context.Context, previous scheduler.Run, policyID string, suppliedKeys ...string) (scheduler.Outcome, error) {- outcome := jobcontext.ConfirmedTarget(previous, policyID)+// watchBackup releases the run's held key once its workflow ends, and fails the run's row when the workflow+// ended without the task completing it, such as a cancel before the task started.+func (s *Service) watchBackup(ctx context.Context, runID, instanceID string) {+ outcome, err := s.flow.Wait(ctx, s.backupWorkflow, instanceID)+ if err != nil {+ return+ }+ if releaseErr := s.recoveryKeys.ReleaseRunKey(ctx, runID); releaseErr != nil {+ slog.WarnContext(ctx, "Failed to release system backup recovery key", "runId", runID, "error", releaseErr)+ }if outcome.Status == scheduler.Succeeded {+ return+ }+ if current, loadErr := s.store.Run(ctx, runID); loadErr == nil && current.Status == backuptypes.SystemBackupStatusRunning {+ if failErr := s.store.FailRun(ctx, runID, cmp.Or(outcome.Message, "Backup did not run")); failErr != nil {+ slog.WarnContext(ctx, "Failed to close a system backup whose workflow ended early", "runId", runID, "error", failErr)+ }+ }+}++// reconcileBackup finishes the backup checkpointed under targetID from the snapshots its interrupted run left behind.+func (s *Service) reconcileBackup(ctx context.Context, previous scheduler.Run, targetID, suppliedKey string) (scheduler.Outcome, error) {+ outcome := jobcontext.ConfirmedTarget(previous, targetID)+ index := slices.IndexFunc(previous.Outcome.Targets, func(target scheduler.TargetOutcome) bool {+ return target.ID == targetID && len(target.RecoveryData) > 0+ })+ if outcome.Status == scheduler.Succeeded || index < 0 {return outcome, nil}- for _, target := range previous.Outcome.Targets {- if target.ID != policyID || len(target.RecoveryData) == 0 {- continue+ target := previous.Outcome.Targets[index]+ var checkpoint systemBackupRecovery+ if err := json.Unmarshal(target.RecoveryData, &checkpoint); err != nil {+ return outcome, err+ }+ confirm := func() (scheduler.Outcome, error) {+ target.Status = scheduler.Succeeded+ if progressErr := jobcontext.Progress(ctx, target); progressErr != nil {+ return outcome, progressErr}- var checkpoint systemBackupRecoveryInternal- if err := json.Unmarshal(target.RecoveryData, &checkpoint); err != nil {+ return scheduler.Outcome{Status: scheduler.Succeeded, Targets: []scheduler.TargetOutcome{target}}, nil+ }+ run, err := s.store.Run(ctx, checkpoint.BackupID)+ if err != nil {+ return outcome, err+ }+ if run.Status == backuptypes.SystemBackupStatusSucceeded {+ return confirm()+ }+ lease, admitted, err := s.engine.AcquireRun(ctx, backup.SystemAdmissionScope, backup.SystemAdmissionID, "")+ if err != nil || !admitted {+ return outcome, err+ }+ defer lease.Release(ctx)+ dockerClient, err := s.dockerService.GetClient(ctx)+ if err != nil {+ return outcome, err+ }+ key, err := s.recoveryKey(ctx, suppliedKey)+ if err != nil {+ return outcome, err+ }+ staged, local, remote, err := s.observeBackupSnapshots(ctx, dockerClient, previous, run, checkpoint, key)+ if err != nil {+ return outcome, err+ }+ if checkpoint.LocalEnabled && staged.ID == "" && run.RemoteSnapshotID != "" {+ if staged, err = s.engine.Replicate(ctx, dockerClient, remote, run.RemoteSnapshotID, local, key, "arcane-system-recovery", backup.RunSnapshotTag(run.ID)); err != nil {return outcome, err}- run, loadRunErr := s.store.Run(ctx, checkpoint.BackupID)- if loadRunErr != nil {- return outcome, loadRunErr- }- if run.Status != backuptypes.SystemBackupStatusSucceeded {- lease, admitted, err := s.engine.AcquireDurableRun(ctx, previous.ID, backup.SystemAdmissionScope, systemAdmissionID)- if err != nil {- return outcome, err- }- if !admitted {- return outcome, nil- }- defer lease.Release(ctx) //nolint:gocritic // The matching recovery target always returns before the loop advances.- key := ""- if len(suppliedKeys) > 0 {- key = suppliedKeys[0]- }- if resumeBackupErr := s.resumeBackupInternal(ctx, previous, run, checkpoint, key); resumeBackupErr != nil {- return outcome, resumeBackupErr- }- }- target.Status = scheduler.Succeeded- if err := jobcontext.Progress(ctx, target); err != nil {+ }+ if staged.ID == "" && run.RemoteSnapshotID == "" {+ // Nothing survived the interruption, so the backup runs again.+ if _, err = s.executeBackup(ctx, &preparedSystemBackup{run: run, recoveryKey: key, checkpoint: checkpoint}); err != nil {return outcome, err}- return scheduler.Outcome{Status: scheduler.Succeeded, Targets: []scheduler.TargetOutcome{target}}, nil+ return confirm()}- return outcome, nil-}--func systemBackupStageEvidenceInternal(previous scheduler.Run, run *backuptypes.SystemBackupRun) (string, bool) {- stageID, attempted := run.LocalSnapshotID, false- for _, evidence := range previous.Outcome.Targets {- if evidence.ID != run.ID+":stage" {- continue- }- attempted = true- if stageID == "" && evidence.Status == scheduler.Succeeded {- stageID = evidence.Message+ if checkpoint.LocalEnabled {+ run.LocalSnapshotID, run.Size = staged.ID, staged.Size+ }+ // A missing remote snapshot implies a staged one; the rerun above handled neither surviving.+ if checkpoint.S3Enabled && run.RemoteSnapshotID == "" {+ snapshot, replicateErr := s.engine.Replicate(ctx, dockerClient, local, staged.ID, remote, key, "arcane-system-recovery", backup.RunSnapshotTag(run.ID))+ if replicateErr != nil {+ return outcome, replicateErr}+ run.RemoteSnapshotID = snapshot.ID}- return stageID, attempted-}--func systemBackupRemoteAttemptedInternal(previous scheduler.Run, runID string) bool {- for _, evidence := range previous.Outcome.Targets {- if evidence.ID == runID+":remote" {- return true+ if !checkpoint.LocalEnabled && staged.ID != "" {+ if forgetErr := s.engine.ForgetSnapshots(ctx, dockerClient, local, key, []string{staged.ID}); forgetErr != nil {+ return outcome, forgetErr}}- return false+ run.Status, run.Error = backuptypes.SystemBackupStatusSucceeded, ""+ if saveErr := s.store.SaveRun(ctx, run); saveErr != nil {+ return outcome, saveErr+ }+ return confirm()}-func (- s *Service,-) observeBackupSnapshotsInternal(- ctx context.Context,- dockerClient *client.Client,- previous scheduler.Run,- run *backuptypes.SystemBackupRun,- checkpoint systemBackupRecoveryInternal,- key string,-) (- backup.Snapshot,- backup.Repository,- backup.Repository,- error,-) {+// observeBackupSnapshots finds the snapshots an interrupted run already wrote and records any remote one on run.+func (s *Service) observeBackupSnapshots(+ ctx context.Context, dockerClient *client.Client, previous scheduler.Run, run *backuptypes.SystemBackupRun, checkpoint systemBackupRecovery, key string,+) (backup.Snapshot, backup.Repository, backup.Repository, error) {var staged backup.Snapshotvar remote backup.Repository- local, err := s.localRepositoryInternal(ctx, dockerClient, false)+ local, err := s.localRepository(ctx, dockerClient, false)if err != nil && checkpoint.LocalEnabled {return staged, local, remote, err}- stageID, stageAttempted := systemBackupStageEvidenceInternal(previous, run)- if err == nil && (stageAttempted || stageID != "") {+ targets := previous.Outcome.Targets+ stageIndex := slices.IndexFunc(targets, func(target scheduler.TargetOutcome) bool { return target.ID == run.ID+":stage" })+ stageID := run.LocalSnapshotID+ if stageIndex >= 0 && stageID == "" && targets[stageIndex].Status == scheduler.Succeeded {+ stageID = targets[stageIndex].Message+ }+ if err == nil && (stageIndex >= 0 || stageID != "") {snapshot, found, findErr := s.engine.FindRunSnapshot(ctx, dockerClient, local, key, run.ID, stageID)if findErr == nil && found {staged = snapshot@@ -1541,163 +1378,96 @@ func (if !checkpoint.S3Enabled {return staged, local, remote, nil}- remote, err = s.remoteRepositoryInternal(ctx, run.S3DestinationID)+ remote, err = s.remoteRepository(ctx, run.S3DestinationID)if err != nil {return staged, local, remote, err}- attempted := (stageAttempted && staged.ID == "") || systemBackupRemoteAttemptedInternal(previous, run.ID)- if attempted || run.RemoteSnapshotID != "" {- snapshot, found, findRunSnapshotErr := s.engine.FindRunSnapshot(ctx, dockerClient, remote, key, run.ID, run.RemoteSnapshotID)- if findRunSnapshotErr != nil {- return staged, local, remote, findRunSnapshotErr+ remoteAttempted := slices.ContainsFunc(targets, func(target scheduler.TargetOutcome) bool { return target.ID == run.ID+":remote" })+ if (stageIndex >= 0 && staged.ID == "") || remoteAttempted || run.RemoteSnapshotID != "" {+ snapshot, found, findErr := s.engine.FindRunSnapshot(ctx, dockerClient, remote, key, run.ID, run.RemoteSnapshotID)+ if findErr != nil {+ return staged, local, remote, findErr}if found {- run.RemoteSnapshotID = snapshot.ID- if run.Size == 0 {- run.Size = snapshot.Size- }+ run.RemoteSnapshotID, run.Size = snapshot.ID, cmp.Or(run.Size, snapshot.Size)}}return staged, local, remote, nil}-func (- s *Service,-) finishRecoveredDestinationsInternal(- ctx context.Context,- dockerClient *client.Client,- run *backuptypes.SystemBackupRun,- checkpoint systemBackupRecoveryInternal,- key string,- staged backup.Snapshot,- local, remote backup.Repository,-) error {- if checkpoint.LocalEnabled && staged.ID == "" && run.RemoteSnapshotID != "" {- var err error- staged, err = s.engine.Replicate(ctx, dockerClient, remote, run.RemoteSnapshotID, local, key, "arcane-system-recovery", backup.RunSnapshotTag(run.ID))- if err != nil {- return err- }- }- if staged.ID == "" && run.RemoteSnapshotID == "" {- _, err := s.executeBackupInternal(ctx, &preparedSystemBackupInternal{run: run, recoveryKey: key, localEnabled: checkpoint.LocalEnabled, s3Enabled: checkpoint.S3Enabled})- return err- }- if checkpoint.LocalEnabled {- run.LocalSnapshotID, run.Size = staged.ID, staged.Size+// runManualBackup is the system-backup step. A delivery that finds the checkpoint it recorded+// resumes that backup; otherwise it takes over the admission StartBackup won and runs it.+func (s *Service) runManualBackup(ctx context.Context, t flow.Task) (_ any, err error) {+ var input manualBackupInput+ if decodeErr := t.Payload(&input); decodeErr != nil {+ return nil, decodeErr}- if checkpoint.S3Enabled && run.RemoteSnapshotID == "" {- if staged.ID == "" {- return errors.New("System backup has no snapshot to replicate") //nolint:staticcheck // Preserve the existing error message.+ runID := input.Checkpoint.BackupID+ defer func() {+ // Shutdown leaves the run, and any key held for it, to the redelivered task.+ if ctx.Err() != nil {+ return}- snapshot, err := s.engine.Replicate(ctx, dockerClient, local, staged.ID, remote, key, "arcane-system-recovery", backup.RunSnapshotTag(run.ID))if err != nil {- return err+ err = errors.Join(err, s.store.FailRun(ctx, runID, err.Error()))}- run.RemoteSnapshotID = snapshot.ID+ err = errors.Join(err, s.recoveryKeys.ReleaseRunKey(ctx, runID))+ }()+ previous, _ := jobcontext.Run(ctx)+ resuming := slices.ContainsFunc(previous.Outcome.Targets, func(target scheduler.TargetOutcome) bool {+ return target.ID == runID && len(target.RecoveryData) > 0+ })+ if !resuming {+ lease, admitted, acquireErr := s.engine.AcquireRun(ctx, backup.SystemAdmissionScope, backup.SystemAdmissionID, runID)+ if acquireErr != nil {+ return nil, acquireErr+ }+ if !admitted {+ return nil, ErrSystemBackupAlreadyRunning+ }+ defer lease.Release(ctx)}- if !checkpoint.LocalEnabled && staged.ID != "" {- return s.engine.ForgetSnapshots(ctx, dockerClient, local, key, []string{staged.ID})+ if authorizeErr := s.engine.Authorize(ctx, input.Requester); authorizeErr != nil {+ return nil, authorizeErr}- return nil-}--func (s *Service) resumeBackupInternal(ctx context.Context, previous scheduler.Run, run *backuptypes.SystemBackupRun, checkpoint systemBackupRecoveryInternal, suppliedKey string) error {- dockerClient, err := s.dockerService.GetClient(ctx)+ succeeded := scheduler.Outcome{Status: scheduler.Succeeded, Message: "Arcane system backup created successfully"}+ run, err := s.store.Run(ctx, runID)if err != nil {- return err+ return nil, err}- key, err := s.recoveryKeyInternal(ctx, suppliedKey)- if err != nil {- return err+ if run.Status == backuptypes.SystemBackupStatusSucceeded {+ return succeeded, nil}- staged, local, remote, err := s.observeBackupSnapshotsInternal(ctx, dockerClient, previous, run, checkpoint, key)+ supplied, err := s.recoveryKeys.RunKey(ctx, runID)if err != nil {- return err- }- if finishRecoveredDestinationsErr := s.finishRecoveredDestinationsInternal(ctx, dockerClient, run, checkpoint, key, staged, local, remote); finishRecoveredDestinationsErr != nil {- return finishRecoveredDestinationsErr- }- run.Status, run.Error = backuptypes.SystemBackupStatusSucceeded, ""- return s.store.SaveRun(ctx, run)-}--type manualSystemBackupInternal struct {- Checkpoint systemBackupRecoveryInternal `json:"checkpoint"`- EncryptedKey string `json:"encryptedKey"`- ActivityID string `json:"activityId"`- UserID string `json:"userId"`-}--func (s *Service) executeDurableBackupInternal(ctx context.Context, runID string, payload []byte, interrupted bool) (err error) {- var command manualSystemBackupInternal- if decodeErr := json.Unmarshal(payload, &command); decodeErr != nil {- return decodeErr- }- defer func() {- if ctx.Err() == nil {- if err != nil {- saveErr := s.store.FailRun(ctx, command.Checkpoint.BackupID, err.Error())- err = errors.Join(err, saveErr)- }- activitylib.CompleteHandlerActivity(ctx, s.activityService, command.ActivityID, "Arcane system backup created successfully", err)- }- }()- run, loadBackupErr := s.store.Run(ctx, command.Checkpoint.BackupID)- if loadBackupErr != nil {- return loadBackupErr- }- if run.Status == backuptypes.SystemBackupStatusSucceeded {- return nil+ return nil, err}- key, err := crypto.Decrypt(command.EncryptedKey)+ key, err := s.recoveryKey(ctx, supplied)if err != nil {- return err+ return nil, err}- if interrupted {- previous, _ := jobcontext.Run(ctx)- outcome, recoveryErr := s.reconcileBackupInternal(ctx, previous, runID, key)- if recoveryErr != nil {- return recoveryErr+ if resuming {+ outcome, reconcileErr := s.reconcileBackup(ctx, previous, runID, key)+ if reconcileErr != nil {+ return nil, reconcileErr}if outcome.Status != scheduler.Succeeded {- return errors.New(outcome.Message)+ return nil, errors.New(outcome.Message)}- return nil- }- lease, err := s.acquireDurableRunInternal(ctx, runID)- if err != nil {- return err+ return succeeded, nil}- defer lease.Release(ctx)- checkpoint, err := json.Marshal(command.Checkpoint)+ checkpoint, err := json.Marshal(input.Checkpoint)if err != nil {- return err+ return nil, err}- if progressErr := jobcontext.Progress(- ctx,- scheduler.TargetOutcome{- ResourceType: "system_backup",- ID: runID,- Status: scheduler.Running,- RecoveryData: checkpoint,- ActivityID: command.ActivityID,- },- ); progressErr != nil {- return progressErr+ target := scheduler.TargetOutcome{ResourceType: "system_backup", ID: runID, Status: scheduler.Running, RecoveryData: checkpoint, ActivityID: t.ActivityID()}+ if progressErr := jobcontext.Progress(ctx, target); progressErr != nil {+ return nil, progressErr}- _, err = s.executeBackupInternal(ctx, &preparedSystemBackupInternal{run: run, recoveryKey: key, localEnabled: command.Checkpoint.LocalEnabled, s3Enabled: command.Checkpoint.S3Enabled})- return err-}--func (s *Service) failDurableBackupInternal(ctx context.Context, _ string, payload []byte, runErr error) error {- var command manualSystemBackupInternal- if err := json.Unmarshal(payload, &command); err != nil {- return err+ if _, executeErr := s.executeBackup(ctx, &preparedSystemBackup{run: run, recoveryKey: key, checkpoint: input.Checkpoint}); executeErr != nil {+ return nil, executeErr}- err := s.store.FailRun(ctx, command.Checkpoint.BackupID, runErr.Error())- activitylib.CompleteHandlerActivity(ctx, s.activityService, command.ActivityID, "Arcane system backup created successfully", runErr)- return err+ return succeeded, nil}// ListBackupHistory returns a unified, server-paginated view of local backup records.@@ -1706,31 +1476,25 @@ func (s *Service) ListBackupHistory(ctx context.Context, params pagination.Queryif err != nil {return nil, pagination.Response{}, fmt.Errorf("list backup history: %w", err)}- decorateHistoryDestinationsInternal(ctx, s.s3Destinations, history)- systemAvailable := backup.RemoteSnapshotChecker(ctx, s.s3Destinations, "arcane-system-recovery")- volumeRoot := ""+ destinations := s.s3Destinations.DestinationsByID(ctx)+ remoteAvailable := backup.RemoteSnapshotChecker(ctx, s.s3Destinations)+ instanceID := ""if s.settingsService != nil {- volumeRoot = "arcane-volume-backups/" + s.settingsService.GetSettingsConfig().InstanceID.Value+ instanceID = s.settingsService.GetSettingsConfig().InstanceID.Value}- volumeAvailable := backup.RemoteSnapshotChecker(ctx, s.s3Destinations, volumeRoot)for i := range history {- check := kit.Ternary(history[i].ResourceType == "volume", volumeAvailable, systemAvailable)- history[i].RemoteAvailable = check(history[i].S3DestinationID, history[i].RemoteSnapshotID)+ history[i].S3DestinationName = destinations[history[i].S3DestinationID].Name+ // Volume backups discovered from another instance live under that instance's root.+ root := "arcane-system-recovery"+ if history[i].ResourceType == "volume" {+ root = ""+ if owner := cmp.Or(history[i].RemoteInstanceID, instanceID); owner != "" {+ root = path.Join(backup.VolumeRoot, owner)+ }+ }+ history[i].RemoteAvailable = remoteAvailable(history[i].S3DestinationID, root, history[i].RemoteSnapshotID)}page.GrandTotalItems = page.TotalItemsreturn history, page, nil}--func decorateHistoryDestinationsInternal(ctx context.Context, service *s3.S3DestinationService, history []backuptypes.HistoryEntry) {- if service == nil {- return- }- destinations, err := service.ListS3DestinationsByID(ctx)- if err != nil {- return- }- for i := range history {- history[i].S3DestinationName = destinations[history[i].S3DestinationID].Name- }-}diff --git a/backend/internal/system/children/backup/service_test.go b/backend/internal/system/children/backup/service_test.goindex 8f772caeb..3657612a3 100644--- a/backend/internal/system/children/backup/service_test.go+++ b/backend/internal/system/children/backup/service_test.go@@ -1,38 +1,56 @@package backupimport (+ "context""testing""github.com/getarcaneapp/arcane/types/v2/backup""github.com/libtnb/sqlite"- "github.com/moby/moby/api/types/container"- "github.com/moby/moby/api/types/mount""github.com/stretchr/testify/require""gorm.io/gorm""github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/s3"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow/flowtest")-func TestRecoveryHelperExecutableInternal(t *testing.T) {- path, executableMount, err := recoveryHelperExecutableInternal(nil, "/app/arcane")- require.NoError(t, err)- require.Equal(t, "/app/arcane", path)- require.Nil(t, executableMount)+func TestProjectsSettingFallsBackToConfig(t *testing.T) {+ service := &Service{config: &config.Config{ProjectsDirectory: "/app/data/projects:/host/projects"}}+ require.Equal(t, "/app/data/projects:/host/projects", service.projectsSetting(t.Context()))+}- path, executableMount, err = recoveryHelperExecutableInternal([]container.MountPoint{{- Type: mount.TypeBind, Source: "/workspace/backend", Destination: "/app/backend", RW: true,- }}, "/app/backend/.bin/arcane")+func TestHistoryDestinationDecoration(t *testing.T) {+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})require.NoError(t, err)- require.Equal(t, systemRecoveryHelperPath, path)- require.Equal(t, mount.TypeBind, executableMount.Type)- require.Equal(t, "/workspace/backend/.bin/arcane", executableMount.Source)- require.Equal(t, systemRecoveryHelperPath, executableMount.Target)- require.True(t, executableMount.ReadOnly)+ require.NoError(t, db.AutoMigrate(&s3.S3Destination{}))+ destination := s3.S3Destination{Name: "Offsite", Bucket: "backups"}+ require.NoError(t, db.Create(&destination).Error)+ service := &Service{+ s3Destinations: s3.NewS3DestinationService(&database.DB{DB: db}, nil),+ store: Store{ListHistory: func(context.Context, pagination.QueryParams) ([]backup.HistoryEntry, pagination.Response, error) {+ return []backup.HistoryEntry{{S3DestinationID: destination.ID}, {S3DestinationID: "missing"}}, pagination.Response{}, nil+ }},+ }+ history, _, err := service.ListBackupHistory(t.Context(), pagination.QueryParams{})+ require.NoError(t, err)+ require.Equal(t, "Offsite", history[0].S3DestinationName)+ require.Empty(t, history[1].S3DestinationName)+ require.NoError(t, db.Migrator().DropTable(&s3.S3Destination{}))+ history, _, err = service.ListBackupHistory(t.Context(), pagination.QueryParams{})+ require.NoError(t, err)+ require.Empty(t, history[0].S3DestinationName)+}++func TestBackupWorkflowDefinitions(t *testing.T) {+ harness := flowtest.New(t, nil)+ require.NoError(t, NewService(Dependencies{}).RegisterWorkflows(harness.Engine))+ harness.Start(t)+ flowtest.AssertDefinitions(t, harness)}-func TestRecoveryEnvironmentInternalIncludesRuntimeSecrets(t *testing.T) {+func TestRecoveryEnvironmentIncludesRuntimeSecrets(t *testing.T) {cfg := &config.Config{JWTSecret: "jwt-secret",EncryptionKey: "encryption-secret",@@ -40,34 +58,10 @@ func TestRecoveryEnvironmentInternalIncludesRuntimeSecrets(t *testing.T) {OidcClientSecret: "oidc-secret",FilePerm: 0o640,}- environment := (&Service{config: cfg}).recoveryEnvironmentInternal(t.Context())+ environment := (&Service{config: cfg}).recoveryEnvironment(t.Context())require.Equal(t, "jwt-secret", environment["JWT_SECRET"])require.Equal(t, "encryption-secret", environment["ENCRYPTION_KEY"])require.Equal(t, "admin-key", environment["ADMIN_STATIC_API_KEY"])require.Equal(t, "oidc-secret", environment["OIDC_CLIENT_SECRET"])require.Equal(t, "0640", environment["FILE_PERM"])}--func TestProjectsDirectoryInternalUsesContainerPathOfMapping(t *testing.T) {- service := &Service{config: &config.Config{ProjectsDirectory: "/app/data/projects:/host/projects"}}- require.Equal(t, "/app/data/projects", service.projectsDirectoryInternal(t.Context()))- require.Equal(t, "/app/data/projects:/host/projects", service.projectsSettingInternal(t.Context()))-}--func TestHistoryDestinationDecorationInternal(t *testing.T) {- db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})- require.NoError(t, err)- require.NoError(t, db.AutoMigrate(&s3.S3Destination{}))- destination := s3.S3Destination{Name: "Offsite", Bucket: "backups"}- require.NoError(t, db.Create(&destination).Error)- service := s3.NewS3DestinationService(&database.DB{DB: db}, nil)- history := []backup.HistoryEntry{{S3DestinationID: destination.ID}, {S3DestinationID: "missing"}}- decorateHistoryDestinationsInternal(t.Context(), service, history)- require.Equal(t, "Offsite", history[0].S3DestinationName)- require.Empty(t, history[1].S3DestinationName)- require.NoError(t, db.Migrator().DropTable(&s3.S3Destination{}))- decorateHistoryDestinationsInternal(t.Context(), service, history)- require.Equal(t, "Offsite", history[0].S3DestinationName)- decorateHistoryDestinationsInternal(t.Context(), nil, history)- require.Equal(t, "Offsite", history[0].S3DestinationName)-}diff --git a/backend/internal/system/children/upgrade/handler.go b/backend/internal/system/children/upgrade/handler.goindex 328cdbf94..12cf3f0ed 100644--- a/backend/internal/system/children/upgrade/handler.go+++ b/backend/internal/system/children/upgrade/handler.go@@ -7,25 +7,23 @@ import ("github.com/danielgtaylor/huma/v2""github.com/getarcaneapp/arcane/types/v2/base"- "go.getarcane.app/kit/pkg"+ kit "go.getarcane.app/kit/pkg""github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/internal/config"- "github.com/getarcaneapp/arcane/backend/v2/internal/environment""github.com/getarcaneapp/arcane/backend/v2/pkg/utils""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil")// Handler serves the self-upgrade and update-all endpoints.type Handler struct {- service *Service- environmentService *environment.EnvironmentService- cfg *config.Config- appCtx context.Context+ service *Service+ cfg *config.Config+ appCtx context.Context}-func NewHandler(service *Service, environmentService *environment.EnvironmentService, cfg *config.Config, appCtx context.Context) *Handler {- return &Handler{service: service, environmentService: environmentService, cfg: cfg, appCtx: appCtx}+func NewHandler(service *Service, cfg *config.Config, appCtx context.Context) *Handler {+ return &Handler{service: service, cfg: cfg, appCtx: appCtx}}type CheckUpgradeInput struct {@@ -52,9 +50,8 @@ type TriggerUpgradeBody struct {TargetVersion string `json:"targetVersion,omitempty" doc:"Release version to upgrade to; overrides this instance's own version check"`}-// TriggerUpgradeData reports the upgrade was accepted. UpToDate lets a client skip-// waiting for a restart: the upgrader still pulls, but when the environment already-// runs the newest image it finds nothing to swap in and no restart follows.+// TriggerUpgradeData reports the upgrade was accepted. UpToDate lets a client skip waiting+// for a restart: an environment already on the newest image is not recreated.type TriggerUpgradeData struct {Message string `json:"message" doc:"Response message"`UpToDate bool `json:"upToDate" doc:"Environment already runs the newest image, so no restart is expected"`@@ -68,14 +65,6 @@ type UpdateAllStatusInput struct {EnvironmentID string `path:"id" doc:"Environment ID"`}-// rejectIfAgentModeInternal blocks manager-only operations when running as an agent.-func (h *Handler) rejectIfAgentModeInternal() error {- if h.cfg != nil && h.cfg.AgentMode {- return huma.Error400BadRequest("update-all is managed on the Arcane manager")- }- return nil-}-// CheckUpgradeAvailable checks if a system upgrade is available.func (h *Handler) CheckUpgradeAvailable(ctx context.Context, input *CheckUpgradeInput) (*CheckUpgradeOutput, error) {canUpgrade, err := h.service.CanUpgrade(ctx)@@ -108,16 +97,10 @@ func (h *Handler) TriggerUpgrade(ctx context.Context, input *TriggerUpgradeInputslog.InfoContext(ctx, "System upgrade triggered", "user", user.Username, "userId", user.ID)- // Resolved before triggering, while the version check still describes the running- // container: the upgrade itself may replace it.+ // Resolved before triggering, while the version check still describes the running container.upToDate := h.service.AlreadyOnNewestImage(ctx)- targetVersion := ""- if input.Body != nil {- targetVersion = input.Body.TargetVersion- }-- err = h.service.TriggerUpgradeAsync(utils.ActivityRuntimeContext(ctx, h.appCtx), *user, targetVersion)+ err = h.service.TriggerUpgradeAsync(utils.ActivityRuntimeContext(ctx, h.appCtx), *user, kit.FromPtr(input.Body).TargetVersion)if err != nil {slog.ErrorContext(ctx, "System upgrade failed", "error", err, "user", user.Username)@@ -145,11 +128,10 @@ func (h *Handler) TriggerUpgrade(ctx context.Context, input *TriggerUpgradeInput}, nil}-// TriggerUpdateAll starts a fleet-wide update, upgrading the manager first and then-// the remote agents (the latter resume after the manager restarts).+// TriggerUpdateAll starts a fleet-wide update: every remote agent in turn, then the manager last.func (h *Handler) TriggerUpdateAll(ctx context.Context, input *TriggerUpdateAllInput) (*handlerutil.Out[EnvironmentUpdateJob], error) {- if err := h.rejectIfAgentModeInternal(); err != nil {- return nil, err+ if h.cfg != nil && h.cfg.AgentMode {+ return nil, huma.Error400BadRequest("update-all is managed on the Arcane manager")}user, err := handlerutil.RequireUser(ctx)@@ -159,11 +141,7 @@ func (h *Handler) TriggerUpdateAll(ctx context.Context, input *TriggerUpdateAllIslog.InfoContext(ctx, "Update-all environments triggered", "user", user.Username, "userId", user.ID)- // Use a runtime context so the agents phase can outlive the request when the- // manager is already up to date.- runtimeCtx := utils.ActivityRuntimeContext(ctx, h.appCtx)-- job, err := h.service.StartUpdateAll(runtimeCtx, *user, h.environmentService)+ job, err := h.service.StartUpdateAll(ctx, *user)if err != nil {if errors.Is(err, common.ErrUpdateAllInProgress) {return nil, huma.Error409Conflict(err.Error())@@ -181,8 +159,8 @@ func (h *Handler) TriggerUpdateAll(ctx context.Context, input *TriggerUpdateAllI// GetUpdateAllStatus returns the latest update-all job for live progress polling.func (h *Handler) GetUpdateAllStatus(ctx context.Context, input *UpdateAllStatusInput) (*handlerutil.Out[EnvironmentUpdateJob], error) {- if err := h.rejectIfAgentModeInternal(); err != nil {- return nil, err+ if h.cfg != nil && h.cfg.AgentMode {+ return nil, huma.Error400BadRequest("update-all is managed on the Arcane manager")}job, err := h.service.GetLatestUpdateAllJob(ctx)diff --git a/backend/internal/system/children/upgrade/model.go b/backend/internal/system/children/upgrade/model.goindex 0dfb3f05e..c713f8be3 100644--- a/backend/internal/system/children/upgrade/model.go+++ b/backend/internal/system/children/upgrade/model.go@@ -12,9 +12,8 @@ import (type EnvironmentUpdateJobStatus stringconst (- // EnvironmentUpdateJobStatusPendingRestart means the remote agents have been- // updated and the manager has triggered its own self-upgrade as the final step;- // it is waiting to restart on the new version, after which the job is finalized.+ // EnvironmentUpdateJobStatusPendingRestart means the agents are done and the manager triggered its own+ // self-upgrade; the job is finalized once it restarts on the new version.EnvironmentUpdateJobStatusPendingRestart EnvironmentUpdateJobStatus = "pending_restart"// EnvironmentUpdateJobStatusRunning means the agents phase is in progress.EnvironmentUpdateJobStatusRunning EnvironmentUpdateJobStatus = "running"@@ -23,9 +22,8 @@ const (// EnvironmentUpdateJobStatusFailed means the job stopped before completing.EnvironmentUpdateJobStatusFailed EnvironmentUpdateJobStatus = "failed"- // EnvironmentUpdateResultStatusPending is the initial state recorded for an- // environment that is seeded into the job but not yet being processed (and for- // the manager entry while its self-upgrade is in flight).+ // EnvironmentUpdateResultStatusPending is the initial state of an environment+ // seeded into the job but not yet being processed.EnvironmentUpdateResultStatusPending EnvironmentUpdateResultStatus = "pending"// EnvironmentUpdateResultStatusUpdating means this environment is being processed// right now; it drives the live per-environment progress indicator in the UI.@@ -65,14 +63,15 @@ type EnvironmentUpdateResult struct {EnvironmentName string `json:"environmentName"`Status EnvironmentUpdateResultStatus `json:"status"`FromVersion string `json:"fromVersion,omitempty"`+ FromDigest string `json:"fromDigest,omitempty"`ToVersion string `json:"toVersion,omitempty"`Stage EnvironmentUpdateStage `json:"stage,omitempty"`StageStartedAt *time.Time `json:"stageStartedAt,omitempty"`Error string `json:"error,omitempty"`}-// clearStageInternal drops live-progress fields once the row leaves updating.-func (r *EnvironmentUpdateResult) clearStageInternal() {+// clearStage drops live-progress fields once the row leaves updating.+func (r *EnvironmentUpdateResult) clearStage() {r.Stage = ""r.StageStartedAt = nil}@@ -91,9 +90,8 @@ func (r *EnvironmentUpdateResults) Scan(value any) error {return database.JSONScan(r, value)}-// EnvironmentUpdateJob is a persisted fleet-wide update orchestration record. It-// survives the manager's final self-upgrade restart so the manager result can be-// finalized on the next boot. See [EnvironmentUpdateJobStatus].+// EnvironmentUpdateJob is a persisted fleet-wide update record. It survives the manager's+// self-upgrade restart so the next boot can finalize the manager result.type EnvironmentUpdateJob struct {database.BaseModeldiff --git a/backend/internal/system/children/upgrade/service.go b/backend/internal/system/children/upgrade/service.goindex b7f455536..2a3b3b034 100644--- a/backend/internal/system/children/upgrade/service.go+++ b/backend/internal/system/children/upgrade/service.go@@ -19,15 +19,17 @@ import ("github.com/containerd/errdefs""github.com/distribution/reference"+ "github.com/getarcaneapp/arcane/types/v2/scheduler"usertypes "github.com/getarcaneapp/arcane/types/v2/user"versiontypes "github.com/getarcaneapp/arcane/types/v2/version"+ "github.com/italypaleale/francis/builtin/workflow""github.com/moby/moby/api/types/container""github.com/moby/moby/api/types/mount""github.com/moby/moby/client""github.com/opencontainers/go-digest""go.getarcane.app/docker""go.getarcane.app/docker/compat"- "go.getarcane.app/kit/pkg"+ kit "go.getarcane.app/kit/pkg""go.getarcane.app/sys/cgroup""go.getarcane.app/updater""go.getarcane.app/updater/labels"@@ -40,27 +42,63 @@ import (dockerInternal "github.com/getarcaneapp/arcane/backend/v2/internal/docker""github.com/getarcaneapp/arcane/backend/v2/internal/environment""github.com/getarcaneapp/arcane/backend/v2/internal/event"+ "github.com/getarcaneapp/arcane/backend/v2/internal/middleware""github.com/getarcaneapp/arcane/backend/v2/internal/project"+ "github.com/getarcaneapp/arcane/backend/v2/internal/role""github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/backend/v2/internal/version"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/authz""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"+ activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts""github.com/getarcaneapp/arcane/backend/v2/pkg/projects""github.com/getarcaneapp/arcane/backend/v2/pkg/remenv"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow""github.com/getarcaneapp/arcane/backend/v2/pkg/utils")-var upgradeLogNameInternal = regexp.MustCompile(`^arcane-upgrade-\d+\.log$`)+var (+ upgradeLogName = regexp.MustCompile(`^arcane-upgrade-\d+\.log$`)+ // exactReleaseTag pins an exact release (vX.Y.Z, prereleases included); channels like v2 or v2.9 are mutable.+ exactReleaseTag = regexp.MustCompile(`^v?\d+\.\d+\.\d+(?:[-+].*)?$`)+ activeUpdateAllStatuses = []EnvironmentUpdateJobStatus{EnvironmentUpdateJobStatusPendingRestart, EnvironmentUpdateJobStatusRunning}+ errUpdateAllInProgress = common.Classify(common.ErrUpdateAllInProgress, errors.New("an update-all job is already in progress"))+)++const (+ // The run never finished observably: nothing may be concluded from it.+ upgraderExitUnobserved upgraderExit = 0+ upgraderExitSucceeded upgraderExit = 1+ upgraderExitFailed upgraderExit = 2+ // The upgrader is gone and its exit code with it: success and failure are indistinguishable.+ upgraderExitCodeLost upgraderExit = 3++ updateAllStaleThreshold = time.Hour+ updateAllAgentRequestTimeout = 15 * time.Second+ updateAllConfirmPollInterval = 10 * time.Second+ updateAllConfirmTimeout = 5 * time.Minute+ updateAllErrorMaxLen = 500+ updateAllWorkflowName = "update-all"+ // Generous: the upgrader pulls the target image before recreating anything.+ updateAllManagerWatchTimeout = 15 * time.Minute+ // How long a recreate already underway gets to stop this container before no restart is assumed.+ updateAllManagerNoRestartGrace = 15 * time.Second+)type Service struct {- upgrading atomic.Bool- updatingAll atomic.Bool- db *database.DB- dockerService *dockerInternal.DockerClientService- versionService *version.VersionService- eventService *event.EventService- settingsService *settings.SettingsService- projectService *project.ProjectService+ upgrading atomic.Bool+ updatingAll atomic.Bool+ flow *flow.Engine+ environments *environment.EnvironmentService+ roles *role.RoleService+ // updateAllWorkflow upgrades every agent in turn, then the manager last.+ updateAllWorkflow *flow.Workflow+ db *database.DB+ dockerService *dockerInternal.DockerClientService+ versionService *version.VersionService+ eventService *event.EventService+ settingsService *settings.SettingsService+ projectService *project.ProjectService// resolveRuntimeOptions determines how the upgrader container reaches the Docker daemon.resolveRuntimeOptions func(ctx context.Context,@@ -72,6 +110,38 @@ type Service struct {) ([]string, []mount.Mount, container.NetworkMode, error)}+// remoteAgent is one environment the update-all workflow upgrades.+type remoteAgent struct {+ ID string `json:"id"`+ Name string `json:"name"`+}++// updateAllInput is the update-all payload: the job and the API key that requested it.+type updateAllInput struct {+ JobID string `json:"jobId"`+ KeyID string `json:"keyId,omitempty"`+}++type preparedUpgrade struct {+ dockerClient *client.Client+ current container.InspectResponse+ containerName string+ binaryPath string+ // targetImage is what the recreated container runs; pullImage is the immutable reference pulled (same unless frozen).+ targetImage string+ pullImage string+ // saveCompose persists a tag change in the container's Compose project.+ saveCompose func(context.Context)+}++type resumeAction struct {+ markStale bool+ managerSucceeded bool+}++// upgraderExit is how much the watcher learned about an upgrader run.+type upgraderExit int+func NewService(db *database.DB,dockerService *dockerInternal.DockerClientService,@@ -99,56 +169,59 @@ func NewService(}}-// CanUpgrade checks if self-upgrade is possible-func (s *Service) CanUpgrade(ctx context.Context) (bool, error) {- // Check if running in Docker- containerId, err := s.getCurrentContainerIDInternal(ctx)- if err != nil {- return false, err- }+// RegisterWorkflows defines the update-all workflow while the host is still unstarted. Its payload names the job;+// the job row holds every result, so a redelivered task resumes from what the row records.+func (s *Service) RegisterWorkflows(engine *flow.Engine, environments *environment.EnvironmentService, roles *role.RoleService) error {+ var err error+ s.flow, s.environments, s.roles = engine, environments, roles+ s.updateAllWorkflow, err = engine.Define(flow.Definition{+ Name: updateAllWorkflowName,+ Version: 1,+ Fingerprint: "65b33dbbe55a5018517b2e8506d9b487f322803ed5a9dfec0edbf5103c96e586",+ Concurrency: 1,+ Timeout: 24 * time.Hour,+ Steps: []workflow.StepSpec{+ workflow.Step("discover", engine.Handler(s.discoverAgents), workflow.WithMaxAttempts(1)),+ workflow.ForEach("agents", engine.Handler(s.upgradeAgentTask), workflow.WithItemsFrom("discover"), workflow.WithMaxParallel(1),+ workflow.WithMaxAttempts(1), workflow.WithFailurePolicy(workflow.TolerateFailures)),+ workflow.Step("manager", engine.Handler(s.upgradeManagerTask), workflow.WithMaxAttempts(1)),+ },+ })+ return err+}- // Verify we can access Docker- _, err = s.dockerService.GetClient(ctx)+// CanUpgrade checks if self-upgrade is possible.+func (s *Service) CanUpgrade(ctx context.Context) (bool, error) {+ dockerClient, err := s.dockerService.GetClient(ctx)if err != nil {return false, errors.New("docker socket is not accessible")}-- // Verify we can find our container- _, err = s.findArcaneContainerInternal(ctx, containerId)- if err != nil {- return false, err+ if _, inspectErr := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient); inspectErr != nil {+ return false, errors.New("arcane is not running in a Docker container")}-return true, nil}-// AlreadyOnNewestImage reports whether this environment's version check is confident-// it already runs the newest image. A triggered upgrade still pulls, but will then-// find nothing to swap in and skip the recreate — so callers can use this to stop-// waiting for a restart that is not coming.+// AlreadyOnNewestImage reports whether the version check is confident this environment runs the newest image,+// so a triggered upgrade will find nothing to swap in and no restart will follow.func (s *Service) AlreadyOnNewestImage(ctx context.Context) bool {return s.versionService.GetAppVersionInfo(ctx).AlreadyOnNewest()}-// TriggerUpgradeViaCLI spawns the upgrade CLI command in a separate container and-// returns that upgrader container's ID. This avoids self-termination issues by running-// the upgrade from outside. A zero-value target resolves the image to upgrade to from-// the version check (see resolveSelfUpgradeTargetImageInternal); the updater engine-// passes an explicit target with the resolved new image, which is used as-is.-// Update-all uses the returned ID to tell an upgrade that recreated this-// container from one that found nothing to do — see watchManagerUpgraderInternal.+// TriggerUpgradeViaCLI runs the upgrade from a separate upgrader container and returns its ID.+// A zero-value target resolves the image from the version check; an explicit target is used as-is.func (s *Service) TriggerUpgradeViaCLI(ctx context.Context, user usertypes.Actor, target updater.SelfUpdateTarget) (string, error) {- prepared, err := s.prepareUpgradeInternal(ctx, user, target, "")+ prepared, err := s.prepareUpgrade(ctx, user, target, "")if err != nil {return "", err}- return s.runPreparedUpgradeInternal(ctx, prepared)+ return s.runPreparedUpgrade(ctx, prepared)}// TriggerUpgradeAsync validates synchronously, then pulls and spawns the upgrader in the background (#3628).// The run follows the app lifecycle with a deadline so a stalled daemon cannot hold the upgrading guard forever.func (s *Service) TriggerUpgradeAsync(ctx context.Context, user usertypes.Actor, targetVersion string) error {- prepared, err := s.prepareUpgradeInternal(ctx, user, updater.SelfUpdateTarget{}, targetVersion)+ prepared, err := s.prepareUpgrade(ctx, user, updater.SelfUpdateTarget{}, targetVersion)if err != nil {return err}@@ -157,27 +230,15 @@ func (s *Service) TriggerUpgradeAsync(ctx context.Context, user usertypes.Actor,runCtx, cancel := context.WithTimeout(utils.ActivityRuntimeContext(ctx, nil), runTimeout)go func() {defer cancel()- if _, runPreparedUpgradeErr := s.runPreparedUpgradeInternal(runCtx, prepared); runPreparedUpgradeErr != nil {+ if _, runPreparedUpgradeErr := s.runPreparedUpgrade(runCtx, prepared); runPreparedUpgradeErr != nil {slog.ErrorContext(ctx, "Background self-upgrade failed", "error", runPreparedUpgradeErr, "targetImage", prepared.targetImage)}}()return nil}-type preparedUpgradeInternal struct {- current container.InspectResponse- containerName string- binaryPath string- // targetImage is the reference the recreated container runs as; pullImage- // is the immutable reference actually pulled (the same unless frozen).- targetImage string- pullImage string- // saveCompose persists a tag change in the container's Compose project.- saveCompose func(context.Context)-}--// prepareUpgradeInternal takes the upgrading guard, released by runPreparedUpgradeInternal (or here on error).-func (s *Service) prepareUpgradeInternal(ctx context.Context, user usertypes.Actor, target updater.SelfUpdateTarget, targetVersion string) (prepared *preparedUpgradeInternal, err error) {+// prepareUpgrade takes the upgrading guard, released by runPreparedUpgrade (or here on error).+func (s *Service) prepareUpgrade(ctx context.Context, user usertypes.Actor, target updater.SelfUpdateTarget, targetVersion string) (prepared *preparedUpgrade, err error) {if !s.upgrading.CompareAndSwap(false, true) {return nil, common.Classify(common.ErrUpgradeInProgress, errors.New("an upgrade is already in progress"))}@@ -187,42 +248,58 @@ func (s *Service) prepareUpgradeInternal(ctx context.Context, user usertypes.Act}}()- containerId := strings.TrimSpace(target.ContainerID)- if containerId == "" {- // Fall back to the container this process runs in- containerId, err = s.getCurrentContainerIDInternal(ctx)- if err != nil {- return nil, fmt.Errorf("get current container: %w", err)- }- }-- currentContainer, err := s.findArcaneContainerInternal(ctx, containerId)+ dockerClient, err := s.dockerService.GetClient(ctx)if err != nil {- return nil, fmt.Errorf("inspect container: %w", err)+ return nil, fmt.Errorf("failed to connect to Docker: %w", err)+ }+ var current container.InspectResponse+ if containerID := strings.TrimSpace(target.ContainerID); containerID != "" {+ inspect, inspectErr := compat.ContainerInspectWithCompatibility(ctx, dockerClient, containerID, client.ContainerInspectOptions{})+ if inspectErr != nil {+ // Fall back to a prefix match across every container.+ containers, listErr := dockerClient.ContainerList(ctx, client.ContainerListOptions{All: true})+ if listErr != nil {+ return nil, fmt.Errorf("inspect container: %w", listErr)+ }+ idx := slices.IndexFunc(containers.Items, func(c container.Summary) bool { return strings.HasPrefix(c.ID, containerID) })+ if idx < 0 {+ return nil, fmt.Errorf("inspect container: %w", common.Classify(common.ErrNotFound, errors.New("could not find Arcane container")))+ }+ if inspect, inspectErr = compat.ContainerInspectWithCompatibility(ctx, dockerClient, containers.Items[idx].ID, client.ContainerInspectOptions{}); inspectErr != nil {+ return nil, fmt.Errorf("inspect container: %w", inspectErr)+ }+ }+ current = inspect.Container+ } else {+ self, inspectErr := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient)+ if inspectErr != nil {+ return nil, errors.New("get current container: arcane is not running in a Docker container")+ }+ current = *self}- containerName := strings.TrimPrefix(currentContainer.Name, "/")-- // Determine binary path based on container type (agent vs main)- binaryPath := "/app/arcane"- if currentContainer.Config != nil {- binaryPath = kit.Ternary(labels.IsArcaneAgentContainer(currentContainer.Config.Labels), "/app/arcane-agent", "/app/arcane")- }+ containerName := strings.TrimPrefix(current.Name, "/")+ config := kit.FromPtr(current.Config)+ binaryPath := kit.Ternary(labels.IsArcaneAgentContainer(config.Labels), "/app/arcane-agent", "/app/arcane")- targetImage, err := s.resolveUpgradeTargetImageInternal(ctx, currentContainer, target.NewImageRef, targetVersion)- if err != nil {- return nil, err+ targetImage := strings.TrimSpace(target.NewImageRef)+ if targetImage == "" {+ // A blank target resolves against the version check; a manager-supplied version outranks this instance's own.+ info := kit.FromPtr(s.versionService.GetAppVersionInfo(ctx))+ info.NewestVersion = cmp.Or(strings.TrimSpace(targetVersion), info.NewestVersion)+ if targetImage, err = resolveSelfUpgradeTargetImage(config.Image, &info); err != nil {+ return nil, fmt.Errorf("resolve upgrade target image: %w", err)+ }}- targetImage, saveCompose, err := s.configuredTargetImageInternal(ctx, currentContainer, targetImage)+ targetImage, saveCompose, err := s.configuredTargetImage(ctx, current, targetImage)if err != nil {return nil, err}pullImage := cmp.Or(strings.TrimSpace(target.PullImageRef), targetImage)- // Log upgrade eventmetadata := database.JSON{"action": "system_upgrade_cli",- "containerId": containerId,+ "containerId": current.ID,"containerName": containerName,"method": "cli","targetImage": targetImage,@@ -232,8 +309,9 @@ func (s *Service) prepareUpgradeInternal(ctx context.Context, user usertypes.Actslog.WarnContext(ctx, "Failed to log upgrade event", "error", logUserEventErr)}- return &preparedUpgradeInternal{- current: currentContainer,+ return &preparedUpgrade{+ dockerClient: dockerClient,+ current: current,containerName: containerName,binaryPath: binaryPath,targetImage: targetImage,@@ -242,34 +320,21 @@ func (s *Service) prepareUpgradeInternal(ctx context.Context, user usertypes.Act}, nil}-func (s *Service) runPreparedUpgradeInternal(ctx context.Context, prepared *preparedUpgradeInternal) (string, error) {+func (s *Service) runPreparedUpgrade(ctx context.Context, prepared *preparedUpgrade) (string, error) {defer s.upgrading.Store(false)- // Run the upgrader from the image we are upgrading to, so the upgrade CLI- // is the new version.- upgraderImage := prepared.pullImage- slog.DebugContext(ctx, "Using upgrader image", "image", upgraderImage)-- slog.InfoContext(ctx, "Spawning upgrade CLI command", "containerName", prepared.containerName, "upgraderImage", upgraderImage)-- // Spawn the upgrade command in a detached container- // This will run independently of the current container- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return "", fmt.Errorf("failed to connect to Docker: %w", err)- }-- // Pull the upgrader image first to ensure it exists- slog.InfoContext(ctx, "Pulling upgrader image", "image", upgraderImage)+ // The upgrader runs from the target image, so the upgrade CLI is the new version.+ dockerClient := prepared.dockerClient+ slog.InfoContext(ctx, "Pulling upgrader image", "containerName", prepared.containerName, "image", prepared.pullImage)localSettings := s.settingsService.GetSettingsConfig()pullCtx, pullCancel := context.WithTimeout(ctx, timeouts.GetDuration(localSettings.DockerImagePullTimeout.AsInt(), timeouts.DefaultDockerImagePull))defer pullCancel()- pullReader, err := dockerClient.ImagePull(pullCtx, upgraderImage, client.ImagePullOptions{})+ pullReader, err := dockerClient.ImagePull(pullCtx, prepared.pullImage, client.ImagePullOptions{})if err != nil {if errors.Is(pullCtx.Err(), context.DeadlineExceeded) {- return "", fmt.Errorf("upgrader image pull timed out for %s (increase DOCKER_IMAGE_PULL_TIMEOUT or setting)", upgraderImage)+ return "", fmt.Errorf("upgrader image pull timed out for %s (increase DOCKER_IMAGE_PULL_TIMEOUT or setting)", prepared.pullImage)}return "", fmt.Errorf("pull upgrader image: %w", err)}@@ -281,12 +346,12 @@ func (s *Service) runPreparedUpgradeInternal(ctx context.Context, prepared *prepif closeErr := pullReader.Close(); closeErr != nil {slog.WarnContext(ctx, "Failed to close upgrader image pull reader", "error", closeErr)}- slog.InfoContext(ctx, "Upgrader image pulled successfully", "image", upgraderImage)+ slog.InfoContext(ctx, "Upgrader image pulled successfully", "image", prepared.pullImage)if prepared.saveCompose != nil {prepared.saveCompose(ctx)}- // Try to get the /app/data mount from current container so upgrade logs persist.+ // Mount /app/data from the current container so upgrade logs persist.appDataMount := docker.MountForDestination(prepared.current.Mounts, libarcane.UpgradeLogDirectory, libarcane.UpgradeLogDirectory)if appDataMount == nil {slog.WarnContext(ctx, "Could not detect /app/data mount; upgrader logs may not persist")@@ -294,7 +359,6 @@ func (s *Service) runPreparedUpgradeInternal(ctx context.Context, prepared *prepslog.DebugContext(ctx, "Mounting /app/data into upgrader container", "type", appDataMount.Type, "source", appDataMount.Source)}- // Create the upgrader container configcontainerEnv, runtimeMounts, networkMode, err := s.resolveRuntimeOptions(ctx,s.dockerService.DockerHost(),@@ -323,11 +387,9 @@ func (s *Service) runPreparedUpgradeInternal(ctx context.Context, prepared *prep}config := &container.Config{- Image: upgraderImage,+ Image: prepared.pullImage,Cmd: upgradeCmd,- // The upgrader needs root for the Docker socket; unlike the server it- // is short-lived and never goes through the runtime-identity drop, so- // don't rely on the image's default user.+ // Root for the Docker socket: the short-lived upgrader never goes through the runtime-identity drop.User: "0:0",Env: containerEnv,Labels: map[string]string{@@ -341,32 +403,33 @@ func (s *Service) runPreparedUpgradeInternal(ctx context.Context, prepared *prepmounts = append(mounts, *appDataMount)}- keepUpgraderContainer := strings.EqualFold(strings.TrimSpace(os.Getenv("ARCANE_UPGRADE_KEEP_CONTAINER")), "true")+ keepUpgraderContainer, _ := kit.ParseBool(os.Getenv("ARCANE_UPGRADE_KEEP_CONTAINER"))if keepUpgraderContainer {slog.InfoContext(ctx, "Keeping upgrader container after exit (ARCANE_UPGRADE_KEEP_CONTAINER=true)")}hostConfig := &container.HostConfig{- AutoRemove: !keepUpgraderContainer, // default: clean up after completion+ AutoRemove: !keepUpgraderContainer,Mounts: mounts,NetworkMode: networkMode,}- // Inherit the security context that lets the running Arcane container reach- // the Docker socket (e.g. SELinux label=disable, privileged); the upgrader- // needs the same access on hardened hosts.+ // Inherit the security context (SELinux label, privileged) that lets Arcane reach the socket on hardened hosts.if prepared.current.HostConfig != nil {hostConfig.SecurityOpt = slices.Clone(prepared.current.HostConfig.SecurityOpt)hostConfig.Privileged = prepared.current.HostConfig.Privileged}- // On SELinux-enforcing hosts the socket carries container_var_run_t, which- // container processes cannot connect to regardless of UID; without an- // explicit label opt the upgrader would exit with EACCES and auto-remove.- if !hostConfig.Privileged && !hasSELinuxLabelOptInternal(hostConfig.SecurityOpt) && daemonHasSELinuxEnabledInternal(ctx, dockerClient) {- hostConfig.SecurityOpt = append(hostConfig.SecurityOpt, "label=disable")+ // On SELinux-enforcing hosts the socket is container_var_run_t, so without a label opt the upgrader exits with EACCES.+ hasLabelOpt := slices.ContainsFunc(hostConfig.SecurityOpt, func(opt string) bool { return strings.HasPrefix(strings.TrimSpace(opt), "label") })+ if !hostConfig.Privileged && !hasLabelOpt {+ infoResult, infoErr := dockerClient.Info(ctx, client.InfoOptions{})+ if infoErr != nil {+ slog.DebugContext(ctx, "Failed to query daemon info for SELinux detection", "error", infoErr)+ } else if slices.Contains(infoResult.Info.SecurityOptions, "name=selinux") {+ hostConfig.SecurityOpt = append(hostConfig.SecurityOpt, "label=disable")+ }}upgraderName := fmt.Sprintf("%s-upgrader-%d", prepared.containerName, time.Now().Unix())-resp, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{Config: config,HostConfig: hostConfig,@@ -376,42 +439,18 @@ func (s *Service) runPreparedUpgradeInternal(ctx context.Context, prepared *prepreturn "", fmt.Errorf("create upgrader container: %w", err)}- // Start the upgrader container - it will run the upgrade and auto-removeif _, containerStartErr := dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); containerStartErr != nil {_, _ = dockerClient.ContainerRemove(ctx, resp.ID, client.ContainerRemoveOptions{Force: true})return "", fmt.Errorf("start upgrader container: %w", containerStartErr)}slog.InfoContext(ctx, "Upgrade container started", "upgraderId", resp.ID[:12], "upgraderName", upgraderName)-return resp.ID, nil}-// hasSELinuxLabelOptInternal reports whether the security options already set-// an SELinux label policy (e.g. "label=disable", "label:disable", "label=type:...").-func hasSELinuxLabelOptInternal(securityOpts []string) bool {- for _, opt := range securityOpts {- if strings.HasPrefix(strings.TrimSpace(opt), "label") {- return true- }- }- return false-}--func daemonHasSELinuxEnabledInternal(ctx context.Context, dockerClient *client.Client) bool {- infoResult, err := dockerClient.Info(ctx, client.InfoOptions{})- if err != nil {- slog.DebugContext(ctx, "Failed to query daemon info for SELinux detection", "error", err)- return false- }- return slices.Contains(infoResult.Info.SecurityOptions, "name=selinux")-}--// configuredTargetImageInternal spells targetImage like the container's Compose-// service, or like its runtime image when Arcane does not manage that project.-// A tag change also returns the Compose edit to save before the upgrader starts;-// sources Arcane cannot edit keep upgrading without it, as they did before.-func (s *Service) configuredTargetImageInternal(ctx context.Context, current container.InspectResponse, targetImage string) (string, func(context.Context), error) {+// configuredTargetImage spells targetImage like the container's Compose service (or its runtime image when unmanaged).+// A tag change also returns the Compose edit to save before the upgrader starts.+func (s *Service) configuredTargetImage(ctx context.Context, current container.InspectResponse, targetImage string) (string, func(context.Context), error) {// Digest targets cannot be written as Compose tags.if refs.NormalizeImageUpdateRef(targetImage) == "" || current.Config == nil {return targetImage, nil, nil@@ -444,43 +483,9 @@ func (s *Service) configuredTargetImageInternal(ctx context.Context, current con}, nil}-// resolveUpgradeTargetImageInternal picks the image the upgrade should move to.-// Explicit targets from the updater engine are authoritative. A blank target-// (manual trigger, update-all) resolves against the version check so a-// version-pinned install actually moves to the newest release (#3687).-func (s *Service) resolveUpgradeTargetImageInternal(ctx context.Context, currentContainer container.InspectResponse, explicitImageRef, targetVersion string) (string, error) {- if targetImage := strings.TrimSpace(explicitImageRef); targetImage != "" {- return targetImage, nil- }-- currentImageRef := ""- if currentContainer.Config != nil {- currentImageRef = strings.TrimSpace(currentContainer.Config.Image)- }- info := s.versionService.GetAppVersionInfo(ctx)- // A manager-supplied target version outranks this instance's own version check.- if targetVersion = strings.TrimSpace(targetVersion); targetVersion != "" {- merged := versiontypes.Info{}- if info != nil {- merged = *info- }- merged.NewestVersion = targetVersion- info = &merged- }- resolved, err := resolveSelfUpgradeTargetImageInternal(currentImageRef, info)- if err != nil {- return "", fmt.Errorf("resolve upgrade target image: %w", err)- }- return resolved, nil-}--// resolveSelfUpgradeTargetImageInternal resolves the image for a blank-target-// self-upgrade from the running container's reference and the version check.-// Exact release tags (vX.Y.Z) move to the newest release; mutable channels and-// untagged references keep theirs; digest pins move only to a resolved newest-// digest. Unresolved or older exact-version targets fail instead of-// reinstalling or downgrading (#3687).-func resolveSelfUpgradeTargetImageInternal(currentImageRef string, info *versiontypes.Info) (string, error) {+// resolveSelfUpgradeTargetImage moves exact release tags to the newest release and digest pins to the newest digest;+// mutable channels keep their reference. Unresolved or older exact-version targets fail (#3687).+func resolveSelfUpgradeTargetImage(currentImageRef string, info *versiontypes.Info) (string, error) {currentImageRef = strings.TrimSpace(currentImageRef)if currentImageRef == "" {return "", errors.New("running container has no image reference to upgrade from")@@ -495,10 +500,9 @@ func resolveSelfUpgradeTargetImageInternal(currentImageRef string, info *versionreturn "", fmt.Errorf("current image reference %q is not a named image", currentImageRef)}- newestDigest, newestVersion := newestTargetIdentifiersInternal(info)+ newest := kit.FromPtr(info)+ newestDigest, newestVersion := strings.TrimSpace(newest.NewestDigest), strings.TrimSpace(newest.NewestVersion)- // Digest-pinned installs only move when the version check resolved a new- // digest; otherwise there is nothing to point them at.if _, isDigested := parsed.(reference.Digested); isDigested {if newestDigest == "" {return "", errors.New("image is digest-pinned but no newest digest could be resolved; refusing an upgrade that could not move it")@@ -515,232 +519,106 @@ func resolveSelfUpgradeTargetImageInternal(currentImageRef string, info *version}tagged, isTagged := parsed.(reference.Tagged)- if !isTagged || !isExactReleaseTagInternal(tagged.Tag()) {- // Mutable channel or untagged: the pull re-resolves whatever the name- // points at, so keeping the reference advances the digest in place.+ if !isTagged || !exactReleaseTag.MatchString(tagged.Tag()) {+ // The pull re-resolves a mutable channel, so keeping the reference advances the digest in place.return currentImageRef, nil}if newestVersion == "" {return "", fmt.Errorf("running exact release %q but the newest release could not be resolved", tagged.Tag())}- newest := kit.EnsurePrefix(newestVersion, "v")- if !semver.IsValid(newest) {+ newestTag := kit.EnsurePrefix(newestVersion, "v")+ if !semver.IsValid(newestTag) {return "", fmt.Errorf("resolved newest version %q is not a valid semver release", newestVersion)}- if semver.Compare(newest, kit.EnsurePrefix(tagged.Tag(), "v")) < 0 {+ if semver.Compare(newestTag, kit.EnsurePrefix(tagged.Tag(), "v")) < 0 {return "", fmt.Errorf("newest release %q is older than the running %q; refusing to downgrade", newestVersion, tagged.Tag())}- withTag, err := reference.WithTag(named, newest)+ withTag, err := reference.WithTag(named, newestTag)if err != nil {return "", fmt.Errorf("build target reference for %q: %w", named.Name(), err)}return withTag.String(), nil}-// isExactReleaseTagInternal reports whether tag pins an exact release version-// (vX.Y.Z, prereleases included). Short channels like v2 or v2.9 are mutable.-// The x/mod/semver parser is deliberately lenient (it accepts v2 and v2.9), so-// the three numeric components are checked explicitly.-func isExactReleaseTagInternal(tag string) bool {- core := strings.TrimPrefix(strings.TrimSpace(tag), "v")- if i := strings.IndexAny(core, "-+"); i >= 0 {- core = core[:i]- }- parts := strings.Split(core, ".")- if len(parts) != 3 {- return false- }- for _, part := range parts {- if part == "" {- return false- }- for _, c := range part {- if c < '0' || c > '9' {- return false- }- }- }- return true-}--// newestTargetIdentifiersInternal reads the version-check result nil-safely: the-// service may be unavailable and the check itself may resolve neither identifier.-func newestTargetIdentifiersInternal(info *versiontypes.Info) (newestDigest, newestVersion string) {- if info == nil {- return "", ""- }- return strings.TrimSpace(info.NewestDigest), strings.TrimSpace(info.NewestVersion)-}--// getCurrentContainerID detects if we're running in Docker and returns container ID-func (s *Service) getCurrentContainerIDInternal(ctx context.Context) (string, error) {- // cgroup detection fails on cgroupv2 with a private namespace, and with- // network_mode: service:<sidecar> the hostname identifies the sidecar —- // InspectCurrentArcaneContainer adds the Arcane-label fallback (#3544).- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return "", err- }- inspect, err := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient)- if err != nil {- return "", errors.New("arcane is not running in a Docker container")- }- return inspect.ID, nil-}--// findArcaneContainer finds the container using the ID-func (s *Service) findArcaneContainerInternal(ctx context.Context, containerId string) (container.InspectResponse, error) {- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return container.InspectResponse{}, err- }-- // Try to inspect the container directly- inspectResult, err := compat.ContainerInspectWithCompatibility(ctx, dockerClient, containerId, client.ContainerInspectOptions{})- if err == nil {- return inspectResult.Container, nil- }-- // Fallback: search for containers with arcane image- filter := make(client.Filters)- filter = filter.Add("ancestor", "ghcr.io/getarcaneapp/arcane")-- containers, err := dockerClient.ContainerList(ctx, client.ContainerListOptions{- All: true,- Filters: filter,- })- if err != nil {- return container.InspectResponse{}, err- }-- for _, c := range containers.Items {- if strings.HasPrefix(c.ID, containerId) {- inspect, inspectErr := compat.ContainerInspectWithCompatibility(ctx, dockerClient, c.ID, client.ContainerInspectOptions{})- if inspectErr != nil {- return container.InspectResponse{}, inspectErr- }- return inspect.Container, nil- }- }-- // Try without filter - search all containers- allContainers, err := dockerClient.ContainerList(ctx, client.ContainerListOptions{All: true})- if err != nil {- return container.InspectResponse{}, err- }-- for _, c := range allContainers.Items {- if strings.HasPrefix(c.ID, containerId) || c.ID == containerId {- inspect, inspectErr := compat.ContainerInspectWithCompatibility(ctx, dockerClient, c.ID, client.ContainerInspectOptions{})- if inspectErr != nil {- return container.InspectResponse{}, inspectErr- }- return inspect.Container, nil- }- }-- return container.InspectResponse{}, common.Classify(common.ErrNotFound, errors.New("could not find Arcane container"))-}--const (- // The run never finished observably: nothing may be concluded from it.- upgraderExitUnobservedInternal upgraderExitInternal = 0- // The upgrader exited 0.- upgraderExitSucceededInternal upgraderExitInternal = 1- // The upgrader exited non-zero.- upgraderExitFailedInternal upgraderExitInternal = 2- // The upgrader is gone, but its exit code was lost with it — success and failure- // are indistinguishable.- upgraderExitCodeLostInternal upgraderExitInternal = 3-- updateAllStaleThresholdInternal = time.Hour- updateAllAgentRequestTimeoutInternal = 15 * time.Second- updateAllConfirmPollIntervalInternal = 10 * time.Second- updateAllConfirmTimeoutInternal = 5 * time.Minute- updateAllErrorMaxLenInternal = 500-- // How long to watch the manager's upgrader container for an exit. Generous: it- // pulls the target image before recreating anything.- updateAllManagerWatchTimeoutInternal = 15 * time.Minute- // How long to let a recreate that is already underway stop this container before- // concluding the manager was up to date and no restart is coming.- updateAllManagerNoRestartGraceInternal = 15 * time.Second-)--// StartUpdateAll begins a fleet-wide update. The agents phase runs first in the-// background (while the manager is up); the agents goroutine triggers the manager-// self-upgrade as its final step (job left pending_restart, finalized at next-// boot). Every environment pulls the latest image, whether or not it reports an-// update available.-func (s *Service) StartUpdateAll(ctx context.Context, user usertypes.Actor, env *environment.EnvironmentService) (*EnvironmentUpdateJob, error) {- // Guard the check-then-create against concurrent callers (e.g. a double-click).- // A dedicated flag rather than s.upgrading, because the manager branch below- // calls TriggerUpgradeViaCLI which acquires s.upgrading itself. The persisted- // job row is the durable guard once committed; this only closes the in-process- // window before that row exists.+// StartUpdateAll records an update-all job and submits the workflow that upgrades every agent, then the manager.+// Every environment pulls the latest image, whether or not it reports an update.+func (s *Service) StartUpdateAll(ctx context.Context, user usertypes.Actor) (*EnvironmentUpdateJob, error) {+ // Closes the in-process check-then-create window (e.g. a double-click); the job row is the durable guard.if !s.updatingAll.CompareAndSwap(false, true) {- return nil, common.Classify(common.ErrUpdateAllInProgress, errors.New("an update-all job is already in progress"))+ return nil, errUpdateAllInProgress}defer s.updatingAll.Store(false)- active, err := s.activeUpdateAllJobInternal(ctx)+ active, err := s.findUpdateAllJob(ctx, "status IN ?", activeUpdateAllStatuses)if err != nil {return nil, fmt.Errorf("check for active update-all job: %w", err)}if active != nil {- return nil, common.Classify(common.ErrUpdateAllInProgress, errors.New("an update-all job is already in progress"))+ return nil, errUpdateAllInProgress}info := s.versionService.GetAppVersionInfo(ctx)-- managerResult := EnvironmentUpdateResult{- EnvironmentID: environment.LocalEnvironmentID,- EnvironmentName: env.ResolveEnvironmentName(ctx, environment.LocalEnvironmentID),- FromVersion: info.CurrentVersion,- }-- // Seed a pending row for every remote environment up front so the dialog can- // show the whole fleet immediately instead of popping rows in as each finishes.- // Best effort: the agents phase re-lists authoritatively and fills any gaps.- remoteResults := s.seedRemoteResultsInternal(ctx, env)-job := &EnvironmentUpdateJob{+ Status: EnvironmentUpdateJobStatusRunning,UserID: user.ID,Username: user.Username,ManagerVersionAtStart: info.CurrentVersion,ManagerDigestAtStart: info.CurrentDigest,- ManagerTargetVersion: updateAllTargetVersionInternal(info),+ ManagerTargetVersion: cmp.Or(info.NewestVersion, info.NewestDigest, info.CurrentVersion, info.CurrentDigest),+ }+ // The manager row stays pending until the final manager step.+ job.Results = EnvironmentUpdateResults{{+ EnvironmentID: environment.LocalEnvironmentID,+ EnvironmentName: s.environments.ResolveEnvironmentName(ctx, environment.LocalEnvironmentID),+ Status: EnvironmentUpdateResultStatusPending,+ FromVersion: info.CurrentVersion,+ ToVersion: job.ManagerTargetVersion,+ }}+ // Seed every remote row so the dialog shows the whole fleet at once; the agents step fills any gaps.+ if remotes, listErr := s.environments.ListRemoteEnvironments(ctx); listErr != nil {+ slog.WarnContext(ctx, "update-all: failed to pre-list remote environments for seeding", "error", listErr)+ } else {+ for _, remote := range remotes {+ upsertPendingResult(job, remote.ID, remote.Name)+ }}-- // Manager upgrades LAST. Seed its row pending; the agents-phase goroutine- // flips it to updating right before triggering the self-upgrade.- managerResult.Status = EnvironmentUpdateResultStatusPending- managerResult.ToVersion = job.ManagerTargetVersion-- // Running from the start: the agents phase happens first, while the backend is- // up and can report progress. The manager self-upgrade fires at the very end of- // the agents goroutine.- job.Status = EnvironmentUpdateJobStatusRunning- job.Results = append(EnvironmentUpdateResults{managerResult}, remoteResults...)if createUpgradeJobErr := s.db.WithContext(ctx).Create(job).Error; createUpgradeJobErr != nil {return nil, fmt.Errorf("create update-all job: %w", createUpgradeJobErr)}+ input := updateAllInput{JobID: job.ID}+ input.KeyID, _ = ctx.Value(middleware.ContextKeyApiKeyID).(string)+ instanceID, submitErr := s.flow.Submit(ctx, s.updateAllWorkflow, input, activitylib.StartRequest{})+ if submitErr != nil {+ s.markUpdateAllFailed(context.WithoutCancel(ctx), job, fmt.Sprintf("failed to start update-all: %v", submitErr))+ return nil, fmt.Errorf("start update-all: %w", submitErr)+ }+ go s.watchUpdateAll(context.WithoutCancel(ctx), job.ID, instanceID)slog.InfoContext(ctx, "Update-all started; upgrading agents first", "jobId", job.ID, "user", user.Username)- go s.runAgentsPhaseInternal(context.WithoutCancel(ctx), job.ID, env, user)-return job, nil}-// ResumeUpdateAllOnStartup is called once at manager startup. When the manager-// self-upgraded as the final step of an update-all (job left pending_restart), the-// agents phase already ran before the restart — so this finalizes the manager's own-// result and closes the job. It is a no-op when there is nothing pending.+// watchUpdateAll fails a job whose workflow ends without reaching the manager step, which would leave it running+// and refuse every later run.+func (s *Service) watchUpdateAll(ctx context.Context, jobID, instanceID string) {+ outcome, err := s.flow.Wait(ctx, s.updateAllWorkflow, instanceID)+ if err != nil || outcome.Status == scheduler.Succeeded {+ return+ }+ current, err := s.loadUpdateAllJob(ctx, jobID)+ if err != nil || current.Status != EnvironmentUpdateJobStatusRunning {+ return+ }+ s.markUpdateAllFailed(ctx, current, cmp.Or(outcome.Message, "update-all did not finish"))+}++// ResumeUpdateAllOnStartup finalizes a job the manager left pending_restart by self-upgrading,+// and fails a running job whose workflow is gone. It is a no-op when nothing is pending.func (s *Service) ResumeUpdateAllOnStartup(ctx context.Context) {- job, err := s.activeUpdateAllJobInternal(ctx)+ job, err := s.findUpdateAllJob(ctx, "status IN ?", activeUpdateAllStatuses)if err != nil {slog.WarnContext(ctx, "Failed to load pending update-all job on startup", "error", err)return@@ -749,492 +627,382 @@ func (s *Service) ResumeUpdateAllOnStartup(ctx context.Context) {return}- // A job left running means the manager died mid-agents-phase, before it reached- // the manager step; we can't safely resume partial progress, so fail it.+ // A running job resumes with its workflow; one without a live workflow cannot resume, so fail it.if job.Status == EnvironmentUpdateJobStatusRunning {- s.markUpdateAllFailedInternal(ctx, job, "interrupted by manager restart")+ standalone, standaloneErr := s.flow.Standalone(ctx)+ if index := slices.IndexFunc(standalone, func(run scheduler.Run) bool { return run.JobID == updateAllWorkflowName }); standaloneErr == nil && index >= 0 {+ go s.watchUpdateAll(context.WithoutCancel(ctx), job.ID, standalone[index].ID)+ return+ }+ s.markUpdateAllFailed(ctx, job, "interrupted by manager restart")return}info := s.versionService.GetAppVersionInfo(ctx)- action := resolveResumeActionInternal(job, info.CurrentVersion, info.CurrentDigest, time.Now())-+ action := resolveResumeAction(job, info.CurrentVersion, info.CurrentDigest, time.Now())if action.markStale {- s.markUpdateAllFailedInternal(ctx, job, "update-all job is stale; manager did not restart in time")+ s.markUpdateAllFailed(ctx, job, "update-all job is stale; manager did not restart in time")return}- // The agents phase already ran before the restart. Finalize the manager's own- // result and close the job — do not re-run the agents phase.+ // The agents ran before the restart; only the manager row is left to settle.managerStatus := kit.Ternary(action.managerSucceeded, EnvironmentUpdateResultStatusUpdated, EnvironmentUpdateResultStatusFailed)- s.recordManagerResultInternal(job, managerStatus, info.CurrentVersion)- s.finalizeUpdateAllJobInternal(ctx, job)-+ s.finalizeUpdateAllJob(ctx, job, managerStatus, info.CurrentVersion)slog.InfoContext(ctx, "Finalized update-all job after manager restart", "jobId", job.ID, "managerUpgraded", action.managerSucceeded)}-type resumeActionInternal struct {- markStale bool- managerSucceeded bool+// resolveResumeAction marks a pending_restart job stale after the threshold, otherwise reports whether the manager upgrade landed.+func resolveResumeAction(job *EnvironmentUpdateJob, currentVersion, currentDigest string, now time.Time) resumeAction {+ if now.Sub(job.CreatedAt) > updateAllStaleThreshold {+ return resumeAction{markStale: true}+ }+ landed := upgradeLanded(job.ManagerVersionAtStart, job.ManagerDigestAtStart, job.ManagerTargetVersion, currentVersion, currentDigest)+ return resumeAction{managerSucceeded: landed}}-// resolveResumeActionInternal is the pure decision for a resumed pending_restart-// job: stale if it has waited too long, otherwise the manager upgrade is considered-// successful when either the version or the digest changed from the at-start values-// (digest covers non-semver, digest-pinned installs), or when the manager is already-// on the recorded target — a force-update of an up-to-date manager recreates the-// container without changing either.-func resolveResumeActionInternal(job *EnvironmentUpdateJob, currentVersion, currentDigest string, now time.Time) resumeActionInternal {- if now.Sub(job.CreatedAt) > updateAllStaleThresholdInternal {- return resumeActionInternal{markStale: true}- }-- versionChanged := job.ManagerVersionAtStart != "" && currentVersion != job.ManagerVersionAtStart- digestChanged := job.ManagerDigestAtStart != "" && currentDigest != job.ManagerDigestAtStart- // ManagerTargetVersion holds the newest version tag when known, otherwise the- // newest digest — compare against both current identifiers.- onTarget := job.ManagerTargetVersion != "" &&- (strings.TrimPrefix(currentVersion, "v") == strings.TrimPrefix(job.ManagerTargetVersion, "v") ||- currentDigest == job.ManagerTargetVersion)-- return resumeActionInternal{managerSucceeded: versionChanged || digestChanged || onTarget}+// upgradeLanded reports whether the version or digest moved off its baseline, or already matches target (a version or+// digest), so a force-update that recreates the same image still counts.+func upgradeLanded(fromVersion, fromDigest, target, currentVersion, currentDigest string) bool {+ versionChanged := fromVersion != "" && currentVersion != fromVersion+ // An agent whose Docker lookup failed reports no digest, which proves nothing.+ digestChanged := fromDigest != "" && currentDigest != "" && currentDigest != fromDigest+ onTarget := target != "" && (strings.TrimPrefix(currentVersion, "v") == strings.TrimPrefix(target, "v") || currentDigest == target)+ return versionChanged || digestChanged || onTarget}-// runAgentsPhaseInternal upgrades every online remote environment sequentially,-// persisting progress after each one so the status endpoint can report live-// progress, then triggers the manager's own self-upgrade as the final step-// (leaving the job pending_restart for the next boot to finalize).-func (s *Service) runAgentsPhaseInternal(ctx context.Context, jobID string, env *environment.EnvironmentService, user usertypes.Actor) {- job, err := s.getUpdateAllJobByIDInternal(ctx, jobID)- if err != nil || job == nil {- slog.WarnContext(ctx, "update-all: failed to reload job for agents phase", "jobId", jobID, "error", err)- return- }-- envs, err := env.ListRemoteEnvironments(ctx)+// discoverAgents lists the remote environments the agents step upgrades in turn.+func (s *Service) discoverAgents(ctx context.Context, t flow.Task) (any, error) {+ envs, err := s.environments.ListRemoteEnvironments(ctx)if err != nil {- s.markUpdateAllFailedInternal(ctx, job, fmt.Sprintf("failed to list remote environments: %v", err))- return- }-- for _, remote := range envs {- // Find the row seeded at job start (or append one if seeding missed this- // environment) and mark it updating so the dialog shows a live indicator on- // the row currently being processed.- idx := upsertPendingResultInternal(job, remote.ID, remote.Name)- job.Results[idx].Status = EnvironmentUpdateResultStatusUpdating- if persistUpdateAllJobErr := s.persistUpdateAllJobInternal(ctx, job); persistUpdateAllJobErr != nil {- slog.WarnContext(ctx, "update-all: failed to persist updating status", "jobId", job.ID, "environmentId", remote.ID, "error", persistUpdateAllJobErr)+ var input updateAllInput+ if payloadErr := t.Payload(&input); payloadErr != nil {+ return nil, errors.Join(err, payloadErr)}-- s.upgradeAgentInternal(ctx, env, remote.ID, job, &job.Results[idx])-- if saveRemoteProgressErr := s.persistUpdateAllJobInternal(ctx, job); saveRemoteProgressErr != nil {- slog.WarnContext(ctx, "update-all: failed to persist progress", "jobId", job.ID, "environmentId", remote.ID, "error", saveRemoteProgressErr)+ if job, loadErr := s.loadUpdateAllJob(ctx, input.JobID); loadErr == nil {+ s.markUpdateAllFailed(ctx, job, fmt.Sprintf("failed to list remote environments: %v", err))}+ return nil, err}-- // All remote agents processed. Handle the manager LAST: flip its row pending ->- // updating and move the job to pending_restart, persisting BEFORE the trigger so- // that if the manager dies the instant the upgrader starts, the next boot sees- // pending_restart and finalizes it.- manager := managerResultInternal(job)- if manager != nil {- manager.Status = EnvironmentUpdateResultStatusUpdating- manager.Stage = EnvironmentUpdateStageStarting- manager.StageStartedAt = new(time.Now())- }- job.Status = EnvironmentUpdateJobStatusPendingRestart- if savePendingRestartErr := s.persistUpdateAllJobInternal(ctx, job); savePendingRestartErr != nil {- slog.WarnContext(ctx, "update-all: failed to persist pending_restart before manager upgrade", "jobId", job.ID, "error", savePendingRestartErr)+ agents := make([]remoteAgent, len(envs))+ for i, remote := range envs {+ agents[i] = remoteAgent{ID: remote.ID, Name: remote.Name}}+ return agents, nil+}- // Snapshot whether the manager was already on the newest image BEFORE triggering:- // only that makes a no-op upgrader run an expected outcome. Checked after the fact- // the same answer proves nothing, since it was already true going in.- wasAlreadyNewest := s.AlreadyOnNewestImage(ctx)-- upgraderID, err := s.TriggerUpgradeViaCLI(ctx, user, updater.SelfUpdateTarget{})- if err != nil {- // Agents already ran and no restart is coming, so finalize now. This flips- // the manager's updating row to failed with the reason.- s.markUpdateAllFailedInternal(ctx, job, fmt.Sprintf("manager upgrade trigger failed: %v", err))- return+// upgradeAgentTask triggers and confirms one remote environment's upgrade, persisting each stage for the status endpoint.+// A redelivered task skips an agent whose result is already terminal.+func (s *Service) upgradeAgentTask(ctx context.Context, t flow.Task) (_ any, err error) {+ var input updateAllInput+ var remote remoteAgent+ if decodeErr := errors.Join(t.Payload(&input), t.DecodeItem(&remote)); decodeErr != nil {+ return nil, decodeErr}-- if manager != nil {- s.setUpdateStageInternal(ctx, job, manager, EnvironmentUpdateStageReconnecting)+ job, err := s.loadUpdateAllJob(ctx, input.JobID)+ if err != nil {+ return nil, err}-- slog.InfoContext(ctx, "Update-all: agents done, manager self-upgrade triggered", "jobId", job.ID, "upgraderId", upgraderID)- s.watchManagerUpgraderInternal(ctx, job.ID, upgraderID, wasAlreadyNewest)-}--// watchManagerUpgraderInternal closes out a job whose manager never restarted. The-// upgrader skips the container recreate when the pull lands on the image already-// running, so this process survives it — and the job, persisted pending_restart for-// the next boot to finalize, would otherwise wait for a restart that never comes.-//-// Runs at the tail of the agents-phase goroutine. When the manager IS recreated this-// process is stopped instead and ResumeUpdateAllOnStartup finalizes on the next boot.-// wasAlreadyNewest is the pre-trigger version check, the only evidence that a run-// which changed nothing was supposed to change nothing.-func (s *Service) watchManagerUpgraderInternal(ctx context.Context, jobID, upgraderID string, wasAlreadyNewest bool) {- exit, exitCode := s.waitForUpgraderExitInternal(ctx, upgraderID)- if exit == upgraderExitUnobservedInternal {- // Never saw it finish, so a recreate may still be coming — but it may equally- // never come, and pending_restart is only ever resolved at boot. Wait the run- // out rather than returning: an unobserved exit must not strand the job.- s.closeOutUnobservedUpgradeInternal(ctx, jobID)- return+ if job.Status != EnvironmentUpdateJobStatusRunning {+ return nil, nil}-- // A recreate stops this container, but not instantly — the stop carries a grace- // period during which this goroutine still runs. Wait it out so a restart that is- // already underway wins: if the manager is being replaced, the process dies here.- time.Sleep(updateAllManagerNoRestartGraceInternal)-- job, err := s.getUpdateAllJobByIDInternal(ctx, jobID)- if err != nil || job == nil {- slog.WarnContext(ctx, "update-all: failed to reload job after upgrader exit", "jobId", jobID, "error", err)- return+ if allowed, authorizeErr := s.authorizeUpdateAll(ctx, job, input.KeyID); !allowed {+ return nil, authorizeErr}- if job.Status != EnvironmentUpdateJobStatusPendingRestart {- return+ result := &job.Results[upsertPendingResult(job, remote.ID, remote.Name)]+ if result.Status != EnvironmentUpdateResultStatusPending && result.Status != EnvironmentUpdateResultStatusUpdating {+ return nil, nil}-- if exit == upgraderExitFailedInternal {- s.markUpdateAllFailedInternal(ctx, job, fmt.Sprintf("manager upgrade failed: upgrader exited with code %d", exitCode))- return+ // A saved starting or later stage means the trigger may have reached the agent; sending it again could start a second upgrader.+ // Only a later stage proves the agent accepted it.+ triggered := result.Status == EnvironmentUpdateResultStatusUpdating && result.Stage != "" && result.Stage != EnvironmentUpdateStageChecking+ unconfirmed := triggered && result.Stage == EnvironmentUpdateStageStarting+ result.Status = EnvironmentUpdateResultStatusUpdating+ if persistErr := s.db.WithContext(ctx).Save(job).Error; persistErr != nil {+ slog.WarnContext(ctx, "update-all: failed to persist updating status", "jobId", job.ID, "environmentId", remote.ID, "error", persistErr)}+ defer func() {+ result.clearStage()+ // Shutdown: leave the row updating at its stage so the redelivered task resumes it.+ if err = ctx.Err(); err != nil {+ return+ }+ if persistErr := s.db.WithContext(ctx).Save(job).Error; persistErr != nil {+ slog.WarnContext(ctx, "update-all: failed to persist progress", "jobId", job.ID, "environmentId", remote.ID, "error", persistErr)+ }+ }()- info := s.versionService.GetAppVersionInfo(ctx)+ var info versiontypes.Info+ if triggered {+ info.CurrentVersion, info.CurrentDigest = result.FromVersion, result.FromDigest+ } else {+ s.setUpdateStage(ctx, job, result, EnvironmentUpdateStageChecking)+ versionCtx, cancel := context.WithTimeout(ctx, updateAllAgentRequestTimeout)+ checkErr := s.environments.ProxyJSONRequest(versionCtx, remote.ID, http.MethodGet, "/api/app-version", nil, &info)+ cancel()+ if checkErr != nil {+ msg := checkErr.Error()+ result.Status, result.Error = updateAllAgentFailureStatus(checkErr), msg[:min(len(msg), updateAllErrorMaxLen)]+ return nil, nil+ }+ // The manager's newest release outranks the agent's own check, so the recorded target tracks what was sent.+ var triggerBody []byte+ if newest := strings.TrimSpace(s.versionService.GetAppVersionInfo(ctx).NewestVersion); newest != "" {+ body, marshalErr := json.Marshal(TriggerUpgradeBody{TargetVersion: newest})+ if marshalErr != nil {+ slog.WarnContext(ctx, "update-all: failed to marshal trigger body", "environmentId", remote.ID, "error", marshalErr)+ } else {+ info.NewestVersion, triggerBody = newest, body+ }+ }+ result.FromVersion, result.FromDigest = info.CurrentVersion, info.CurrentDigest+ result.ToVersion = cmp.Or(info.NewestVersion, info.NewestDigest, info.CurrentVersion, info.CurrentDigest)++ // The saved starting stage keeps a replay from sending the trigger again, so the trigger waits on it.+ result.Stage, result.StageStartedAt = EnvironmentUpdateStageStarting, new(time.Now())+ if saveErr := s.db.WithContext(ctx).Save(job).Error; saveErr != nil {+ result.Status, result.Error = EnvironmentUpdateResultStatusFailed, "Could not save upgrade progress: "+saveErr.Error()+ return nil, nil+ }+ triggerCtx, cancel := context.WithTimeout(ctx, updateAllAgentRequestTimeout)+ resp, triggerErr := s.environments.ExecuteRemoteRequest(triggerCtx, remote.ID, http.MethodPost, "/api/environments/0/system/upgrade", triggerBody)+ cancel()+ if triggerErr == nil {+ triggerErr = resp.RequireSuccess()+ }+ if triggerErr != nil {+ msg := triggerErr.Error()+ result.Status, result.Error = EnvironmentUpdateResultStatusFailed, msg[:min(len(msg), updateAllErrorMaxLen)]+ return nil, nil+ }- // Still running here means the container was never recreated, so the upgrader either- // found the image already current or died before it got that far. With no exit code- // to tell those apart, a no-op is only credible when the pre-trigger check already- // said there was nothing to swap in and the post-run check still agrees: an after-- // the-fact "already newest" on its own restates what was true before the run, so it- // cannot vouch for it — and on a manager that was due an update it would launder a- // dead upgrader (stale newest-image state, a check that regressed to the running- // image) into a green row and zero logged failures, leaving a broken upgrade with no- // retry path. Anything less is a failure; the job closes either way, since no restart- // is coming.- if exit == upgraderExitCodeLostInternal && (!wasAlreadyNewest || !info.AlreadyOnNewest()) {- s.markUpdateAllFailedInternal(ctx, job, "manager upgrade could not be confirmed: the upgrader exited without a readable status and this manager was not a confirmed no-op upgrade")- return+ // An agent already on the target finds nothing to swap in; confirming would only burn the poll window.+ if info.AlreadyOnNewest() {+ result.Status, result.ToVersion = EnvironmentUpdateResultStatusUpToDate, info.CurrentVersion+ return nil, nil+ }}- s.recordManagerResultInternal(job, EnvironmentUpdateResultStatusUpToDate, info.CurrentVersion)- slog.InfoContext(ctx, "Update-all: manager was already up to date; no restart needed", "jobId", job.ID, "version", info.CurrentVersion)- s.finalizeUpdateAllJobInternal(ctx, job)-}--// closeOutUnobservedUpgradeInternal fails a job whose upgrader outcome was never-// observed and whose manager then never restarted. Waiting is what makes the answer-// safe: a recreate kills this process, so still being here once the job would be-// considered stale proves no restart is coming. Nothing else would close it — only a-// boot resolves pending_restart — and until it closes, every later update-all is-// refused as already in progress.-func (s *Service) closeOutUnobservedUpgradeInternal(ctx context.Context, jobID string) {- job, err := s.getUpdateAllJobByIDInternal(ctx, jobID)- if err != nil || job == nil {- slog.WarnContext(ctx, "update-all: failed to reload job after unobserved upgrader exit", "jobId", jobID, "error", err)- return- }- if job.Status != EnvironmentUpdateJobStatusPendingRestart {- return+ // Poll until the agent's version lands; past the window the upgrade is only recorded as triggered.+ // An unconfirmed request keeps its starting stage, so another restart still knows nothing confirmed it.+ stage := func(next EnvironmentUpdateStage) {+ if !unconfirmed {+ s.setUpdateStage(ctx, job, result, next)+ }}-- // Hold until the job hits the same staleness bound a resumed job is judged by, so a- // slow upgrader still gets its full window to recreate this container.- if remaining := time.Until(job.CreatedAt.Add(updateAllStaleThresholdInternal)); remaining > 0 {+ stage(EnvironmentUpdateStageReconnecting)+ ticker := time.NewTicker(updateAllConfirmPollInterval)+ defer ticker.Stop()+ for deadline := time.Now().Add(updateAllConfirmTimeout); time.Now().Before(deadline); {select {case <-ctx.Done():- return- case <-time.After(remaining):+ return nil, nil+ case <-ticker.C:+ }+ pollCtx, pollCancel := context.WithTimeout(ctx, updateAllAgentRequestTimeout)+ var current versiontypes.Info+ pollErr := s.environments.ProxyJSONRequest(pollCtx, remote.ID, http.MethodGet, "/api/app-version", nil, ¤t)+ pollCancel()+ if pollErr != nil {+ stage(EnvironmentUpdateStageReconnecting)+ continue+ }+ stage(EnvironmentUpdateStageVerifying)+ // An unconfirmed request proves nothing by the agent merely matching the target, so it needs a real change.+ if upgradeLanded(info.CurrentVersion, info.CurrentDigest, kit.Ternary(unconfirmed, "", result.ToVersion), current.CurrentVersion, current.CurrentDigest) {+ result.Status = EnvironmentUpdateResultStatusUpdated+ return nil, nil}}-- job, err = s.getUpdateAllJobByIDInternal(ctx, jobID)- if err != nil || job == nil {- slog.WarnContext(ctx, "update-all: failed to reload stale job", "jobId", jobID, "error", err)- return- }- if job.Status != EnvironmentUpdateJobStatusPendingRestart {- return+ if unconfirmed {+ result.Status, result.Error = EnvironmentUpdateResultStatusFailed, "Upgrade request was interrupted before the agent confirmed it"+ return nil, nil}-- slog.WarnContext(ctx, "update-all: manager upgrade was never observed and no restart followed", "jobId", job.ID)- s.markUpdateAllFailedInternal(ctx, job, "manager upgrade could not be observed: the upgrader run was never seen to finish and the manager did not restart")+ result.Status = EnvironmentUpdateResultStatusTriggered+ return nil, nil}-// upgraderExitInternal is how much the watcher managed to learn about an upgrader run.-type upgraderExitInternal int--// waitForUpgraderExitInternal blocks until the upgrader container stops, reporting what-// could be learned about how it ended along with its exit code when one was observed.-func (s *Service) waitForUpgraderExitInternal(ctx context.Context, upgraderID string) (upgraderExitInternal, int64) {- dockerClient, err := s.dockerService.GetClient(ctx)+// upgradeManagerTask triggers the manager's own self-upgrade last. The job is persisted pending_restart BEFORE the+// trigger, so a redelivery never triggers twice and the next boot finalizes it.+func (s *Service) upgradeManagerTask(ctx context.Context, t flow.Task) (any, error) {+ var input updateAllInput+ if err := t.Payload(&input); err != nil {+ return nil, err+ }+ job, err := s.loadUpdateAllJob(ctx, input.JobID)if err != nil {- slog.WarnContext(ctx, "update-all: cannot watch upgrader container", "upgraderId", upgraderID, "error", err)- return upgraderExitUnobservedInternal, 0+ return nil, err}-- waitCtx, cancel := context.WithTimeout(ctx, updateAllManagerWatchTimeoutInternal)- defer cancel()-- wait := dockerClient.ContainerWait(waitCtx, upgraderID, client.ContainerWaitOptions{- Condition: container.WaitConditionNotRunning,- })- select {- case result, ok := <-wait.Result:- // A closed channel hands back a zero-value response, which is indistinguishable- // from a clean exit 0 — nothing was delivered, so nothing may be concluded.- if !ok {- slog.WarnContext(ctx, "update-all: upgrader wait ended without a status", "upgraderId", upgraderID)- return upgraderExitUnobservedInternal, 0+ allowed := false+ if job.Status == EnvironmentUpdateJobStatusRunning {+ if allowed, err = s.authorizeUpdateAll(ctx, job, input.KeyID); err != nil {+ return nil, err}- if result.StatusCode != 0 {- return upgraderExitFailedInternal, result.StatusCode+ }+ if allowed {+ // Every agent task has finished, so an agent row still in progress belongs to a task that failed first.+ for i := range job.Results {+ pending := job.Results[i].Status == EnvironmentUpdateResultStatusPending || job.Results[i].Status == EnvironmentUpdateResultStatusUpdating+ if job.Results[i].EnvironmentID != environment.LocalEnvironmentID && pending {+ job.Results[i].clearStage()+ job.Results[i].Status, job.Results[i].Error = EnvironmentUpdateResultStatusFailed, "Agent upgrade did not finish"+ }}- // The daemon fills in status code 0 alongside a wait error (the container was- // removed mid-wait, the wait itself broke): the exit is real but its code is not.- if result.Error != nil {- slog.WarnContext(ctx, "update-all: upgrader wait reported an error", "upgraderId", upgraderID, "error", result.Error.Message)- return upgraderExitCodeLostInternal, 0+ manager := slices.IndexFunc(job.Results, func(r EnvironmentUpdateResult) bool { return r.EnvironmentID == environment.LocalEnvironmentID })+ if manager >= 0 {+ job.Results[manager].Status = EnvironmentUpdateResultStatusUpdating+ job.Results[manager].Stage = EnvironmentUpdateStageStarting+ job.Results[manager].StageStartedAt = new(time.Now())}- return upgraderExitSucceededInternal, 0- case waitUpgraderErr, ok := <-wait.Error:- // The upgrader runs with AutoRemove, so a fast run can be gone before the wait- // is acknowledged. It exited, but the code went with it — a failed upgrade must- // not be mistaken for a clean no-op, so report the ambiguity rather than a code.- if errdefs.IsNotFound(waitUpgraderErr) {- return upgraderExitCodeLostInternal, 0+ job.Status = EnvironmentUpdateJobStatusPendingRestart+ // A replay that misses pending_restart would trigger the manager upgrade again, so the trigger waits on it.+ if savePendingRestartErr := s.db.WithContext(ctx).Save(job).Error; savePendingRestartErr != nil {+ return nil, fmt.Errorf("persist pending_restart before manager upgrade: %w", savePendingRestartErr)}- if !ok || waitUpgraderErr == nil {- slog.WarnContext(ctx, "update-all: upgrader wait ended without a status", "upgraderId", upgraderID)+ // Only a manager already on the newest image makes a no-op upgrader run expected; checked after the fact it proves nothing.+ wasAlreadyNewest := s.AlreadyOnNewestImage(ctx)+ upgraderID, triggerErr := s.TriggerUpgradeViaCLI(ctx, usertypes.Actor{ID: job.UserID, Username: job.Username}, updater.SelfUpdateTarget{})+ if triggerErr != nil {+ // No restart is coming, so this flips the manager's updating row to failed now.+ s.markUpdateAllFailed(ctx, job, fmt.Sprintf("manager upgrade trigger failed: %v", triggerErr))} else {- slog.WarnContext(ctx, "update-all: failed waiting for upgrader container", "upgraderId", upgraderID, "error", waitUpgraderErr)+ if manager >= 0 {+ s.setUpdateStage(ctx, job, &job.Results[manager], EnvironmentUpdateStageReconnecting)+ }+ slog.InfoContext(ctx, "Update-all: agents done, manager self-upgrade triggered", "jobId", job.ID, "upgraderId", upgraderID)+ s.watchManagerUpgrader(ctx, job, upgraderID, wasAlreadyNewest)+ }+ if ctx.Err() != nil {+ return nil, ctx.Err()+ }+ if job, err = s.loadUpdateAllJob(ctx, input.JobID); err != nil {+ return nil, err}- return upgraderExitUnobservedInternal, 0- case <-waitCtx.Done():- slog.WarnContext(ctx, "update-all: timed out waiting for upgrader container", "upgraderId", upgraderID)- return upgraderExitUnobservedInternal, 0- }-}--// seedRemoteResultsInternal builds a pending result row for every remote environment-// so the dialog can render the whole fleet immediately. Best effort: on error it-// returns nil and the agents phase appends rows as it processes them.-func (s *Service) seedRemoteResultsInternal(ctx context.Context, env *environment.EnvironmentService) EnvironmentUpdateResults {- envs, err := env.ListRemoteEnvironments(ctx)- if err != nil {- slog.WarnContext(ctx, "update-all: failed to pre-list remote environments for seeding", "error", err)- return nil- }- results := make(EnvironmentUpdateResults, 0, len(envs))- for _, remote := range envs {- results = append(results, EnvironmentUpdateResult{- EnvironmentID: remote.ID,- EnvironmentName: remote.Name,- Status: EnvironmentUpdateResultStatusPending,- })- }- return results-}--// upsertPendingResultInternal returns the index of the existing result row for envID,-// appending a new pending row when seeding missed it (e.g. the seed list failed, or a-// new environment was registered after the job started).-func upsertPendingResultInternal(job *EnvironmentUpdateJob, envID, envName string) int {- if idx := slices.IndexFunc(job.Results, func(r EnvironmentUpdateResult) bool { return r.EnvironmentID == envID }); idx >= 0 {- return idx}- job.Results = append(job.Results, EnvironmentUpdateResult{- EnvironmentID: envID,- EnvironmentName: envName,- Status: EnvironmentUpdateResultStatusPending,- })- return len(job.Results) - 1-}--// managerResultInternal returns the manager's (env "0") row, or nil when the job has none.-func managerResultInternal(job *EnvironmentUpdateJob) *EnvironmentUpdateResult {- idx := slices.IndexFunc(job.Results, func(r EnvironmentUpdateResult) bool { return r.EnvironmentID == environment.LocalEnvironmentID })- if idx < 0 {- return nil+ if job.Status == EnvironmentUpdateJobStatusFailed {+ return scheduler.Outcome{Status: scheduler.Failed, Message: kit.FromPtr(job.Error)}, nil}- return &job.Results[idx]+ return scheduler.Outcome{Status: scheduler.Succeeded, Message: "Update-all finished; " + string(job.Status)}, nil}-// upgradeAgentInternal triggers and confirms a single remote environment's-// self-upgrade, recording the outcome on result. The upgrade always runs — the-// agent pulls the latest image even when it reports no update available.-func (s *Service) upgradeAgentInternal(ctx context.Context, env *environment.EnvironmentService, envID string, job *EnvironmentUpdateJob, result *EnvironmentUpdateResult) {- defer result.clearStageInternal()- s.setUpdateStageInternal(ctx, job, result, EnvironmentUpdateStageChecking)- versionCtx, cancel := context.WithTimeout(ctx, updateAllAgentRequestTimeoutInternal)- var info versiontypes.Info- err := env.ProxyJSONRequest(versionCtx, envID, http.MethodGet, "/api/app-version", nil, &info)- cancel()+// watchManagerUpgrader closes out a pending_restart job whose manager was never recreated, which a recreate would have+// prevented by stopping this process. wasAlreadyNewest is the pre-trigger check that a no-op run was expected.+func (s *Service) watchManagerUpgrader(ctx context.Context, job *EnvironmentUpdateJob, upgraderID string, wasAlreadyNewest bool) {+ exit, exitCode := upgraderExitUnobserved, int64(0)+ dockerClient, err := s.dockerService.GetClient(ctx)if err != nil {- result.Status = updateAllAgentFailureStatusInternal(err)- result.Error = truncateUpdateAllErrorInternal(err)- return- }- // The manager's newest release outranks the agent's own check, mirrored into info- // so the recorded target, up-to-date check and confirm poll track what was sent.- var triggerBody []byte- if managerInfo := s.versionService.GetAppVersionInfo(ctx); managerInfo != nil {- if newest := strings.TrimSpace(managerInfo.NewestVersion); newest != "" {- body, marshalErr := json.Marshal(TriggerUpgradeBody{TargetVersion: newest})- if marshalErr != nil {- slog.WarnContext(ctx, "update-all: failed to marshal trigger body", "environmentId", envID, "error", marshalErr)+ slog.WarnContext(ctx, "update-all: cannot watch upgrader container", "upgraderId", upgraderID, "error", err)+ } else {+ waitCtx, cancel := context.WithTimeout(ctx, updateAllManagerWatchTimeout)+ wait := dockerClient.ContainerWait(waitCtx, upgraderID, client.ContainerWaitOptions{Condition: container.WaitConditionNotRunning})+ select {+ case result, ok := <-wait.Result:+ // A closed channel yields a zero response that looks like exit 0; a wait error comes with a filler code 0.+ switch {+ case !ok:+ slog.WarnContext(ctx, "update-all: upgrader wait ended without a status", "upgraderId", upgraderID)+ case result.StatusCode != 0:+ exit, exitCode = upgraderExitFailed, result.StatusCode+ case result.Error != nil:+ slog.WarnContext(ctx, "update-all: upgrader wait reported an error", "upgraderId", upgraderID, "error", result.Error.Message)+ exit = upgraderExitCodeLost+ default:+ exit = upgraderExitSucceeded+ }+ case waitErr := <-wait.Error:+ // AutoRemove can delete a fast upgrader before the wait lands: it exited, but its code went with it.+ if errdefs.IsNotFound(waitErr) {+ exit = upgraderExitCodeLost} else {- info.NewestVersion = newest- triggerBody = body+ slog.WarnContext(ctx, "update-all: failed waiting for upgrader container", "upgraderId", upgraderID, "error", waitErr)}+ case <-waitCtx.Done():+ slog.WarnContext(ctx, "update-all: timed out waiting for upgrader container", "upgraderId", upgraderID)}+ cancel()}- result.FromVersion = info.CurrentVersion- result.ToVersion = updateAllTargetVersionInternal(&info)- s.setUpdateStageInternal(ctx, job, result, EnvironmentUpdateStageStarting)- triggerCtx, cancel := context.WithTimeout(ctx, updateAllAgentRequestTimeoutInternal)- resp, err := env.ExecuteRemoteRequest(triggerCtx, envID, http.MethodPost, "/api/environments/0/system/upgrade", triggerBody)- cancel()+ // Let a recreate already underway stop this process; an unobserved run waits out the stale window instead.+ hold := updateAllManagerNoRestartGrace+ if exit == upgraderExitUnobserved {+ hold = time.Until(job.CreatedAt.Add(updateAllStaleThreshold))+ }+ select {+ case <-ctx.Done():+ return+ case <-time.After(hold):+ }++ job, err = s.loadUpdateAllJob(ctx, job.ID)if err != nil {- result.Status = EnvironmentUpdateResultStatusFailed- result.Error = truncateUpdateAllErrorInternal(err)+ slog.WarnContext(ctx, "update-all: failed to reload job after upgrader exit", "upgraderId", upgraderID, "error", err)return}- if requireSuccessErr := resp.RequireSuccess(); requireSuccessErr != nil {- result.Status = EnvironmentUpdateResultStatusFailed- result.Error = truncateUpdateAllErrorInternal(requireSuccessErr)+ if job.Status != EnvironmentUpdateJobStatusPendingRestart {return}- // An agent that reports no update available and already runs the target has- // nothing to swap in: its upgrader pulls, finds the same image and skips the- // recreate. Confirming that would only burn the poll window waiting for a version- // change that cannot come, so record it directly.- if info.AlreadyOnNewest() {- result.Status = EnvironmentUpdateResultStatusUpToDate- result.ToVersion = info.CurrentVersion- return+ info := s.versionService.GetAppVersionInfo(ctx)+ switch {+ case exit == upgraderExitUnobserved:+ slog.WarnContext(ctx, "update-all: manager upgrade was never observed and no restart followed", "jobId", job.ID)+ s.markUpdateAllFailed(ctx, job, "manager upgrade could not be observed: the upgrader run was never seen to finish and the manager did not restart")+ case exit == upgraderExitFailed:+ s.markUpdateAllFailed(ctx, job, fmt.Sprintf("manager upgrade failed: upgrader exited with code %d", exitCode))+ case exit == upgraderExitCodeLost && (!wasAlreadyNewest || !info.AlreadyOnNewest()):+ // Without an exit code a no-op is only credible when the check said so both before and after the run.+ s.markUpdateAllFailed(ctx, job, "manager upgrade could not be confirmed: the upgrader exited without a readable status and this manager was not a confirmed no-op upgrade")+ default:+ slog.InfoContext(ctx, "Update-all: manager was already up to date; no restart needed", "jobId", job.ID, "version", info.CurrentVersion)+ s.finalizeUpdateAllJob(ctx, job, EnvironmentUpdateResultStatusUpToDate, info.CurrentVersion)}+}- s.setUpdateStageInternal(ctx, job, result, EnvironmentUpdateStageReconnecting)- if s.confirmAgentUpgradedInternal(ctx, env, envID, info, job, result) {- result.Status = EnvironmentUpdateResultStatusUpdated- } else {- // Upgrade fired but the new version was not confirmed within the wait window.- result.Status = EnvironmentUpdateResultStatusTriggered+// upsertPendingResult returns the index of envID's result row, appending a pending row when seeding missed it.+func upsertPendingResult(job *EnvironmentUpdateJob, envID, envName string) int {+ if idx := slices.IndexFunc(job.Results, func(r EnvironmentUpdateResult) bool { return r.EnvironmentID == envID }); idx >= 0 {+ return idx}+ job.Results = append(job.Results, EnvironmentUpdateResult{+ EnvironmentID: envID,+ EnvironmentName: envName,+ Status: EnvironmentUpdateResultStatusPending,+ })+ return len(job.Results) - 1}-// updateAllAgentFailureStatusInternal classifies a failed agent pre-check. An-// environment we actually reached but whose request failed or timed out is a real-// failure, not an offline skip: poll-mode agents connect on demand, so a slow-// tunnel round-trip surfaces here as a deadline even though the agent is online.-// Only errors that look like the environment was never reachable (no tunnel,-// connection refused, DNS failure, …) stay an offline skip.-func updateAllAgentFailureStatusInternal(err error) EnvironmentUpdateResultStatus {- // The tunnel/connection was established but the request did not finish: it- // either timed out or was canceled (e.g. the parent context was aborted).+// updateAllAgentFailureStatus classifies a failed agent pre-check: a reached environment whose request failed or timed+// out (poll-mode tunnels connect on demand) is a failure; only a never-reachable one is an offline skip.+func updateAllAgentFailureStatus(err error) EnvironmentUpdateResultStatus {if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {return EnvironmentUpdateResultStatusFailed}- // The environment answered with a non-success status — reached, not offline.if _, ok := errors.AsType[*remenv.StatusError](err); ok {return EnvironmentUpdateResultStatusFailed}return EnvironmentUpdateResultStatusSkippedOffline}-// confirmAgentUpgradedInternal polls the agent's version until it moves off the-// pre-upgrade baseline or reports the upgrade target, or the wait window elapses.-// The target is the same fallback-resolved identifier recorded in ToVersion, so a-// force-update of an already-latest agent — including one whose version check-// could not determine the latest release — confirms on a same-image recreation-// instead of timing out to triggered.-func (s *Service) confirmAgentUpgradedInternal(- ctx context.Context, env *environment.EnvironmentService, envID string,- baseline versiontypes.Info, job *EnvironmentUpdateJob, result *EnvironmentUpdateResult,-) bool {- target := updateAllTargetVersionInternal(&baseline)- deadline := time.Now().Add(updateAllConfirmTimeoutInternal)- ticker := time.NewTicker(updateAllConfirmPollIntervalInternal)- defer ticker.Stop()-- for {- select {- case <-ctx.Done():- return false- case <-ticker.C:- reqCtx, cancel := context.WithTimeout(ctx, updateAllAgentRequestTimeoutInternal)- var info versiontypes.Info- err := env.ProxyJSONRequest(reqCtx, envID, http.MethodGet, "/api/app-version", nil, &info)- cancel()- if err == nil {- s.setUpdateStageInternal(ctx, job, result, EnvironmentUpdateStageVerifying)- versionChanged := baseline.CurrentVersion != "" && info.CurrentVersion != baseline.CurrentVersion- digestChanged := baseline.CurrentDigest != "" && info.CurrentDigest != baseline.CurrentDigest- onTarget := target != "" &&- (strings.TrimPrefix(info.CurrentVersion, "v") == strings.TrimPrefix(target, "v") ||- info.CurrentDigest == target)- if versionChanged || digestChanged || onTarget {- return true- }- } else {- s.setUpdateStageInternal(ctx, job, result, EnvironmentUpdateStageReconnecting)- }- if time.Now().After(deadline) {- return false- }- }- }-}--// setUpdateStageInternal records which step result is in and persists the job so-// the status endpoint reflects it; re-entering the current stage keeps its start time.-func (s *Service) setUpdateStageInternal(ctx context.Context, job *EnvironmentUpdateJob, result *EnvironmentUpdateResult, stage EnvironmentUpdateStage) {+// setUpdateStage records result's step and persists the job; re-entering the current stage keeps its start time.+func (s *Service) setUpdateStage(ctx context.Context, job *EnvironmentUpdateJob, result *EnvironmentUpdateResult, stage EnvironmentUpdateStage) {if result.Stage == stage {return}result.Stage = stageresult.StageStartedAt = new(time.Now())- if err := s.persistUpdateAllJobInternal(ctx, job); err != nil {+ if err := s.db.WithContext(ctx).Save(job).Error; err != nil {slog.WarnContext(ctx, "update-all: failed to persist stage", "jobId", job.ID, "environmentId", result.EnvironmentID, "stage", stage, "error", err)}}// GetLatestUpdateAllJob returns the most recently created update-all job, or nil.func (s *Service) GetLatestUpdateAllJob(ctx context.Context) (*EnvironmentUpdateJob, error) {- var jobs []EnvironmentUpdateJob- if err := s.db.WithContext(ctx).- Order("created_at DESC").- Limit(1).- Find(&jobs).Error; err != nil {- return nil, err- }- if len(jobs) == 0 {- return nil, nil- }- return &jobs[0], nil+ return s.findUpdateAllJob(ctx)}-func (s *Service) activeUpdateAllJobInternal(ctx context.Context) (*EnvironmentUpdateJob, error) {+// findUpdateAllJob returns the newest job matching the inline GORM conditions, or nil when none does.+func (s *Service) findUpdateAllJob(ctx context.Context, conds ...any) (*EnvironmentUpdateJob, error) {var jobs []EnvironmentUpdateJob- if err := s.db.WithContext(ctx).- Where("status IN ?", []string{- string(EnvironmentUpdateJobStatusPendingRestart),- string(EnvironmentUpdateJobStatusRunning),- }).- Order("created_at DESC").- Limit(1).- Find(&jobs).Error; err != nil {+ if err := s.db.WithContext(ctx).Order("created_at DESC").Limit(1).Find(&jobs, conds...).Error; err != nil {return nil, err}if len(jobs) == 0 {@@ -1243,42 +1011,46 @@ func (s *Service) activeUpdateAllJobInternal(ctx context.Context) (*EnvironmentUreturn &jobs[0], nil}-func (s *Service) getUpdateAllJobByIDInternal(ctx context.Context, id string) (*EnvironmentUpdateJob, error) {- var jobs []EnvironmentUpdateJob- if err := s.db.WithContext(ctx).- Where("id = ?", id).- Limit(1).- Find(&jobs).Error; err != nil {- return nil, err- }- if len(jobs) == 0 {- return nil, nil+// loadUpdateAllJob loads a job by ID, failing when it no longer exists.+func (s *Service) loadUpdateAllJob(ctx context.Context, id string) (*EnvironmentUpdateJob, error) {+ job, err := s.findUpdateAllJob(ctx, "id = ?", id)+ if err == nil && job == nil {+ err = fmt.Errorf("update-all job %s not found", id)}- return &jobs[0], nil+ return job, err}-func (s *Service) persistUpdateAllJobInternal(ctx context.Context, job *EnvironmentUpdateJob) error {- return s.db.WithContext(ctx).Save(job).Error+// authorizeUpdateAll reports whether the job's requester may still upgrade, failing the job when not.+// A task recovered after a restart never passed the request's permission check. Shutdown returns its+// error so the task resumes instead of failing the job.+func (s *Service) authorizeUpdateAll(ctx context.Context, job *EnvironmentUpdateJob, keyID string) (bool, error) {+ permissions, err := s.roles.ResolveExecutionPermissions(ctx, job.UserID, keyID)+ if ctx.Err() != nil {+ return false, ctx.Err()+ }+ if err == nil && permissions.Allows(authz.PermSystemUpgrade, environment.LocalEnvironmentID) {+ return true, nil+ }+ s.markUpdateAllFailed(ctx, job, "requesting user no longer has permission to upgrade")+ return false, nil}-func (s *Service) markUpdateAllFailedInternal(ctx context.Context, job *EnvironmentUpdateJob, reason string) {+func (s *Service) markUpdateAllFailed(ctx context.Context, job *EnvironmentUpdateJob, reason string) {job.Status = EnvironmentUpdateJobStatusFailedjob.Error = &reasonjob.CompletedAt = new(time.Now())for i := range job.Results {- job.Results[i].clearStageInternal()+ job.Results[i].clearStage()if job.Results[i].Status == EnvironmentUpdateResultStatusUpdating {job.Results[i].Status = EnvironmentUpdateResultStatusFailedjob.Results[i].Error = reason}}- if err := s.persistUpdateAllJobInternal(ctx, job); err != nil {+ if err := s.db.WithContext(ctx).Save(job).Error; err != nil {slog.WarnContext(ctx, "update-all: failed to mark job failed", "jobId", job.ID, "error", err)}- // Surface the failure in the events audit log; the success path logs via- // logUpdateAllEventInternal. LogUserEvent hardcodes an info-severity "completed"- // title, so create the event directly with error severity and the reason.+ // LogUserEvent always records an info-severity "completed" event, so create the error event directly.if _, err := s.eventService.CreateEvent(ctx, event.CreateEventRequest{Type: event.EventTypeSystemUpgrade,Severity: event.EventSeverityError,@@ -1299,51 +1071,29 @@ func (s *Service) markUpdateAllFailedInternal(ctx context.Context, job *Environmslog.WarnContext(ctx, "Update-all job failed", "jobId", job.ID, "reason", reason)}-// recordManagerResultInternal sets the manager (env "0") entry to its final status:-// updated after a confirmed restart, up_to_date when the pull found nothing to swap-// in, or failed otherwise.-func (s *Service) recordManagerResultInternal(job *EnvironmentUpdateJob, status EnvironmentUpdateResultStatus, currentVersion string) {- manager := managerResultInternal(job)- if manager == nil {- return- }- manager.Status = status- switch status {- case EnvironmentUpdateResultStatusUpdated, EnvironmentUpdateResultStatusUpToDate:- manager.ToVersion = currentVersion- case EnvironmentUpdateResultStatusFailed:- manager.Error = "manager version did not change after upgrade"- case EnvironmentUpdateResultStatusPending,- EnvironmentUpdateResultStatusUpdating,- EnvironmentUpdateResultStatusTriggered,- EnvironmentUpdateResultStatusSkippedOffline:- // Nothing more to record: the row keeps the target version it was seeded- // with, since none of these outcomes establishes what it ended up running.+// finalizeUpdateAllJob settles the manager row, completes the job and records the audit event.+func (s *Service) finalizeUpdateAllJob(ctx context.Context, job *EnvironmentUpdateJob, managerStatus EnvironmentUpdateResultStatus, managerVersion string) {+ if idx := slices.IndexFunc(job.Results, func(r EnvironmentUpdateResult) bool { return r.EnvironmentID == environment.LocalEnvironmentID }); idx >= 0 {+ job.Results[idx].Status = managerStatus+ if managerStatus == EnvironmentUpdateResultStatusFailed {+ job.Results[idx].Error = "manager version did not change after upgrade"+ } else {+ job.Results[idx].ToVersion = managerVersion+ }}-}--// finalizeUpdateAllJobInternal closes a job out as completed and records the audit-// event. The per-environment rows must already carry their final statuses.-func (s *Service) finalizeUpdateAllJobInternal(ctx context.Context, job *EnvironmentUpdateJob) {job.Status = EnvironmentUpdateJobStatusCompletedjob.CompletedAt = new(time.Now())+ failed := 0for i := range job.Results {- job.Results[i].clearStageInternal()+ job.Results[i].clearStage()+ if job.Results[i].Status == EnvironmentUpdateResultStatusFailed {+ failed+++ }}- if err := s.persistUpdateAllJobInternal(ctx, job); err != nil {+ if err := s.db.WithContext(ctx).Save(job).Error; err != nil {slog.WarnContext(ctx, "update-all: failed to finalize job", "jobId", job.ID, "error", err)return}- s.logUpdateAllEventInternal(ctx, job)-}--func (s *Service) logUpdateAllEventInternal(ctx context.Context, job *EnvironmentUpdateJob) {- failed := 0- for _, r := range job.Results {- if r.Status == EnvironmentUpdateResultStatusFailed {- failed++- }- }metadata := database.JSON{"action": "update_all_environments",@@ -1351,17 +1101,13 @@ func (s *Service) logUpdateAllEventInternal(ctx context.Context, job *Environmen"environments": len(job.Results),"failed": failed,}-- // All environments succeeded: log the standard completed (info) event.if failed == 0 {if err := s.eventService.LogUserEvent(ctx, event.EventTypeSystemUpgrade, job.UserID, job.Username, metadata); err != nil {slog.WarnContext(ctx, "Failed to log update-all event", "jobId", job.ID, "error", err)}return}-- // The job ran to completion but some environments failed to update — record a- // warning-severity event so those failures still show in the audit log.+ // Completed with failures: a warning event keeps them visible in the audit log.if _, err := s.eventService.CreateEvent(ctx, event.CreateEventRequest{Type: event.EventTypeSystemUpgrade,Severity: event.EventSeverityWarning,@@ -1375,40 +1121,6 @@ func (s *Service) logUpdateAllEventInternal(ctx context.Context, job *Environmen}}-// updateAllTargetVersionInternal picks the best human-readable target identifier:-// the newest version tag if known, otherwise the newest digest. When the version-// check could not determine the latest release (offline, rate-limited) it falls-// back to the current identifiers: updates run unconditionally, so the target of a-// force-update with an unknown latest is wherever the pull lands — recording the-// current version keeps the resume check able to recognize a same-image recreation-// as success instead of finalizing it as failed.-func updateAllTargetVersionInternal(info *versiontypes.Info) string {- if info == nil {- return ""- }- if info.NewestVersion != "" {- return info.NewestVersion- }- if info.NewestDigest != "" {- return info.NewestDigest- }- if info.CurrentVersion != "" {- return info.CurrentVersion- }- return info.CurrentDigest-}--func truncateUpdateAllErrorInternal(err error) string {- if err == nil {- return ""- }- msg := err.Error()- if len(msg) > updateAllErrorMaxLenInternal {- return msg[:updateAllErrorMaxLenInternal]- }- return msg-}-// PruneUpgradeLogs removes expired upgrade logs from this instance's data directory.func (s *Service) PruneUpgradeLogs(ctx context.Context, dataDir string, now time.Time) (int, error) {retentionDays := s.settingsService.GetIntSetting(ctx, "upgradeLogRetentionDays", 3)@@ -1446,7 +1158,7 @@ func (s *Service) PruneUpgradeLogs(ctx context.Context, dataDir string, now timeif cancellationErr := ctx.Err(); cancellationErr != nil {return removed, errors.Join(append(failures, cancellationErr)...)}- if !upgradeLogNameInternal.MatchString(entry.Name()) {+ if !upgradeLogName.MatchString(entry.Name()) {continue}info, lstatErr := root.Lstat(entry.Name())diff --git a/backend/internal/system/children/upgrade/service_test.go b/backend/internal/system/children/upgrade/service_test.goindex b131734ff..9c2350cd5 100644--- a/backend/internal/system/children/upgrade/service_test.go+++ b/backend/internal/system/children/upgrade/service_test.go@@ -23,6 +23,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/backend/v2/internal/version""github.com/getarcaneapp/arcane/backend/v2/pkg/remenv"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow/flowtest")// TestService_UpgradeFlag tests the upgrading flag behavior@@ -192,14 +193,14 @@ func TestUpdateAllAgentFailureStatus(t *testing.T) {for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {- require.Equal(t, tt.want, updateAllAgentFailureStatusInternal(tt.err))+ require.Equal(t, tt.want, updateAllAgentFailureStatus(tt.err))})}}// A blank-target self-upgrade resolves its image from the version check (#3687).// Explicit targets from the updater engine never reach this function.-func TestResolveSelfUpgradeTargetImageInternal(t *testing.T) {+func TestResolveSelfUpgradeTargetImage(t *testing.T) {tests := []struct {name stringcurrentImage string@@ -259,7 +260,7 @@ func TestResolveSelfUpgradeTargetImageInternal(t *testing.T) {for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {- got, err := resolveSelfUpgradeTargetImageInternal(tt.currentImage, tt.info)+ got, err := resolveSelfUpgradeTargetImage(tt.currentImage, tt.info)if tt.wantErr != "" {require.ErrorContains(t, err, tt.wantErr)return@@ -342,7 +343,7 @@ func TestResolveSelfUpgradeTargetImageInternal(t *testing.T) {Labels: map[string]string{"com.docker.compose.project": "arcane", "com.docker.compose.service": "server"},}}- got, saveCompose, err := svc.configuredTargetImageInternal(t.Context(), current, tt.target)+ got, saveCompose, err := svc.configuredTargetImage(t.Context(), current, tt.target)if tt.wantErr != "" {require.ErrorContains(t, err, tt.wantErr)require.Nil(t, saveCompose)@@ -425,7 +426,7 @@ func TestUpdateAllResolveResumeAction(t *testing.T) {for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {- got := resolveResumeActionInternal(tt.job, tt.currentVersion, tt.currentDigest, now)+ got := resolveResumeAction(tt.job, tt.currentVersion, tt.currentDigest, now)require.Equal(t, tt.wantStale, got.markStale,"markStale = %v, want %v", got.markStale, tt.wantStale)@@ -436,15 +437,6 @@ func TestUpdateAllResolveResumeAction(t *testing.T) {}}-// A force-update with an unknown latest (offline or rate-limited version check)-// must still record a target — the current identifiers — so the resume check can-// recognize a same-image recreation as success instead of finalizing it as failed.-func TestUpdateAllTargetVersionFallsBackToCurrent(t *testing.T) {- require.Equal(t, "v2.0.0", updateAllTargetVersionInternal(&versiontypes.Info{NewestVersion: "v2.0.0", CurrentVersion: "v1.2.3"}))- require.Equal(t, "v1.2.3", updateAllTargetVersionInternal(&versiontypes.Info{CurrentVersion: "v1.2.3", CurrentDigest: "sha256:a"}))- require.Equal(t, "sha256:a", updateAllTargetVersionInternal(&versiontypes.Info{CurrentDigest: "sha256:a"}))-}-func TestUpsertPendingResult(t *testing.T) {job := &EnvironmentUpdateJob{Results: EnvironmentUpdateResults{@@ -455,7 +447,7 @@ func TestUpsertPendingResult(t *testing.T) {{// A seeded environment resolves to its existing row without appending.- idx := upsertPendingResultInternal(job, "abc", "palladium")+ idx := upsertPendingResult(job, "abc", "palladium")require.Equal(t, 1, idx,"existing env index = %d, want 1", idx)}@@ -465,7 +457,7 @@ func TestUpsertPendingResult(t *testing.T) {// A missing environment (seeding raced or a new env was registered) appends a// fresh pending row and returns the new index.- idx := upsertPendingResultInternal(job, "xyz", "oracle-cloud")+ idx := upsertPendingResult(job, "xyz", "oracle-cloud")require.Equal(t, 2, idx,"new env index = %d, want 2", idx)@@ -505,7 +497,7 @@ func TestUpdateAllFailedJobMarksUpdatingResultsFailed(t *testing.T) {require.NoError(t, db.WithContext(ctx).Create(job).Error)reason := "interrupted by manager restart"- svc.markUpdateAllFailedInternal(ctx, job, reason)+ svc.markUpdateAllFailed(ctx, job, reason)var got EnvironmentUpdateJobrequire.NoError(t, db.WithContext(ctx).First(&got, "id = ?", job.ID).Error)@@ -555,8 +547,7 @@ func TestUpdateAllFinalizesUpToDateManagerWithoutRestart(t *testing.T) {}require.NoError(t, db.WithContext(ctx).Create(job).Error)- svc.recordManagerResultInternal(job, EnvironmentUpdateResultStatusUpToDate, "v1.0.0")- svc.finalizeUpdateAllJobInternal(ctx, job)+ svc.finalizeUpdateAllJob(ctx, job, EnvironmentUpdateResultStatusUpToDate, "v1.0.0")var got EnvironmentUpdateJobrequire.NoError(t, db.WithContext(ctx).First(&got, "id = ?", job.ID).Error)@@ -594,15 +585,15 @@ func TestUpdateAllStageChangesPersist(t *testing.T) {}require.NoError(t, db.WithContext(ctx).Create(job).Error)- svc.setUpdateStageInternal(ctx, job, &job.Results[0], EnvironmentUpdateStageChecking)+ svc.setUpdateStage(ctx, job, &job.Results[0], EnvironmentUpdateStageChecking)started := job.Results[0].StageStartedAtrequire.NotNil(t, started)require.WithinDuration(t, time.Now(), *started, time.Second)- svc.setUpdateStageInternal(ctx, job, &job.Results[0], EnvironmentUpdateStageChecking)+ svc.setUpdateStage(ctx, job, &job.Results[0], EnvironmentUpdateStageChecking)require.Same(t, started, job.Results[0].StageStartedAt)- svc.setUpdateStageInternal(ctx, job, &job.Results[0], EnvironmentUpdateStageStarting)+ svc.setUpdateStage(ctx, job, &job.Results[0], EnvironmentUpdateStageStarting)var got EnvironmentUpdateJobrequire.NoError(t, db.WithContext(ctx).First(&got, "id = ?", job.ID).Error)require.Equal(t, EnvironmentUpdateStageStarting, got.Results[0].Stage)@@ -688,6 +679,10 @@ func TestResumeUpdateAllFinalizesManagerWithoutRerunningAgents(t *testing.T) {// is judged successful.versionSvc := version.NewVersionService(nil, true, "v9.9.9-new", "", nil, nil, nil, nil)svc := NewService(db, nil, versionSvc, event.NewEventService(db, nil, nil), nil, nil, nil)+ harness := flowtest.New(t, nil)+ require.NoError(t, svc.RegisterWorkflows(harness.Engine, nil, nil))+ harness.Start(t)+ flowtest.AssertDefinitions(t, harness)job := &EnvironmentUpdateJob{Status: EnvironmentUpdateJobStatusPendingRestart,diff --git a/backend/internal/system/module.go b/backend/internal/system/module.goindex e5f52fdb2..27df6978c 100644--- a/backend/internal/system/module.go+++ b/backend/internal/system/module.go@@ -10,7 +10,6 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/activity""github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/docker"- "github.com/getarcaneapp/arcane/backend/v2/internal/environment""github.com/getarcaneapp/arcane/backend/v2/internal/middleware""github.com/getarcaneapp/arcane/backend/v2/internal/system/children/backup""github.com/getarcaneapp/arcane/backend/v2/internal/system/children/prune"@@ -21,22 +20,20 @@ import (// Module wires the system domain and mounts its routes.type Module struct {- service *SystemService- dockerService *docker.DockerClientService- environmentService *environment.EnvironmentService- activityService *activity.ActivityService- cfg *config.Config+ service *SystemService+ dockerService *docker.DockerClientService+ activityService *activity.ActivityService+ cfg *config.Config}// New wires system routes around an existing service.func New(service *SystemService,dockerService *docker.DockerClientService,- environmentService *environment.EnvironmentService,activityService *activity.ActivityService,cfg *config.Config,) *Module {- return &Module{service: service, dockerService: dockerService, environmentService: environmentService, activityService: activityService, cfg: cfg}+ return &Module{service: service, dockerService: dockerService, activityService: activityService, cfg: cfg}}// Service exposes the system service to collaborators.@@ -60,7 +57,7 @@ func (m *Module) RegisterRoutes(api huma.API, appCtx handlerutil.ActivityAppContRegisterSystem(api, NewHandler(m.dockerService, m.service, appCtx.Context()))prune.RegisterRoutes(api, prune.NewHandler(service.prune, appCtx.Context()))- upgrade.RegisterRoutes(api, upgrade.NewHandler(service.upgrade, m.environmentService, m.cfg, appCtx.Context()))+ upgrade.RegisterRoutes(api, upgrade.NewHandler(service.upgrade, m.cfg, appCtx.Context()))backup.RegisterRoutes(api, backup.NewHandler(service.backup, m.activityService, appCtx.Context()))}diff --git a/backend/internal/system/service.go b/backend/internal/system/service.goindex d8cea128a..d3519925d 100644--- a/backend/internal/system/service.go+++ b/backend/internal/system/service.go@@ -37,10 +37,12 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/container""github.com/getarcaneapp/arcane/backend/v2/internal/database"dockerInternal "github.com/getarcaneapp/arcane/backend/v2/internal/docker"+ "github.com/getarcaneapp/arcane/backend/v2/internal/environment""github.com/getarcaneapp/arcane/backend/v2/internal/event""github.com/getarcaneapp/arcane/backend/v2/internal/imageupdate""github.com/getarcaneapp/arcane/backend/v2/internal/network""github.com/getarcaneapp/arcane/backend/v2/internal/project"+ "github.com/getarcaneapp/arcane/backend/v2/internal/role""github.com/getarcaneapp/arcane/backend/v2/internal/s3""github.com/getarcaneapp/arcane/backend/v2/internal/settings"systembackup "github.com/getarcaneapp/arcane/backend/v2/internal/system/children/backup"@@ -50,6 +52,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/volume""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/vuln""github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs""github.com/getarcaneapp/arcane/backend/v2/pkg/utils""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis"@@ -126,13 +129,22 @@ func NewSystemService(return s}-// ReconcileInterruptedBackups fails system backup runs left running by the previous process.+// RegisterWorkflows defines the backup workflows while the host is still unstarted.+func (s *SystemService) RegisterWorkflows(engine *flow.Engine, environments *environment.EnvironmentService, roles *role.RoleService) error {+ return errors.Join(s.backup.RegisterWorkflows(engine), s.upgrade.RegisterWorkflows(engine, environments, roles))+}++// ReconcileInterruptedBackups fails system backup runs left running by the previous process and watches the+// manual backups that saved workflows resume.func (s *SystemService) ReconcileInterruptedBackups(ctx context.Context, protectedIDs ...string) error {query := s.db.WithContext(ctx).Model(&SystemBackupRun{}).Where("status = ?", SystemBackupStatusRunning)if len(protectedIDs) > 0 {query = query.Where("id NOT IN ?", protectedIDs)}- return query.Updates(map[string]any{"status": SystemBackupStatusFailed, "error": "Backup interrupted by application restart"}).Error+ if err := query.Updates(map[string]any{"status": SystemBackupStatusFailed, "error": "Backup interrupted by application restart"}).Error; err != nil {+ return err+ }+ return s.backup.WatchAcceptedBackups(ctx)}// SetBackupScheduler injects the dynamic scheduler and admission gate for@@ -472,9 +484,9 @@ func resolveUpgraderRuntimeOptionsInternal(}const systemBackupHistoryUnion = "SELECT id, size, created_at, status, trigger, destination, '' AS format, local_snapshot_id, " +- "remote_snapshot_id, s3_destination_id, policy_id, error, 'system' AS type, 'system' AS " ++ "remote_snapshot_id, '' AS remote_instance_id, s3_destination_id, policy_id, error, 'system' AS type, 'system' AS " +"resource_type, 'Arcane' AS resource_name FROM system_backup_runs UNION ALL SELECT id, size, " +- "created_at, status, trigger, destination, format, local_snapshot_id, remote_snapshot_id, " ++ "created_at, status, trigger, destination, format, local_snapshot_id, remote_snapshot_id, COALESCE(remote_instance_id, '') AS remote_instance_id, " +"s3_destination_id, policy_id, error, CASE WHEN policy_id LIKE 'system-volume:%' THEN 'system' ELSE " +"'volume' END AS type, 'volume' AS resource_type, volume_name AS resource_name FROM volume_backups"@@ -622,15 +634,8 @@ func (s *SystemService) backupStoreInternal(s3Destinations *s3.S3DestinationServ},}.Run(ctx, updates)},- DisablePolicyRemote: func(ctx context.Context, policy *backuptypes.SystemBackupPolicy) (bool, error) {- column := "enabled"- if policy.LocalEnabled {- column = "s3_enabled"- }- result := s.db.WithContext(ctx).Model(&SystemBackupPolicy{}).- Where("id = ? AND s3_destination_id = ? AND enabled = ? AND s3_enabled = ? AND local_enabled = ?", policy.ID, policy.S3DestinationID, true, true, policy.LocalEnabled).- Update(column, false)- return result.Error == nil && result.RowsAffected > 0, result.Error+ DisablePolicyRemote: func(ctx context.Context, policy *backuptypes.SystemBackupPolicy, field string) (bool, error) {+ return backup.DisableStoredRemote(ctx, s.db, &SystemBackupPolicy{}, policy.ID, policy.S3DestinationID, policy.LocalEnabled, field)},CheckScheduledRemote: func(ctx context.Context, destinationID string) error {return backup.CheckScheduledRemote(ctx, s.db, s3Destinations, "system_backup_runs", destinationID, "arcane-system-recovery")diff --git a/backend/internal/system/service_test.go b/backend/internal/system/service_test.goindex 440f23207..bc2799023 100644--- a/backend/internal/system/service_test.go+++ b/backend/internal/system/service_test.go@@ -20,6 +20,7 @@ import ("go.getarcane.app/sys/crypto""gorm.io/gorm"+ "github.com/getarcaneapp/arcane/backend/v2/internal/activity""github.com/getarcaneapp/arcane/backend/v2/internal/backup""github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/database"@@ -27,6 +28,7 @@ import (systembackup "github.com/getarcaneapp/arcane/backend/v2/internal/system/children/backup""github.com/getarcaneapp/arcane/backend/v2/internal/volume""github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow/flowtest""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing")@@ -196,7 +198,7 @@ func (s *systemBackupPolicySchedulerInternal) Submit(_ context.Context, requestfunc TestSystemBackupPoliciesRegisterIndependentJobs(t *testing.T) {gormDB, err := gorm.Open(sqlite.Open("file:system-backup-schedules?mode=memory&cache=shared"), &gorm.Config{})require.NoError(t, err)- require.NoError(t, gormDB.AutoMigrate(&SystemBackupPolicy{}, &SystemBackupRun{}, &backup.SystemBackupRecoveryConfig{}, &s3.S3Destination{}))+ require.NoError(t, gormDB.AutoMigrate(&SystemBackupPolicy{}, &SystemBackupRun{}, &backup.SystemBackupRecoveryConfig{}, &s3.S3Destination{}, &activity.Activity{}, &activity.ActivityMessage{}))crypto.InitEncryption(&crypto.Config{EncryptionKey: "system-backup-policy-test-key-32bytes", Environment: "test"})db := &database.DB{DB: gormDB}recoveryKeys := backup.NewRecoveryKeyStore(db)@@ -210,6 +212,9 @@ func TestSystemBackupPoliciesRegisterIndependentJobs(t *testing.T) {Config: &config.Config{DatabaseURL: "file:system-backup-schedules-test.db"},RecoveryKeys: recoveryKeys,})+ harness := flowtest.New(t, activity.NewActivityService(db, nil))+ require.NoError(t, service.backup.RegisterWorkflows(harness.Engine))+ harness.Start(t)jobScheduler := &systemBackupPolicySchedulerInternal{jobs: make(map[string]scheduler.Job)}require.NoError(t, service.SetBackupScheduler(t.Context(), jobScheduler, newSystemBackupAdmissionGateForTestInternal(t)))@@ -260,10 +265,11 @@ func TestSystemBackupPoliciesRegisterIndependentJobs(t *testing.T) {// The scheduled run checks the remote first; a local policy then// attempts its backup, which has no engine in this test.outcome, runErr := job.Run(t.Context())+ require.NoError(t, runErr)if local {- require.ErrorContains(t, runErr, "backup engine is unavailable")+ require.Equal(t, scheduler.Failed, outcome.Status)+ require.Contains(t, outcome.Message, "backup engine is unavailable")} else {- require.NoError(t, runErr)require.Equal(t, scheduler.NeedsAttention, outcome.Status)require.Equal(t, backup.RemoteDisabledMessage, outcome.Message)}diff --git a/backend/internal/template/handler.go b/backend/internal/template/handler.goindex 073ea9d09..d4b1d5119 100644--- a/backend/internal/template/handler.go+++ b/backend/internal/template/handler.go@@ -257,14 +257,7 @@ func (h *TemplateHandler) DeleteTemplate(ctx context.Context, input *DeleteTemplreturn nil, huma.Error500InternalServerError("Failed to delete template: " + err.Error())}- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{- Message: "Template deleted successfully",- },- },- }, nil+ return handlerutil.MessageOutput("Template deleted successfully", ""), nil}// DownloadTemplate downloads a remote template to local storage.@@ -332,14 +325,7 @@ func (h *TemplateHandler) SaveDefaultTemplates(ctx context.Context, input *SaveDreturn nil, huma.Error500InternalServerError("Failed to save default template: " + err.Error())}- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{- Message: "Default templates saved successfully",- },- },- }, nil+ return handlerutil.MessageOutput("Default templates saved successfully", ""), nil}// GetRegistries returns all template registries.@@ -416,14 +402,7 @@ func (h *TemplateHandler) UpdateRegistry(ctx context.Context, input *UpdateTemplreturn nil, huma.Error500InternalServerError("Failed to update registry: " + err.Error())}- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{- Message: "Registry updated successfully",- },- },- }, nil+ return handlerutil.MessageOutput("Registry updated successfully", ""), nil}// DeleteRegistry deletes a template registry.@@ -439,14 +418,7 @@ func (h *TemplateHandler) DeleteRegistry(ctx context.Context, input *DeleteTemplreturn nil, huma.Error500InternalServerError("Failed to delete registry: " + err.Error())}- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{- Message: "Registry deleted successfully",- },- },- }, nil+ return handlerutil.MessageOutput("Registry deleted successfully", ""), nil}// FetchRegistry fetches templates from a remote registry URL.diff --git a/backend/internal/updater/children/execution/service.go b/backend/internal/updater/children/execution/service.goindex b803368bb..5573dadcf 100644--- a/backend/internal/updater/children/execution/service.go+++ b/backend/internal/updater/children/execution/service.go@@ -2,493 +2,177 @@ package executionimport ("context"+ "encoding/json/v2""errors"- "fmt"- "log/slog"- "sync"+ "slices""time"activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"+ "github.com/getarcaneapp/arcane/types/v2/scheduler""github.com/getarcaneapp/arcane/types/v2/updater""github.com/italypaleale/francis/actor"- "github.com/italypaleale/francis/host/local"- "gorm.io/gorm"+ "github.com/italypaleale/francis/builtin/workflow"- "github.com/getarcaneapp/arcane/backend/v2/internal/activity""github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/middleware""github.com/getarcaneapp/arcane/backend/v2/internal/role""github.com/getarcaneapp/arcane/backend/v2/pkg/authz"- "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"+ activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext""github.com/getarcaneapp/arcane/backend/v2/pkg/utils"- "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/userctx")-// Dependencies are the updater operations single-container delivery runs on.-type Dependencies struct {- Coordinator *runs.Coordinator- Config *config.Config- Roles *role.RoleService- Activity *activity.ActivityService- Logger func() *slog.Logger- AcquireUpdate func(ctx context.Context) (context.Context, func(), error)- UpdateBusy error- TrackActivity func(ctx context.Context, activityID string) context.Context- RunUpdate func(ctx context.Context, containerID, activityID string) (*updater.Result, error)- FinishUpdate func(ctx context.Context, activityID string, result *updater.Result, runErr error)- FreezeContainer func(ctx context.Context, containerID string) (*updater.FrozenUpdateTarget, error)- ConfirmTarget func(ctx context.Context, target updater.FrozenUpdateTarget) (bool, bool, error)- WithFrozenTarget func(ctx context.Context, target *updater.FrozenUpdateTarget, persist func() error) context.Context-}--// Service delivers accepted single-container updates through durable Francis-// jobs and repairs interrupted deliveries.-type Service struct {- coordinator *runs.Coordinator- config *config.Config- roles *role.RoleService- activity *activity.ActivityService- logger func() *slog.Logger- acquireUpdate func(ctx context.Context) (context.Context, func(), error)- updateBusy error- trackActivity func(ctx context.Context, activityID string) context.Context- runUpdate func(ctx context.Context, containerID, activityID string) (*updater.Result, error)- finishUpdate func(ctx context.Context, activityID string, result *updater.Result, runErr error)- freezeContainer func(ctx context.Context, containerID string) (*updater.FrozenUpdateTarget, error)- confirmTarget func(ctx context.Context, target updater.FrozenUpdateTarget) (bool, bool, error)- withFrozenTarget func(ctx context.Context, target *updater.FrozenUpdateTarget, persist func() error) context.Context-- singleUpdates *actor.Service- singleMu sync.Mutex- singleContext context.Context- singleCancel context.CancelFunc- singleWorkers sync.WaitGroup- singleDone chan struct{}- singleStopping bool-}--func NewService(deps Dependencies) *Service {- return &Service{- coordinator: deps.Coordinator,- config: deps.Config,- roles: deps.Roles,- activity: deps.Activity,- logger: deps.Logger,- acquireUpdate: deps.AcquireUpdate,- updateBusy: deps.UpdateBusy,- trackActivity: deps.TrackActivity,- runUpdate: deps.RunUpdate,- finishUpdate: deps.FinishUpdate,- freezeContainer: deps.FreezeContainer,- confirmTarget: deps.ConfirmTarget,- withFrozenTarget: deps.WithFrozenTarget,- }-}--// Ready reports whether the actor host has been registered.-func (s *Service) Ready() bool {- return s.singleUpdates != nil-}--// Submit persists an accepted update intent and dispatches it without waiting-// for the update to run.-func (s *Service) Submit(ctx, workCtx context.Context, containerID, activityID string) error {- command := singleUpdateCommandInternal(ctx, containerID, activityID)- state := updater.SingleUpdateState{Command: command, Status: "queued"}- if setStateErr := s.singleUpdates.SetState(workCtx, StateType, activityID, state, nil); setStateErr != nil {- s.finishUpdate(workCtx, activityID, nil, setStateErr)- return fmt.Errorf("persist container update: %w", setStateErr)+type (+ // Dependencies are the updater operations a single-container update runs on.+ Dependencies struct {+ Config *config.Config+ Roles *role.RoleService+ AcquireUpdate func(ctx context.Context) (context.Context, func(), error)+ UpdateBusy error+ RunUpdate func(ctx context.Context, containerID, activityID string) (*updater.Result, error)+ // RecordUpdate saves the result on the activity and summarizes it as the run outcome.+ RecordUpdate func(ctx context.Context, activityID string, result *updater.Result, runErr error) scheduler.Outcome+ FreezeContainer func(ctx context.Context, containerID string) (*updater.FrozenUpdateTarget, error)+ ConfirmTarget func(ctx context.Context, target updater.FrozenUpdateTarget) (bool, bool, error)+ WithFrozenTarget func(ctx context.Context, target *updater.FrozenUpdateTarget, persist func() error) context.Context+ }++ // Service runs accepted single-container updates as container-update workflows.+ Service struct {+ deps Dependencies+ flow *flow.Engine+ workflow *flow.Workflow+ }++ // updateInput is an accepted update: the container and who asked for it.+ updateInput struct {+ ContainerID string `json:"containerId"`+ UserID string `json:"userId,omitempty"`+ KeyID string `json:"keyId,omitempty"`}- if dispatchSingleErr := s.dispatchSingleInternal(workCtx, state); dispatchSingleErr != nil {- s.logger().WarnContext(workCtx, "container update dispatch deferred", "activityId", activityID, "error", dispatchSingleErr)- }- return nil-}--// ActorType and StateType are the durable Francis identifiers of-// single-container update jobs and their persisted delivery state.-const (- ActorType = "single-container-update"- StateType = "single-container-update-state")-type singleUpdateActorInternal struct{ service *Service }--func (s *Service) RegisterActors(runtime *francis.Runtime) error {- s.singleUpdates = runtime.Service()- return runtime.RegisterActor(- ActorType,- func(- _ string,- _ *actor.Service,- ) actor.Actor {- return &singleUpdateActorInternal{- service: s,- }+func NewService(deps Dependencies) *Service {+ return &Service{deps: deps}+}++// RegisterWorkflows defines the container-update workflow while the host is still unstarted.+func (s *Service) RegisterWorkflows(engine *flow.Engine) error {+ var err error+ s.flow = engine+ s.workflow, err = engine.Define(flow.Definition{+ Name: "container-update",+ Version: 1,+ Fingerprint: "1b1bad80429a7061e5541b6fa57eb43b187046c3987bec83ea39f0496589ccf8",+ Concurrency: 4,+ Timeout: 2 * time.Hour,+ Activity: activitylib.StartRequest{+ Type: activitytypes.TypeAutoUpdate, EnvironmentID: "0", Queue: true, DeferSlot: true,+ ResourceType: new("container"), Step: "Updating container", LatestMessage: "Container update started",},- local.WithCapacityGroup(- "jobs",- 4,- ),- local.WithCompletedJobRetention(- 7*24*time.Hour,- ),- )-}--// Start repairs persisted single-container delivery intents until shutdown.-func (s *Service) Start(ctx context.Context) error {- s.singleMu.Lock()- defer s.singleMu.Unlock()- if s.singleContext != nil {- return nil- }- s.singleContext, s.singleCancel = context.WithCancel(ctx)- s.singleDone = make(chan struct{})- go func() {- defer close(s.singleDone)- ticker := time.NewTicker(5 * time.Second)- defer ticker.Stop()- for {- if s.coordinator == nil || s.coordinator.Active() {- if err := s.repairSinglesInternal(s.singleContext); err != nil && s.singleContext.Err() == nil {- s.logger().WarnContext(s.singleContext, "repair container update dispatch", "error", err)- }- }- select {- case <-s.singleContext.Done():- return- case <-ticker.C:- }- }- }()- return nil+ Labels: map[string]string{"update": "Updating container"},+ Steps: []workflow.StepSpec{workflow.Step("update", engine.Handler(s.update), workflow.WithMaxAttempts(1))},+ })+ return err}-// Stop cancels business execution before the actor host drains jobs.-func (s *Service) Stop(ctx context.Context) error {- s.singleMu.Lock()- s.singleStopping = true- if s.singleCancel != nil {- s.singleCancel()- }- done := s.singleDone- s.singleMu.Unlock()- joined := make(chan struct{})- go func() {- if done != nil {- <-done- }- s.singleWorkers.Wait()- close(joined)- }()- select {- case <-joined:- return nil- case <-ctx.Done():- return ctx.Err()+// Submit accepts an update of the container and returns its activity ID; activity describes that activity.+func (s *Service) Submit(ctx context.Context, containerID string, activity activitylib.StartRequest) (string, error) {+ input := updateInput{ContainerID: containerID}+ if user, ok := userctx.CurrentUserFromContext(ctx); ok && user != nil {+ input.UserID = user.ID}+ input.KeyID, _ = ctx.Value(middleware.ContextKeyApiKeyID).(string)+ return s.flow.Submit(ctx, s.workflow, input, activity)}-func (s *Service) dispatchSingleInternal(ctx context.Context, state updater.SingleUpdateState) error {- if s.coordinator != nil && !s.coordinator.Active() {- return nil+// update is the container-update step. A delivery that finds the target it froze confirms that target's effect+// before touching Docker again; while another update holds the updater, the task is handed back.+func (s *Service) update(ctx context.Context, t flow.Task) (any, error) {+ var input updateInput+ if err := t.Payload(&input); err != nil {+ return nil, err}- id, _, err := s.singleUpdates.Dispatch(ctx, ActorType, "single-container", "update", state.Command, actor.WithIdempotencyKey(state.Command.ActivityID))- if err != nil {- return err+ ctx, release, err := s.deps.AcquireUpdate(ctx)+ if errors.Is(err, s.deps.UpdateBusy) {+ return nil, actor.ErrJobRejected}- job, err := s.singleUpdates.GetJob(ctx, id)if err != nil {- return err+ return nil, err}- if !job.Status.IsTerminal() {- return nil+ defer release()+ activityID := t.ActivityID()+ failed := func(reason string) scheduler.Outcome {+ return scheduler.Outcome{Status: scheduler.Failed, Message: reason}}- // A terminal transport record cannot prove the durable business outcome.- // Its replacement reconciles the saved effect before doing any further work.- if deleteJobErr := s.singleUpdates.DeleteJob(ctx, ActorType, "single-container", id); deleteJobErr != nil {- return deleteJobErr+ if input.UserID != "" && (s.deps.Config == nil || !s.deps.Config.AgentMode || input.UserID != "agent" || input.KeyID != "") {+ if s.deps.Roles == nil {+ return failed("requesting user permissions unavailable"), nil+ }+ permissions, permissionsErr := s.deps.Roles.ResolveExecutionPermissions(ctx, input.UserID, input.KeyID)+ if permissionsErr != nil || !permissions.Allows(authz.PermImageUpdatesCheck, "0") {+ return failed("requesting user no longer has permission to update containers"), nil+ }}- _, _, err = s.singleUpdates.Dispatch(ctx, ActorType, "single-container", "update", state.Command, actor.WithIdempotencyKey(state.Command.ActivityID))- return err-}-// repairSinglesInternal scans every persisted update, collecting per-entry-// failures so one bad entry cannot block repair of the rest.-func (s *Service) repairSinglesInternal(ctx context.Context) error {- var failures []error- for cursor := ""; ; {- page, err := s.singleUpdates.ListStates(ctx, StateType, &actor.ListStatesOpts{IncludeData: true, After: cursor, Limit: 100})- if err != nil {- return errors.Join(append(failures, err)...)+ var target updater.FrozenUpdateTarget+ previous, _ := jobcontext.Run(ctx)+ if index := slices.IndexFunc(previous.Outcome.Targets, func(evidence scheduler.TargetOutcome) bool {+ return evidence.ID == input.ContainerID && len(evidence.RecoveryData) > 0+ }); index >= 0 {+ if decodeErr := json.Unmarshal(previous.Outcome.Targets[index].RecoveryData, &target); decodeErr != nil {+ return nil, decodeErr}- for _, entry := range page.States {- if ctxErr := ctx.Err(); ctxErr != nil {- return errors.Join(append(failures, ctxErr)...)- }- state, decodeErr := decodeSingleStateInternal(entry)- if decodeErr == nil {- decodeErr = s.repairSingleInternal(ctx, state)- }- if decodeErr != nil {- failures = append(failures, fmt.Errorf("repair container update %s: %w", entry.ActorID, decodeErr))+ confirmed, unchanged, confirmErr := s.deps.ConfirmTarget(ctx, target)+ switch {+ case confirmErr != nil:+ return nil, confirmErr+ case confirmed:+ item := updater.ResourceResult{+ ResourceID: target.ContainerID, ResourceName: target.ContainerName, ResourceType: "container",+ Status: updater.StatusUpdated, UpdateApplied: true,}+ return s.deps.RecordUpdate(ctx, activityID, &updater.Result{Updated: 1, Items: []updater.ResourceResult{item}}, nil), nil+ case !unchanged:+ return failed("Interrupted container update effect cannot be confirmed"), nil}- cursor = page.AfterID()- if cursor == "" {- return errors.Join(failures...)+ } else {+ frozen, freezeErr := s.deps.FreezeContainer(ctx, input.ContainerID)+ if freezeErr != nil {+ return failed(freezeErr.Error()), nil}+ target = *frozen}-}--func (s *Service) repairSingleInternal(ctx context.Context, state updater.SingleUpdateState) error {- if state.Status == "queued" || state.Status == "running" {- return s.dispatchSingleInternal(ctx, state)- }- if s.activity == nil {- return nil- }- detail, err := s.activity.GetActivityDetail(ctx, "0", state.Command.ActivityID, 1)- // Pruned or deleted history needs no projection repair.- if errors.Is(err, gorm.ErrRecordNotFound) {- return nil- }- if err != nil {- return err- }- if detail.Activity.Status != activitytypes.StatusQueued && detail.Activity.Status != activitytypes.StatusRunning {- return nil- }- var outcomeErr error- if state.Failure != "" {- outcomeErr = errors.New(state.Failure)- }- s.finishUpdate(ctx, state.Command.ActivityID, state.Result, outcomeErr)- return nil-}--func decodeSingleStateInternal(entry actor.StateInfo) (updater.SingleUpdateState, error) {- var state updater.SingleUpdateState- if entry.Data == nil {- return state, errors.New("empty update state")- }- err := entry.Data.Decode(&state)- return state, err-}-func (a *singleUpdateActorInternal) Job(ctx context.Context, _ string, data actor.Envelope) error {- s := a.service- if s.coordinator != nil && !s.coordinator.Active() {- return actor.ErrJobRejected- }- s.singleMu.Lock()- if s.singleStopping {- s.singleMu.Unlock()- return actor.ErrJobRejected- }- s.singleWorkers.Add(1)- lifetime := s.singleContext- s.singleMu.Unlock()- defer s.singleWorkers.Done()- ctx, cancel := context.WithCancel(ctx)- defer cancel()- if lifetime != nil {- //nolint:contextcheck // The service lifetime cancels the inherited Francis job context before host drain.- stop := context.AfterFunc(lifetime, cancel)- defer stop()- }- ctx, release, err := s.acquireUpdate(ctx)- if err != nil {- if errors.Is(err, s.updateBusy) {- return actor.ErrJobRejected+ // The frozen target is the evidence a redelivery confirms, so it is recorded before and as the updater pins its image.+ record := func() error {+ data, marshalErr := json.Marshal(target)+ if marshalErr != nil {+ return marshalErr}- return err- }- defer release()- var command updater.SingleUpdateCommand- if decodeErr := data.Decode(&command); decodeErr != nil {- return errors.Join(decodeErr, actor.ErrJobPermanentFailure)- }- var state updater.SingleUpdateState- if getStateErr := s.singleUpdates.GetState(ctx, StateType, command.ActivityID, &state); getStateErr != nil {- return getStateErr- }- if state.Status == "completed" || state.Status == "needs_attention" {- return nil- }- if state.Command != command {- return actor.ErrJobPermanentFailure+ return jobcontext.Progress(ctx, scheduler.TargetOutcome{ResourceType: "container", ID: input.ContainerID, Status: scheduler.Running, RecoveryData: data, ActivityID: activityID})}- if authorizeSingleErr := s.authorizeSingleInternal(ctx, command); authorizeSingleErr != nil {- return s.failSingleInternal(ctx, &state, authorizeSingleErr.Error())+ if recordErr := record(); recordErr != nil {+ return nil, recordErr}- ctx = s.trackActivity(ctx, command.ActivityID)- canceled, err := s.awaitSingleActivityInternal(ctx, lifetime, &state)- if err != nil || canceled {- return err- }- completed, err := s.prepareSingleTargetInternal(ctx, &state)- if err != nil || completed {- return err- }- ctx = s.withFrozenTarget(ctx, state.Target, func() error {- return s.singleUpdates.SetState(ctx, StateType, command.ActivityID, state, nil)- })+ ctx = s.deps.WithFrozenTarget(ctx, &target, record)result, runErr := func() (out *updater.Result, err error) {defer utils.RecoverToError(&err, "single container update")- return s.runUpdate(ctx, state.Target.ContainerID, command.ActivityID)+ return s.deps.RunUpdate(ctx, target.ContainerID, activityID)}()- return s.completeSingleInternal(ctx, &state, result, runErr)-}--func (s *Service) failSingleInternal(ctx context.Context, state *updater.SingleUpdateState, reason string) error {- state.Status = "needs_attention"- state.Failure = reason- if err := s.singleUpdates.SetState(ctx, StateType, state.Command.ActivityID, *state, nil); err != nil {- return err- }- s.finishUpdate(ctx, state.Command.ActivityID, nil, errors.New(reason))- return actor.ErrJobPermanentFailure-}--func (s *Service) authorizeSingleInternal(ctx context.Context, command updater.SingleUpdateCommand) error {- if command.UserID == "" {- return nil- }- if s.config != nil && s.config.AgentMode && command.UserID == "agent" && command.KeyID == "" {- return nil- }- if s.roles == nil {- return errors.New("requesting user permissions unavailable")- }- permissions, err := s.roles.ResolveExecutionPermissions(ctx, command.UserID, command.KeyID)- if err != nil || !permissions.Allows(authz.PermImageUpdatesCheck, "0") {- return errors.New("requesting user no longer has permission to update containers")- }- return nil-}--func (s *Service) awaitSingleActivityInternal(ctx, lifetime context.Context, state *updater.SingleUpdateState) (bool, error) {- if s.activity == nil {- return false, nil- }- detail, err := s.activity.GetActivityDetail(ctx, "0", state.Command.ActivityID, 1)- if err != nil {- return false, err- }- if detail.Activity.Status == activitytypes.StatusCancelled {- state.Status = "needs_attention"- state.Failure = "Container update cancelled"- return true, s.singleUpdates.SetState(ctx, StateType, state.Command.ActivityID, *state, nil)- }- err = s.activity.AwaitActivitySlotBounded(ctx, state.Command.ActivityID, "0")- if err == nil {- return false, nil- }- if ctx.Err() == nil {- return false, s.failSingleInternal(ctx, state, err.Error())- }- if lifetime != nil && lifetime.Err() != nil {- return false, ctx.Err()- }- state.Status = "needs_attention"- state.Failure = err.Error()- writeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)- defer cancel()- return true, s.singleUpdates.SetState(writeCtx, StateType, state.Command.ActivityID, *state, nil)-}--func (s *Service) prepareSingleTargetInternal(ctx context.Context, state *updater.SingleUpdateState) (bool, error) {- if state.Status != "running" {- target, err := s.freezeContainer(ctx, state.Command.ContainerID)- if err != nil {- return false, s.failSingleInternal(ctx, state, err.Error())- }- state.Target = target- state.Status = "running"- return false, s.singleUpdates.SetState(ctx, StateType, state.Command.ActivityID, *state, nil)- }- if state.Target == nil {- return false, s.failSingleInternal(ctx, state, "Interrupted container update has no saved target evidence")- }- confirmed, unchanged, err := s.confirmTarget(ctx, *state.Target)- if err != nil {- return false, err- }- if confirmed {- result := &updater.Result{- Updated: 1,- Items: []updater.ResourceResult{- {- ResourceID: state.Target.ContainerID,- ResourceName: state.Target.ContainerName,- ResourceType: "container",- Status: updater.StatusUpdated,- UpdateApplied: true,- },- },- }- return true, s.completeSingleInternal(ctx, state, result, nil)- }- if !unchanged {- return false, s.failSingleInternal(ctx, state, "Interrupted container update effect cannot be confirmed")- }- return false, nil-}--func (s *Service) completeSingleInternal(ctx context.Context, state *updater.SingleUpdateState, result *updater.Result, runErr error) error {if ctx.Err() != nil {- return ctx.Err()+ return nil, ctx.Err()}- if runErr != nil {- return s.failSingleInternal(ctx, state, runErr.Error())- }- if state.Target.DesiredImageRef != "" && result != nil && result.Updated > 0 {- confirmed, _, err := s.confirmTarget(ctx, *state.Target)- if err != nil {- return err+ if runErr == nil && target.DesiredImageRef != "" && result != nil && result.Updated > 0 {+ confirmed, _, confirmErr := s.deps.ConfirmTarget(ctx, target)+ if confirmErr != nil {+ return nil, confirmErr}if !confirmed {- return s.failSingleInternal(ctx, state, "Updater result does not confirm the frozen desired image")+ return failed("Updater result does not confirm the frozen desired image"), nil}}- state.Status = "completed"- state.Result = result- if err := s.singleUpdates.SetState(ctx, StateType, state.Command.ActivityID, *state, nil); err != nil {- return err- }- s.finishUpdate(ctx, state.Command.ActivityID, result, nil)- return nil-}--func (s *Service) ActiveUpdateActivityIDs(ctx context.Context) ([]string, error) {- ids := []string{}- for cursor := ""; ; {- page, err := s.singleUpdates.ListStates(ctx, StateType, &actor.ListStatesOpts{IncludeData: true, After: cursor, Limit: 100})- if err != nil {- return nil, err- }- for _, entry := range page.States {- // States are keyed by activity ID, so an undecodable entry is still protected.- if _, decodeErr := decodeSingleStateInternal(entry); decodeErr != nil {- s.logger().WarnContext(ctx, "decode container update state", "activityId", entry.ActorID, "error", decodeErr)- }- ids = append(ids, entry.ActorID)- }- cursor = page.AfterID()- if cursor == "" {- return ids, nil- }- }-}--func singleUpdateCommandInternal(ctx context.Context, containerID, activityID string) updater.SingleUpdateCommand {- command := updater.SingleUpdateCommand{ContainerID: containerID, ActivityID: activityID}- if user, ok := userctx.CurrentUserFromContext(ctx); ok && user != nil {- command.UserID = user.ID- }- command.KeyID, _ = ctx.Value(middleware.ContextKeyApiKeyID).(string)- return command+ return s.deps.RecordUpdate(ctx, activityID, result, runErr), nil}diff --git a/backend/internal/updater/children/execution/service_test.go b/backend/internal/updater/children/execution/service_test.godeleted file mode 100644index 0f79a4034..000000000--- a/backend/internal/updater/children/execution/service_test.go+++ /dev/null@@ -1,67 +0,0 @@-package execution--import (- "context"- "errors"- "log/slog"- "testing"- "time"-- "github.com/getarcaneapp/arcane/types/v2/updater"- "github.com/stretchr/testify/require"-- francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing"-)--// testDependenciesInternal wires delivery to stand-ins: freezing fails as it-// does without Docker.-func testDependenciesInternal() Dependencies {- return Dependencies{- Logger: slog.Default,- AcquireUpdate: func(ctx context.Context) (context.Context, func(), error) { return ctx, func() {}, nil },- UpdateBusy: errors.New("busy"),- TrackActivity: func(ctx context.Context, _ string) context.Context { return ctx },- RunUpdate: func(context.Context, string, string) (*updater.Result, error) {- return nil, errors.New("unexpected update run")- },- FinishUpdate: func(context.Context, string, *updater.Result, error) {},- FreezeContainer: func(context.Context, string) (*updater.FrozenUpdateTarget, error) {- return nil, errors.New("docker service unavailable")- },- ConfirmTarget: func(context.Context, updater.FrozenUpdateTarget) (bool, bool, error) { return false, false, nil },- WithFrozenTarget: func(ctx context.Context, _ *updater.FrozenUpdateTarget, _ func() error) context.Context {- return ctx- },- }-}--func TestService_RepairsAcceptedIntentBeforeDispatch(t *testing.T) {- svc := NewService(testDependenciesInternal())- runtime := francistest.New(t)- require.NoError(t, svc.RegisterActors(runtime))- francistest.Start(t, runtime)- command := updater.SingleUpdateCommand{ContainerID: "missing", ActivityID: "accepted"}- require.NoError(- t,- runtime.Service().SetState(- t.Context(),- StateType,- command.ActivityID,- updater.SingleUpdateState{- Command: command,- Status: "queued",- },- nil,- ),- )- require.NoError(t, svc.Start(t.Context()))- t.Cleanup(func() {- ctx, cancel := context.WithTimeout(context.WithoutCancel(t.Context()), time.Second)- defer cancel()- require.NoError(t, svc.Stop(ctx))- })- require.Eventually(t, func() bool {- var state updater.SingleUpdateState- return runtime.Service().GetState(t.Context(), StateType, command.ActivityID, &state) == nil && state.Status == "needs_attention"- }, 3*time.Second, 10*time.Millisecond)-}diff --git a/backend/internal/updater/children/recovery/service.go b/backend/internal/updater/children/recovery/service.goindex a1503e508..db440c3ee 100644--- a/backend/internal/updater/children/recovery/service.go+++ b/backend/internal/updater/children/recovery/service.go@@ -1,6 +1,7 @@package recoveryimport (+ "cmp""context""encoding/json/v2""errors"@@ -96,29 +97,27 @@ type frozenSingleInternal struct {Persist func() error}-// FreezePending persists identities and desired images before any pull.-func (s *Service) FreezePending(ctx context.Context) (context.Context, error) {+// errNoFrozenPlanInternal reports a run without a persisted frozen plan.+var errNoFrozenPlanInternal = errors.New("interrupted update has no frozen target plan")++// FreezePending persists identities and desired images before any pull. A run+// that already holds a plan keeps it. Image refs and container IDs in excluded are left+// out, and a target that cannot be frozen fails alone instead of aborting the batch.+func (s *Service) FreezePending(ctx context.Context, run scheduler.Run, excluded map[string]bool) (context.Context, error) {if _, ok := ctx.Value(frozenPendingKeyInternal{}).(*frozenUpdatePlanInternal); ok {return ctx, nil}- run, ok := jobcontext.Run(ctx)- if !ok {- return ctx, nil- }- for _, target := range run.Outcome.Targets {- if target.ID == "auto-update" && len(target.RecoveryData) > 0 {- var plan frozenUpdatePlanInternal- if err := json.Unmarshal(target.RecoveryData, &plan); err != nil {- return ctx, err- }- return context.WithValue(ctx, frozenPendingKeyInternal{}, &plan), nil- }+ if plan, found, err := frozenPlanInternal(run); err != nil || found {+ return context.WithValue(ctx, frozenPendingKeyInternal{}, &plan), err}records, err := s.pendingUpdates(ctx)if err != nil {return ctx, err}- plan, err := s.buildFrozenPlanInternal(ctx, records)+ records = slices.DeleteFunc(records, func(record updater.ImageUpdateRecord) bool {+ return excluded[refs.NormalizeImageUpdateRef(record.ImageRef())] || excluded[record.ContainerID]+ })+ plan, failures, err := s.buildFrozenPlanInternal(ctx, records, excluded)if err != nil {return ctx, err}@@ -126,18 +125,16 @@ func (s *Service) FreezePending(ctx context.Context) (context.Context, error) {if err != nil {return ctx, err}- if progressErr := jobcontext.Progress(- ctx,- scheduler.TargetOutcome{- ID: "auto-update",- ResourceType: "update-batch",- Status: scheduler.Running,- RecoveryData: raw,- ActivityID: s.activityID(- ctx,- ),- },- ); progressErr != nil {+ activityID := s.activityID(ctx)+ // Failures are recorded before the plan, since a delivery that finds the plan never plans again.+ for containerID, freezeErr := range failures {+ failure := scheduler.TargetOutcome{ID: containerID, ResourceType: "container", Status: scheduler.Failed, Message: "Update could not be planned: " + freezeErr.Error(), ActivityID: activityID}+ if failedProgressErr := jobcontext.Progress(ctx, failure); failedProgressErr != nil {+ return ctx, failedProgressErr+ }+ }+ batch := scheduler.TargetOutcome{ID: "auto-update", ResourceType: "update-batch", Status: scheduler.Running, RecoveryData: raw, ActivityID: activityID}+ if progressErr := jobcontext.Progress(ctx, batch); progressErr != nil {return ctx, progressErr}for _, target := range plan.Targets {@@ -148,25 +145,26 @@ func (s *Service) FreezePending(ctx context.Context) (context.Context, error) {return context.WithValue(ctx, frozenPendingKeyInternal{}, &plan), nil}-func (s *Service) buildFrozenPlanInternal(ctx context.Context, records []updater.ImageUpdateRecord) (frozenUpdatePlanInternal, error) {+func (s *Service) buildFrozenPlanInternal(ctx context.Context, records []updater.ImageUpdateRecord, excluded map[string]bool) (frozenUpdatePlanInternal, map[string]error, error) {plan := frozenUpdatePlanInternal{Records: make([]updater.ImageUpdateRecord, 0, len(records))}+ failures := map[string]error{}if len(records) == 0 {- return plan, nil+ return plan, failures, nil}dockerClient, err := s.dockerClient(ctx)if err != nil {- return plan, err+ return plan, nil, err}listed, err := dockerClient.ContainerList(ctx, client.ContainerListOptions{All: false})if err != nil {- return plan, err+ return plan, nil, err}for _, record := range records {if !record.HasUpdate {continue}for _, candidate := range listed.Items {- if record.ContainerID != "" && record.ContainerID != candidate.ID {+ if excluded[candidate.ID] || (record.ContainerID != "" && record.ContainerID != candidate.ID) {continue}if record.ContainerID == "" && refs.NormalizeImageUpdateRef(candidate.Image) != refs.NormalizeImageUpdateRef(record.ImageRef()) {@@ -174,13 +172,14 @@ func (s *Service) buildFrozenPlanInternal(ctx context.Context, records []updater}target, selected, freezeRecordTargetErr := s.freezeRecordTargetInternal(ctx, record, candidate.ID)if freezeRecordTargetErr != nil {- return plan, freezeRecordTargetErr+ failures[candidate.ID] = freezeRecordTargetErr+ continue}plan.Records = append(plan.Records, selected)plan.Targets = append(plan.Targets, *target)}}- return plan, nil+ return plan, failures, nil}func (s *Service) freezeRecordTargetInternal(ctx context.Context, record updater.ImageUpdateRecord, containerID string) (*arcaneupdater.FrozenUpdateTarget, updater.ImageUpdateRecord, error) {@@ -364,18 +363,40 @@ func (s *Service) ReconcilePending(ctx context.Context, run scheduler.Run) (schereturn scheduler.Outcome{Status: scheduler.Waiting}, err}defer release()- var plan frozenUpdatePlanInternal- found := false- for _, target := range run.Outcome.Targets {- if target.ID == "auto-update" && len(target.RecoveryData) > 0 {- if unmarshalErr := json.Unmarshal(target.RecoveryData, &plan); unmarshalErr != nil {- return scheduler.Outcome{Status: scheduler.Failed}, unmarshalErr- }- found = true+ ctx, remaining, unresolved, err := s.ResumePlan(ctx, run)+ if errors.Is(err, errNoFrozenPlanInternal) {+ return scheduler.Outcome{Status: scheduler.Failed, Message: "Interrupted update has no frozen target plan", Targets: run.Outcome.Targets}, nil+ }+ if err != nil {+ return scheduler.Outcome{Status: scheduler.Waiting}, err+ }+ if remaining > 0 {+ result, applyPendingErr := s.applyPending(ctx, arcaneupdater.Options{})+ if applyPendingErr != nil || result == nil || result.Failed > 0 {+ return scheduler.Outcome{Status: scheduler.Failed, Message: "Frozen update recovery could not confirm completion"}, applyPendingErr}}+ outcome := scheduler.Outcome{Status: scheduler.Succeeded, Message: "Frozen update targets confirmed"}+ // Containers that could not be planned were never updated, so recovery reports them as finalize does.+ if outcome.Targets = s.Unplanned(run); len(outcome.Targets) > 0 {+ outcome.Status, outcome.Message = scheduler.Partial, "Some updates failed"+ }+ if unresolved {+ outcome.Status, outcome.Message = scheduler.Failed, "Some update effects could not be confirmed"+ }+ return outcome, nil+}++// ResumePlan confirms each unsettled frozen target before anything repeats and+// scopes ctx to the targets still to apply. Targets that changed without+// reaching the desired image are marked for review and reported as unresolved.+func (s *Service) ResumePlan(ctx context.Context, run scheduler.Run) (context.Context, int, bool, error) {+ plan, found, err := frozenPlanInternal(run)+ if err != nil {+ return ctx, 0, false, err+ }if !found {- return scheduler.Outcome{Status: scheduler.Failed, Message: "Interrupted update has no frozen target plan", Targets: run.Outcome.Targets}, nil+ return ctx, 0, false, errNoFrozenPlanInternal}remaining := frozenUpdatePlanInternal{}unresolved := false@@ -383,44 +404,85 @@ func (s *Service) ReconcilePending(ctx context.Context, run scheduler.Run) (scheif frozenTargetSettledInternal(run, target.ContainerID) {continue}- confirmed, unchanged, confirmFrozenTargetErr := s.ConfirmTarget(ctx, target)- if confirmFrozenTargetErr != nil {- return scheduler.Outcome{Status: scheduler.Waiting}, confirmFrozenTargetErr- }- if confirmed {- if progressErr := jobcontext.Progress(- ctx,- scheduler.TargetOutcome{- ID: target.ContainerID,- ResourceType: "container",- Status: scheduler.Succeeded,- Message: "Frozen desired image confirmed after restart",- },- ); progressErr != nil {- return scheduler.Outcome{}, progressErr- }- continue+ confirmed, unchanged, confirmErr := s.ConfirmTarget(ctx, target)+ if confirmErr != nil {+ return ctx, 0, false, confirmErr}- if !unchanged {+ progress := scheduler.TargetOutcome{ID: target.ContainerID, ResourceType: "container"}+ switch {+ case confirmed:+ progress.Status, progress.Message = scheduler.Succeeded, "Frozen desired image confirmed"+ case !unchanged:unresolved = true+ progress.Status, progress.Message = scheduler.NeedsAttention, "Update effect could not be confirmed"+ default:+ remaining.Targets = append(remaining.Targets, target)+ remaining.Records = appendFrozenRecordInternal(remaining.Records, plan.Records, target.ContainerID)continue}- remaining.Targets = append(remaining.Targets, target)- remaining.Records = appendFrozenRecordInternal(remaining.Records, plan.Records, target.ContainerID)+ if progressErr := jobcontext.Progress(ctx, progress); progressErr != nil {+ return ctx, 0, false, progressErr+ }}- if len(remaining.Records) > 0 {- result, applyPendingErr := s.applyPending(context.WithValue(ctx, frozenPendingKeyInternal{}, &remaining), arcaneupdater.Options{})- if applyPendingErr != nil || result == nil || result.Failed > 0 {- return scheduler.Outcome{Status: scheduler.Failed, Message: "Frozen update recovery could not confirm completion"}, applyPendingErr+ return context.WithValue(ctx, frozenPendingKeyInternal{}, &remaining), len(remaining.Records), unresolved, nil+}++// Planned reports whether the run already holds a frozen plan, as a retry does.+func (s *Service) Planned(run scheduler.Run) bool {+ _, found, err := frozenPlanInternal(run)+ return found && err == nil+}++// Started reports whether applying the frozen plan began: a planned container moved past queued.+func (s *Service) Started(run scheduler.Run) bool {+ plan, found, err := frozenPlanInternal(run)+ if err != nil || !found {+ return false+ }+ return slices.ContainsFunc(plan.Targets, func(planned arcaneupdater.FrozenUpdateTarget) bool {+ return slices.ContainsFunc(run.Outcome.Targets, func(target scheduler.TargetOutcome) bool {+ return target.ID == planned.ContainerID && target.Status != scheduler.Queued+ })+ })+}++// Unsettled counts the frozen targets a retry would still resume.+func (s *Service) Unsettled(run scheduler.Run) (int, error) {+ plan, found, err := frozenPlanInternal(run)+ if err != nil || !found {+ return 0, cmp.Or(err, errNoFrozenPlanInternal)+ }+ unsettled := 0+ for _, target := range plan.Targets {+ if !frozenTargetSettledInternal(run, target.ContainerID) {+ unsettled++}}- status := scheduler.Succeeded- message := "Frozen update targets confirmed"- if unresolved {- status = scheduler.Failed- message = "Some update effects could not be confirmed"+ return unsettled, nil+}++// Unplanned lists the images and containers that never reached the frozen plan: failed checks and failed freezes.+func (s *Service) Unplanned(run scheduler.Run) []scheduler.TargetOutcome {+ plan, found, err := frozenPlanInternal(run)+ if err != nil || !found {+ return nil+ }+ return slices.DeleteFunc(slices.Clone(run.Outcome.Targets), func(target scheduler.TargetOutcome) bool {+ return (target.ResourceType != "container" && target.ResourceType != "image") || target.Status != scheduler.Failed ||+ slices.ContainsFunc(plan.Targets, func(planned arcaneupdater.FrozenUpdateTarget) bool { return planned.ContainerID == target.ID })+ })+}++// frozenPlanInternal decodes the plan persisted on the run's batch target.+func frozenPlanInternal(run scheduler.Run) (frozenUpdatePlanInternal, bool, error) {+ var plan frozenUpdatePlanInternal+ for _, target := range run.Outcome.Targets {+ if target.ID == "auto-update" && len(target.RecoveryData) > 0 {+ err := json.Unmarshal(target.RecoveryData, &plan)+ return plan, err == nil, err+ }}- return scheduler.Outcome{Status: status, Message: message}, nil+ return plan, false, nil}func appendFrozenRecordInternal(remaining, records []updater.ImageUpdateRecord, id string) []updater.ImageUpdateRecord {diff --git a/backend/internal/updater/children/recovery/service_test.go b/backend/internal/updater/children/recovery/service_test.goindex 96061cf32..f3d1a567e 100644--- a/backend/internal/updater/children/recovery/service_test.go+++ b/backend/internal/updater/children/recovery/service_test.go@@ -100,7 +100,7 @@ func TestFrozenTargetsPreserveAllSelectedContainers(t *testing.T) {}return nil})- frozenCtx, err := svc.FreezePending(ctx)+ frozenCtx, err := svc.FreezePending(ctx, scheduler.Run{ID: "run"}, nil)require.NoError(t, err)plan := frozenCtx.Value(frozenPendingKeyInternal{}).(*frozenUpdatePlanInternal)require.Len(t, plan.Records, 2)@@ -170,3 +170,109 @@ func TestFrozenTargetConfirmsReplacementAndRejectsUnknownEffect(t *testing.T) {})}}++func TestFreezePendingExcludesFailedChecksAndIsolatesFreezeFailures(t *testing.T) {+ desired := "sha256:desired"+ pending := []updater.ImageUpdateRecord{+ {ID: "app", Repository: "registry.example.com/app", Tag: "latest", HasUpdate: true, LatestDigest: &desired},+ {ID: "unresolvable", Repository: "registry.example.com/unresolvable", Tag: "latest", HasUpdate: true},+ {ID: "unchecked", Repository: "registry.example.com/unchecked", Tag: "latest", HasUpdate: true, LatestDigest: &desired},+ }+ running := []container.Summary{+ {ID: "app", Names: []string{"/app"}, Image: "registry.example.com/app:latest"},+ {ID: "unresolvable", Names: []string{"/unresolvable"}, Image: "registry.example.com/unresolvable:latest"},+ {ID: "unchecked", Names: []string{"/unchecked"}, Image: "registry.example.com/unchecked:latest"},+ }+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ if strings.HasSuffix(r.URL.Path, "/containers/json") {+ assert.NoError(t, json.MarshalWrite(w, running))+ return+ }+ for _, summary := range running {+ if strings.HasSuffix(r.URL.Path, "/containers/"+summary.ID+"/json") {+ assert.NoError(t, json.MarshalWrite(w, container.InspectResponse{+ ID: summary.ID, Name: summary.Names[0], Image: "sha256:old",+ Config: &container.Config{Image: summary.Image}, State: &container.State{StartedAt: "baseline"},+ }))+ return+ }+ }+ http.NotFound(w, r)+ }))+ defer server.Close()+ svc := newTestServiceInternal(t, server, func(context.Context) ([]updater.ImageUpdateRecord, error) { return pending, nil })+ targets := map[string]scheduler.TargetOutcome{}+ ctx := jobcontext.WithExecution(t.Context(), scheduler.Run{ID: "run"}, func(target scheduler.TargetOutcome) error {+ targets[target.ID] = target+ return nil+ })++ frozenCtx, err := svc.FreezePending(ctx, scheduler.Run{ID: "run"}, map[string]bool{"registry.example.com/unchecked:latest": true})+ require.NoError(t, err)+ plan := frozenCtx.Value(frozenPendingKeyInternal{}).(*frozenUpdatePlanInternal)+ require.Len(t, plan.Targets, 1)+ require.Equal(t, "app", plan.Targets[0].ContainerID)+ require.Equal(t, scheduler.Queued, targets["app"].Status)+ require.Equal(t, scheduler.Failed, targets["unresolvable"].Status, "a target that cannot be frozen fails alone")+ require.Contains(t, targets["unresolvable"].Message, "Update could not be planned")+ require.NotContains(t, targets, "unchecked", "images whose check failed stay out of the plan")+}++func TestResumePlanConfirmsRedeliveredChangeWithoutReapplying(t *testing.T) {+ target := arcaneupdater.FrozenUpdateTarget{+ ContainerID: "old", ContainerName: "app", BaselineImageID: "sha256:baseline", BaselineStartedAt: "baseline",+ DesiredImageRef: "registry.example.com/app:latest", DesiredDigest: "sha256:desired",+ }+ raw, err := json.Marshal(frozenUpdatePlanInternal{+ Records: []updater.ImageUpdateRecord{{ID: "image", ContainerID: "old", Repository: "registry.example.com/app", Tag: "latest", HasUpdate: true}},+ Targets: []arcaneupdater.FrozenUpdateTarget{target},+ })+ require.NoError(t, err)+ run := scheduler.Run{ID: "run", Outcome: scheduler.Outcome{Targets: []scheduler.TargetOutcome{+ {ID: "auto-update", ResourceType: "update-batch", Status: scheduler.Running, RecoveryData: raw},+ {ID: "old", ResourceType: "container", Status: scheduler.Running},+ }}}+ for _, test := range []struct {+ name, id, image string+ remaining int+ status scheduler.RunStatus+ }{+ {"applied before the redelivery", "new", "sha256:desired", 0, scheduler.Succeeded},+ {"not yet applied", "old", "sha256:baseline", 1, ""},+ } {+ t.Run(test.name, func(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ switch {+ case strings.HasSuffix(r.URL.Path, "/containers/json"):+ assert.NoError(t, json.MarshalWrite(w, []container.Summary{{ID: test.id, Names: []string{"/app"}}}))+ case strings.Contains(r.URL.Path, "/images/"):+ assert.NoError(t, json.MarshalWrite(w, image.InspectResponse{ID: "sha256:desired"}))+ case strings.Contains(r.URL.Path, "/containers/"):+ assert.NoError(t, json.MarshalWrite(w, container.InspectResponse{+ ID: test.id, Name: "/app", Image: test.image, Config: &container.Config{}, State: &container.State{StartedAt: "baseline"},+ }))+ default:+ http.NotFound(w, r)+ }+ }))+ defer server.Close()+ svc := newTestServiceInternal(t, server, nil)+ var status scheduler.RunStatus+ ctx := jobcontext.WithExecution(t.Context(), run, func(progress scheduler.TargetOutcome) error {+ status = progress.Status+ return nil+ })++ resumed, remaining, unresolved, resumeErr := svc.ResumePlan(ctx, run)+ require.NoError(t, resumeErr)+ require.False(t, unresolved)+ require.Equal(t, test.remaining, remaining)+ require.Equal(t, test.status, status)+ records, ok := svc.FrozenRecords(resumed)+ require.True(t, ok)+ require.Len(t, records, test.remaining)+ })+ }+}diff --git a/backend/internal/updater/handler.go b/backend/internal/updater/handler.goindex 0d8f23c75..2c4e50883 100644--- a/backend/internal/updater/handler.go+++ b/backend/internal/updater/handler.go@@ -102,7 +102,7 @@ func (h *UpdaterHandler) GetUpdaterHistory(ctx context.Context, input *GetUpdate}, nil}-func (h *UpdaterHandler) updateContainerInternal(ctx context.Context, input *UpdateContainerInput) (*updateContainerOutput, error) {+func (h *UpdaterHandler) updateContainer(ctx context.Context, input *UpdateContainerInput) (*updateContainerOutput, error) {runtimeCtx := utils.ActivityRuntimeContext(ctx, h.appCtx)if input.Async {acceptedActivity, err := h.updaterService.AcceptSingleContainerUpdate(runtimeCtx, input.ContainerID)diff --git a/backend/internal/updater/module.go b/backend/internal/updater/module.goindex 93c2f8ebd..74eeadf8d 100644--- a/backend/internal/updater/module.go+++ b/backend/internal/updater/module.go@@ -38,11 +38,7 @@ func (m *Module) Service() *UpdaterService {// RegisterRoutes mounts the updater endpoints. A nil module still registers, so// OpenAPI spec generation can discover the routes without a service graph.func (m *Module) RegisterRoutes(api huma.API, appCtx handlerutil.ActivityAppContext) {- if m == nil {- RegisterUpdater(api, nil, appCtx)- return- }- RegisterUpdater(api, m.service, appCtx)+ RegisterUpdater(api, m.Service(), appCtx)}// RegisterUpdater registers updater management routes using Huma.@@ -101,5 +97,5 @@ func RegisterUpdater(api huma.API, updaterService *UpdaterService, appCtx handle"200": {Description: "Container update completed", Content: map[string]*huma.MediaType{"application/json": {Schema: completedSchema}}},"202": {Description: "Container update accepted", Content: map[string]*huma.MediaType{"application/json": {Schema: acceptedSchema}}},},- }, authz.PermImageUpdatesCheck, h.updateContainerInternal)+ }, authz.PermImageUpdatesCheck, h.updateContainer)}diff --git a/backend/internal/updater/service.go b/backend/internal/updater/service.goindex c9fb41c06..32b527981 100644--- a/backend/internal/updater/service.go+++ b/backend/internal/updater/service.go@@ -17,16 +17,18 @@ import ("github.com/containerd/errdefs"activitytypes "github.com/getarcaneapp/arcane/types/v2/activity""github.com/getarcaneapp/arcane/types/v2/containerregistry"+ imageupdatetypes "github.com/getarcaneapp/arcane/types/v2/imageupdate"projecttypes "github.com/getarcaneapp/arcane/types/v2/project""github.com/getarcaneapp/arcane/types/v2/scheduler"arcaneupdater "github.com/getarcaneapp/arcane/types/v2/updater"usertypes "github.com/getarcaneapp/arcane/types/v2/user"+ "github.com/italypaleale/francis/builtin/workflow""github.com/moby/moby/api/types/container""github.com/moby/moby/client""github.com/samber/mo""go.getarcane.app/docker""go.getarcane.app/docker/compat"- "go.getarcane.app/kit/pkg"+ kit "go.getarcane.app/kit/pkg""go.getarcane.app/sys/cgroup""go.getarcane.app/updater""go.getarcane.app/updater/labels"@@ -53,10 +55,10 @@ import (activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts""github.com/getarcaneapp/arcane/backend/v2/pkg/projects"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs""github.com/getarcaneapp/arcane/backend/v2/pkg/utils"- "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/imageref""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/notifications""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/userctx"@@ -64,22 +66,18 @@ import (// UpdaterService is Arcane's handler-facing service for the standalone updater engine.type UpdaterService struct {- admission *runs.Admission- config *config.Config- coordinator *runs.Coordinator- roles *role.RoleService- deps updaterDependenciesInternal- engine *updater.Service- execution *execution.Service- recovery *recovery.Service- // updateMu serializes per-container updates. docker compose's recreate- // pipeline is not concurrency-safe for sibling containers sharing a- // namespace. ponytail: global lock ceiling — all updates serialize; fine- // for a UI, upgrade to per-project if batch throughput ever matters.+ admission *runs.Admission+ deps updaterDependencies+ engine *updater.Service+ execution *execution.Service+ recovery *recovery.Service+ autoUpdate *flow.Workflow+ // updateMu serializes container updates: compose's recreate pipeline is not safe for+ // sibling containers sharing a namespace. Upgrade to per-project if throughput matters.updateMu sync.Mutex}-type updaterDependenciesInternal struct {+type updaterDependencies struct {DB *database.DBDocker *dockerInternal.DockerClientServiceSettings *settings.SettingsService@@ -89,18 +87,66 @@ type updaterDependenciesInternal struct {RegistryDigestResolver *registry.ContainerRegistryServiceEvents *event.EventServiceNotifications *notification.NotificationService- SelfUpgrade selfUpgradeServiceInternal+ SelfUpgrade selfUpgradeServiceActivity *activity.ActivityServiceSystemUser usertypes.Actor- Logger *slog.Logger}-type selfUpgradeServiceInternal interface {+type selfUpgradeService interface {// TriggerUpgradeViaCLI returns the spawned upgrader container's ID, which this// service does not need — only update-all's manager step uses it.TriggerUpgradeViaCLI(ctx context.Context, user usertypes.Actor, target updater.SelfUpdateTarget) (string, error)}+type (+ // containerUpdateBatch accumulates per-container update notifications+ // so a single update batch produces one batched notification.+ containerUpdateBatch struct {+ sync.Mutex++ entries []notifications.ContainerUpdateBatchEntry+ }++ containerUpdateBatchContextKey struct{}+ activityIDContextKey struct{}+ updateAdmissionKey struct{}+ updateProgressKey struct{}++ updateProgress struct {+ mu sync.Mutex+ err error+ selfTriggered bool+ selfID string+ }++ // autoUpdateApplied is the apply step's output, read back by finalize.+ autoUpdateApplied struct {+ Result arcaneupdater.Result `json:"result"`+ Error string `json:"error,omitempty"`+ RecordingError string `json:"recordingError,omitempty"`+ SelfTriggered bool `json:"selfTriggered,omitempty"`+ SelfID string `json:"selfId,omitempty"`+ Unresolved bool `json:"unresolved,omitempty"`+ Completed bool `json:"completed"`+ // Busy means another update held the updater, so this run applied nothing; Started means an earlier+ // delivery had begun applying its plan.+ Busy bool `json:"busy,omitempty"`+ Started bool `json:"started,omitempty"`+ }+)++var (+ errUpdateBusy = errors.New("another container update is running")++ containerEventTypes = map[string]event.EventType{+ "container_stop": event.EventTypeContainerStop,+ "container_delete": event.EventTypeContainerDelete,+ "container_create": event.EventTypeContainerCreate,+ "container_start": event.EventTypeContainerStart,+ "container_update": event.EventTypeContainerUpdate,+ }+)+// NewUpdaterService constructs the Arcane updater facade.func NewUpdaterService(db *database.DB,@@ -112,16 +158,16 @@ func NewUpdaterService(events *event.EventService,imageSvc *image.ImageService,localNotifications *notification.NotificationService,- upgrade selfUpgradeServiceInternal,+ upgrade selfUpgradeService,activityService *activity.ActivityService,cfg *config.Config,- coordinator *runs.Coordinator,+ _ *runs.Coordinator,admission *runs.Admission,roles *role.RoleService,) (*UpdaterService, error) {service := &UpdaterService{- config: cfg, coordinator: coordinator, admission: admission, roles: roles,- deps: updaterDependenciesInternal{+ admission: admission,+ deps: updaterDependencies{DB: db,Docker: localDocker,Settings: localSettings,@@ -136,30 +182,49 @@ func NewUpdaterService(SystemUser: usertypes.SystemUser,},}+ // A nil registry service must reach the engine as a nil interface, not a typed nil.+ var digestResolver updater.RegistryDigestResolver+ if registries != nil {+ digestResolver = registries+ }service.recovery = recovery.NewService(service.DockerClient,- service.registryDigestResolverInternal,+ func() updater.RegistryDigestResolver { return digestResolver },service.PendingImageUpdates,- activityIDFromContextInternal,- service.acquireUpdateInternal,+ activityIDFromContext,+ service.acquireUpdate,service.ApplyPending,)service.execution = execution.NewService(execution.Dependencies{- Coordinator: coordinator,Config: cfg,Roles: roles,- Activity: activityService,- Logger: service.loggerInternal,- AcquireUpdate: service.acquireUpdateInternal,- UpdateBusy: errUpdateBusyInternal,- TrackActivity: service.trackActivityInternal,- RunUpdate: service.runSingleContainerUpdateInternal,- FinishUpdate: service.finishSingleContainerUpdateInternal,+ AcquireUpdate: service.acquireUpdate,+ UpdateBusy: errUpdateBusy,+ RunUpdate: service.runSingleContainerUpdate,+ RecordUpdate: service.recordSingleContainerUpdate,FreezeContainer: service.recovery.FreezeContainer,ConfirmTarget: service.recovery.ConfirmTarget,WithFrozenTarget: service.recovery.WithSingleTarget,})- engine, err := updater.New(service.configInternal())+ // The self container ID routes Arcane through the CLI self-updater even without its labels.+ selfContainerID, selfErr := cgroup.CurrentContainerID()+ engine, err := updater.New(updater.Config{+ DockerClientProvider: service,+ ImagePuller: service,+ PendingStore: service,+ RunRecorder: service,+ Settings: service,+ RegistryDigestResolver: digestResolver,+ RegistryTagLister: service,+ ProjectUpdater: service,+ SelfUpdater: service,+ Notifier: service,+ EventRecorder: service,+ UsedImageCollector: updater.UsedImageCollectorFunc(service.CollectUsedImages),+ LabelPolicy: updater.DefaultLabelPolicy(),+ SelfContainerID: kit.Ternary(selfErr != nil, "", selfContainerID),+ Logger: slog.Default(),+ })if err != nil {return nil, fmt.Errorf("configure updater engine: %w", err)}@@ -168,109 +233,88 @@ func NewUpdaterService(return service, nil}-func (s *UpdaterService) configInternal() updater.Config {- return updater.Config{- DockerClientProvider: s,- ImagePuller: s,- PendingStore: s,- RunRecorder: s,- Settings: s,- RegistryDigestResolver: s.registryDigestResolverInternal(),- RegistryTagLister: s,- ProjectUpdater: s,- SelfUpdater: s,- Notifier: s,- EventRecorder: s,- UsedImageCollector: updater.UsedImageCollectorFunc(s.CollectUsedImages),- LabelPolicy: updater.DefaultLabelPolicy(),- SelfContainerID: selfContainerIDInternal(),- Logger: s.loggerInternal(),- }-}--// selfContainerIDInternal returns the ID of the container Arcane runs in, so-// the updater engine routes it through the CLI self-updater even when the-// container is missing the Arcane labels. Empty when not running in Docker.-func selfContainerIDInternal() string {- id, err := cgroup.CurrentContainerID()- return kit.Ternary(err != nil, "", id)-}--func (s *UpdaterService) engineInternal() *updater.Service {- return s.engine-}--func (s *UpdaterService) loggerInternal() *slog.Logger {- if s.deps.Logger != nil {- return s.deps.Logger- }- return slog.Default()-}--func (s *UpdaterService) registryDigestResolverInternal() updater.RegistryDigestResolver {- if s == nil || s.deps.RegistryDigestResolver == nil {- return nil- }- return s.deps.RegistryDigestResolver-}--// ApplyPending executes pending image updates. When the options carry-// resource IDs the run is scoped: the engine's ApplyPending has no resource-// filtering (it would apply every pending update), so scoped requests resolve-// to concrete containers and go through the engine's single-container path-// instead — same activity, events, and cleanup either way.+// ApplyPending executes pending image updates. The engine's ApplyPending applies every+// pending update, so a scoped request runs its containers through the single-container path.func (s *UpdaterService) ApplyPending(ctx context.Context, options arcaneupdater.Options) (out *arcaneupdater.Result, err error) {var release func()- ctx, release, err = s.acquireUpdateInternal(ctx)+ ctx, release, err = s.acquireUpdate(ctx)if err != nil {return nil, err}defer release()start := time.Now()batchCompleted := false- activityID := s.startAutoUpdateActivityInternal(ctx, options.DryRun)- out = &arcaneupdater.Result{Items: []arcaneupdater.ResourceResult{}, ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()}- ctx = s.trackActivityInternal(ctx, activityID)- ctx = contextWithActivityIDInternal(ctx, activityID)- progress := &updateProgressInternal{}- ctx = context.WithValue(ctx, updateProgressKeyInternal{}, progress)- notifyBatch := &containerUpdateBatchInternal{}- ctx = context.WithValue(ctx, containerUpdateBatchContextKeyInternal{}, notifyBatch)+ activityID := ""+ if s.deps.Activity != nil {+ started, startErr := s.deps.Activity.StartActivity(ctx, activitylib.StartRequest{+ EnvironmentID: "0",+ Type: activitytypes.TypeAutoUpdate,+ Queue: true,+ ResourceType: new("system"),+ ResourceName: new("Auto update"),+ Step: "Planning updates",+ LatestMessage: "Auto-update run started",+ Metadata: database.JSON{"dryRun": options.DryRun},+ })+ if startErr != nil {+ slog.DebugContext(ctx, "failed to start auto-update activity", "error", startErr)+ } else {+ activityID = started.ID+ ctx = contextWithActivityID(s.deps.Activity.Track(ctx, activityID), activityID)+ }+ }+ out = &arcaneupdater.Result{Items: []arcaneupdater.ResourceResult{}}+ progress := &updateProgress{}+ ctx = context.WithValue(ctx, updateProgressKey{}, progress)defer func() {- s.flushBatchedContainerUpdatesInternal(ctx, notifyBatch)- if out == nil {- out = &arcaneupdater.Result{Items: []arcaneupdater.ResourceResult{}}- }- if out.Duration == "" {- out.Duration = time.Since(start).String()+ out.Duration = cmp.Or(out.Duration, time.Since(start).String())+ out.ActivityID = mo.EmptyableToOption(activityID).ToPointer()+ err = progress.completeBatch(ctx, options, out, batchCompleted, err)+ failure := ""+ if err != nil {+ failure = err.Error()+ } else if out.Failed > 0 {+ failure = fmt.Sprintf("%d update action(s) failed", out.Failed)}- out.ActivityID = mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()- err = progress.completeBatchInternal(ctx, options, out, batchCompleted, err)- s.completeAutoUpdateActivityInternal(ctx, activityID, out, err)+ s.completeAutoUpdateActivity(ctx, activityID, "Auto-update run completed", failure)}()- ctx, err = s.recovery.FreezePending(ctx)- if err != nil {- return out, err- }-- if activityID != "" && s.deps.Activity != nil {- // Bounded slot wait: an unbounded wait behind other long-running runs- // would strand the queued activity row (the completion defer above- // flips it to failed on timeout instead).+ if activityID != "" {+ // Bounded so a long queue fails the activity instead of stranding it queued.if slotErr := s.deps.Activity.AwaitActivitySlotBounded(ctx, activityID, "0"); slotErr != nil {return out, slotErr}}- // The engine's per-container docker operations carry no timeouts, so cap- // the whole run; the Track ctx stays unbounded for the deferred completion- // so user cancellation is still detected there.+ if applyErr := s.applyBatch(ctx, options, out); applyErr != nil {+ return out, applyErr+ }+ batchCompleted = true+ return out, nil+}++// applyBatch runs one update batch into out, records its events, and sends its notifications+// as one batch. The engine's per-container docker operations carry no timeouts, so the run is capped.+func (s *UpdaterService) applyBatch(ctx context.Context, options arcaneupdater.Options, out *arcaneupdater.Result) error {+ batch := &containerUpdateBatch{}+ ctx = context.WithValue(ctx, containerUpdateBatchContextKey{}, batch)+ defer func() {+ batch.Lock()+ entries := batch.entries+ batch.Unlock()+ if len(entries) == 0 || s.deps.Notifications == nil {+ return+ }+ if err := s.deps.Notifications.SendBatchContainerUpdateNotification(ctx, entries); err != nil {+ slog.ErrorContext(ctx, "failed to send batched container update notification", "error", err, "count", len(entries))+ }+ }()runCtx, cancelRun := context.WithTimeout(ctx, timeouts.DefaultAutoUpdateApply)defer cancelRun()+ activityID := activityIDFromContext(ctx)- s.recordAutoUpdateEventInternal(ctx, event.EventSeverityInfo, database.JSON{+ s.recordAutoUpdateEvent(ctx, event.EventSeverityInfo, database.JSON{"phase": "start","dryRun": options.DryRun,"forceUpdate": options.ForceUpdate,@@ -278,36 +322,33 @@ func (s *UpdaterService) ApplyPending(ctx context.Context, options arcaneupdater"scopedCount": len(options.ResourceIds),"time": time.Now().UTC().Format(time.RFC3339),})- s.appendAutoUpdateActivityMessageInternal(ctx, activityID, "Planning pending updates", "Planning updates", 5)+ s.appendAutoUpdateActivityMessage(ctx, activityID, "Planning pending updates", "Planning updates", 5)if len(options.ResourceIds) > 0 {- if applyErr := s.applyScopedUpdatesInternal(runCtx, options, out); applyErr != nil {- return out, applyErr+ if applyErr := s.applyScopedUpdates(runCtx, options, out); applyErr != nil {+ return applyErr}} else {- moduleResult, engineErr := s.engineInternal().ApplyPending(runCtx, moduleOptionsFromUpdaterOptionsInternal(options))+ moduleResult, engineErr := s.engine.ApplyPending(runCtx, updater.Options{Force: options.ForceUpdate, DryRun: options.DryRun})if moduleResult != nil {- out = resultFromModuleInternal(moduleResult)- out.ActivityID = mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()- s.logResultItemsInternal(ctx, out)- }- if moduleResult == nil && engineErr == nil {+ *out = *resultFromModule(moduleResult)+ s.logResultItems(ctx, out)+ } else if engineErr == nil {engineErr = errors.New("updater returned no batch result")}if engineErr != nil {- err = engineErr- return out, err+ return engineErr}}if !options.DryRun && s.deps.ImageUpdates != nil {- s.appendAutoUpdateActivityMessageInternal(ctx, activityID, "Cleaning up update records", "Cleaning up", 95)+ s.appendAutoUpdateActivityMessage(ctx, activityID, "Cleaning up update records", "Cleaning up", 95)if cleanupErr := s.deps.ImageUpdates.CleanupOrphanedRecords(runCtx); cleanupErr != nil {- s.loggerInternal().WarnContext(ctx, "cleanup orphaned update records failed", "error", cleanupErr)+ slog.WarnContext(ctx, "cleanup orphaned update records failed", "error", cleanupErr)}}- s.recordAutoUpdateEventInternal(ctx, event.EventSeverityInfo, database.JSON{+ s.recordAutoUpdateEvent(ctx, event.EventSeverityInfo, database.JSON{"phase": "complete","checked": out.Checked,"updated": out.Updated,@@ -317,15 +358,60 @@ func (s *UpdaterService) ApplyPending(ctx context.Context, options arcaneupdater"duration": out.Duration,"time": time.Now().UTC().Format(time.RFC3339),})- batchCompleted = true- return out, nil+ return nil+}++// scopedContainerIDs resolves the requested containers, projects, or images to container IDs.+func (s *UpdaterService) scopedContainerIDs(ctx context.Context, options arcaneupdater.Options) ([]string, error) {+ requested := kit.TrimNonEmpty(options.ResourceIds)+ scope := strings.ToLower(strings.TrimSpace(options.Type))+ containerIDs := requested+ switch {+ case len(requested) == 0, scope == "", scope == "container":+ case scope == "project", scope == "image":+ // Projects match by lowercase Compose project name; images by ID or normalized reference.+ wanted := make(map[string]bool, len(requested)*2)+ for _, ref := range requested {+ if scope == "image" {+ wanted[ref], wanted[refs.NormalizeImageUpdateRef(ref)] = true, true+ continue+ }+ name, discovered := strings.CutPrefix(ref, "compose:")+ if !discovered && s.deps.Projects != nil {+ if project, lookupErr := s.deps.Projects.GetProjectFromDatabaseByID(ctx, ref); lookupErr == nil && project != nil {+ name = cmp.Or(strings.TrimSpace(kit.FromPtr(project.ComposeProjectName)), strings.TrimSpace(project.Name), name)+ }+ }+ wanted[strings.ToLower(strings.TrimSpace(name))] = true+ }+ delete(wanted, "")+ if s.deps.Docker == nil {+ return nil, errors.New("docker service unavailable")+ }+ containers, _, _, _, err := s.deps.Docker.GetAllContainers(ctx)+ if err != nil {+ return nil, fmt.Errorf("list containers: %w", err)+ }+ containerIDs = nil+ for _, summary := range containers {+ matched := wanted[summary.ImageID] || wanted[refs.NormalizeImageUpdateRef(summary.Image)]+ if scope == "project" {+ project := strings.ToLower(strings.TrimSpace(docker.ComposeProjectLabel(summary.Labels)))+ matched = docker.ComposeServiceLabel(summary.Labels) != "" && wanted[project]+ }+ if matched {+ containerIDs = append(containerIDs, summary.ID)+ }+ }+ default:+ return nil, fmt.Errorf("unsupported scoped update type %q", options.Type)+ }+ return containerIDs, nil}-// applyScopedUpdatesInternal runs a scoped update into the caller's result:-// resolves the requested resources to container IDs and updates each through-// the engine's single-container path.-func (s *UpdaterService) applyScopedUpdatesInternal(ctx context.Context, options arcaneupdater.Options, out *arcaneupdater.Result) error {- containerIDs, err := s.resolveScopedContainerIDsInternal(ctx, options)+// applyScopedUpdates updates each requested container through the engine's single-container path.+func (s *UpdaterService) applyScopedUpdates(ctx context.Context, options arcaneupdater.Options, out *arcaneupdater.Result) error {+ containerIDs, err := s.scopedContainerIDs(ctx, options)if err != nil {return err}@@ -333,17 +419,17 @@ func (s *UpdaterService) applyScopedUpdatesInternal(ctx context.Context, optionsreturn common.ErrUpdaterNoContainersMatched}- engineOpts := moduleOptionsFromUpdaterOptionsInternal(options)+ engineOpts := updater.Options{Force: options.ForceUpdate, DryRun: options.DryRun}var engineErrs []errorfor _, containerID := range containerIDs {target := scheduler.TargetOutcome{ID: containerID, ResourceType: "container", Status: scheduler.Running}if progressErr := jobcontext.Progress(ctx, target); progressErr != nil {return progressErr}- moduleResult, engineErr := s.engineInternal().UpdateContainer(ctx, containerID, engineOpts)+ moduleResult, engineErr := s.engine.UpdateContainer(ctx, containerID, engineOpts)target.Status = scheduler.NeedsAttentionif moduleResult != nil {- partial := resultFromModuleInternal(moduleResult)+ partial := resultFromModule(moduleResult)out.Checked += partial.Checkedout.Updated += partial.Updatedout.Restarted += partial.Restarted@@ -369,205 +455,94 @@ func (s *UpdaterService) applyScopedUpdatesInternal(ctx context.Context, optionstarget.Status = scheduler.Failedtarget.Message = engineErr.Error()}- if progressErr2 := jobcontext.Progress(ctx, target); progressErr2 != nil {- return errors.Join(progressErr2, errors.Join(engineErrs...))+ if progressErr := jobcontext.Progress(ctx, target); progressErr != nil {+ return errors.Join(progressErr, errors.Join(engineErrs...))}}- s.logResultItemsInternal(ctx, out)+ s.logResultItems(ctx, out)out.Success = out.Failed == 0- // Engine errors propagate like the unscoped path's engine error does —- // the remaining containers were still attempted and recorded above.+ // Like the unscoped engine error, these surface after every container was attempted.return errors.Join(engineErrs...)}-// resolveScopedContainerIDsInternal maps a scoped options payload to the-// container IDs it covers.-func (s *UpdaterService) resolveScopedContainerIDsInternal(ctx context.Context, options arcaneupdater.Options) ([]string, error) {- requested := kit.TrimNonEmpty(options.ResourceIds)- if len(requested) == 0 {- return nil, nil- }-- switch strings.ToLower(strings.TrimSpace(options.Type)) {- case "", "container":- return requested, nil- case "project":- return s.containerIDsForProjectsInternal(ctx, requested)- case "image":- return s.containerIDsForImagesInternal(ctx, requested)- default:- return nil, fmt.Errorf("unsupported scoped update type %q", options.Type)- }-}--// containerIDsForProjectsInternal resolves project IDs or compose names to-// the IDs of the containers that belong to those projects.-func (s *UpdaterService) containerIDsForProjectsInternal(ctx context.Context, projectRefs []string) ([]string, error) {- if s.deps.Docker == nil {- return nil, errors.New("docker service unavailable")- }-- names := make(map[string]struct{}, len(projectRefs))- for _, ref := range projectRefs {- name := strings.TrimSpace(ref)- if discoveredName, discovered := strings.CutPrefix(name, "compose:"); discovered {- name = discoveredName- } else if s.deps.Projects != nil {- if project, lookupErr := s.deps.Projects.GetProjectFromDatabaseByID(ctx, ref); lookupErr == nil && project != nil {- switch {- case project.ComposeProjectName != nil && strings.TrimSpace(*project.ComposeProjectName) != "":- name = *project.ComposeProjectName- case strings.TrimSpace(project.Name) != "":- name = project.Name- }- }- }- names[strings.ToLower(strings.TrimSpace(name))] = struct{}{}- }-- containers, _, _, _, err := s.deps.Docker.GetAllContainers(ctx)- if err != nil {- return nil, fmt.Errorf("list containers: %w", err)- }-- var ids []string- for _, summary := range containers {- if docker.ComposeServiceLabel(summary.Labels) == "" {- continue- }- project := strings.ToLower(strings.TrimSpace(docker.ComposeProjectLabel(summary.Labels)))- if project == "" {- continue- }- if _, ok := names[project]; ok {- ids = append(ids, summary.ID)- }- }- return ids, nil-}--// containerIDsForImagesInternal resolves image IDs or references to the IDs-// of the containers currently running those images.-func (s *UpdaterService) containerIDsForImagesInternal(ctx context.Context, imageRefs []string) ([]string, error) {- if s.deps.Docker == nil {- return nil, errors.New("docker service unavailable")- }-- wanted := make(map[string]struct{}, len(imageRefs)*2)- for _, ref := range imageRefs {- trimmed := strings.TrimSpace(ref)- if trimmed == "" {- continue- }- wanted[trimmed] = struct{}{}- if normalized := refs.NormalizeImageUpdateRef(trimmed); normalized != "" {- wanted[normalized] = struct{}{}- }- }-- containers, _, _, _, err := s.deps.Docker.GetAllContainers(ctx)- if err != nil {- return nil, fmt.Errorf("list containers: %w", err)- }-- var ids []string- for _, summary := range containers {- if _, ok := wanted[summary.ImageID]; ok {- ids = append(ids, summary.ID)- continue- }- if normalized := refs.NormalizeImageUpdateRef(summary.Image); normalized != "" {- if _, ok := wanted[normalized]; ok {- ids = append(ids, summary.ID)- }- }- }- return ids, nil-}-// UpdateSingleContainer updates a single container by ID to the latest available image.func (s *UpdaterService) UpdateSingleContainer(ctx context.Context, containerID string) (out *arcaneupdater.Result, err error) {- localActivity, workCtx, err := s.startSingleContainerUpdateActivityInternal(ctx, containerID, true)- if err != nil {- return nil, err- }- activityID := ""- if localActivity != nil {- activityID = localActivity.ID+ activityID, workCtx := "", ctx+ if s.deps.Activity != nil {+ localActivity, trackedCtx, startErr := s.deps.Activity.StartTrackedActivity(ctx, s.singleContainerActivityRequest(ctx, containerID))+ if startErr != nil {+ return nil, fmt.Errorf("start container update activity: %w", startErr)+ }+ activityID, workCtx = localActivity.ID, trackedCtx}defer func() {- s.finishSingleContainerUpdateInternal(workCtx, activityID, out, err)+ s.finishSingleContainerUpdate(workCtx, activityID, out, err)}()defer utils.RecoverToError(&err, "single container update")- return s.runSingleContainerUpdateInternal(workCtx, containerID, activityID)+ return s.runSingleContainerUpdate(workCtx, containerID, activityID)}-// AcceptSingleContainerUpdate persists and submits a cancellable update activity.+// AcceptSingleContainerUpdate submits a cancellable container-update workflow and returns its activity.func (s *UpdaterService) AcceptSingleContainerUpdate(ctx context.Context, containerID string) (*activitytypes.Activity, error) {- if s.deps.Activity == nil || !s.execution.Ready() {+ if s.deps.Activity == nil {return nil, errors.New("asynchronous container updates unavailable")}- localActivity, workCtx, err := s.startSingleContainerUpdateActivityInternal(ctx, containerID, false)+ activityID, err := s.execution.Submit(ctx, containerID, s.singleContainerActivityRequest(ctx, containerID))if err != nil {return nil, err}- if submitErr := s.execution.Submit(ctx, workCtx, containerID, localActivity.ID); submitErr != nil {- return nil, submitErr+ detail, err := s.deps.Activity.GetActivityDetail(ctx, "0", activityID, 1)+ if err != nil {+ return nil, err}- return localActivity, nil+ return &detail.Activity, nil}-func (s *UpdaterService) runSingleContainerUpdateInternal(ctx context.Context, containerID, activityID string) (out *arcaneupdater.Result, err error) {+func (s *UpdaterService) runSingleContainerUpdate(ctx context.Context, containerID, activityID string) (out *arcaneupdater.Result, err error) {var release func()- ctx, release, err = s.acquireUpdateInternal(ctx)+ ctx, release, err = s.acquireUpdate(ctx)if err != nil {return nil, err}defer release()start := time.Now()- out = &arcaneupdater.Result{Items: []arcaneupdater.ResourceResult{}, ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()}- ctx = contextWithActivityIDInternal(ctx, activityID)+ out = &arcaneupdater.Result{Items: []arcaneupdater.ResourceResult{}}+ defer func() {+ out.Duration = cmp.Or(out.Duration, time.Since(start).String())+ out.ActivityID = mo.EmptyableToOption(activityID).ToPointer()+ }()+ ctx = contextWithActivityID(ctx, activityID)if s.deps.Activity != nil && activityID != "" {- if awaitActivitySlotBoundedErr := s.deps.Activity.AwaitActivitySlotBounded(ctx, activityID, "0"); awaitActivitySlotBoundedErr != nil {- return out, awaitActivitySlotBoundedErr+ if slotErr := s.deps.Activity.AwaitActivitySlotBounded(ctx, activityID, "0"); slotErr != nil {+ return out, slotErr}}- defer func() {- if out == nil {- out = &arcaneupdater.Result{Items: []arcaneupdater.ResourceResult{}}- }- if out.Duration == "" {- out.Duration = time.Since(start).String()- }- out.ActivityID = mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()- }()-s.updateMu.Lock()defer s.updateMu.Unlock()- if errErr := ctx.Err(); errErr != nil {- return out, errErr+ if ctxErr := ctx.Err(); ctxErr != nil {+ return out, ctxErr}- // The caller picked this container, so the autoUpdateExcludedContainers- // setting does not apply: it only governs automatic and pending runs, which- // keep skipping the container. Labels and immutable references still do.- moduleResult, engineErr := s.engineInternal().UpdateContainer(ctx, containerID, updater.Options{IgnoreSettingsExclusions: true})+ // The caller picked this container, so the autoUpdateExcludedContainers setting does not+ // apply here; labels and immutable references still do.+ moduleResult, err := s.engine.UpdateContainer(ctx, containerID, updater.Options{IgnoreSettingsExclusions: true})if moduleResult != nil {- out = resultFromModuleInternal(moduleResult)- out.ActivityID = mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()- s.logResultItemsInternal(ctx, out)+ out = resultFromModule(moduleResult)+ s.logResultItems(ctx, out)}- if engineErr != nil {- err = engineErr- return out, err- }- return out, nil+ return out, err}// GetStatus returns the current in-memory update activity snapshot.func (s *UpdaterService) GetStatus() arcaneupdater.Status {- return statusFromModuleInternal(s.engineInternal().Status())+ status := s.engine.Status()+ return arcaneupdater.Status{+ UpdatingContainers: status.UpdatingContainers,+ UpdatingProjects: status.UpdatingProjects,+ ContainerIds: status.ContainerIDs,+ ProjectIds: status.ProjectIDs,+ }}// GetHistory returns the most recent auto-update history records, newest first.@@ -585,8 +560,8 @@ func (s *UpdaterService) GetHistory(ctx context.Context, limit int) ([]AutoUpdat// RestartContainersUsingOldIDs restarts containers matching old image IDs or refs.func (s *UpdaterService) RestartContainersUsingOldIDs(ctx context.Context, oldIDToNewRef, oldRefToNewRef map[string]string) ([]arcaneupdater.ResourceResult, error) {- results, err := s.engineInternal().RestartContainersUsingOldImages(ctx, oldIDToNewRef, oldRefToNewRef)- return resourceResultsFromModuleInternal(results), err+ results, err := s.engine.RestartContainersUsingOldImages(ctx, oldIDToNewRef, oldRefToNewRef)+ return resultFromModule(&updater.Result{Items: results}).Items, err}// TriggerSelfUpdateViaCLI triggers Arcane's detached CLI self-update path.@@ -604,53 +579,49 @@ func (s *UpdaterService) TriggerSelfUpdateViaCLI(ctx context.Context, source, co// BeginContainerUpdate marks a container as updating.func (s *UpdaterService) BeginContainerUpdate(containerID string) func() {- return s.engineInternal().BeginContainerUpdate(containerID)+ return s.engine.BeginContainerUpdate(containerID)}// BeginProjectUpdate marks a project as updating.func (s *UpdaterService) BeginProjectUpdate(projectID string) func() {- return s.engineInternal().BeginProjectUpdate(projectID)+ return s.engine.BeginProjectUpdate(projectID)}-func (s *UpdaterService) recordAutoUpdateEventInternal(ctx context.Context, severity event.EventSeverity, metadata database.JSON) {+func (s *UpdaterService) recordAutoUpdateEvent(ctx context.Context, severity event.EventSeverity, metadata database.JSON) {if s.deps.Events == nil {return}phase, _ := metadata["phase"].(string)- _, err := s.deps.Events.CreateEvent(ctx, event.CreateEventRequest{- Type: event.EventTypeSystemAutoUpdate,- Severity: severity,- Title: autoUpdateEventTitleInternal(phase, metadata),- ResourceType: mo.EmptyableToOption(strings.TrimSpace("system")).ToPointer(),- ResourceName: mo.EmptyableToOption(strings.TrimSpace("auto_updater")).ToPointer(),- EnvironmentID: mo.EmptyableToOption(strings.TrimSpace("0")).ToPointer(),- Metadata: metadata,- })- if err != nil {- s.loggerInternal().DebugContext(ctx, "failed to record auto-update event", "error", err)- }-}--func autoUpdateEventTitleInternal(phase string, metadata database.JSON) string {+ title, subject := kit.Ternary(phase != "", "Auto-update: "+phase, "Auto-update"), ""switch phase {case "start":- return "Auto-update run started"+ title = "Auto-update run started"+ case "complete":+ title = "Auto-update run completed"case "image_pull", "image":- localImage := cmp.Or(kit.ToString(metadata["imageNew"]), kit.ToString(metadata["imageOld"]))- return kit.Ternary(localImage != "", "Auto-update: image pull "+localImage, "Auto-update: image pull")+ title, subject = "Auto-update: image pull", cmp.Or(kit.ToString(metadata["imageNew"]), kit.ToString(metadata["imageOld"]))case "image_prune":- imageID := kit.ToString(metadata["imageId"])- return kit.Ternary(imageID != "", "Auto-update: image prune "+imageID, "Auto-update: image prune")+ title, subject = "Auto-update: image prune", kit.ToString(metadata["imageId"])case "container":- name := cmp.Or(kit.ToString(metadata["resourceName"]), kit.ToString(metadata["container"]), kit.ToString(metadata["containerId"]))- return kit.Ternary(name != "", "Auto-update: container "+name, "Auto-update: container")+ title = "Auto-update: container"+ subject = cmp.Or(kit.ToString(metadata["resourceName"]), kit.ToString(metadata["container"]), kit.ToString(metadata["containerId"]))case "project":- name := cmp.Or(kit.ToString(metadata["projectName"]), kit.ToString(metadata["projectId"]))- return kit.Ternary(name != "", "Auto-update: project "+name, "Auto-update: project")- case "complete":- return "Auto-update run completed"- default:- return kit.Ternary(phase != "", "Auto-update: "+phase, "Auto-update")+ title, subject = "Auto-update: project", cmp.Or(kit.ToString(metadata["projectName"]), kit.ToString(metadata["projectId"]))+ }+ if subject != "" {+ title += " " + subject+ }+ _, err := s.deps.Events.CreateEvent(ctx, event.CreateEventRequest{+ Type: event.EventTypeSystemAutoUpdate,+ Severity: severity,+ Title: title,+ ResourceType: new("system"),+ ResourceName: new("auto_updater"),+ EnvironmentID: new("0"),+ Metadata: metadata,+ })+ if err != nil {+ slog.DebugContext(ctx, "failed to record auto-update event", "error", err)}}@@ -662,10 +633,8 @@ func (s *UpdaterService) DockerClient(ctx context.Context) (*client.Client, erroreturn s.deps.Docker.GetClient(ctx)}-// PullImage pulls an image through Arcane's image service. The pull is-// bounded by the dockerImagePullTimeout setting — image.ImageService.PullImage does-// not bound itself, and an unbounded engine pull would hold the auto-update-// run (and its activity slot) indefinitely.+// PullImage pulls an image through Arcane's image service, bounded by the+// dockerImagePullTimeout setting so a stuck pull cannot hold the auto-update run.func (s *UpdaterService) PullImage(ctx context.Context, imageRef string, progress io.Writer) error {if s == nil || s.deps.ImagePuller == nil {return common.Classify(common.ErrUnavailable, errors.New("image service unavailable"))@@ -674,8 +643,7 @@ func (s *UpdaterService) PullImage(ctx context.Context, imageRef string, progresif err != nil {return err}- activityID := activityIDFromContextInternal(ctx)- writer := activitylib.NewWriter(ctx, s.deps.Activity, activityID, progress, "Pulling updated images")+ writer := activitylib.NewWriter(ctx, s.deps.Activity, activityIDFromContext(ctx), progress, "Pulling updated images")defer activitylib.FlushWriter(writer)pullTimeoutSeconds := 0@@ -687,22 +655,24 @@ func (s *UpdaterService) PullImage(ctx context.Context, imageRef string, progresvar credentials []containerregistry.Credentialif s.deps.Projects != nil {- resolved, resolveRegistryCredentialsErr := s.deps.Projects.ResolveRegistryCredentials(pullCtx)- if resolveRegistryCredentialsErr != nil {- return fmt.Errorf("resolve registry credentials: %w", resolveRegistryCredentialsErr)+ if credentials, err = s.deps.Projects.ResolveRegistryCredentials(pullCtx); err != nil {+ return fmt.Errorf("resolve registry credentials: %w", err)}- credentials = resolved}- if pullImageErr := s.deps.ImagePuller.PullImage(pullCtx, pulledRef, writer, s.deps.SystemUser, credentials); pullImageErr != nil {- return pullImageErr+ if pullErr := s.deps.ImagePuller.PullImage(pullCtx, pulledRef, writer, s.deps.SystemUser, credentials); pullErr != nil {+ return pullErr}- if tagErr := s.tagFrozenPullInternal(pullCtx, pulledRef, imageRef); tagErr != nil {- return tagErr+ // A frozen digest pull is tagged back to the reference the engine asked for.+ if pulledRef != imageRef {+ dockerClient, clientErr := s.DockerClient(pullCtx)+ if clientErr != nil {+ return clientErr+ }+ if _, tagErr := dockerClient.ImageTag(pullCtx, client.ImageTagOptions{Source: pulledRef, Target: imageRef}); tagErr != nil {+ return tagErr+ }}- // Reconcile by tag so the new image's record and any project preview rows- // for this reference stop reporting the update just installed (#4306).- // This runs on the run context, not the pull deadline, so a pull that- // finished late still records its result.+ // Reconcile by tag on the run context so a late pull still clears its update records (#4306).if reconcileErr := s.deps.ImagePuller.ReconcilePulledImageUpdate(ctx, imageRef); reconcileErr != nil {slog.WarnContext(ctx, "failed to reconcile pulled image update state", "image", imageRef, "error", reconcileErr)}@@ -723,24 +693,62 @@ func (s *UpdaterService) PendingImageUpdates(ctx context.Context) ([]updater.Imareturn nil, fmt.Errorf("query pending image updates: %w", err)}- // Flush pending "Updates Available" notifications before the engine- // consumes and clears these records, otherwise the notification for an- // update applied here is silently lost (#3132). A flush that cannot- // determine what to send aborts the run so the records survive for retry.+ // Flush "Updates Available" notifications before the engine clears these records (#3132);+ // a flush that cannot decide what to send aborts so the records survive for retry.if s.deps.ImageUpdates != nil {if err := s.deps.ImageUpdates.SendBatchUpdateNotifications(ctx); err != nil {return nil, err}}- s.appendAutoUpdateActivityMessageInternal(- ctx,- activityIDFromContextInternal(ctx),- fmt.Sprintf("Found %d pending image update records", len(records)),- "Planning updates",- 10,- )+ s.appendAutoUpdateActivityMessage(ctx, activityIDFromContext(ctx), fmt.Sprintf("Found %d pending image update records", len(records)), "Planning updates", 10)- return s.scopedPendingRecordsInternal(ctx, records)+ // Once container-scoped records exist, an unscoped record applies per running non-tag-policy container of its image.+ var scopedCount int64+ if err := s.deps.DB.WithContext(ctx).Model(&imageupdate.ImageUpdateRecord{}).Where("container_id <> ?", "").Limit(1).Count(&scopedCount).Error; err != nil {+ return nil, err+ }+ var running []container.Summary+ if scopedCount > 0 {+ dockerClient, err := s.DockerClient(ctx)+ if err != nil {+ return nil, err+ }+ listed, err := dockerClient.ContainerList(ctx, client.ContainerListOptions{})+ if err != nil {+ return nil, err+ }+ running = listed.Items+ }+ out := make([]updater.ImageUpdateRecord, 0, len(records))+ for _, record := range records {+ converted := updater.ImageUpdateRecord{+ ID: record.ID,+ ContainerID: record.ContainerID,+ Repository: record.Repository,+ Tag: record.Tag,+ HasUpdate: record.HasUpdate,+ UpdateType: updater.UpdateType(record.UpdateType),+ CurrentVersion: record.CurrentVersion,+ LatestVersion: record.LatestVersion,+ CurrentDigest: record.CurrentDigest,+ LatestDigest: record.LatestDigest,+ CheckTime: record.CheckTime,+ LastError: record.LastError,+ }+ if scopedCount == 0 || record.ContainerID != "" {+ out = append(out, converted)+ continue+ }+ for _, cnt := range running {+ resolved, policyErr := tagpolicy.Resolve(cnt.Image, updater.DefaultLabelPolicy().TagPolicy(cnt.Labels))+ if policyErr != nil || resolved.Strategy == "tag" || refs.NormalizeImageUpdateRef(cnt.Image) != refs.NormalizeImageUpdateRef(converted.ImageRef()) {+ continue+ }+ converted.ContainerID = cnt.ID+ out = append(out, converted)+ }+ }+ return out, nil}// ClearImageUpdateRecord clears a pending image update record after it is handled.@@ -748,15 +756,46 @@ func (s *UpdaterService) ClearImageUpdateRecord(ctx context.Context, record updaif s == nil {return common.Classify(common.ErrUnavailable, errors.New("updater service unavailable"))}- return s.clearImageUpdateRecordForModuleInternal(ctx, record)+ if s.deps.DB == nil {+ return nil+ }+ query := s.deps.DB.WithContext(ctx).Model(&imageupdate.ImageUpdateRecord{})+ if record.ContainerID == "" || strings.HasPrefix(record.ID, "container::") {+ query = query.Where("container_id = ?", record.ContainerID)+ if record.ID != "" {+ return query.Where("id = ?", record.ID).Update("has_update", false).Error+ }+ return query.Where("repository = ? AND tag = ?", record.Repository, record.Tag).Update("has_update", false).Error+ }++ // A shared record stays pending while another non-tag-policy container still runs an older image.+ dockerClient, err := s.DockerClient(ctx)+ if err != nil {+ return err+ }+ target, err := dockerClient.ImageInspect(ctx, record.NewImageRef())+ if err != nil {+ return err+ }+ listed, err := dockerClient.ContainerList(ctx, client.ContainerListOptions{})+ if err != nil {+ return err+ }+ for _, cnt := range listed.Items {+ resolved, policyErr := tagpolicy.Resolve(cnt.Image, updater.DefaultLabelPolicy().TagPolicy(cnt.Labels))+ if policyErr != nil || resolved.Strategy == "tag" {+ continue+ }+ if refs.NormalizeImageUpdateRef(cnt.Image) == refs.NormalizeImageUpdateRef(record.ImageRef()) && cnt.ImageID != target.ID {+ return nil+ }+ }+ return query.Where("id = ? AND container_id = ?", record.ID, "").Update("has_update", false).Error}// ExcludedContainers returns auto-update exclusions from Arcane settings.func (s *UpdaterService) ExcludedContainers(ctx context.Context) ([]string, error) {- if s == nil {- return nil, nil- }- if s.deps.Settings == nil {+ if s == nil || s.deps.Settings == nil {return nil, nil}return kit.Unique(kit.TrimNonEmpty(strings.Split(s.deps.Settings.GetStringSetting(ctx, "autoUpdateExcludedContainers", ""), ","))), nil@@ -801,17 +840,23 @@ func (s *UpdaterService) TriggerSelfUpdate(ctx context.Context, target updater.Sreturn fmt.Errorf("%s self-update requires CLI upgrade service", instanceType)}- // A server self-update stops this process before the run can complete its- // activity, so annotate the activity first; startup reconciliation uses- // the metadata flag to finalize it after the restart.- if target.InstanceType != "agent" {- s.markSelfUpdateTriggeredInternal(ctx, target.NewImageRef)+ // A server self-update stops this process before the run completes, so the+ // activity is flagged first and startup reconciliation finalizes it.+ if activityID := activityIDFromContext(ctx); target.InstanceType != "agent" && activityID != "" && s.deps.Activity != nil {+ message := "Self-update initiated — Arcane will restart"+ if target.NewImageRef != "" {+ message += " with " + target.NewImageRef+ }+ s.appendAutoUpdateActivityMessage(ctx, activityID, message, "Self-update", 90)+ if err := s.deps.Activity.PatchActivityMetadata(ctx, activityID, database.JSON{"selfUpdateTriggered": true}); err != nil {+ slog.DebugContext(ctx, "failed to mark self-update on activity", "activityId", activityID, "error", err)+ }}if _, err := s.deps.SelfUpgrade.TriggerUpgradeViaCLI(ctx, s.deps.SystemUser, target); err != nil {return fmt.Errorf("CLI upgrade failed: %w", err)}- if progress, ok := ctx.Value(updateProgressKeyInternal{}).(*updateProgressInternal); ok {+ if progress, ok := ctx.Value(updateProgressKey{}).(*updateProgress); ok {progress.mu.Lock()progress.selfTriggered = trueprogress.selfID = target.ContainerID@@ -820,30 +865,13 @@ func (s *UpdaterService) TriggerSelfUpdate(ctx context.Context, target updater.Sreturn nil}-func (s *UpdaterService) markSelfUpdateTriggeredInternal(ctx context.Context, imageRef string) {- activityID := activityIDFromContextInternal(ctx)- if s.deps.Activity == nil || activityID == "" {- return- }- message := "Self-update initiated — Arcane will restart"- if ref := strings.TrimSpace(imageRef); ref != "" {- message = "Self-update initiated — Arcane will restart with " + ref- }- s.appendAutoUpdateActivityMessageInternal(ctx, activityID, message, "Self-update", 90)- if err := s.deps.Activity.PatchActivityMetadata(ctx, activityID, database.JSON{"selfUpdateTriggered": true}); err != nil {- slog.DebugContext(ctx, "failed to mark self-update on activity", "activityId", activityID, "error", err)- }-}--// Notify buffers Arcane's container update notification when called within an-// auto-update run (see withBatchedNotificationsInternal); buffered entries are-// flushed as one batched notification when the run completes. Outside a run it-// sends the legacy per-container notification immediately.+// Notify buffers the container update notification inside an update batch, which sends+// them as one notification when it ends; outside a batch it sends immediately.func (s *UpdaterService) Notify(ctx context.Context, localNotification updater.Notification) error {if s == nil || s.deps.Notifications == nil {return nil}- if buffer := batchedContainerUpdatesFromContextInternal(ctx); buffer != nil {+ if buffer, ok := ctx.Value(containerUpdateBatchContextKey{}).(*containerUpdateBatch); ok {buffer.Lock()buffer.entries = append(buffer.entries, notifications.ContainerUpdateBatchEntry{ContainerName: localNotification.ContainerName,@@ -863,47 +891,13 @@ func (s *UpdaterService) Notify(ctx context.Context, localNotification updater.N)}-// containerUpdateBatchInternal accumulates per-container update notifications-// so a single auto-update run produces one batched notification.-type containerUpdateBatchInternal struct {- sync.Mutex-- entries []notifications.ContainerUpdateBatchEntry-}--type containerUpdateBatchContextKeyInternal struct{}--func batchedContainerUpdatesFromContextInternal(ctx context.Context) *containerUpdateBatchInternal {- batch, _ := ctx.Value(containerUpdateBatchContextKeyInternal{}).(*containerUpdateBatchInternal)- return batch-}--// flushBatchedContainerUpdatesInternal delivers the accumulated container-// update notifications as one batched notification.-func (s *UpdaterService) flushBatchedContainerUpdatesInternal(ctx context.Context, batch *containerUpdateBatchInternal) {- if s == nil || s.deps.Notifications == nil || batch == nil {- return- }- batch.Lock()- entries := batch.entries- batch.entries = nil- batch.Unlock()- if len(entries) == 0 {- return- }- if err := s.deps.Notifications.SendBatchContainerUpdateNotification(ctx, entries); err != nil {- s.loggerInternal().ErrorContext(ctx, "failed to send batched container update notification", "error", err, "count", len(entries))- }-}-// RecordEvent records updater lifecycle events in Arcane's event stream.func (s *UpdaterService) RecordEvent(ctx context.Context, evt updater.Event) error {if s == nil {return nil}- eventType, ok := containerEventTypeInternal(evt.Phase).Get()- if ok {+ if eventType, ok := containerEventTypes[evt.Phase]; ok {if s.deps.Events == nil {return nil}@@ -920,7 +914,7 @@ func (s *UpdaterService) RecordEvent(ctx context.Context, evt updater.Event) err}severity := kit.Ternary(strings.EqualFold(evt.Severity, "error"), event.EventSeverityError, event.EventSeverityInfo)- s.recordAutoUpdateEventInternal(ctx, severity, database.JSON{+ s.recordAutoUpdateEvent(ctx, severity, database.JSON{"phase": evt.Phase,"resourceId": evt.ResourceID,"resourceName": evt.ResourceName,@@ -930,73 +924,28 @@ func (s *UpdaterService) RecordEvent(ctx context.Context, evt updater.Event) errreturn nil}-func containerEventTypeInternal(phase string) mo.Option[event.EventType] {- switch phase {- case "container_stop":- return mo.Some(event.EventTypeContainerStop)- case "container_delete":- return mo.Some(event.EventTypeContainerDelete)- case "container_create":- return mo.Some(event.EventTypeContainerCreate)- case "container_start":- return mo.Some(event.EventTypeContainerStart)- case "container_update":- return mo.Some(event.EventTypeContainerUpdate)- default:- return mo.None[event.EventType]()- }-}--type activityIDContextKeyInternal struct{}--func contextWithActivityIDInternal(ctx context.Context, activityID string) context.Context {- activityID = strings.TrimSpace(activityID)+func contextWithActivityID(ctx context.Context, activityID string) context.Context {if activityID == "" {return ctx}- return context.WithValue(ctx, activityIDContextKeyInternal{}, activityID)+ return context.WithValue(ctx, activityIDContextKey{}, activityID)}-func activityIDFromContextInternal(ctx context.Context) string {+func activityIDFromContext(ctx context.Context) string {if ctx == nil {return ""}- activityID, _ := ctx.Value(activityIDContextKeyInternal{}).(string)- return strings.TrimSpace(activityID)+ activityID, _ := ctx.Value(activityIDContextKey{}).(string)+ return activityID}-func (s *UpdaterService) startAutoUpdateActivityInternal(ctx context.Context, dryRun bool) string {- if s.deps.Activity == nil {- return ""- }- localActivity, err := s.deps.Activity.StartActivity(ctx, activitylib.StartRequest{- EnvironmentID: "0",- Type: activitytypes.TypeAutoUpdate,- Queue: true,- ResourceType: mo.EmptyableToOption(strings.TrimSpace("system")).ToPointer(),- ResourceName: mo.EmptyableToOption(strings.TrimSpace("Auto update")).ToPointer(),- Step: "Planning updates",- LatestMessage: "Auto-update run started",- Metadata: database.JSON{"dryRun": dryRun},- })- if err != nil {- slog.DebugContext(ctx, "failed to start auto-update activity", "error", err)- return ""- }- return localActivity.ID-}--func (s *UpdaterService) startSingleContainerUpdateActivityInternal(ctx context.Context, containerID string, tracked bool) (*activitytypes.Activity, context.Context, error) {- if s.deps.Activity == nil {- return nil, ctx, nil- }+// singleContainerActivityRequest describes a container update's activity, named after the container when it can be inspected.+func (s *UpdaterService) singleContainerActivityRequest(ctx context.Context, containerID string) activitylib.StartRequest {name := containerIDlookupCtx, cancelLookup := context.WithTimeout(ctx, 2*time.Second)- if dockerClient, dockerErr := s.DockerClient(lookupCtx); dockerErr == nil && dockerClient != nil {+ if dockerClient, dockerErr := s.DockerClient(lookupCtx); dockerErr == nil {if inspected, inspectErr := compat.ContainerInspectWithCompatibility(lookupCtx, dockerClient, containerID, client.ContainerInspectOptions{}); inspectErr == nil {- if actualName := strings.TrimPrefix(strings.TrimSpace(inspected.Container.Name), "/"); actualName != "" {- name = actualName- }+ name = cmp.Or(strings.TrimPrefix(inspected.Container.Name, "/"), containerID)}}cancelLookup()@@ -1008,12 +957,12 @@ func (s *UpdaterService) startSingleContainerUpdateActivityInternal(ctx context.user.DisplayName = &initiator.DisplayName}}- request := activitylib.StartRequest{+ return activitylib.StartRequest{EnvironmentID: "0",Type: activitytypes.TypeAutoUpdate,Queue: true,DeferSlot: true,- ResourceType: mo.EmptyableToOption(strings.TrimSpace("container")).ToPointer(),+ ResourceType: new("container"),ResourceID: &containerID,ResourceName: &name,StartedBy: user,@@ -1021,92 +970,72 @@ func (s *UpdaterService) startSingleContainerUpdateActivityInternal(ctx context.LatestMessage: "Container update started",Metadata: database.JSON{"containerID": containerID},}- var item *activitytypes.Activity- workCtx := ctx- var err error- if tracked {- item, workCtx, err = s.deps.Activity.StartTrackedActivity(ctx, request)- } else {- item, err = s.deps.Activity.StartActivity(ctx, request)- }- if err != nil {- return nil, nil, fmt.Errorf("start container update activity: %w", err)- }- return item, workCtx, nil}-func (s *UpdaterService) finishSingleContainerUpdateInternal(ctx context.Context, activityID string, result *arcaneupdater.Result, runErr error) {+func (s *UpdaterService) finishSingleContainerUpdate(ctx context.Context, activityID string, result *arcaneupdater.Result, runErr error) {if s.deps.Activity == nil || activityID == "" {return}- metadata, message := singleContainerActivitySummaryInternal(result)- writeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)- defer cancel()- if containerName, ok := metadata["containerName"].(string); ok && containerName != "" {- if _, err := s.deps.Activity.UpdateActivity(writeCtx, activityID, activitylib.UpdateRequest{ResourceName: &containerName}); err != nil {- slog.WarnContext(writeCtx, "failed to update container activity name", "activityId", activityID, "error", err)+ outcome := s.recordSingleContainerUpdate(ctx, activityID, result, runErr)+ s.completeAutoUpdateActivity(ctx, activityID, outcome.Message, kit.Ternary(outcome.Status != scheduler.Succeeded, outcome.Message, ""))+}++// recordSingleContainerUpdate saves a container update's result on its activity and summarizes it as an outcome.+func (s *UpdaterService) recordSingleContainerUpdate(ctx context.Context, activityID string, result *arcaneupdater.Result, runErr error) scheduler.Outcome {+ metadata, containerName := database.JSON{}, ""+ if result != nil {+ metadata = database.JSON{"updated": result.Updated, "restarted": result.Restarted, "skipped": result.Skipped, "failed": result.Failed}+ if len(result.Items) > 0 {+ item := result.Items[0]+ metadata["updateOutcome"] = item.Status+ if containerName = item.ResourceName; containerName != "" {+ metadata["containerName"] = containerName+ }+ if item.Error != "" {+ metadata["updateReason"] = item.Error+ }}}- if err := s.deps.Activity.PatchActivityMetadata(writeCtx, activityID, metadata); err != nil {- slog.WarnContext(writeCtx, "failed to persist container update result", "activityId", activityID, "error", err)+ if s.deps.Activity != nil && activityID != "" {+ writeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)+ defer cancel()+ if containerName != "" {+ if _, err := s.deps.Activity.UpdateActivity(writeCtx, activityID, activitylib.UpdateRequest{ResourceName: &containerName}); err != nil {+ slog.WarnContext(writeCtx, "failed to update container activity name", "activityId", activityID, "error", err)+ }+ }+ if err := s.deps.Activity.PatchActivityMetadata(writeCtx, activityID, metadata); err != nil {+ slog.WarnContext(writeCtx, "failed to persist container update result", "activityId", activityID, "error", err)+ }}if runErr == nil && result != nil && result.Failed == 0 {- if result.Updated > 0 || result.Restarted > 0 {+ message := "Container already current"+ switch {+ case result.Updated > 0 || result.Restarted > 0:message = "Container updated"+ case len(result.Items) > 0 && result.Items[0].Status == string(updater.StatusSkipped) && result.Items[0].Error != "":+ message = "Container update skipped: " + result.Items[0].Error+ case result.Skipped > 0:+ message = "Container update skipped"}- if _, err := s.deps.Activity.CompleteActivity(writeCtx, activityID, activitytypes.StatusSuccess, message, nil); err != nil {- slog.ErrorContext(writeCtx, "failed to complete container update activity", "activityId", activityID, "error", err)- }- return+ return scheduler.Outcome{Status: scheduler.Succeeded, Message: message}}- if runErr == nil && result == nil {+ switch {+ case runErr != nil:+ case result == nil:runErr = errors.New("container update produced no result")- }- if runErr == nil && result != nil && len(result.Items) > 0 && result.Items[0].Error != "" {+ case len(result.Items) > 0 && result.Items[0].Error != "":runErr = errors.New(result.Items[0].Error)+ default:+ runErr = fmt.Errorf("%d update action(s) failed", result.Failed)}- s.completeAutoUpdateActivityInternal(ctx, activityID, result, runErr)-}--func singleContainerActivitySummaryInternal(result *arcaneupdater.Result) (database.JSON, string) {- metadata := database.JSON{}- message := "Container update completed"- if result == nil {- return metadata, message- }- metadata["updated"] = result.Updated- metadata["restarted"] = result.Restarted- metadata["skipped"] = result.Skipped- metadata["failed"] = result.Failed- if len(result.Items) > 0 {- item := result.Items[0]- metadata["updateOutcome"] = item.Status- if item.ResourceName != "" {- metadata["containerName"] = item.ResourceName- }- if item.Error != "" {- metadata["updateReason"] = item.Error- if item.Status == string(updater.StatusSkipped) {- return metadata, "Container update skipped: " + item.Error- }- }- }- if result.Skipped > 0 {- return metadata, "Container update skipped"- }- if result.Updated == 0 && result.Restarted == 0 && result.Failed == 0 {- message = "Container already current"- }- return metadata, message+ return scheduler.Outcome{Status: scheduler.Failed, Message: runErr.Error()}}-func (s *UpdaterService) appendAutoUpdateActivityMessageInternal(ctx context.Context, activityID, message, step string, progress int) {- if s.deps.Activity == nil || strings.TrimSpace(activityID) == "" {+func (s *UpdaterService) appendAutoUpdateActivityMessage(ctx context.Context, activityID, message, step string, progress int) {+ if s.deps.Activity == nil || activityID == "" {return}- if strings.TrimSpace(step) == "" {- step = message- }if _, err := s.deps.Activity.AppendMessage(ctx, activityID, activitylib.AppendMessageRequest{Level: activitytypes.MessageLevelInfo,Message: message,@@ -1117,207 +1046,69 @@ func (s *UpdaterService) appendAutoUpdateActivityMessageInternal(ctx context.Con}}-func (s *UpdaterService) completeAutoUpdateActivityInternal(ctx context.Context, activityID string, result *arcaneupdater.Result, applyErr error) {- if s.deps.Activity == nil || strings.TrimSpace(activityID) == "" {+// completeAutoUpdateActivity ends the activity with message, or as failed when failure is set;+// a failure caused by cancelling the activity ends it as cancelled.+func (s *UpdaterService) completeAutoUpdateActivity(ctx context.Context, activityID, message, failure string) {+ if s.deps.Activity == nil || activityID == "" {return}-status := activitytypes.StatusSuccess- message := "Auto-update run completed"var errMessage *string- if applyErr != nil {- status = activitytypes.StatusFailed- errText := applyErr.Error()- errMessage = &errText- message = errText- } else if result != nil && result.Failed > 0 {- status = activitytypes.StatusFailed- errText := fmt.Sprintf("%d update action(s) failed", result.Failed)- errMessage = &errText- message = errText- }- if status == activitytypes.StatusFailed && activitylib.CancelledByContext(ctx) {- status = activitytypes.StatusCancelled- message = "Auto-update cancelled"- errMessage = nil- }-- if _, err := s.deps.Activity.CompleteActivity(utils.ActivityRuntimeContext(ctx, nil), activityID, status, message, errMessage); err != nil {- // A lost terminal write strands the activity in running forever, so it- // must be loud enough to correlate with a stuck activity panel entry.- slog.ErrorContext(ctx, "failed to complete auto-update activity", "activityId", activityID, "error", err)- }-}--func (s *UpdaterService) trackActivityInternal(ctx context.Context, activityID string) context.Context {- if s.deps.Activity == nil || strings.TrimSpace(activityID) == "" {- return ctx- }- return s.deps.Activity.Track(ctx, activityID)-}--func imageUpdateRecordToModuleInternal(record imageupdate.ImageUpdateRecord) updater.ImageUpdateRecord {- return updater.ImageUpdateRecord{- ID: record.ID,- ContainerID: record.ContainerID,- Repository: record.Repository,- Tag: record.Tag,- HasUpdate: record.HasUpdate,- UpdateType: updater.UpdateType(record.UpdateType),- CurrentVersion: record.CurrentVersion,- LatestVersion: record.LatestVersion,- CurrentDigest: record.CurrentDigest,- LatestDigest: record.LatestDigest,- CheckTime: record.CheckTime,- LastError: record.LastError,+ switch {+ case failure == "":+ case activitylib.CancelledByContext(ctx):+ status, message = activitytypes.StatusCancelled, "Auto-update cancelled"+ default:+ status, message, errMessage = activitytypes.StatusFailed, failure, &failure}-}--// moduleOptionsFromUpdaterOptionsInternal narrows Arcane's request options to-// what the engine acts on. Options.Type and Options.ResourceIds stay behind:-// the engine never read them, and ApplyPending already routes a scoped request-// through applyScopedUpdatesInternal before reaching the engine. Settings-// exclusions always apply to these runs; only UpdateSingleContainer overrides-// them for its explicitly requested container.-func moduleOptionsFromUpdaterOptionsInternal(options arcaneupdater.Options) updater.Options {- return updater.Options{- Force: options.ForceUpdate,- DryRun: options.DryRun,+ // A lost terminal write strands the activity as running, so it is written past cancellation and logged loudly.+ if _, err := s.deps.Activity.CompleteActivity(context.WithoutCancel(ctx), activityID, status, message, errMessage); err != nil {+ slog.ErrorContext(ctx, "failed to complete auto-update activity", "activityId", activityID, "error", err)}}-// resultFromModuleInternal converts an engine result to Arcane's wire type. The-// engine reports times as time.Time and a Duration method; Arcane's API has-// always carried them as strings, so they are formatted here. ActivityID is not-// an engine concept; every caller sets it on the returned value.-func resultFromModuleInternal(result *updater.Result) *arcaneupdater.Result {+// resultFromModule converts an engine result to Arcane's wire type, which carries times as+// strings and images as maps holding only the "main" entry. Callers set ActivityID.+func resultFromModule(result *updater.Result) *arcaneupdater.Result {if result == nil {return &arcaneupdater.Result{Items: []arcaneupdater.ResourceResult{}}}- return &arcaneupdater.Result{+ mainImage := func(ref string) map[string]string {+ if ref == "" {+ return nil+ }+ return map[string]string{"main": ref}+ }+ out := &arcaneupdater.Result{Success: result.Success,Checked: result.Checked,Updated: result.Updated,Restarted: result.Restarted,Skipped: result.Skipped,Failed: result.Failed,- StartTime: formatModuleTimeInternal(result.StartTime),- EndTime: formatModuleTimeInternal(result.EndTime),+ StartTime: kit.Ternary(result.StartTime.IsZero(), "", result.StartTime.UTC().Format(time.RFC3339)),+ EndTime: kit.Ternary(result.EndTime.IsZero(), "", result.EndTime.UTC().Format(time.RFC3339)),Duration: result.Duration().String(),- Items: resourceResultsFromModuleInternal(result.Items),- }-}--func formatModuleTimeInternal(value time.Time) string {- if value.IsZero() {- return ""- }- return value.UTC().Format(time.RFC3339)-}--func resourceResultsFromModuleInternal(results []updater.ResourceResult) []arcaneupdater.ResourceResult {- out := make([]arcaneupdater.ResourceResult, 0, len(results))- for _, result := range results {- out = append(out, resourceResultFromModuleInternal(result))- }- return out-}--// resourceResultFromModuleInternal converts one engine result to Arcane's wire-// type. The engine now reports a single old/new image; Arcane's API carries-// maps, which only ever held the "main" entry, so that shape is rebuilt here.-func resourceResultFromModuleInternal(result updater.ResourceResult) arcaneupdater.ResourceResult {- mainImage := func(ref string) map[string]string {- if ref == "" {- return nil- }- return map[string]string{"main": ref}- }- return arcaneupdater.ResourceResult{- ResourceID: result.ResourceID,- ResourceName: result.ResourceName,- ResourceType: string(result.ResourceType),- Status: string(result.Status),- UpdateAvailable: result.UpdateAvailable,- UpdateApplied: result.UpdateApplied,- OldImages: mainImage(result.OldImage),- NewImages: mainImage(result.NewImage),- Error: result.Error,- Details: result.Details,- }-}--func statusFromModuleInternal(status updater.Status) arcaneupdater.Status {- return arcaneupdater.Status{- UpdatingContainers: status.UpdatingContainers,- UpdatingProjects: status.UpdatingProjects,- ContainerIds: status.ContainerIDs,- ProjectIds: status.ProjectIDs,- }-}--func (s *UpdaterService) recordRunInternal(ctx context.Context, item arcaneupdater.ResourceResult) error {- now := time.Now()- record := &AutoUpdateRecord{- ResourceID: item.ResourceID,- ResourceType: item.ResourceType,- ResourceName: item.ResourceName,- Status: AutoUpdateStatus(item.Status),- StartTime: now,- EndTime: &now,- UpdateAvailable: item.UpdateAvailable || item.Status == string(updater.StatusUpdated) || item.Status == string(updater.StatusUpdateAvailable),- UpdateApplied: item.UpdateApplied,- OldImageVersions: mapToJSONInternal(item.OldImages),- NewImageVersions: mapToJSONInternal(item.NewImages),- Details: detailsToJSONInternal(item.Details),- }- if item.Error != "" {- record.Error = &item.Error- }- return s.deps.DB.WithContext(ctx).Create(record).Error-}--func (s *UpdaterService) clearImageUpdateRecordForModuleInternal(ctx context.Context, record updater.ImageUpdateRecord) error {- if s.deps.DB == nil {- return nil- }-- if record.ContainerID != "" && !strings.HasPrefix(record.ID, "container::") {- return s.clearUnscopedRecordInternal(ctx, record)- }- query := s.deps.DB.WithContext(ctx).Model(&imageupdate.ImageUpdateRecord{})- if record.ContainerID != "" {- query = query.Where("container_id = ?", record.ContainerID)- } else {- query = query.Where("container_id = ?", "")- }- if strings.TrimSpace(record.ID) != "" {- return query.Where("id = ?", record.ID).Update("has_update", false).Error- }- return query.Where("repository = ? AND tag = ?", record.Repository, record.Tag).Update("has_update", false).Error-}--func mapToJSONInternal(values map[string]string) database.JSON {- if len(values) == 0 {- return nil+ Items: make([]arcaneupdater.ResourceResult, 0, len(result.Items)),}- out := make(database.JSON, len(values))- for key, value := range values {- out[key] = value- }- return out-}--func detailsToJSONInternal(values map[string]any) database.JSON {- if len(values) == 0 {- return nil+ for _, item := range result.Items {+ out.Items = append(out.Items, arcaneupdater.ResourceResult{+ ResourceID: item.ResourceID,+ ResourceName: item.ResourceName,+ ResourceType: string(item.ResourceType),+ Status: string(item.Status),+ UpdateAvailable: item.UpdateAvailable,+ UpdateApplied: item.UpdateApplied,+ OldImages: mainImage(item.OldImage),+ NewImages: mainImage(item.NewImage),+ Error: item.Error,+ Details: item.Details,+ })}- out := make(database.JSON, len(values))- maps.Copy(out, values)return out}-func (s *UpdaterService) logResultItemsInternal(ctx context.Context, result *arcaneupdater.Result) {+func (s *UpdaterService) logResultItems(ctx context.Context, result *arcaneupdater.Result) {if result == nil {return}@@ -1329,7 +1120,7 @@ func (s *UpdaterService) logResultItemsInternal(ctx context.Context, result *arccase string(updater.StatusUpdated):severity = event.EventSeveritySuccess}- s.recordAutoUpdateEventInternal(ctx, severity, database.JSON{+ s.recordAutoUpdateEvent(ctx, severity, database.JSON{"phase": item.ResourceType,"resourceId": item.ResourceID,"resourceName": item.ResourceName,@@ -1347,28 +1138,17 @@ func (s *UpdaterService) CollectUsedImages(ctx context.Context) (map[string]struvar errs []errorsuccessfulSources := 0- if s.deps.Docker == nil {- errs = append(errs, common.Classify(common.ErrUnavailable, errors.New("docker service unavailable")))+ if err := s.collectUsedImagesFromContainers(ctx, out); err != nil {+ errs = append(errs, err)+ slog.DebugContext(ctx, "collectUsedImages: failed collecting from containers", "error", err)} else {- dcli, err := s.deps.Docker.GetClient(ctx)- if err != nil || dcli == nil {- if err == nil {- err = common.Classify(common.ErrUnavailable, errors.New("docker client unavailable"))- }- errs = append(errs, err)- s.loggerInternal().DebugContext(ctx, "collectUsedImages: docker connection unavailable", "error", err)- } else if collectUsedImagesFromContainersErr := s.collectUsedImagesFromContainersInternal(ctx, dcli, out); collectUsedImagesFromContainersErr != nil {- errs = append(errs, collectUsedImagesFromContainersErr)- s.loggerInternal().DebugContext(ctx, "collectUsedImages: failed collecting from containers", "error", collectUsedImagesFromContainersErr)- } else {- successfulSources++- }+ successfulSources++}if s.deps.Projects != nil {- if err := s.collectUsedImagesFromProjectsInternal(ctx, out); err != nil {+ if err := s.collectUsedImagesFromProjects(ctx, out); err != nil {errs = append(errs, err)- s.loggerInternal().DebugContext(ctx, "collectUsedImages: failed collecting from projects", "error", err)+ slog.DebugContext(ctx, "collectUsedImages: failed collecting from projects", "error", err)} else {successfulSources++}@@ -1378,116 +1158,84 @@ func (s *UpdaterService) CollectUsedImages(ctx context.Context) (map[string]strureturn nil, errors.Join(errs...)}- s.loggerInternal().DebugContext(ctx, "collectUsedImages: collected used images", "count", len(out))+ slog.DebugContext(ctx, "collectUsedImages: collected used images", "count", len(out))return out, nil}-func (s *UpdaterService) collectUsedImagesFromContainersInternal(ctx context.Context, dcli *client.Client, out map[string]struct{}) error {- if dcli == nil {- return nil+func (s *UpdaterService) collectUsedImagesFromContainers(ctx context.Context, out map[string]struct{}) error {+ dcli, err := s.DockerClient(ctx)+ if err != nil {+ return err}-- excludedContainers := s.buildExcludedContainerSetInternal(ctx)- listResult, err := dcli.ContainerList(ctx, client.ContainerListOptions{All: false})+ var excludedContainers map[string]bool+ if s.deps.Settings != nil {+ excludedContainers = docker.ExcludedContainerNameSet(s.deps.Settings.GetStringSetting(ctx, "autoUpdateExcludedContainers", ""))+ }+ listResult, err := dcli.ContainerList(ctx, client.ContainerListOptions{})if err != nil {return err}for _, summary := range listResult.Items {if labels.IsUpdateDisabled(summary.Labels) {- s.loggerInternal().DebugContext(ctx, "collectUsedImagesFromContainers: container opted out by labels", "containerId", summary.ID)+ slog.DebugContext(ctx, "collectUsedImagesFromContainers: container opted out by labels", "containerId", summary.ID)continue}if docker.ContainerNameExcluded(summary.Names, excludedContainers) {- s.loggerInternal().DebugContext(ctx, "collectUsedImagesFromContainers: skipping excluded container", "containerId", summary.ID, "names", summary.Names)+ slog.DebugContext(ctx, "collectUsedImagesFromContainers: skipping excluded container", "containerId", summary.ID, "names", summary.Names)continue}imageRef := strings.TrimSpace(summary.Image)if imageRef != "" && !refs.IsImageIDLikeReference(imageRef) {- addNormalizedImageUpdateRefInternal(ctx, out, imageRef, "collectUsedImagesFromContainers", "containerId", summary.ID)+ addNormalizedImageUpdateRef(ctx, out, imageRef, "collectUsedImagesFromContainers", "containerId", summary.ID)continue}+ // The summary only names an image ID, so the image's tags and the configured reference stand in.inspectResult, inspectErr := compat.ContainerInspectWithCompatibility(ctx, dcli, summary.ID, client.ContainerInspectOptions{})if inspectErr != nil {- s.loggerInternal().DebugContext(ctx, "collectUsedImagesFromContainers: container inspect failed", "containerId", summary.ID, "error", inspectErr)+ slog.DebugContext(ctx, "collectUsedImagesFromContainers: container inspect failed", "containerId", summary.ID, "error", inspectErr)continue}inspect := inspectResult.Containerif inspect.Config != nil && labels.IsUpdateDisabled(inspect.Config.Labels) {- s.loggerInternal().DebugContext(ctx, "collectUsedImagesFromContainers: container inspect labels opted out", "containerId", summary.ID)- continue- }- for _, tag := range s.normalizedTagsForContainerInternal(ctx, dcli, inspect) {- out[tag] = struct{}{}- }- }- return nil-}--func (s *UpdaterService) collectUsedImagesFromComposeContainersInternal(ctx context.Context, composeContainers []container.Summary, activeProjectNames, out map[string]struct{}) {- for _, summary := range composeContainers {- projectName := docker.ComposeProjectLabel(summary.Labels)- if projectName == "" {- continue- }- if _, isActive := activeProjectNames[projectName]; !isActive {- continue- }- if labels.IsUpdateDisabled(summary.Labels) {- continue- }-- imageRef := strings.TrimSpace(summary.Image)- if imageRef == "" || refs.IsImageIDLikeReference(imageRef) {+ slog.DebugContext(ctx, "collectUsedImagesFromContainers: container inspect labels opted out", "containerId", summary.ID)continue}- addNormalizedImageUpdateRefInternal(ctx, out, imageRef, "collectUsedImagesFromComposeContainers", "containerId", summary.ID)- }-}--func (s *UpdaterService) normalizedTagsForContainerInternal(ctx context.Context, dcli *client.Client, inspect container.InspectResponse) []string {- seen := map[string]struct{}{}-- if dcli != nil {- if imageInspect, err := dcli.ImageInspect(ctx, inspect.Image); err == nil {+ if imageInspect, imageErr := dcli.ImageInspect(ctx, inspect.Image); imageErr == nil {for _, tag := range imageInspect.RepoTags {- if strings.TrimSpace(tag) == "" || tag == "<none>:<none>" {- continue+ if strings.TrimSpace(tag) != "" && tag != "<none>:<none>" {+ addNormalizedImageUpdateRef(ctx, out, tag, "normalizedTagsForContainer repo tag", "imageId", inspect.Image)}- addNormalizedImageUpdateRefInternal(ctx, seen, tag, "normalizedTagsForContainer repo tag", "imageId", inspect.Image)}}+ if inspect.Config != nil && inspect.Config.Image != "" {+ addNormalizedImageUpdateRef(ctx, out, inspect.Config.Image, "normalizedTagsForContainer config image", "imageId", inspect.Image)+ }}-- if inspect.Config != nil && inspect.Config.Image != "" {- addNormalizedImageUpdateRefInternal(ctx, seen, inspect.Config.Image, "normalizedTagsForContainer config image", "imageId", inspect.Image)- }-- out := slices.Collect(maps.Keys(seen))- return out-}--func (s *UpdaterService) buildExcludedContainerSetInternal(ctx context.Context) map[string]bool {- if s.deps.Settings == nil {- return nil- }- return docker.ExcludedContainerNameSet(s.deps.Settings.GetStringSetting(ctx, "autoUpdateExcludedContainers", ""))+ return nil}-func (s *UpdaterService) collectUsedImagesFromProjectsInternal(ctx context.Context, out map[string]struct{}) error {- if s.deps.Projects == nil {- return nil- }-+func (s *UpdaterService) collectUsedImagesFromProjects(ctx context.Context, out map[string]struct{}) error {allProjects, err := s.deps.Projects.ListAllProjects(ctx)if err != nil {return err}- activeProjectNames := activeComposeProjectNameSetInternal(allProjects)+ activeProjectNames := map[string]struct{}{}+ for _, project := range allProjects {+ name := strings.TrimSpace(project.Name)+ running := project.Status == projectpkg.ProjectStatusRunning || project.Status == projectpkg.ProjectStatusPartiallyRunning+ if project.IsArchived || !running || name == "" {+ continue+ }+ activeProjectNames[name] = struct{}{}+ if normalized := projects.NormalizeProjectName(name); normalized != "" {+ activeProjectNames[normalized] = struct{}{}+ }+ }if len(activeProjectNames) == 0 {return nil}@@ -1505,62 +1253,29 @@ func (s *UpdaterService) collectUsedImagesFromProjectsInternal(ctx context.Contereturn err}- s.collectUsedImagesFromComposeContainersInternal(ctx, composeContainers, activeProjectNames, out)- return nil-}--func activeComposeProjectNameSetInternal(items []projectpkg.Project) map[string]struct{} {- active := make(map[string]struct{})- for _, project := range items {- if project.IsArchived {- continue- }- if project.Status != projectpkg.ProjectStatusRunning && project.Status != projectpkg.ProjectStatusPartiallyRunning {+ for _, summary := range composeContainers {+ if _, isActive := activeProjectNames[docker.ComposeProjectLabel(summary.Labels)]; !isActive || labels.IsUpdateDisabled(summary.Labels) {continue}-- name := strings.TrimSpace(project.Name)- if name == "" {+ imageRef := strings.TrimSpace(summary.Image)+ if imageRef == "" || refs.IsImageIDLikeReference(imageRef) {continue}- active[name] = struct{}{}- if normalized := projects.NormalizeProjectName(name); normalized != "" {- active[normalized] = struct{}{}- }+ addNormalizedImageUpdateRef(ctx, out, imageRef, "collectUsedImagesFromComposeContainers", "containerId", summary.ID)}- return active+ return nil}-func addNormalizedImageUpdateRefInternal(ctx context.Context, out map[string]struct{}, imageRef, source string, attrs ...any) {- normalizedRef := refs.NormalizeImageUpdateRef(imageRef)- if normalizedRef != "" {+func addNormalizedImageUpdateRef(ctx context.Context, out map[string]struct{}, imageRef, source string, attrs ...any) {+ if normalizedRef := refs.NormalizeImageUpdateRef(imageRef); normalizedRef != "" {out[normalizedRef] = struct{}{}return}-- args := slices.Clone(attrs)- args = append(args, "source", source, "imageRef", imageRef)- slog.DebugContext(ctx, "skipping invalid image reference", args...)-}--func (s *UpdaterService) tagFrozenPullInternal(ctx context.Context, pulledRef, imageRef string) error {- if pulledRef == imageRef {- return nil- }- dockerClient, err := s.DockerClient(ctx)- if err != nil {- return err- }- _, err = dockerClient.ImageTag(ctx, client.ImageTagOptions{Source: pulledRef, Target: imageRef})- return err+ slog.DebugContext(ctx, "skipping invalid image reference", append(attrs, "source", source, "imageRef", imageRef)...)}-type updateAdmissionKeyInternal struct{}--var errUpdateBusyInternal = errors.New("another container update is running")--func (s *UpdaterService) acquireUpdateInternal(ctx context.Context) (context.Context, func(), error) {- if s.admission == nil || ctx.Value(updateAdmissionKeyInternal{}) == s {+func (s *UpdaterService) acquireUpdate(ctx context.Context) (context.Context, func(), error) {+ if s.admission == nil || ctx.Value(updateAdmissionKey{}) == s {return ctx, func() {}, nil}lease, admitted, err := s.admission.TryAcquire(ctx, scheduler.AdmissionKey{Scope: "updater"})@@ -1568,26 +1283,39 @@ func (s *UpdaterService) acquireUpdateInternal(ctx context.Context) (context.Conreturn ctx, nil, err}if !admitted {- return ctx, nil, errUpdateBusyInternal+ return ctx, nil, errUpdateBusy}- return context.WithValue(ctx, updateAdmissionKeyInternal{}, s), func() { lease.Release(ctx) }, nil+ return context.WithValue(ctx, updateAdmissionKey{}, s), func() { lease.Release(ctx) }, nil}-type (- updateProgressKeyInternal struct{}- updateProgressInternal struct {- mu sync.Mutex- err error- selfTriggered bool- selfID string- }-)-// RecordUpdateRun persists one updater resource result into Arcane history.func (s *UpdaterService) RecordUpdateRun(ctx context.Context, result updater.ResourceResult) error {var recordErr errorif s != nil && s.deps.DB != nil {- recordErr = s.recordRunInternal(ctx, resourceResultFromModuleInternal(result))+ now := time.Now()+ record := &AutoUpdateRecord{+ ResourceID: result.ResourceID,+ ResourceType: string(result.ResourceType),+ ResourceName: result.ResourceName,+ Status: AutoUpdateStatus(result.Status),+ StartTime: now,+ EndTime: &now,+ UpdateAvailable: result.UpdateAvailable || result.Status == updater.StatusUpdated || result.Status == updater.StatusUpdateAvailable,+ UpdateApplied: result.UpdateApplied,+ }+ if result.OldImage != "" {+ record.OldImageVersions = database.JSON{"main": result.OldImage}+ }+ if result.NewImage != "" {+ record.NewImageVersions = database.JSON{"main": result.NewImage}+ }+ if len(result.Details) > 0 {+ record.Details = maps.Clone(result.Details)+ }+ if result.Error != "" {+ record.Error = &result.Error+ }+ recordErr = s.deps.DB.WithContext(ctx).Create(record).Error}name := cmp.Or(strings.TrimSpace(result.ResourceName), result.ResourceID)message := name + ": " + string(result.Status)@@ -1614,38 +1342,22 @@ func (s *UpdaterService) RecordUpdateRun(ctx context.Context, result updater.Resmessage += " (" + reason + ")"}}- if activityID := activityIDFromContextInternal(ctx); activityID != "" && s != nil && s.deps.Activity != nil {+ activityID := activityIDFromContext(ctx)+ if activityID != "" && s != nil && s.deps.Activity != nil {if _, appendErr := s.deps.Activity.AppendMessage(ctx, activityID, activitylib.AppendMessageRequest{Level: level, Message: message, Step: "Applying updates"}); appendErr != nil {slog.DebugContext(ctx, "failed to append update result activity message", "activityId", activityID, "resource", name, "error", appendErr)}}+ progressMessage := result.Errorvar evidenceErr errorif status == scheduler.Succeeded {- evidenceErr = s.recovery.VerifyResult(ctx, result.ResourceID)- }- if evidenceErr != nil {- status = scheduler.NeedsAttention- }- progressMessage := result.Error- if evidenceErr != nil {- progressMessage = evidenceErr.Error()+ if evidenceErr = s.recovery.VerifyResult(ctx, result.ResourceID); evidenceErr != nil {+ status, progressMessage = scheduler.NeedsAttention, evidenceErr.Error()+ }}- progressErr := jobcontext.Progress(- ctx,- scheduler.TargetOutcome{- ID: result.ResourceID,- ResourceType: string(- result.ResourceType,- ),- Status: status,- Message: progressMessage,- ActivityID: activityIDFromContextInternal(- ctx,- ),- },- )- err := errors.Join(recordErr, progressErr, evidenceErr)- if progress, ok := ctx.Value(updateProgressKeyInternal{}).(*updateProgressInternal); ok && err != nil {+ target := scheduler.TargetOutcome{ID: result.ResourceID, ResourceType: string(result.ResourceType), Status: status, Message: progressMessage, ActivityID: activityID}+ err := errors.Join(recordErr, jobcontext.Progress(ctx, target), evidenceErr)+ if progress, ok := ctx.Value(updateProgressKey{}).(*updateProgress); ok && err != nil {progress.mu.Lock()progress.err = errors.Join(progress.err, err)progress.mu.Unlock()@@ -1653,12 +1365,12 @@ func (s *UpdaterService) RecordUpdateRun(ctx context.Context, result updater.Resreturn err}-func (p *updateProgressInternal) completeBatchInternal(ctx context.Context, options arcaneupdater.Options, out *arcaneupdater.Result, batchCompleted bool, err error) error {- activityID := activityIDFromContextInternal(ctx)+func (p *updateProgress) completeBatch(ctx context.Context, options arcaneupdater.Options, out *arcaneupdater.Result, batchCompleted bool, err error) error {+ activityID := activityIDFromContext(ctx)p.mu.Lock()err = errors.Join(err, p.err)recordingFailed := p.err != nil- _, durableRun := jobcontext.Run(ctx)+ previous, durableRun := jobcontext.Run(ctx)selfTriggered := p.selfTriggered && durableRunselfID := p.selfIDp.mu.Unlock()@@ -1674,23 +1386,26 @@ func (p *updateProgressInternal) completeBatchInternal(ctx context.Context, optistatus = scheduler.NeedsAttention}if selfTriggered {- err = errors.Join(- err,- jobcontext.Progress(- ctx,- scheduler.TargetOutcome{- ID: selfID,- ResourceType: "container",- Status: scheduler.NeedsAttention,- ActivityID: activityID,- Message: "Self-update completion requires review",- },- ),- )+ selfTarget := scheduler.TargetOutcome{+ ID: selfID,+ ResourceType: "container",+ Status: scheduler.NeedsAttention,+ ActivityID: activityID,+ Message: "Self-update completion requires review",+ }+ err = errors.Join(err, jobcontext.Progress(ctx, selfTarget), errors.New("self-update was triggered but completion requires review"))status = scheduler.NeedsAttention- err = errors.Join(err, errors.New("self-update was triggered but completion requires review"))}- batchType := updateBatchTypeInternal(ctx, options, batchCompleted)+ // A scoped rerun of a durable run is a retry unless that run already recorded a finished batch.+ batchType := "update-batch"+ if durableRun && len(options.ResourceIds) > 0 && !slices.ContainsFunc(previous.Outcome.Targets, func(target scheduler.TargetOutcome) bool {+ return target.ID == "auto-update" && target.ResourceType == "update-batch" && (target.Status == scheduler.Succeeded || target.Status == scheduler.Partial)+ }) {+ batchType = "update-retry"+ }+ if !batchCompleted {+ batchType = "update-interrupted"+ }checkpoint := scheduler.TargetOutcome{ID: "auto-update", ResourceType: batchType, Status: status, ActivityID: activityID}if err != nil {checkpoint.Message = err.Error()@@ -1715,7 +1430,7 @@ func (s *UpdaterService) ListTags(ctx context.Context, imageRef string) ([]strinif s.deps.Projects != nil {resolved, err := s.deps.Projects.ResolveRegistryCredentials(ctx)if err != nil {- return nil, err+ return nil, fmt.Errorf("resolve registry credentials: %w", err)}credentials = resolved}@@ -1727,78 +1442,10 @@ func (s *UpdaterService) UpdateServiceImages(ctx context.Context, projectID striif s.deps.Projects == nil {return errors.New("project service unavailable")}- s.appendAutoUpdateActivityMessageInternal(ctx, activityIDFromContextInternal(ctx), "Updating Compose image references", "Persisting image updates", 50)+ s.appendAutoUpdateActivityMessage(ctx, activityIDFromContext(ctx), "Updating Compose image references", "Persisting image updates", 50)return s.deps.Projects.UpdateProjectServiceImages(ctx, projectID, changes, s.deps.SystemUser)}-func (s *UpdaterService) scopedPendingRecordsInternal(ctx context.Context, records []imageupdate.ImageUpdateRecord) ([]updater.ImageUpdateRecord, error) {- var scopedCount int64- if err := s.deps.DB.WithContext(ctx).Model(&imageupdate.ImageUpdateRecord{}).Where("container_id <> ?", "").Limit(1).Count(&scopedCount).Error; err != nil {- return nil, err- }- hasScoped := scopedCount > 0- if !hasScoped {- out := make([]updater.ImageUpdateRecord, 0, len(records))- for _, record := range records {- out = append(out, imageUpdateRecordToModuleInternal(record))- }- return out, nil- }- dockerClient, err := s.DockerClient(ctx)- if err != nil {- return nil, err- }- listed, err := dockerClient.ContainerList(ctx, client.ContainerListOptions{All: false})- if err != nil {- return nil, err- }- out := make([]updater.ImageUpdateRecord, 0, len(records))- for _, record := range records {- converted := imageUpdateRecordToModuleInternal(record)- if record.ContainerID != "" {- out = append(out, converted)- continue- }- for _, cnt := range listed.Items {- resolved, policyErr := tagpolicy.Resolve(cnt.Image, updater.DefaultLabelPolicy().TagPolicy(cnt.Labels))- if policyErr != nil || resolved.Strategy == "tag" {- continue- }- if refs.NormalizeImageUpdateRef(cnt.Image) != refs.NormalizeImageUpdateRef(converted.ImageRef()) {- continue- }- converted.ContainerID = cnt.ID- out = append(out, converted)- }- }- return out, nil-}--func (s *UpdaterService) clearUnscopedRecordInternal(ctx context.Context, record updater.ImageUpdateRecord) error {- dockerClient, err := s.DockerClient(ctx)- if err != nil {- return err- }- target, err := dockerClient.ImageInspect(ctx, record.NewImageRef())- if err != nil {- return err- }- listed, err := dockerClient.ContainerList(ctx, client.ContainerListOptions{All: false})- if err != nil {- return err- }- for _, cnt := range listed.Items {- resolved, policyErr := tagpolicy.Resolve(cnt.Image, updater.DefaultLabelPolicy().TagPolicy(cnt.Labels))- if policyErr != nil || resolved.Strategy == "tag" {- continue- }- if refs.NormalizeImageUpdateRef(cnt.Image) == refs.NormalizeImageUpdateRef(record.ImageRef()) && cnt.ImageID != target.ID {- return nil- }- }- return s.deps.DB.WithContext(ctx).Model(&imageupdate.ImageUpdateRecord{}).Where("id = ? AND container_id = ?", record.ID, "").Update("has_update", false).Error-}-// CheckProjectUpdates checks effective Compose service policies without deploying containers.func (s *UpdaterService) CheckProjectUpdates(ctx context.Context, projectID string) (*projecttypes.UpdateInfo, error) {if s.deps.Projects == nil || s.deps.DB == nil || s.engine == nil {@@ -1816,7 +1463,7 @@ func (s *UpdaterService) CheckProjectUpdates(ctx context.Context, projectID striif strings.TrimSpace(service.Image) == "" {continue}- records = append(records, s.checkProjectServiceInternal(ctx, projectID, service))+ records = append(records, s.checkProjectService(ctx, projectID, service))}if transactionErr := s.deps.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {if deleteProjectUpdatesErr := tx.Where("project_id = ?", projectID).Delete(&imageupdate.ImageUpdateRecord{}).Error; deleteProjectUpdatesErr != nil {@@ -1832,16 +1479,13 @@ func (s *UpdaterService) CheckProjectUpdates(ctx context.Context, projectID strireturn projectpkg.BuildConfiguredUpdateInfo(projectID, details.Services, nil, records), nil}-func (s *UpdaterService) checkProjectServiceInternal(ctx context.Context, projectID string, service types.ServiceConfig) imageupdate.ImageUpdateRecord {+func (s *UpdaterService) checkProjectService(ctx context.Context, projectID string, service types.ServiceConfig) imageupdate.ImageUpdateRecord {record := imageupdate.ImageUpdateRecord{ID: "project::" + projectID + "::" + service.Name,ProjectID: projectID,ServiceName: service.Name,- PolicyKey: imageref.UpdatePolicyKey(- service.Image,- service.Labels,- ),- CheckTime: time.Now().UTC(),+ PolicyKey: imageref.UpdatePolicyKey(service.Image, service.Labels),+ CheckTime: time.Now().UTC(),}parsed, err := refs.NormalizeReference(service.Image)if err != nil {@@ -1890,40 +1534,222 @@ func (s *UpdaterService) checkProjectServiceInternal(ctx context.Context, projecreturn record}-func updateBatchTypeInternal(ctx context.Context, options arcaneupdater.Options, batchCompleted bool) string {- batchType := "update-batch"- if previous, ok := jobcontext.Run(ctx); ok && len(options.ResourceIds) > 0 {- batchType = "update-retry"- for _, target := range previous.Outcome.Targets {- if target.ID == "auto-update" && target.ResourceType == "update-batch" && (target.Status == scheduler.Succeeded || target.Status == scheduler.Partial) {- batchType = "update-batch"- }- }+// RegisterWorkflows defines the auto-update workflow (fresh candidate check, frozen plan without+// failed checks, confirm-first apply) and the container-update workflow. Call it before the host starts.+func (s *UpdaterService) RegisterWorkflows(engine *flow.Engine) error {+ autoUpdate, err := engine.Define(flow.Definition{+ Name: "auto-update",+ Version: 5,+ Fingerprint: "91e27dc48df6737110ec334dfa9c2e2e032a6cc4285e8f1dd08dc40741e6d5b9",+ Concurrency: 1,+ Timeout: 2 * time.Hour,+ Activity: activitylib.StartRequest{+ Type: activitytypes.TypeAutoUpdate,+ Queue: true,+ ResourceType: new("system"),+ ResourceName: new("Auto update"),+ Step: "Planning updates",+ LatestMessage: "Auto-update run started",+ Metadata: database.JSON{"dryRun": false},+ },+ Labels: map[string]string{+ "resume": "Checking for a saved plan",+ "candidates": "Selecting update candidates",+ "check": "Checking images",+ "plan": "Planning updates",+ "apply": "Applying updates",+ "finalize": "Finishing auto-update",+ },+ Steps: []workflow.StepSpec{+ workflow.Step("resume", engine.Handler(s.autoUpdateResume)),+ // A retry keeps its frozen plan, so it skips selecting and checking images.+ workflow.Step("candidates", engine.Handler(s.autoUpdateCandidates), workflow.WithSkipIf("resume", true)),+ workflow.Child("check", workflow.WithDefinition(s.deps.ImageUpdates.CheckWorkflow().Francis()), workflow.WithSkipIf("resume", true)),+ workflow.Step("plan", engine.Handler(s.planAutoUpdate)),+ workflow.Step("apply", engine.Handler(s.applyAutoUpdate),+ workflow.WithMaxAttempts(5), workflow.WithRetryBackoff(30*time.Second, 5*time.Minute)),+ workflow.Step("finalize", engine.Handler(s.finalizeAutoUpdate)),+ },+ })+ if err != nil {+ return err}- if !batchCompleted {- batchType = "update-interrupted"+ s.autoUpdate = autoUpdate+ return s.execution.RegisterWorkflows(engine)+}++// AutoUpdateWorkflow is the durable auto-update workflow.+func (s *UpdaterService) AutoUpdateWorkflow() *flow.Workflow { return s.autoUpdate }++// autoUpdateCandidates selects the monitored images of auto-update eligible containers.+func (s *UpdaterService) autoUpdateCandidates(ctx context.Context, t flow.Task) (any, error) {+ used, err := s.CollectUsedImages(ctx)+ if err != nil {+ return nil, common.Classify(common.ErrUnavailable, err)+ }+ monitored, err := s.deps.ImageUpdates.MonitoredImageRefs(ctx, slices.Sorted(maps.Keys(used)))+ if err != nil {+ return nil, common.Classify(common.ErrUnavailable, err)}- return batchType+ return t.ChildInput(imageupdatetypes.CheckRequest{ImageRefs: monitored})}-// RegisterActors registers durable single-container update delivery.-func (s *UpdaterService) RegisterActors(runtime *francis.Runtime) error {- return s.execution.RegisterActors(runtime)+// autoUpdateResume reports whether the run already holds a frozen plan.+func (s *UpdaterService) autoUpdateResume(ctx context.Context, _ flow.Task) (any, error) {+ run, ok := jobcontext.Run(ctx)+ return ok && s.recovery.Planned(run), nil}-// Start repairs persisted single-container delivery intents until shutdown.-func (s *UpdaterService) Start(ctx context.Context) error {- return s.execution.Start(ctx)+// planAutoUpdate freezes the update plan without the images and containers whose check failed.+func (s *UpdaterService) planAutoUpdate(ctx context.Context, t flow.Task) (any, error) {+ run, ok := jobcontext.Run(ctx)+ if !ok {+ return nil, errors.New("auto-update requires a job run")+ }+ var checked scheduler.Outcome+ if err := t.DecodeOutput("check", &checked); err != nil && !errors.Is(err, workflow.ErrStepSkipped) {+ return nil, err+ }+ excluded := map[string]bool{}+ for _, target := range checked.Targets {+ // Failed checks are run evidence, so a retry or recovery that skips the check still reports them.+ if err := jobcontext.Progress(ctx, target); err != nil {+ return nil, common.Classify(common.ErrUnavailable, err)+ }+ switch target.ResourceType {+ case "image":+ excluded[refs.NormalizeImageUpdateRef(target.ID)] = true+ case "container":+ excluded[target.ID] = true+ }+ }+ if _, err := s.recovery.FreezePending(contextWithActivityID(ctx, t.ActivityID()), run, excluded); err != nil {+ return nil, common.Classify(common.ErrUnavailable, err)+ }+ return nil, nil}-// Stop cancels business execution before the actor host drains jobs.-func (s *UpdaterService) Stop(ctx context.Context) error {- return s.execution.Stop(ctx)+// applyAutoUpdate confirms every frozen target on each delivery before+// applying the ones still unchanged, so redelivery never repeats a Docker change.+func (s *UpdaterService) applyAutoUpdate(ctx context.Context, t flow.Task) (any, error) {+ run, ok := jobcontext.Run(ctx)+ if !ok {+ return nil, errors.New("auto-update requires a job run")+ }+ ctx, release, err := s.acquireUpdate(ctx)+ if errors.Is(err, errUpdateBusy) {+ // A plan whose every target already settled needs no updater, so it finishes normally.+ if unsettled, unsettledErr := s.recovery.Unsettled(run); unsettledErr == nil && unsettled == 0 {+ return autoUpdateApplied{Completed: true}, nil+ }+ // Waiting would hold this run's activity slot against the update holding the updater, so the run ends now:+ // a plan that already started applying stays retryable, and an untouched one skips as the job always has.+ return autoUpdateApplied{Busy: true, Started: s.recovery.Started(run)}, nil+ }+ if err != nil {+ return nil, common.Classify(common.ErrUnavailable, err)+ }+ defer release()+ start := time.Now()+ ctx = contextWithActivityID(ctx, t.ActivityID())+ progress := &updateProgress{}+ ctx = context.WithValue(ctx, updateProgressKey{}, progress)++ ctx, remaining, unresolved, err := s.recovery.ResumePlan(ctx, run)+ if err != nil {+ return nil, common.Classify(common.ErrUnavailable, err)+ }+ applied := autoUpdateApplied{Unresolved: unresolved, Completed: true}+ if remaining > 0 {+ if applyErr := s.applyBatch(ctx, arcaneupdater.Options{}, &applied.Result); applyErr != nil {+ // A shutdown hands the task back so the redelivery finishes the remaining frozen targets.+ if ctx.Err() != nil {+ return nil, ctx.Err()+ }+ applied.Error, applied.Completed = applyErr.Error(), false+ }+ }+ applied.Result.Items = nil+ applied.Result.Duration = time.Since(start).String()+ progress.mu.Lock()+ if progress.err != nil {+ applied.RecordingError = progress.err.Error()+ }+ applied.SelfTriggered, applied.SelfID = progress.selfTriggered, progress.selfID+ progress.mu.Unlock()+ return applied, nil}-// ActiveUpdateActivityIDs lists activities owned by persisted single-container updates.-func (s *UpdaterService) ActiveUpdateActivityIDs(ctx context.Context) ([]string, error) {- return s.execution.ActiveUpdateActivityIDs(ctx)+// finalizeAutoUpdate records the batch checkpoint and returns the run outcome.+func (s *UpdaterService) finalizeAutoUpdate(ctx context.Context, t flow.Task) (any, error) {+ var applied autoUpdateApplied+ if err := t.DecodeOutput("apply", &applied); err != nil {+ return nil, err+ }+ activityID := t.ActivityID()+ ctx = contextWithActivityID(ctx, activityID)+ if applied.Busy {+ status, message := scheduler.Skipped, "Another update is active"+ if applied.Started {+ status, message = scheduler.Failed, "Another update is active; the remaining targets were not updated"+ }+ batch := scheduler.TargetOutcome{ID: "auto-update", ResourceType: "update-batch", Status: status, ActivityID: activityID, Message: message}+ if err := jobcontext.Progress(ctx, batch); err != nil {+ return nil, common.Classify(common.ErrUnavailable, err)+ }+ return scheduler.Outcome{Status: status, Message: message, ActivityID: activityID}, nil+ }+ progress := &updateProgress{selfTriggered: applied.SelfTriggered, selfID: applied.SelfID}+ if applied.RecordingError != "" {+ progress.err = errors.New(applied.RecordingError)+ }+ var applyErr error+ if applied.Error != "" {+ applyErr = errors.New(applied.Error)+ }+ result := &applied.Result+ // Containers that could not be planned never reached apply, so they count as failed here.+ run, _ := jobcontext.Run(ctx)+ unplanned := s.recovery.Unplanned(run)+ result.Failed += len(unplanned)+ err := progress.completeBatch(ctx, arcaneupdater.Options{}, result, applied.Completed, applyErr)+ slog.InfoContext(ctx, "auto-update run completed",+ "checked", result.Checked,+ "updated", result.Updated,+ "restarted", result.Restarted,+ "skipped", result.Skipped,+ "failed", result.Failed,+ )+ outcome := scheduler.Outcome{Status: scheduler.Succeeded, Message: "Auto-update run completed", ActivityID: activityID}+ if result.Failed > 0 || err != nil {+ outcome.Status = scheduler.Partial+ outcome.Message = "Some updates failed"+ if err != nil {+ outcome.Message += ": " + err.Error()+ }+ }+ outcome.Targets = append(outcome.Targets, unplanned...)+ if applied.Unresolved {+ outcome.Status = scheduler.Failed+ outcome.Message = "Some target outcomes could not be confirmed"+ }+ if outcomeErr, ok := errors.AsType[*scheduler.OutcomeError](err); ok {+ outcome.Status = outcomeErr.Outcome.Status+ outcome.Message = outcomeErr.Outcome.Message+ }+ return outcome, nil+}++// ValidateAutoUpdateRetry allows a retry only while frozen targets remain unsettled.+func (s *UpdaterService) ValidateAutoUpdateRetry(_ context.Context, run scheduler.Run) error {+ unsettled, err := s.recovery.Unsettled(run)+ if err != nil {+ return fmt.Errorf("auto-update has no frozen plan to retry: %w", err)+ }+ if unsettled == 0 {+ return errors.New("auto-update has no unsettled targets to retry")+ }+ return nil}// ReconcilePending resumes only frozen targets whose original container is unchanged.diff --git a/backend/internal/updater/service_test.go b/backend/internal/updater/service_test.goindex 41483489c..51c11f429 100644--- a/backend/internal/updater/service_test.go+++ b/backend/internal/updater/service_test.go@@ -32,7 +32,7 @@ import ("github.com/moby/moby/client""github.com/stretchr/testify/assert""github.com/stretchr/testify/require"- "go.getarcane.app/kit/pkg"+ kit "go.getarcane.app/kit/pkg""go.getarcane.app/sys/crypto""go.getarcane.app/updater""go.getarcane.app/updater/labels"@@ -53,21 +53,20 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/project""github.com/getarcaneapp/arcane/backend/v2/internal/registry""github.com/getarcaneapp/arcane/backend/v2/internal/settings"- "github.com/getarcaneapp/arcane/backend/v2/internal/updater/children/execution"activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity""github.com/getarcaneapp/arcane/backend/v2/pkg/projects"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow/flowtest""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs""github.com/getarcaneapp/arcane/backend/v2/pkg/utils"- francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/notifications")-type fakeDockerClientProviderInternal struct {+type fakeDockerClientProvider struct {client any}-func (f fakeDockerClientProviderInternal) DockerClient(context.Context) (*client.Client, error) {+func (f fakeDockerClientProvider) DockerClient(context.Context) (*client.Client, error) {cli, _ := f.client.(*client.Client)if cli == nil {return nil, errors.New("docker client unavailable")@@ -75,12 +74,12 @@ func (f fakeDockerClientProviderInternal) DockerClient(context.Context) (*clientreturn cli, nil}-type fakeImagePullerInternal struct {+type fakeImagePuller struct {pulled []stringfail map[string]error}-func (f *fakeImagePullerInternal) PullImage(_ context.Context, imageRef string, _ io.Writer) error {+func (f *fakeImagePuller) PullImage(_ context.Context, imageRef string, _ io.Writer) error {f.pulled = append(f.pulled, imageRef)if f.fail == nil {return nil@@ -88,44 +87,44 @@ func (f *fakeImagePullerInternal) PullImage(_ context.Context, imageRef string,return f.fail[imageRef]}-type fakeRunRecorderInternal struct {+type fakeRunRecorder struct {results []updater.ResourceResult}-func (f *fakeRunRecorderInternal) RecordUpdateRun(_ context.Context, result updater.ResourceResult) error {+func (f *fakeRunRecorder) RecordUpdateRun(_ context.Context, result updater.ResourceResult) error {f.results = append(f.results, result)return nil}-type fakeSettingsProviderInternal struct{}+type fakeSettingsProvider struct{}-func (fakeSettingsProviderInternal) ExcludedContainers(context.Context) ([]string, error) {+func (fakeSettingsProvider) ExcludedContainers(context.Context) ([]string, error) {return nil, nil}-type fakeUsedImageCollectorInternal struct {+type fakeUsedImageCollector struct {images map[string]struct{}}-func (f fakeUsedImageCollectorInternal) UsedImages(context.Context) (map[string]struct{}, error) {+func (f fakeUsedImageCollector) UsedImages(context.Context) (map[string]struct{}, error) {return f.images, nil}-type fakeProjectUpdaterInternal struct {+type fakeProjectUpdater struct {projects map[string]updater.ComposeProjectupdateErrs map[string]errorcalls []stringimageChanges map[string]map[string]updatertypes.ServiceImageChange}-func (f *fakeProjectUpdaterInternal) ProjectByComposeName(_ context.Context, composeName string) (updater.ComposeProject, error) {+func (f *fakeProjectUpdater) ProjectByComposeName(_ context.Context, composeName string) (updater.ComposeProject, error) {if localProject, ok := f.projects[composeName]; ok {return localProject, nil}return updater.ComposeProject{}, errors.New("project not found")}-func (f *fakeProjectUpdaterInternal) UpdateServices(_ context.Context, projectID string, services []string) error {+func (f *fakeProjectUpdater) UpdateServices(_ context.Context, projectID string, services []string) error {f.calls = append(f.calls, projectID+":"+strings.Join(services, ","))if f.updateErrs == nil {return nil@@ -133,7 +132,7 @@ func (f *fakeProjectUpdaterInternal) UpdateServices(_ context.Context, projectIDreturn f.updateErrs[projectID]}-func (f *fakeProjectUpdaterInternal) UpdateServiceImages(_ context.Context, projectID string, changes map[string]updatertypes.ServiceImageChange) error {+func (f *fakeProjectUpdater) UpdateServiceImages(_ context.Context, projectID string, changes map[string]updatertypes.ServiceImageChange) error {if f.imageChanges == nil {f.imageChanges = map[string]map[string]updatertypes.ServiceImageChange{}}@@ -143,7 +142,7 @@ func (f *fakeProjectUpdaterInternal) UpdateServiceImages(_ context.Context, projreturn f.updateErrs[projectID]}-func newUpdaterApplyPendingDockerServerInternal(+func newUpdaterApplyPendingDockerServer(t *testing.T,containers []container.Summary,verificationByService map[string][]container.Summary,@@ -224,32 +223,31 @@ func newUpdaterApplyPendingDockerServerInternal(return server}-type mockSystemUpgradeServiceInternal struct {+type mockSystemUpgradeService struct {triggerCalled booltriggerError errorcapturedUser *usertypes.ActorcapturedTarget *updater.SelfUpdateTarget}-func (m *mockSystemUpgradeServiceInternal) TriggerUpgradeViaCLI(_ context.Context, user usertypes.Actor, target updater.SelfUpdateTarget) (string, error) {+func (m *mockSystemUpgradeService) TriggerUpgradeViaCLI(_ context.Context, user usertypes.Actor, target updater.SelfUpdateTarget) (string, error) {m.triggerCalled = truem.capturedUser = &userm.capturedTarget = &targetreturn "upgrader-container-id", m.triggerError}-func TestUpdaterService_ApplyPendingNoRecordsInternal(t *testing.T) {+func TestUpdaterService_ApplyPendingNoRecords(t *testing.T) {ctx := t.Context()- db := setupProjectTestDBInternal(t)+ db := setupProjectTestDB(t)svc, svcErr := NewUpdaterService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)require.NoError(t, svcErr)var checkpoints []scheduler.TargetOutcomectx = jobcontext.WithExecution(ctx, scheduler.Run{AttemptCount: 1}, func(target scheduler.TargetOutcome) error { checkpoints = append(checkpoints, target); return nil })result, err := svc.ApplyPending(ctx, arcaneupdater.Options{DryRun: true})- require.Len(t, checkpoints, 2)- assert.Equal(t, scheduler.Running, checkpoints[0].Status)- assert.Equal(t, scheduler.Succeeded, checkpoints[1].Status)+ require.Len(t, checkpoints, 1)+ assert.Equal(t, scheduler.Succeeded, checkpoints[0].Status)require.NoError(t, err)require.NotNil(t, result)@@ -265,41 +263,18 @@ func TestUpdaterService_ApplyPendingNoRecordsInternal(t *testing.T) {require.ErrorAs(t, err, &outcomeErr)assert.Equal(t, scheduler.NeedsAttention, outcomeErr.Outcome.Status)checkpoints = nil- ctx = jobcontext.WithExecution(- t.Context(),- scheduler.Run{AttemptCount: 1},- func(target scheduler.TargetOutcome) error {- checkpoints = append(- checkpoints,- target,- )- return nil- },- )+ ctx = jobcontext.WithExecution(t.Context(), scheduler.Run{AttemptCount: 1}, func(target scheduler.TargetOutcome) error {+ checkpoints = append(checkpoints, target)+ return nil+ })svc.engine = nilrequire.Panics(t, func() { _, _ = svc.ApplyPending(ctx, arcaneupdater.Options{}) })require.NotEmpty(t, checkpoints)assert.Equal(t, scheduler.NeedsAttention, checkpoints[len(checkpoints)-1].Status)}-// Type and ResourceIds are Arcane-side scoping the engine never acted on;-// ApplyPending handles a scoped request itself before reaching the engine.-// A pending run, forced or scoped, never lifts settings exclusions: that-// override belongs to UpdateSingleContainer alone.-func TestUpdaterService_ModuleOptionsFromUpdaterOptionsInternal(t *testing.T) {- got := moduleOptionsFromUpdaterOptionsInternal(arcaneupdater.Options{- Type: string(updater.ResourceTypeContainer),- ResourceIds: []string{"container-1", "container-2"},- ForceUpdate: true,- DryRun: true,- })-- assert.Equal(t, updater.Options{Force: true, DryRun: true}, got)- assert.False(t, got.IgnoreSettingsExclusions)-}--func TestUpdaterService_ResultFromModulePreservesRestartedInternal(t *testing.T) {- got := resultFromModuleInternal(&updater.Result{+func TestUpdaterService_ResultFromModulePreservesRestarted(t *testing.T) {+ got := resultFromModule(&updater.Result{Success: true,Checked: 2,Updated: 1,@@ -322,11 +297,11 @@ func TestUpdaterService_ResultFromModulePreservesRestartedInternal(t *testing.T)assert.True(t, got.Items[0].UpdateApplied)}-func TestUpdaterService_TriggerSelfUpdateViaCLIInternal(t *testing.T) {+func TestUpdaterService_TriggerSelfUpdateViaCLI(t *testing.T) {ctx := t.Context()t.Run("server label triggers upgrade with system user", func(t *testing.T) {- mockUpgrade := &mockSystemUpgradeServiceInternal{}+ mockUpgrade := &mockSystemUpgradeService{}svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil, nil)require.NoError(t, svcErr)@@ -347,7 +322,7 @@ func TestUpdaterService_TriggerSelfUpdateViaCLIInternal(t *testing.T) {})t.Run("engine target keeps the selected tag and pulls the frozen digest", func(t *testing.T) {- mockUpgrade := &mockSystemUpgradeServiceInternal{}+ mockUpgrade := &mockSystemUpgradeService{}svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil, nil)require.NoError(t, svcErr)@@ -370,7 +345,7 @@ func TestUpdaterService_TriggerSelfUpdateViaCLIInternal(t *testing.T) {})t.Run("agent label triggers upgrade", func(t *testing.T) {- mockUpgrade := &mockSystemUpgradeServiceInternal{}+ mockUpgrade := &mockSystemUpgradeService{}svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil, nil)require.NoError(t, svcErr)@@ -383,7 +358,7 @@ func TestUpdaterService_TriggerSelfUpdateViaCLIInternal(t *testing.T) {})t.Run("non Arcane labels fail without triggering upgrade", func(t *testing.T) {- mockUpgrade := &mockSystemUpgradeServiceInternal{}+ mockUpgrade := &mockSystemUpgradeService{}svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil, nil)require.NoError(t, svcErr)@@ -409,7 +384,7 @@ func TestUpdaterService_TriggerSelfUpdateViaCLIInternal(t *testing.T) {})t.Run("upgrade errors are wrapped", func(t *testing.T) {- mockUpgrade := &mockSystemUpgradeServiceInternal{triggerError: errors.New("upgrade failed")}+ mockUpgrade := &mockSystemUpgradeService{triggerError: errors.New("upgrade failed")}svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil, nil)require.NoError(t, svcErr)@@ -422,7 +397,7 @@ func TestUpdaterService_TriggerSelfUpdateViaCLIInternal(t *testing.T) {})}-func TestUpdaterService_StatusTrackingInternal(t *testing.T) {+func TestUpdaterService_StatusTracking(t *testing.T) {svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)require.NoError(t, svcErr)@@ -445,13 +420,13 @@ func TestUpdaterService_StatusTrackingInternal(t *testing.T) {assert.Empty(t, status.ProjectIds)}-func TestUpdaterService_DockerClientAdapterInternal(t *testing.T) {+func TestUpdaterService_DockerClientAdapter(t *testing.T) {ctx := t.Context()t.Run("active engine updates correlate before the stop event is recorded", func(t *testing.T) {events := event.NewEventService(nil, nil, nil)svc, err := NewUpdaterService(nil, nil, nil, nil, nil, nil, events, nil, nil, nil, nil, nil, nil, nil, nil)require.NoError(t, err)- end := svc.engineInternal().BeginContainerUpdate("updating-container")+ end := svc.engine.BeginContainerUpdate("updating-container")require.True(t, events.ShouldSuppressDaemonEvent("container", "updating-container", "", ""))require.False(t, events.ShouldSuppressDaemonEvent("container", "unrelated-container", "", ""))require.False(t, events.ShouldSuppressDaemonEvent("image", "updating-container", "", ""))@@ -471,8 +446,8 @@ func TestUpdaterService_DockerClientAdapterInternal(t *testing.T) {})t.Run("delegates to configured docker service", func(t *testing.T) {- server := newImagePullServerWithObserverInternal(t, nil, nil)- wantClient := newTestDockerClientInternal(t, server)+ server := newImagePullServerWithObserver(t, nil, nil)+ wantClient := newTestDockerClient(t, server)dockerSvc := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(wantClient)svc, svcErr := NewUpdaterService(nil, nil, dockerSvc, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)require.NoError(t, svcErr)@@ -484,7 +459,7 @@ func TestUpdaterService_DockerClientAdapterInternal(t *testing.T) {})}-func TestUpdaterService_PullImageAdapterInternal(t *testing.T) {+func TestUpdaterService_PullImageAdapter(t *testing.T) {ctx := t.Context()t.Run("missing image service returns unavailable error", func(t *testing.T) {@@ -498,9 +473,9 @@ func TestUpdaterService_PullImageAdapterInternal(t *testing.T) {})t.Run("delegates to Arcane image puller", func(t *testing.T) {- db := setupProjectTestDBInternal(t)- server := newImagePullServerWithObserverInternal(t, nil, nil)- dockerSvc := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClientInternal(t, server))+ db := setupProjectTestDB(t)+ server := newImagePullServerWithObserver(t, nil, nil)+ dockerSvc := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClient(t, server))imageSvc := image.NewImageService(db, dockerSvc, nil, nil, nil, event.NewEventService(db, nil, nil), nil, nil)svc, svcErr := NewUpdaterService(db, nil, dockerSvc, nil, nil, nil, nil, imageSvc, nil, nil, nil, nil, nil, nil, nil)require.NoError(t, svcErr)@@ -513,22 +488,22 @@ func TestUpdaterService_PullImageAdapterInternal(t *testing.T) {})t.Run("passes database registry credentials to Arcane image puller", func(t *testing.T) {- db := setupProjectTestDBInternal(t)+ db := setupProjectTestDB(t)require.NoError(t, db.AutoMigrate(®istry.ContainerRegistry{}, &kv.KVEntry{}))crypto.InitEncryption(&crypto.Config{Environment: string(config.AppEnvironmentTest),EncryptionKey: "test-encryption-key-for-testing-32bytes-min",})- createTestPullRegistryInternal(t, db, "https://registry.example.com", "db-user", "db-token")+ createTestPullRegistry(t, db, "https://registry.example.com", "db-user", "db-token")imageRef := "registry.example.com/team/app:latest"var gotAuth string- server := newImagePullServerWithObserverInternal(t, nil, func(fullRef, authHeader string) {+ server := newImagePullServerWithObserver(t, nil, func(fullRef, authHeader string) {if fullRef == imageRef {gotAuth = authHeader}})- dockerSvc := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClientInternal(t, server))+ dockerSvc := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClient(t, server))registrySvc := registry.NewContainerRegistryService(db, nil, kv.NewKVService(db), nil)imageSvc := image.NewImageService(db, dockerSvc, registrySvc, nil, nil, event.NewEventService(db, nil, nil), nil, nil)envSvc := environment.NewEnvironmentService(db, nil, nil, nil, nil, nil)@@ -542,16 +517,16 @@ func TestUpdaterService_PullImageAdapterInternal(t *testing.T) {require.NoError(t, err)require.NotEmpty(t, gotAuth)- authConfig := decodeRegistryAuthInternal(t, gotAuth)+ authConfig := decodeRegistryAuth(t, gotAuth)assert.Equal(t, "db-user", authConfig.Username)assert.Equal(t, "db-token", authConfig.Password)assert.Contains(t, progress.String(), "Pulled")})}-func TestUpdaterService_PendingImageUpdatesAdapterInternal(t *testing.T) {+func TestUpdaterService_PendingImageUpdatesAdapter(t *testing.T) {ctx := t.Context()- db := setupProjectTestDBInternal(t)+ db := setupProjectTestDB(t)latest := "1.2.4"currentDigest := "sha256:1111111111111111111111111111111111111111111111111111111111111111"latestDigest := "sha256:2222222222222222222222222222222222222222222222222222222222222222"@@ -597,10 +572,10 @@ func TestUpdaterService_PendingImageUpdatesAdapterInternal(t *testing.T) {assert.Equal(t, &lastError, records[0].LastError)}-// TestUpdaterService_PendingImageUpdatesFlushesPendingNotificationsInternal guards-// newEmptyContainerListDockerServiceInternal stubs the container listing the+// TestUpdaterService_PendingImageUpdatesFlushesPendingNotifications guards+// newEmptyContainerListDockerService stubs the container listing the// notification flush uses to resolve update-check eligibility.-func newEmptyContainerListDockerServiceInternal(t *testing.T) *docker.DockerClientService {+func newEmptyContainerListDockerService(t *testing.T) *docker.DockerClientService {t.Helper()server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {w.Header().Set("Content-Type", "application/json")@@ -613,14 +588,14 @@ func newEmptyContainerListDockerServiceInternal(t *testing.T) *docker.DockerCliehttp.NotFound(w, r)}))t.Cleanup(server.Close)- return &docker.DockerClientService{Client: newTestDockerClientInternal(t, server)}+ return &docker.DockerClientService{Client: newTestDockerClient(t, server)}}// PendingImageUpdates must not hand records to the engine when the flush could// not determine what to send (#3532): consuming them would lose the pending// notification for good instead of retrying once Docker is reachable again.-func TestUpdaterService_PendingImageUpdatesAbortsWhenNotificationEligibilityUnresolvedInternal(t *testing.T) {- db := setupProjectTestDBInternal(t)+func TestUpdaterService_PendingImageUpdatesAbortsWhenNotificationEligibilityUnresolved(t *testing.T) {+ db := setupProjectTestDB(t)require.NoError(t, db.AutoMigrate(&imageupdate.ImageUpdateRecord{}, ¬ification.NotificationSettings{}))notif := notification.NewNotificationService(db, nil, nil, nil, nil)imageUpdates := imageupdate.NewImageUpdateService(db, nil, nil, nil, nil, notif, nil)@@ -640,9 +615,9 @@ func TestUpdaterService_PendingImageUpdatesAbortsWhenNotificationEligibilityUnre// issue #3132: the updater consumes and clears has_update records without checking// notification_sent, so an "Updates Available" notification pending at consumption// time was silently lost. PendingImageUpdates must flush it before the engine runs.-func TestUpdaterService_PendingImageUpdatesFlushesPendingNotificationsInternal(t *testing.T) {+func TestUpdaterService_PendingImageUpdatesFlushesPendingNotifications(t *testing.T) {ctx := t.Context()- db := setupProjectTestDBInternal(t)+ db := setupProjectTestDB(t)require.NoError(t, db.AutoMigrate(&imageupdate.ImageUpdateRecord{}, ¬ification.NotificationSettings{}))var calls atomic.Int32@@ -662,7 +637,7 @@ func TestUpdaterService_PendingImageUpdatesFlushesPendingNotificationsInternal(t}).Error)notif := notification.NewNotificationService(db, nil, nil, nil, nil)- imageUpdates := imageupdate.NewImageUpdateService(db, nil, nil, newEmptyContainerListDockerServiceInternal(t), nil, notif, nil)+ imageUpdates := imageupdate.NewImageUpdateService(db, nil, nil, newEmptyContainerListDockerService(t), nil, notif, nil)svc, svcErr := NewUpdaterService(db, nil, nil, nil, imageUpdates, nil, nil, nil, notif, nil, nil, nil, nil, nil, nil)require.NoError(t, svcErr)@@ -687,13 +662,13 @@ func TestUpdaterService_PendingImageUpdatesFlushesPendingNotificationsInternal(t// With no providers configured, PendingImageUpdates still returns the records and// leaves notification_sent false (preserves the #3079 "don't mark when nothing// delivered" semantics).-func TestUpdaterService_PendingImageUpdatesNoProvidersLeavesUnnotifiedInternal(t *testing.T) {+func TestUpdaterService_PendingImageUpdatesNoProvidersLeavesUnnotified(t *testing.T) {ctx := t.Context()- db := setupProjectTestDBInternal(t)+ db := setupProjectTestDB(t)require.NoError(t, db.AutoMigrate(&imageupdate.ImageUpdateRecord{}, ¬ification.NotificationSettings{}))notif := notification.NewNotificationService(db, nil, nil, nil, nil)- imageUpdates := imageupdate.NewImageUpdateService(db, nil, nil, newEmptyContainerListDockerServiceInternal(t), nil, notif, nil)+ imageUpdates := imageupdate.NewImageUpdateService(db, nil, nil, newEmptyContainerListDockerService(t), nil, notif, nil)svc, svcErr := NewUpdaterService(db, nil, nil, nil, imageUpdates, nil, nil, nil, notif, nil, nil, nil, nil, nil, nil)require.NoError(t, svcErr)@@ -714,9 +689,9 @@ func TestUpdaterService_PendingImageUpdatesNoProvidersLeavesUnnotifiedInternal(tassert.False(t, reloaded.NotificationSent)}-func TestUpdaterService_RecordUpdateRunAdapterInternal(t *testing.T) {+func TestUpdaterService_RecordUpdateRunAdapter(t *testing.T) {ctx := t.Context()- db := setupProjectTestDBInternal(t)+ db := setupProjectTestDB(t)require.NoError(t, db.AutoMigrate(&AutoUpdateRecord{}))svc, svcErr := NewUpdaterService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)require.NoError(t, svcErr)@@ -744,8 +719,8 @@ func TestUpdaterService_RecordUpdateRunAdapterInternal(t *testing.T) {assert.Equal(t, "nginx:1.2.3", record.OldImageVersions["main"])assert.Equal(t, "nginx:1.2.4", record.NewImageVersions["main"])assert.Equal(t, "test", record.Details["source"])- progress := &updateProgressInternal{}- ctx = context.WithValue(ctx, updateProgressKeyInternal{}, progress)+ progress := &updateProgress{}+ ctx = context.WithValue(ctx, updateProgressKey{}, progress)ctx = jobcontext.WithExecution(ctx, scheduler.Run{AttemptCount: 1}, func(target scheduler.TargetOutcome) error {assert.Equal(t, scheduler.Failed, target.Status)return errors.New("progress unavailable")@@ -755,9 +730,9 @@ func TestUpdaterService_RecordUpdateRunAdapterInternal(t *testing.T) {require.ErrorContains(t, progress.err, "progress unavailable")}-func TestUpdaterService_RecordUpdateRunAppendsActivityMessageInternal(t *testing.T) {+func TestUpdaterService_RecordUpdateRunAppendsActivityMessage(t *testing.T) {ctx := t.Context()- db := setupProjectTestDBInternal(t)+ db := setupProjectTestDB(t)require.NoError(t, db.AutoMigrate(&AutoUpdateRecord{}, &activity.Activity{}, &activity.ActivityMessage{}))activityService := activity.NewActivityService(db, nil)svc, svcErr := NewUpdaterService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, activityService, nil, nil, nil, nil)@@ -816,7 +791,7 @@ func TestUpdaterService_RecordUpdateRunAppendsActivityMessageInternal(t *testingstarted, err := activityService.StartActivity(ctx, activitylib.StartRequest{EnvironmentID: "0", Type: activitytypes.TypeAutoUpdate})require.NoError(t, err)- require.NoError(t, svc.RecordUpdateRun(contextWithActivityIDInternal(ctx, started.ID), tt.result))+ require.NoError(t, svc.RecordUpdateRun(contextWithActivityID(ctx, started.ID), tt.result))detail, err := activityService.GetActivityDetail(ctx, "0", started.ID, 10)require.NoError(t, err)@@ -827,26 +802,25 @@ func TestUpdaterService_RecordUpdateRunAppendsActivityMessageInternal(t *testing}}-func TestUpdaterService_AcceptSingleContainerUpdateUsesQueuedActivityInternal(t *testing.T) {+func TestUpdaterService_AcceptSingleContainerUpdateUsesQueuedActivity(t *testing.T) {for _, cancelActivity := range []bool{false, true} {name := kit.Ternary(cancelActivity, "activity cancellation while queued", "request disconnect")t.Run(name, func(t *testing.T) {- db := setupProjectTestDBInternal(t)+ db := setupProjectTestDB(t)require.NoError(t, db.AutoMigrate(&activity.Activity{}, &activity.ActivityMessage{}))sqlDB, err := db.DB.DB()require.NoError(t, err)sqlDB.SetMaxOpenConns(1)activityService := activity.NewActivityService(db, nil)- runtime := francistest.New(t)- admission := runs.NewAdmission(runtime.Service(), t.Name())- require.NoError(t, admission.Register(runtime))+ harness := flowtest.New(t, activityService)+ admission := runs.NewAdmission(harness.Runtime.Service(), t.Name())+ require.NoError(t, admission.Register(harness.Runtime))svc, err := NewUpdaterService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, activityService, nil, nil, admission, nil)require.NoError(t, err)- require.NoError(t, svc.RegisterActors(runtime))- francistest.Start(t, runtime)+ require.NoError(t, svc.execution.RegisterWorkflows(harness.Engine))+ harness.Start(t)- // Holding the update admission makes the executor decline the- // dispatched job until the lease is released.+ // Holding the update admission makes the workflow hand its task back until the lease is released.held, admitted, err := admission.TryAcquire(t.Context(), scheduler.AdmissionKey{Scope: "updater"})require.NoError(t, err)require.True(t, admitted)@@ -856,14 +830,14 @@ func TestUpdaterService_AcceptSingleContainerUpdateUsesQueuedActivityInternal(trequestCtx, cancelRequest := context.WithCancel(t.Context())workCtx := utils.ActivityRuntimeContext(requestCtx, t.Context())- type acceptedInternal struct {+ type acceptResult struct {activity *activitytypes.Activityerr error}- accepted := make(chan acceptedInternal, 1)+ accepted := make(chan acceptResult, 1)go func() {item, acceptErr := svc.AcceptSingleContainerUpdate(workCtx, "missing-container")- accepted <- acceptedInternal{activity: item, err: acceptErr}+ accepted <- acceptResult{activity: item, err: acceptErr}}()var item *activitytypes.Activityselect {@@ -890,31 +864,30 @@ func TestUpdaterService_AcceptSingleContainerUpdateUsesQueuedActivityInternal(t}}-func TestUpdaterService_AcceptSingleContainerUpdateFailureFinalizesInternal(t *testing.T) {- db := setupProjectTestDBInternal(t)+func TestUpdaterService_AcceptSingleContainerUpdateFailureFinalizes(t *testing.T) {+ db := setupProjectTestDB(t)require.NoError(t, db.AutoMigrate(&activity.Activity{}, &activity.ActivityMessage{}))activityService := activity.NewActivityService(db, nil)svc, err := NewUpdaterService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, activityService, nil, nil, nil, nil)require.NoError(t, err)- runtime := francistest.New(t)- require.NoError(t, svc.RegisterActors(runtime))- // A service whose host has not started rejects durable dispatch.+ // A host that has not started cannot start the workflow.+ require.NoError(t, svc.execution.RegisterWorkflows(flowtest.New(t, activityService).Engine))accepted, err := svc.AcceptSingleContainerUpdate(t.Context(), "container-1")require.Nil(t, accepted)- require.ErrorContains(t, err, "persist container update")+ require.ErrorContains(t, err, "start container-update")var saved activity.Activityrequire.NoError(t, db.First(&saved, "resource_id = ?", "container-1").Error)require.Equal(t, activitytypes.StatusFailed, saved.Status)- unmigrated := setupProjectTestDBInternal(t)- svc.deps.Activity = activity.NewActivityService(unmigrated, nil)+ unmigrated := setupProjectTestDB(t)+ require.NoError(t, svc.execution.RegisterWorkflows(flowtest.New(t, activity.NewActivityService(unmigrated, nil)).Engine))accepted, err = svc.AcceptSingleContainerUpdate(t.Context(), "container-2")require.Nil(t, accepted)- require.ErrorContains(t, err, "start container update activity")+ require.ErrorContains(t, err, "start container-update activity")}-func TestUpdaterService_SingleContainerActivityResultMessagesInternal(t *testing.T) {+func TestUpdaterService_SingleContainerActivityResultMessages(t *testing.T) {for _, tt := range []struct {name stringresult *arcaneupdater.Result@@ -980,7 +953,7 @@ func TestUpdaterService_SingleContainerActivityResultMessagesInternal(t *testing},} {t.Run(tt.name, func(t *testing.T) {- db := setupProjectTestDBInternal(t)+ db := setupProjectTestDB(t)require.NoError(t, db.AutoMigrate(&activity.Activity{}, &activity.ActivityMessage{}))activityService := activity.NewActivityService(db, nil)svc, err := NewUpdaterService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, activityService, nil, nil, nil, nil)@@ -994,7 +967,7 @@ func TestUpdaterService_SingleContainerActivityResultMessagesInternal(t *testing},)require.NoError(t, err)- svc.finishSingleContainerUpdateInternal(t.Context(), started.ID, tt.result, nil)+ svc.finishSingleContainerUpdate(t.Context(), started.ID, tt.result, nil)var saved activity.Activityrequire.NoError(t, db.First(&saved, "id = ?", started.ID).Error)require.Equal(t, tt.wantStatus, saved.Status)@@ -1004,7 +977,7 @@ func TestUpdaterService_SingleContainerActivityResultMessagesInternal(t *testing}}-func TestUpdaterService_ApplyPending_ProjectFailureDoesNotBlockOtherProjectsInternal(t *testing.T) {+func TestUpdaterService_ApplyPending_ProjectFailureDoesNotBlockOtherProjects(t *testing.T) {ctx := t.Context()oldRefFailed := "registry.example.com/team/fail:1.0.0"@@ -1098,11 +1071,11 @@ func TestUpdaterService_ApplyPending_ProjectFailureDoesNotBlockOtherProjectsInte},}- server := newUpdaterApplyPendingDockerServerInternal(t, containers, verificationByService, inspectByID, imageInspectByRef)- dockerProvider := fakeDockerClientProviderInternal{client: newTestDockerClientInternal(t, server)}- puller := &fakeImagePullerInternal{}- recorder := &fakeRunRecorderInternal{}- projectUpdater := &fakeProjectUpdaterInternal{+ server := newUpdaterApplyPendingDockerServer(t, containers, verificationByService, inspectByID, imageInspectByRef)+ dockerProvider := fakeDockerClientProvider{client: newTestDockerClient(t, server)}+ puller := &fakeImagePuller{}+ recorder := &fakeRunRecorder{}+ projectUpdater := &fakeProjectUpdater{projects: map[string]updater.ComposeProject{"proj-fail": {ID: "project-fail", Name: "proj-fail"},"proj-success": {ID: "project-success", Name: "proj-success"},@@ -1138,9 +1111,9 @@ func TestUpdaterService_ApplyPending_ProjectFailureDoesNotBlockOtherProjectsInte},),RunRecorder: recorder,- Settings: fakeSettingsProviderInternal{},+ Settings: fakeSettingsProvider{},ProjectUpdater: projectUpdater,- UsedImageCollector: fakeUsedImageCollectorInternal{images: map[string]struct{}{+ UsedImageCollector: fakeUsedImageCollector{images: map[string]struct{}{oldRefFailed: {},oldRefUpdated: {},}},@@ -1181,7 +1154,7 @@ func TestUpdaterService_ApplyPending_ProjectFailureDoesNotBlockOtherProjectsInteassert.Contains(t, recordedStatuses, updater.StatusUpdated)}-func TestUpdaterService_ApplyPending_RoutesLegacyArcaneServerThroughSelfUpgradeInternal(t *testing.T) {+func TestUpdaterService_ApplyPending_RoutesLegacyArcaneServerThroughSelfUpgrade(t *testing.T) {ctx := t.Context()oldRef := "ghcr.io/getarcaneapp/arcane:1.0.0"@@ -1235,15 +1208,15 @@ func TestUpdaterService_ApplyPending_RoutesLegacyArcaneServerThroughSelfUpgradeI},}- server := newUpdaterApplyPendingDockerServerInternal(t, containers, verificationByService, inspectByID, imageInspectByRef)- dockerProvider := fakeDockerClientProviderInternal{client: newTestDockerClientInternal(t, server)}- puller := &fakeImagePullerInternal{}- projectUpdater := &fakeProjectUpdaterInternal{+ server := newUpdaterApplyPendingDockerServer(t, containers, verificationByService, inspectByID, imageInspectByRef)+ dockerProvider := fakeDockerClientProvider{client: newTestDockerClient(t, server)}+ puller := &fakeImagePuller{}+ projectUpdater := &fakeProjectUpdater{projects: map[string]updater.ComposeProject{"arcane": {ID: "project-arcane", Name: "arcane"},},}- mockUpgrade := &mockSystemUpgradeServiceInternal{}+ mockUpgrade := &mockSystemUpgradeService{}svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil, nil)require.NoError(t, svcErr)@@ -1259,10 +1232,10 @@ func TestUpdaterService_ApplyPending_RoutesLegacyArcaneServerThroughSelfUpgradeICurrentVersion: "1.0.0",LatestVersion: new("1.0.1"),}),- Settings: fakeSettingsProviderInternal{},+ Settings: fakeSettingsProvider{},ProjectUpdater: projectUpdater,SelfUpdater: svc,- UsedImageCollector: fakeUsedImageCollectorInternal{images: map[string]struct{}{+ UsedImageCollector: fakeUsedImageCollector{images: map[string]struct{}{oldRef: {},}},LabelPolicy: updater.DefaultLabelPolicy(),@@ -1291,7 +1264,7 @@ func TestUpdaterService_ApplyPending_RoutesLegacyArcaneServerThroughSelfUpgradeI// The engine picks the reference Arcane's updates are applied to, so its choice// is worth pinning here as well as in the library.-func TestPullableImageRefInternal(t *testing.T) {+func TestPullableImageRef(t *testing.T) {tests := []struct {name stringsummaryImage string@@ -1333,8 +1306,8 @@ func TestPullableImageRefInternal(t *testing.T) {}}-// createTestPullRegistryInternal inserts an enabled generic registry with an encrypted token.-func createTestPullRegistryInternal(t *testing.T, db *database.DB, localUrl, username, token string) {+// createTestPullRegistry inserts an enabled generic registry with an encrypted token.+func createTestPullRegistry(t *testing.T, db *database.DB, localUrl, username, token string) {t.Helper()encryptedToken, err := crypto.Encrypt(token)@@ -1350,8 +1323,8 @@ func createTestPullRegistryInternal(t *testing.T, db *database.DB, localUrl, userequire.NoError(t, db.WithContext(t.Context()).Create(reg).Error)}-// decodeRegistryAuthInternal decodes a base64 X-Registry-Auth header.-func decodeRegistryAuthInternal(t *testing.T, encoded string) dockerregistry.AuthConfig {+// decodeRegistryAuth decodes a base64 X-Registry-Auth header.+func decodeRegistryAuth(t *testing.T, encoded string) dockerregistry.AuthConfig {t.Helper()cfg, err := authconfig.Decode(encoded)@@ -1359,8 +1332,8 @@ func decodeRegistryAuthInternal(t *testing.T, encoded string) dockerregistry.Autreturn *cfg}-// newTestDockerClientInternal builds a Docker client pointed at a fake Docker HTTP server.-func newTestDockerClientInternal(t *testing.T, server *httptest.Server) *client.Client {+// newTestDockerClient builds a Docker client pointed at a fake Docker HTTP server.+func newTestDockerClient(t *testing.T, server *httptest.Server) *client.Client {t.Helper()httpClient := server.Client()@@ -1378,8 +1351,8 @@ func newTestDockerClientInternal(t *testing.T, server *httptest.Server) *client.return cli}-// newImagePullServerWithObserverInternal is NewImagePullServer with a pull callback.-func newImagePullServerWithObserverInternal(t *testing.T, inspectByRef map[string]dockertypesimage.InspectResponse, onPull func(fullRef, authHeader string)) *httptest.Server {+// newImagePullServerWithObserver is NewImagePullServer with a pull callback.+func newImagePullServerWithObserver(t *testing.T, inspectByRef map[string]dockertypesimage.InspectResponse, onPull func(fullRef, authHeader string)) *httptest.Server {t.Helper()server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {@@ -1434,8 +1407,8 @@ func newImagePullServerWithObserverInternal(t *testing.T, inspectByRef map[strinreturn server}-// setupProjectTestDBInternal builds an in-memory DB migrated for project-related tests.-func setupProjectTestDBInternal(t *testing.T) *database.DB {+// setupProjectTestDB builds an in-memory DB migrated for project-related tests.+func setupProjectTestDB(t *testing.T) *database.DB {t.Helper()db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})require.NoError(t, err)@@ -1443,8 +1416,8 @@ func setupProjectTestDBInternal(t *testing.T) *database.DB {return &database.DB{DB: db}}-func TestUpdaterService_ScopedPendingRecordsIsolationInternal(t *testing.T) {- db := setupProjectTestDBInternal(t)+func TestUpdaterService_ScopedPendingRecordsIsolation(t *testing.T) {+ db := setupProjectTestDB(t)imageRef := "registry.example.com/team/app:1.2.3"stored := []imageupdate.ImageUpdateRecord{{ID: "shared-image", Repository: "registry.example.com/team/app", Tag: "1.2.3", HasUpdate: true, UpdateType: imageupdate.UpdateTypeDigest},@@ -1476,8 +1449,8 @@ func TestUpdaterService_ScopedPendingRecordsIsolationInternal(t *testing.T) {{ID: "digest-a", Labels: map[string]string{labels.LabelUpdateStrategy: "digest"}, Image: imageRef, ImageID: "shared-image", State: container.StateRunning},{ID: "digest-b", Labels: map[string]string{labels.LabelUpdateStrategy: "digest"}, Image: imageRef, ImageID: "shared-image", State: container.StateRunning},}- server := newUpdaterApplyPendingDockerServerInternal(t, containers, nil, nil, nil)- dockerSvc := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClientInternal(t, server))+ server := newUpdaterApplyPendingDockerServer(t, containers, nil, nil, nil)+ dockerSvc := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClient(t, server))svc, err := NewUpdaterService(db, nil, dockerSvc, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)require.NoError(t, err)records, err := svc.PendingImageUpdates(t.Context())@@ -1503,7 +1476,7 @@ func TestUpdaterService_ScopedPendingRecordsIsolationInternal(t *testing.T) {assert.ElementsMatch(t, []string{"shared-image", "container::tag-b"}, pendingIDs)}-func TestUpdaterService_SharedDigestRecordClearingInternal(t *testing.T) {+func TestUpdaterService_SharedDigestRecordClearing(t *testing.T) {for _, tt := range []struct {name stringremainingImage string@@ -1513,7 +1486,7 @@ func TestUpdaterService_SharedDigestRecordClearingInternal(t *testing.T) {{name: "all digest containers updated", remainingImage: "new-image", wantPending: false},} {t.Run(tt.name, func(t *testing.T) {- db := setupProjectTestDBInternal(t)+ db := setupProjectTestDB(t)imageRef := "registry.example.com/team/app:1.2.3"record := imageupdate.ImageUpdateRecord{ID: "shared-image", Repository: "registry.example.com/team/app", Tag: "1.2.3", HasUpdate: true, UpdateType: imageupdate.UpdateTypeDigest}require.NoError(t, db.Create(&record).Error)@@ -1522,12 +1495,14 @@ func TestUpdaterService_SharedDigestRecordClearingInternal(t *testing.T) {{ID: "remaining", Labels: map[string]string{labels.LabelUpdateStrategy: "digest"}, Image: imageRef, ImageID: tt.remainingImage, State: container.StateRunning},{ID: "separate-tag-policy", Image: imageRef, ImageID: "old-image", State: container.StateRunning, Labels: map[string]string{labels.LabelUpdateStrategy: "tag"}},}- server := newUpdaterApplyPendingDockerServerInternal(t, containers, nil, nil, map[string]dockertypesimage.InspectResponse{imageRef: {ID: "new-image"}})- dockerSvc := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClientInternal(t, server))+ server := newUpdaterApplyPendingDockerServer(t, containers, nil, nil, map[string]dockertypesimage.InspectResponse{imageRef: {ID: "new-image"}})+ dockerSvc := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClient(t, server))svc, err := NewUpdaterService(db, nil, dockerSvc, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)require.NoError(t, err)- converted := imageUpdateRecordToModuleInternal(record)- converted.ContainerID = "updated"+ converted := updater.ImageUpdateRecord{+ ID: record.ID, ContainerID: "updated", Repository: record.Repository, Tag: record.Tag,+ HasUpdate: record.HasUpdate, UpdateType: updater.UpdateType(record.UpdateType),+ }require.NoError(t, svc.ClearImageUpdateRecord(t.Context(), converted))var actual imageupdate.ImageUpdateRecordrequire.NoError(t, db.First(&actual, "id = ?", record.ID).Error)@@ -1536,50 +1511,33 @@ func TestUpdaterService_SharedDigestRecordClearingInternal(t *testing.T) {}}-type projectCheckRegistryInternal struct{}+type projectCheckRegistry struct{}-func (projectCheckRegistryInternal) ListTags(context.Context, string) ([]string, error) {+func (projectCheckRegistry) ListTags(context.Context, string) ([]string, error) {return []string{"3.20.0", "3.20.1", "3.20.2"}, nil}-func (projectCheckRegistryInternal) ImageDigest(context.Context, string) (string, error) {+func (projectCheckRegistry) ImageDigest(context.Context, string) (string, error) {return "sha256:" + strings.Repeat("a", 64), nil}-func TestUpdaterProjectChecksWithoutContainersInternal(t *testing.T) {+func TestUpdaterProjectChecksWithoutContainers(t *testing.T) {server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {if !assert.Contains(t, r.URL.Path, "/images/", "project checks must not need containers") {return}w.Header().Set("Content-Type", "application/json")- if !assert.NoError(- t,- json.MarshalWrite(- w,- dockertypesimage.InspectResponse{- ID: "sha256:" + strings.Repeat(- "a",- 64,- ),- RepoDigests: []string{"alpine@sha256:" + strings.Repeat(- "a",- 64,- )},- },- ),- ) {+ digest := "sha256:" + strings.Repeat("a", 64)+ if !assert.NoError(t, json.MarshalWrite(w, dockertypesimage.InspectResponse{ID: digest, RepoDigests: []string{"alpine@" + digest}})) {return}}))defer server.Close()- puller := &fakeImagePullerInternal{}+ puller := &fakeImagePuller{}engine, err := updater.New(updater.Config{- DockerClientProvider: fakeDockerClientProviderInternal{client: newTestDockerClientInternal(- t,- server,- )},- RegistryTagLister: projectCheckRegistryInternal{},- RegistryDigestResolver: projectCheckRegistryInternal{},+ DockerClientProvider: fakeDockerClientProvider{client: newTestDockerClient(t, server)},+ RegistryTagLister: projectCheckRegistry{},+ RegistryDigestResolver: projectCheckRegistry{},ImagePuller: puller,})require.NoError(t, err)@@ -1595,23 +1553,16 @@ func TestUpdaterProjectChecksWithoutContainersInternal(t *testing.T) {} {localConfig := types.ServiceConfig{Name: tt.name, Image: "alpine:3.20.0", Labels: map[string]string{labels.LabelUpdateConstraint: tt.constraint}}if metadata {- model, loadErr := projects.LoadComposeProjectFromContent(- t.Context(),- projecttypes.ComposeContentOptions{- ProjectName: "metadata",- WorkingDir: t.TempDir(),- ComposeContent: fmt.Sprintf(- "x-arcane:\n updater:\n strategy: auto\nservices:\n %s:\n image: alpine:3.20.0\n x-arcane:\n "+- " updater:\n constraint: %q\n",- tt.name,- tt.constraint,- ),- },- )+ compose := "x-arcane:\n updater:\n strategy: auto\nservices:\n %s:\n image: alpine:3.20.0\n x-arcane:\n updater:\n constraint: %q\n"+ model, loadErr := projects.LoadComposeProjectFromContent(t.Context(), projecttypes.ComposeContentOptions{+ ProjectName: "metadata",+ WorkingDir: t.TempDir(),+ ComposeContent: fmt.Sprintf(compose, tt.name, tt.constraint),+ })require.NoError(t, loadErr)localConfig = model.Services[tt.name]}- record := service.checkProjectServiceInternal(t.Context(), "project", localConfig)+ record := service.checkProjectService(t.Context(), "project", localConfig)require.Nil(t, record.LastError)require.Equal(t, tt.available, record.HasUpdate)require.Equal(t, tt.target, *record.LatestVersion)@@ -1623,7 +1574,7 @@ func TestUpdaterProjectChecksWithoutContainersInternal(t *testing.T) {}func TestSingleUpdatePersistsSuccessfulActivity(t *testing.T) {- db := setupProjectTestDBInternal(t)+ db := setupProjectTestDB(t)require.NoError(t, db.AutoMigrate(&activity.Activity{}, &activity.ActivityMessage{}, &AutoUpdateRecord{}))pool, err := db.DB.DB()require.NoError(t, err)@@ -1637,22 +1588,14 @@ func TestSingleUpdatePersistsSuccessfulActivity(t *testing.T) {return}case strings.HasSuffix(r.URL.Path, "/containers/app/json"):- if !assert.NoError(- t,- json.MarshalWrite(- w,- container.InspectResponse{- ID: "app",- Name: "/app",- Image: "sha256:baseline",- Config: &container.Config{Image: immutableRef},- State: &container.State{- StartedAt: "baseline",- Running: true,- },- },- ),- ) {+ inspect := container.InspectResponse{+ ID: "app",+ Name: "/app",+ Image: "sha256:baseline",+ Config: &container.Config{Image: immutableRef},+ State: &container.State{StartedAt: "baseline", Running: true},+ }+ if !assert.NoError(t, json.MarshalWrite(w, inspect)) {return}default:@@ -1661,47 +1604,33 @@ func TestSingleUpdatePersistsSuccessfulActivity(t *testing.T) {}))defer server.Close()activityService := activity.NewActivityService(db, nil)- svc, err := NewUpdaterService(- db,- nil,- (&docker.DockerClientService{}).WithClient(newTestDockerClientInternal(- t,- server,- )),- nil,- nil,- nil,- nil,- nil,- nil,- nil,- activityService,- nil,- nil,- nil,- nil,- )+ dockerSvc := (&docker.DockerClientService{}).WithClient(newTestDockerClient(t, server))+ svc, err := NewUpdaterService(db, nil, dockerSvc, nil, nil, nil, nil, nil, nil, nil, activityService, nil, nil, nil, nil)require.NoError(t, err)- runtime := francistest.New(t)- require.NoError(t, svc.RegisterActors(runtime))- francistest.Start(t, runtime)- require.NoError(t, svc.Start(t.Context()))- t.Cleanup(func() {- ctx, cancel := context.WithTimeout(context.WithoutCancel(t.Context()), time.Second)- defer cancel()- require.NoError(t, svc.Stop(ctx))- })+ harness := flowtest.New(t, activityService)+ require.NoError(t, svc.execution.RegisterWorkflows(harness.Engine))+ harness.Start(t)accepted, err := svc.AcceptSingleContainerUpdate(t.Context(), "app")require.NoError(t, err)require.Eventually(t, func() bool {- var state arcaneupdater.SingleUpdateState- if runtime.Service().GetState(t.Context(), execution.StateType, accepted.ID, &state) != nil || state.Status != "completed" {- return false- }detail, getActivityDetailErr := activityService.GetActivityDetail(t.Context(), "0", accepted.ID, 1)return getActivityDetailErr == nil && detail.Activity.Status == activitytypes.StatusSuccess- }, 3*time.Second, 10*time.Millisecond)+ }, 5*time.Second, 10*time.Millisecond)detail, err := activityService.GetActivityDetail(t.Context(), "0", accepted.ID, 1)require.NoError(t, err)require.Equal(t, activitytypes.StatusSuccess, detail.Activity.Status)}++func TestUpdaterService_WorkflowDefinitionsAreVersioned(t *testing.T) {+ db := setupProjectTestDB(t)+ harness := flowtest.New(t, activity.NewActivityService(db, nil))+ imageUpdates := imageupdate.NewImageUpdateService(db, nil, nil, nil, nil, nil, nil)+ require.NoError(t, imageUpdates.RegisterWorkflows(harness.Engine))+ svc, err := NewUpdaterService(db, nil, nil, nil, imageUpdates, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)+ require.NoError(t, err)+ require.NoError(t, svc.RegisterWorkflows(harness.Engine))+ harness.Start(t)++ // auto-update embeds image-check, so bumping image-check also requires bumping auto-update.+ flowtest.AssertDefinitions(t, harness)+}diff --git a/backend/internal/upload/handler.go b/backend/internal/upload/handler.goindex d2675980f..476e48bb7 100644--- a/backend/internal/upload/handler.go+++ b/backend/internal/upload/handler.go@@ -146,12 +146,7 @@ func (h *UploadHandler) DeleteSession(ctx context.Context, input *DeleteUploadSe}return nil, huma.Error500InternalServerError(err.Error())}- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{Message: "Upload session deleted"},- },- }, nil+ return handlerutil.MessageOutput("Upload session deleted", ""), nil}// LegacyMultipartMiddleware converts a deprecated multipart/form-data requestdiff --git a/backend/internal/user/handler.go b/backend/internal/user/handler.goindex 1f2487695..52775e2a6 100644--- a/backend/internal/user/handler.go+++ b/backend/internal/user/handler.go@@ -279,14 +279,7 @@ func (h *UserHandler) DeleteUser(ctx context.Context, input *DeleteUserInput) (*h.invalidateUserTokenCache(input.UserID)}- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{- Message: "User deleted successfully",- },- },- }, nil+ return handlerutil.MessageOutput("User deleted successfully", ""), nil}// GetUserAvatar returns the custom profile picture for a usertypes.diff --git a/backend/internal/variable/handler.go b/backend/internal/variable/handler.goindex 161413cb0..f4ecb62e1 100644--- a/backend/internal/variable/handler.go+++ b/backend/internal/variable/handler.go@@ -171,14 +171,7 @@ func (h *VariableHandler) UpdateMaterializedVariables(ctx context.Context, inputreturn nil, huma.Error500InternalServerError("Failed to update global variables: " + err.Error())}- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{- Message: "Global variables updated successfully",- },- },- }, nil+ return handlerutil.MessageOutput("Global variables updated successfully", ""), nil}func variableMutationHTTPErrorInternal(err error) error {diff --git a/backend/internal/volume/children/backup/children/policies/service.go b/backend/internal/volume/children/backup/children/policies/service.goindex 46ff57cb5..c64497942 100644--- a/backend/internal/volume/children/backup/children/policies/service.go+++ b/backend/internal/volume/children/backup/children/policies/service.go@@ -5,23 +5,25 @@ import ("errors""fmt""log/slog"+ "path"+ "slices""strings"+ "time"activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"- backuptypes "github.com/getarcaneapp/arcane/types/v2/backup""github.com/getarcaneapp/arcane/types/v2/scheduler""github.com/getarcaneapp/arcane/types/v2/user""github.com/getarcaneapp/arcane/types/v2/volume"- "go.getarcane.app/kit/pkg"+ "github.com/italypaleale/francis/builtin/workflow""gorm.io/gorm"- "github.com/getarcaneapp/arcane/backend/v2/internal/activity""github.com/getarcaneapp/arcane/backend/v2/internal/backup""github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/s3""github.com/getarcaneapp/arcane/backend/v2/internal/settings"activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/entityjobs"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs")@@ -33,7 +35,6 @@ type Dependencies struct {DB *database.DBS3Destinations *s3.S3DestinationServiceSettings *settings.SettingsService- Activity *activity.ActivityServiceAlreadyRunning errorLatestRun func(ctx context.Context, policyID string) (*volume.BackupEntry, error)CreateBackup func(ctx context.Context, volumeName, policyID string) (*volume.Backup, error)@@ -42,8 +43,15 @@ type Dependencies struct {// Service owns per-volume backup policies and their scheduled jobs.type Service struct {- deps Dependencies- jobs *entityjobs.Registry+ deps Dependencies+ jobs *entityjobs.Registry+ flow *flow.Engine+ workflow *flow.Workflow+}++// scheduledBackupInput is the policy a scheduled backup job runs for.+type scheduledBackupInput struct {+ PolicyID string `json:"policyId"`}func NewService(deps Dependencies) *Service {@@ -56,13 +64,32 @@ func (s *Service) SetScheduler(ctx context.Context, dynamicScheduler scheduler.Dreturn s.jobs.SetScheduler(ctx, dynamicScheduler, admissionGate)}-func (s *Service) policiesInternal(ctx context.Context, volumeName string) ([]VolumeBackupPolicy, error) {+// RegisterWorkflows defines the scheduled backup workflow while the host is still unstarted.+func (s *Service) RegisterWorkflows(engine *flow.Engine) error {+ var err error+ s.flow = engine+ s.workflow, err = engine.Define(flow.Definition{+ Name: "volume-backup-policy",+ Version: 1,+ Fingerprint: "cbf39d2f46876dd35ec9c1a792e7946097827079c8fc33c00b0f8034e8044534",+ Concurrency: 4,+ Timeout: 24 * time.Hour,+ Activity: activitylib.StartRequest{+ Type: activitytypes.TypeResourceAction, ResourceType: new("volume_backup"), StartedBy: &user.SystemUser,+ Step: "Creating scheduled backup", LatestMessage: "Creating scheduled volume backup",+ },+ Labels: map[string]string{"backup": "Creating scheduled backup"},+ Steps: []workflow.StepSpec{workflow.Step("backup", engine.Handler(s.runScheduledBackup), workflow.WithMaxAttempts(1))},+ })+ return err+}++func (s *Service) policies(ctx context.Context, volumeName string) ([]VolumeBackupPolicy, error) {var policies []VolumeBackupPolicyif err := s.deps.DB.WithContext(ctx).Where("volume_name = ?", volumeName).Order("created_at ASC").Find(&policies).Error; err != nil {return nil, fmt.Errorf("failed to load volume backup policies: %w", err)}- return policies,- nil+ return policies, nil}// Policy loads one policy of the volume; an empty or unknown ID returns nil.@@ -82,19 +109,13 @@ func (s *Service) Policy(ctx context.Context, volumeName, policyID string) (*Vol}func (s *Service) GetBackupPolicies(ctx context.Context, volumeName string) (*volume.BackupPolicyCollection, error) {- policies, err := s.policiesInternal(ctx, volumeName)+ policies, err := s.policies(ctx, volumeName)if err != nil {return nil, err}result := &volume.BackupPolicyCollection{Policies: make([]volume.BackupPolicy, 0, len(policies))}- destinations := make(map[string]backuptypes.S3Destination)- if s.deps.S3Destinations != nil {- available, listErr := s.deps.S3Destinations.ListS3DestinationsByID(ctx)- if listErr == nil {- result.S3Available = len(available) > 0- destinations = available- }- }+ destinations := s.deps.S3Destinations.DestinationsByID(ctx)+ result.S3Available = len(destinations) > 0for i := range policies {lastRun, runErr := s.deps.LatestRun(ctx, policies[i].ID)if runErr != nil {@@ -117,7 +138,7 @@ func (s *Service) GetBackupPolicies(ctx context.Context, volumeName string) (*vo}func (s *Service) UpdateBackupPolicies(ctx context.Context, volumeName string, updates []volume.UpdateBackupPolicy) (*volume.BackupPolicyCollection, error) {- existing, err := s.policiesInternal(ctx, volumeName)+ existing, err := s.policies(ctx, volumeName)if err != nil {return nil, err}@@ -140,7 +161,7 @@ func (s *Service) UpdateBackupPolicies(ctx context.Context, volumeName string, ureturn nil},Unregister: s.jobs.Unregister,- Reschedule: s.rescheduleInternal,+ Reschedule: s.reschedule,}if runErr := reconcile.Run(ctx, updates); runErr != nil {return nil, runErr@@ -158,70 +179,68 @@ func (s *Service) HasEnabledBackupPolicy(ctx context.Context, volumeName string)return count > 0, nil}-func (s *Service) runScheduledBackupInternal(ctx context.Context, policyID string) (scheduler.Outcome, error) {+// runScheduledBackup is the volume-backup-policy step. A delivery that finds the checkpoint+// it recorded resumes that backup instead of starting another.+func (s *Service) runScheduledBackup(ctx context.Context, t flow.Task) (any, error) {+ var input scheduledBackupInput+ if err := t.Payload(&input); err != nil {+ return nil, err+ }+ // The run belongs to one policy, so its checkpoint is found by kind; it names the volume as the backup did,+ // even if the volume was renamed since the job was registered.+ // A retry after a rename can leave an older checkpoint first, so the latest one is the run's own.+ previous, _ := jobcontext.Run(ctx)+ for _, target := range slices.Backward(previous.Outcome.Targets) {+ if target.ResourceType != "volume_backup" || len(target.RecoveryData) == 0 {+ continue+ }+ // A retry of a backup that already failed runs a new one below.+ if outcome, err := s.deps.Reconcile(ctx, previous, target.ID); !errors.Is(err, backup.ErrBackupSettled) {+ return outcome, err+ }+ break+ }var policy VolumeBackupPolicy- if err := s.deps.DB.WithContext(ctx).Where("id = ? AND enabled = ?", policyID, true).First(&policy).Error; err != nil {+ if err := s.deps.DB.WithContext(ctx).Where("id = ? AND enabled = ?", input.PolicyID, true).First(&policy).Error; err != nil {if errors.Is(err, gorm.ErrRecordNotFound) {return scheduler.Outcome{Status: scheduler.Canceled, Message: "Backup policy disabled or deleted"}, nil}- return scheduler.Outcome{}, err+ return nil, err}- if previous, ok := jobcontext.Run(ctx); ok {- outcome := jobcontext.ConfirmedTarget(previous, policy.VolumeName)- if outcome.Status == scheduler.Succeeded {- return outcome, nil- }+ var remoteErr error+ if policy.S3Enabled {+ root := path.Join(backup.VolumeRoot, s.deps.Settings.GetSettingsConfig().InstanceID.Value)+ remoteErr = backup.CheckScheduledRemote(ctx, s.deps.DB, s.deps.S3Destinations, "volume_backups", policy.S3DestinationID, root)}- remoteDisabled, checkErr := s.disableMissingS3Internal(ctx, &policy)- if checkErr != nil {- return scheduler.Outcome{}, checkErr+ remoteDisabled, disableErr := backup.DisableMissingRemote(remoteErr, policy.LocalEnabled, &policy.S3Enabled, &policy.Enabled, func(field string) (bool, error) {+ return backup.DisableStoredRemote(ctx, s.deps.DB, &VolumeBackupPolicy{}, policy.ID, policy.S3DestinationID, policy.LocalEnabled, field)+ })+ if disableErr != nil {+ return nil, disableErr+ }+ if remoteDisabled {+ s.reschedule(ctx, &policy)}if remoteDisabled && !policy.LocalEnabled {return scheduler.Outcome{Status: scheduler.NeedsAttention, Message: backup.RemoteDisabledMessage}, nil}- var entry *volume.Backup- activityID, err := activitylib.RunHandlerActivity(ctx, s.deps.Activity, activitylib.HandlerOptions{- EnvironmentID: "0",- Type: activitytypes.TypeResourceAction,- ResourceType: "volume_backup",- ResourceID: policy.VolumeName,- ResourceName: policy.VolumeName,- User: &user.SystemUser,- Step: "Creating scheduled backup",- Message: "Creating scheduled volume backup",- SuccessMessage: "Scheduled volume backup created successfully",- Metadata: database.JSON{- "action": "scheduled_volume_backup",- "policyId": policy.ID,- "schedule": policy.Schedule,- "volumeName": policy.VolumeName,- "retentionCount": policy.RetentionCount,- "stopContainers": policy.StopContainers,- "localEnabled": policy.LocalEnabled,- "s3Enabled": policy.S3Enabled,- "s3DestinationId": policy.S3DestinationID,- },- }, func(activityCtx context.Context) error {- var backupErr error- entry, backupErr = s.deps.CreateBackup(activityCtx, policy.VolumeName, policy.ID)- return backupErr- })+ entry, err := s.deps.CreateBackup(ctx, policy.VolumeName, policy.ID)if errors.Is(err, s.deps.AlreadyRunning) {slog.InfoContext(ctx, "Scheduled volume backup skipped; another backup is running", "volume", policy.VolumeName)- return scheduler.Outcome{Status: scheduler.Skipped}, nil+ return scheduler.Outcome{Status: scheduler.Skipped, Message: "Skipped: another backup is running for this volume"}, nil}if err != nil {slog.ErrorContext(ctx, "Scheduled volume backup failed", "volume", policy.VolumeName, "error", err)- return scheduler.Outcome{}, err+ return nil, err}slog.InfoContext(ctx, "Scheduled volume backup completed", "volume", policy.VolumeName, "backupId", entry.ID, "remoteSnapshotId", entry.RemoteSnapshotID)if remoteDisabled {- return scheduler.Outcome{Status: scheduler.Partial, ActivityID: activityID, Message: backup.RemoteDisabledMessage}, nil+ return scheduler.Outcome{Status: scheduler.Partial, Message: backup.RemoteDisabledMessage}, nil}- return scheduler.Outcome{Status: scheduler.Succeeded, ActivityID: activityID}, nil+ return scheduler.Outcome{Status: scheduler.Succeeded, Message: "Scheduled volume backup created successfully"}, nil}-func (s *Service) rescheduleInternal(ctx context.Context, policy *VolumeBackupPolicy) {+func (s *Service) reschedule(ctx context.Context, policy *VolumeBackupPolicy) {if policy == nil {return}@@ -229,22 +248,31 @@ func (s *Service) rescheduleInternal(ctx context.Context, policy *VolumeBackupPos.jobs.Unregister(ctx, policy.ID)return}- policyID := policy.ID- s.jobs.Register(ctx, policyID,- func(ctx context.Context) string {+ policyID, volumeName := policy.ID, policy.VolumeName+ s.jobs.Add(ctx, &flow.Job{+ Engine: s.flow,+ Workflow: s.workflow,+ JobName: s.jobs.JobName(policyID),+ Payload: scheduledBackupInput{PolicyID: policyID},+ Activity: activitylib.StartRequest{+ ResourceID: new(volumeName), ResourceName: new(volumeName),+ Metadata: database.JSON{+ "action": "scheduled_volume_backup", "policyId": policyID, "schedule": policy.Schedule, "volumeName": volumeName,+ "retentionCount": policy.RetentionCount, "stopContainers": policy.StopContainers,+ "localEnabled": policy.LocalEnabled, "s3Enabled": policy.S3Enabled, "s3DestinationId": policy.S3DestinationID,+ },+ },+ ScheduleFn: func(ctx context.Context) string {var current VolumeBackupPolicyif err := s.deps.DB.WithContext(ctx).Where("id = ?", policyID).First(¤t).Error; err != nil {return defaultSchedule}return current.Schedule},- func(ctx context.Context) (scheduler.Outcome, error) {- return s.runScheduledBackupInternal(ctx, policyID)- },- func(ctx context.Context, previous scheduler.Run) (scheduler.Outcome, error) {- return s.deps.Reconcile(ctx, previous, policy.VolumeName)+ FallbackFn: func(ctx context.Context, previous scheduler.Run) (scheduler.Outcome, error) {+ return s.deps.Reconcile(ctx, previous, volumeName)},- )+ })}func (s *Service) RegisterJobsOnStartup(ctx context.Context) {@@ -257,14 +285,14 @@ func (s *Service) RegisterJobsOnStartup(ctx context.Context) {return}for i := range policies {- s.rescheduleInternal(ctx, &policies[i])+ s.reschedule(ctx, &policies[i])}slog.InfoContext(ctx, "Registered scheduled volume backup jobs", "count", len(policies))}// Remove unregisters and deletes every policy of a removed volume.func (s *Service) Remove(ctx context.Context, volumeName string) {- policies, err := s.policiesInternal(ctx, volumeName)+ policies, err := s.policies(ctx, volumeName)if err != nil {return}@@ -280,27 +308,3 @@ func (s *Service) Remove(ctx context.Context, volumeName string) {func (s *Service) Rename(tx *gorm.DB, oldName, newName string) error {return tx.Model(&VolumeBackupPolicy{}).Where("volume_name = ?", oldName).Update("volume_name", newName).Error}--func (s *Service) disableMissingS3Internal(ctx context.Context, policy *VolumeBackupPolicy) (bool, error) {- if !policy.S3Enabled {- return false, nil- }- err := backup.CheckScheduledRemote(ctx, s.deps.DB, s.deps.S3Destinations, "volume_backups", policy.S3DestinationID, "arcane-volume-backups/"+s.deps.Settings.GetSettingsConfig().InstanceID.Value)- if !errors.Is(err, backup.ErrRemoteRepositoryMissing) {- return false, nil- }- column := kit.Ternary(policy.LocalEnabled, "s3_enabled", "enabled")- result := s.deps.DB.WithContext(ctx).Model(&VolumeBackupPolicy{}).- Where("id = ? AND s3_destination_id = ? AND enabled = ? AND s3_enabled = ? AND local_enabled = ?", policy.ID, policy.S3DestinationID, true, true, policy.LocalEnabled).- Update(column, false)- if result.Error != nil || result.RowsAffected == 0 {- return false, result.Error- }- if policy.LocalEnabled {- policy.S3Enabled = false- } else {- policy.Enabled = false- }- s.rescheduleInternal(ctx, policy)- return true, nil-}diff --git a/backend/internal/volume/children/backup/children/policies/service_test.go b/backend/internal/volume/children/backup/children/policies/service_test.goindex 1e2c3227b..389c88412 100644--- a/backend/internal/volume/children/backup/children/policies/service_test.go+++ b/backend/internal/volume/children/backup/children/policies/service_test.go@@ -18,11 +18,11 @@ import (francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing")-func noLatestRunInternal(context.Context, string) (*volume.BackupEntry, error) {+func noLatestRun(context.Context, string) (*volume.BackupEntry, error) {return nil, nil}-func newVolumeAdmissionForTestInternal(t testing.TB) *runs.Admission {+func newVolumeAdmissionForTest(t testing.TB) *runs.Admission {t.Helper()runtime := francistest.New(t)gate := runs.NewAdmission(runtime.Service(), t.Name())@@ -31,21 +31,21 @@ func newVolumeAdmissionForTestInternal(t testing.TB) *runs.Admission {return gate}-type volumeBackupPolicySchedulerInternal struct {+type volumeBackupPolicyScheduler struct {submitted []scheduler.Requestjobs map[string]scheduler.Job}-func (s *volumeBackupPolicySchedulerInternal) AddJob(_ context.Context, job scheduler.Job) error {+func (s *volumeBackupPolicyScheduler) AddJob(_ context.Context, job scheduler.Job) error {s.jobs[job.Name()] = jobreturn nil}-func (s *volumeBackupPolicySchedulerInternal) RemoveJob(_ context.Context, name string) {+func (s *volumeBackupPolicyScheduler) RemoveJob(_ context.Context, name string) {delete(s.jobs, name)}-func (s *volumeBackupPolicySchedulerInternal) HasJob(name string) bool {+func (s *volumeBackupPolicyScheduler) HasJob(name string) bool {_, ok := s.jobs[name]return ok}@@ -55,9 +55,9 @@ func TestVolumeBackupPolicy_UpdateRegistersIndependentJobsAndSettings(t *testingrequire.NoError(t, err)require.NoError(t, gormDB.AutoMigrate(&VolumeBackupPolicy{}))db := &database.DB{DB: gormDB}- jobScheduler := &volumeBackupPolicySchedulerInternal{jobs: make(map[string]scheduler.Job)}- service := NewService(Dependencies{DB: db, LatestRun: noLatestRunInternal})- gate := newVolumeAdmissionForTestInternal(t)+ jobScheduler := &volumeBackupPolicyScheduler{jobs: make(map[string]scheduler.Job)}+ service := NewService(Dependencies{DB: db, LatestRun: noLatestRun})+ gate := newVolumeAdmissionForTest(t)require.NoError(t, service.SetScheduler(t.Context(), jobScheduler, gate))collection, err := service.UpdateBackupPolicies(t.Context(), "app-data", []volume.UpdateBackupPolicy{@@ -131,7 +131,7 @@ func TestVolumeBackupPolicy_UpdateUsesSelectedS3Destination(t *testing.T) {service := NewService(Dependencies{DB: db,S3Destinations: s3.NewS3DestinationService(db, nil),- LatestRun: noLatestRunInternal,+ LatestRun: noLatestRun,})collection, err := service.UpdateBackupPolicies(t.Context(), "app-data", []volume.UpdateBackupPolicy{{Schedule: "0 0 2 * * *",@@ -156,13 +156,13 @@ func TestVolumeBackupPolicy_UpdateUsesSelectedS3Destination(t *testing.T) {require.Equal(t, destination.ID, stored.S3DestinationID)}-func TestBackupPolicyDestinationLookupFailureInternal(t *testing.T) {+func TestBackupPolicyDestinationLookupFailure(t *testing.T) {db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})require.NoError(t, err)require.NoError(t, db.AutoMigrate(&VolumeBackupPolicy{}, &s3.S3Destination{}))policy := &VolumeBackupPolicy{VolumeName: "app-data", Schedule: "0 0 2 * * *", S3DestinationID: "missing"}require.NoError(t, db.Create(policy).Error)- service := NewService(Dependencies{DB: &database.DB{DB: db}, S3Destinations: s3.NewS3DestinationService(&database.DB{DB: db}, nil), LatestRun: noLatestRunInternal})+ service := NewService(Dependencies{DB: &database.DB{DB: db}, S3Destinations: s3.NewS3DestinationService(&database.DB{DB: db}, nil), LatestRun: noLatestRun})for _, drop := range []bool{false, true} {if drop {require.NoError(t, db.Migrator().DropTable(&s3.S3Destination{}))@@ -176,7 +176,7 @@ func TestBackupPolicyDestinationLookupFailureInternal(t *testing.T) {}}-func (s *volumeBackupPolicySchedulerInternal) Submit(_ context.Context, request scheduler.Request) (scheduler.Run, error) {+func (s *volumeBackupPolicyScheduler) Submit(_ context.Context, request scheduler.Request) (scheduler.Run, error) {s.submitted = append(s.submitted, request)return scheduler.Run{ID: request.RunID, JobID: request.JobID, EnvironmentID: request.EnvironmentID, Status: scheduler.Queued}, nil}diff --git a/backend/internal/volume/children/backup/children/restore/handler.go b/backend/internal/volume/children/backup/children/restore/handler.goindex 5306408f7..d7cf5201b 100644--- a/backend/internal/volume/children/backup/children/restore/handler.go+++ b/backend/internal/volume/children/backup/children/restore/handler.go@@ -3,14 +3,12 @@ package restoreimport ("context""errors"- "strings""github.com/danielgtaylor/huma/v2"activitytypes "github.com/getarcaneapp/arcane/types/v2/activity""github.com/getarcaneapp/arcane/types/v2/backup""github.com/getarcaneapp/arcane/types/v2/base"uploadtypes "github.com/getarcaneapp/arcane/types/v2/upload"- "github.com/samber/mo""github.com/getarcaneapp/arcane/backend/v2/internal/activity""github.com/getarcaneapp/arcane/backend/v2/internal/common"@@ -78,12 +76,7 @@ func (h *Handler) RestoreBackup(ctx context.Context, input *RestoreBackupInput)if err != nil {return nil, huma.Error500InternalServerError(err.Error())}- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{Message: "Restore initiated successfully", ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()},- },- }, nil+ return handlerutil.MessageOutput("Restore initiated successfully", activityID), nil}func (h *Handler) RestoreBackupFiles(ctx context.Context, input *RestoreBackupFilesInput) (*handlerutil.Out[base.MessageResponse], error) {@@ -120,12 +113,7 @@ func (h *Handler) RestoreBackupFiles(ctx context.Context, input *RestoreBackupFireturn nil, huma.Error500InternalServerError(err.Error())}- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{Message: "Restore initiated successfully", ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()},- },- }, nil+ return handlerutil.MessageOutput("Restore initiated successfully", activityID), nil}func (h *Handler) UploadAndRestore(ctx context.Context, input *UploadAndRestoreInput) (*handlerutil.Out[base.MessageResponse], error) {@@ -164,10 +152,5 @@ func (h *Handler) UploadAndRestore(ctx context.Context, input *UploadAndRestoreIif err != nil {return nil, huma.Error500InternalServerError(err.Error())}- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{Message: "Backup uploaded and restored successfully", ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()},- },- }, nil+ return handlerutil.MessageOutput("Backup uploaded and restored successfully", activityID), nil}diff --git a/backend/internal/volume/children/backup/handler.go b/backend/internal/volume/children/backup/handler.goindex e8d0fecbc..3f9bbd1b6 100644--- a/backend/internal/volume/children/backup/handler.go+++ b/backend/internal/volume/children/backup/handler.go@@ -13,6 +13,7 @@ import ("github.com/getarcaneapp/arcane/types/v2/base""github.com/getarcaneapp/arcane/types/v2/volume""github.com/samber/mo"+ kit "go.getarcane.app/kit/pkg""github.com/getarcaneapp/arcane/backend/v2/internal/activity""github.com/getarcaneapp/arcane/backend/v2/internal/database"@@ -136,19 +137,14 @@ func (h *Handler) ListBackups(ctx context.Context, input *ListBackupsInput) (*Lireturn nil, huma.Error500InternalServerError(err.Error())}- warning := h.service.backupMountWarningInternal(ctx)+ warning := h.service.backupMountWarning(ctx)return &ListBackupsOutput{Body: VolumeBackupPaginatedResponse{Success: true,Data: backups,Pagination: handlerutil.PaginationResponse(paginationResp),- Warnings: func() []string {- if warning == "" {- return nil- }- return []string{warning}- }(),+ Warnings: kit.Ternary(warning == "", nil, []string{warning}),},}, nil}@@ -158,11 +154,7 @@ func (h *Handler) CreateBackup(ctx context.Context, input *CreateBackupInput) (*if err != nil {return nil, err}- request := volume.CreateBackupRequest{}- if input.Body != nil {- request = *input.Body- }- entry, err := h.service.StartBackup(utils.ActivityRuntimeContext(ctx, h.appCtx), input.EnvironmentID, input.VolumeName, *user, request)+ entry, err := h.service.StartBackup(utils.ActivityRuntimeContext(ctx, h.appCtx), input.EnvironmentID, input.VolumeName, *user, kit.FromPtr(input.Body))if errors.Is(err, h.service.deps.AlreadyRunning) {return nil, huma.Error409Conflict(err.Error())}@@ -197,12 +189,7 @@ func (h *Handler) DeleteBackup(ctx context.Context, input *DeleteBackupInput) (*if err != nil {return nil, huma.Error500InternalServerError(err.Error())}- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{Message: "Backup deleted successfully", ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()},- },- }, nil+ return handlerutil.MessageOutput("Backup deleted successfully", activityID), nil}func (h *Handler) DiscoverBackups(ctx context.Context, input *DiscoverVolumeBackupsInput) (*DiscoverVolumeBackupsOutput, error) {@@ -225,9 +212,6 @@ func (h *Handler) DiscoverBackups(ctx context.Context, input *DiscoverVolumeBack}func (h *Handler) UploadBackup(ctx context.Context, input *UploadBackupInput) (*handlerutil.Out[*volume.Backup], error) {- if h.service == nil {- return nil, huma.Error500InternalServerError("service not available")- }user, _ := userctx.CurrentUserFromContext(ctx)var uploaded *volume.BackupruntimeCtx := utils.ActivityRuntimeContext(ctx, h.appCtx)diff --git a/backend/internal/volume/children/backup/helpers.go b/backend/internal/volume/children/backup/helpers.goindex 1fbb97a39..13d82fede 100644--- a/backend/internal/volume/children/backup/helpers.go+++ b/backend/internal/volume/children/backup/helpers.go@@ -1,185 +1,105 @@package backupimport (+ "cmp""context""errors""fmt""log/slog"- "path"+ "slices""strings""time"- "github.com/getarcaneapp/arcane/types/v2/scheduler"usertypes "github.com/getarcaneapp/arcane/types/v2/user""github.com/getarcaneapp/arcane/types/v2/volume""github.com/moby/moby/api/types/container""github.com/moby/moby/api/types/mount""github.com/moby/moby/client"- "github.com/samber/mo""go.getarcane.app/docker"- "go.getarcane.app/kit/pkg"+ kit "go.getarcane.app/kit/pkg""github.com/getarcaneapp/arcane/backend/v2/internal/backup"+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"+ "github.com/getarcaneapp/arcane/backend/v2/internal/event""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/volumehelper""github.com/getarcaneapp/arcane/backend/v2/pkg/projects"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs")-type backupStorageMode string-const (- // backupStorageModeArcaneMount means backup helpers mirror an existing Arcane- // container mount at /backups. This intentionally covers any mount the Arcane- // container already has at /backups, not exclusively bind mounts.- backupStorageModeArcaneMount backupStorageMode = "arcane_mount"- // backupStorageModeNamedVolumeFallback means no suitable Arcane container- // mount was found, so Arcane's dedicated named backup volume is used.- backupStorageModeNamedVolumeFallback backupStorageMode = "named_volume_fallback"-backupMountMissingWarning = "No volume is mounted at /backups in the Arcane container. Backups will only live inside Docker unless you mount a host path."volumeBackupContainerRecoveryTimeout = 30 * time.SecondvolumeBackupContainerRecoveryInterval = 500 * time.Millisecond- volumeRusticRepositoryPath = "/repository/volumes"- localVolumeRepositoryID = "volumes:local"- legacyVolumePasswordSaltInternal = "arcane-volume-backups:"+ volumeRusticRepositoryPath = "/repository/volumes"+ localVolumeRepositoryID = "volumes:local"+ legacyVolumePasswordSalt = "arcane-volume-backups:"systemRecoverySnapshotLabel = "arcane-system-recovery")-type backupStorageMountInternal struct {- mode backupStorageMode- mount mount.Mount- requiresEnsure bool-}--func resolveBackupStorageMountFromMountsInternal(ctx context.Context, mounts []container.MountPoint, target string, readOnly bool) mo.Option[backupStorageMountInternal] {+// resolveBackupStorageMountFromMounts mirrors the Arcane container's /backups mount at target, or returns nil.+func resolveBackupStorageMountFromMounts(ctx context.Context, mounts []container.MountPoint, target string, readOnly bool) *mount.Mount {mirroredMount := docker.MountForDestination(mounts, "/backups", target)if mirroredMount == nil {- return mo.None[backupStorageMountInternal]()+ return nil}- // MountForDestination only returns non-nil for bind and named volume mounts.-if !readOnly && mirroredMount.ReadOnly {slog.WarnContext(ctx, "volume service: requested writable backup mount but source is read-only; writes may fail")}mirroredMount.ReadOnly = readOnly-- return mo.Some(backupStorageMountInternal{- mode: backupStorageModeArcaneMount,- mount: *mirroredMount,- })-}--func (s *Service) resolveBackupStorageMountInternal(ctx context.Context, dockerClient *client.Client, target string, readOnly bool) backupStorageMountInternal {- if dockerClient != nil {- inspect, err := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient)- if err != nil {- slog.WarnContext(ctx, "volume service: failed to inspect arcane container for backup mount resolution, falling back to named volume", "error", err.Error())- } else if resolved, ok := resolveBackupStorageMountFromMountsInternal(ctx, inspect.Mounts, target, readOnly).Get(); ok {- return resolved- }- }-- return backupStorageMountInternal{- mode: backupStorageModeNamedVolumeFallback,- mount: mount.Mount{- Type: mount.TypeVolume,- Source: s.deps.BackupVolumeName,- Target: target,- ReadOnly: readOnly,- },- requiresEnsure: true,- }-}--func (s *Service) resolveUsableBackupStorageMountInternal(ctx context.Context, dockerClient *client.Client, target string, readOnly bool) (backupStorageMountInternal, error) {- backupStorage := s.resolveBackupStorageMountInternal(ctx, dockerClient, target, readOnly)- if backupStorage.requiresEnsure {- if err := s.ensureBackupVolumeInternal(ctx); err != nil {- return backupStorageMountInternal{}, err- }- }- return backupStorage, nil+ return mirroredMount}-func backupMountWarningForStorageInternal(storage backupStorageMountInternal) string {- return kit.Ternary(storage.mode == backupStorageModeArcaneMount, "", backupMountMissingWarning)-}--func backupMountWarningFromArcaneMountsInternal(ctx context.Context, mounts []container.MountPoint) string {- backupStorage, ok := resolveBackupStorageMountFromMountsInternal(ctx, mounts, "/backups", true).Get()- if ok {- return backupMountWarningForStorageInternal(backupStorage)- }-- // Backward compatibility: historically either /backups or /restores mount- // suppressed the warning. Preserve that user-visible behavior.- for _, m := range mounts {- if m.Destination == "/restores" {- return ""- }+// backupMountWarningFromArcaneMounts warns unless Arcane mounts /backups; a /restores mount also suppresses it for compatibility.+func backupMountWarningFromArcaneMounts(mounts []container.MountPoint) string {+ restoresMounted := slices.ContainsFunc(mounts, func(m container.MountPoint) bool { return m.Destination == "/restores" })+ if docker.MountForDestination(mounts, "/backups", "/backups") != nil || restoresMounted {+ return ""}-return backupMountMissingWarning}-func (s *Service) backupMountWarningInternal(ctx context.Context) string {+func (s *Service) backupMountWarning(ctx context.Context) string {dockerClient, err := s.deps.Docker.GetClient(ctx)if err != nil {return ""}-// Cannot determine Arcane mount status (e.g. running outside Docker); suppress warning.inspect, err := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient)if err != nil {return ""}-- return backupMountWarningFromArcaneMountsInternal(ctx, inspect.Mounts)+ return backupMountWarningFromArcaneMounts(inspect.Mounts)}// StorageMount resolves the backup repository mount shared with system-backup and workspace operations.+// It mirrors Arcane's own /backups mount and otherwise falls back to the dedicated named backup volume.func (s *Service) StorageMount(ctx context.Context, dockerClient *client.Client, target string, readOnly bool) (mount.Mount, error) {- storage, err := s.resolveUsableBackupStorageMountInternal(ctx, dockerClient, target, readOnly)- if err != nil {- return mount.Mount{}, err+ if dockerClient != nil {+ inspect, err := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient)+ if err != nil {+ slog.WarnContext(ctx, "volume service: failed to inspect arcane container for backup mount resolution, falling back to named volume", "error", err.Error())+ } else if mirrored := resolveBackupStorageMountFromMounts(ctx, inspect.Mounts, target, readOnly); mirrored != nil {+ return *mirrored, nil+ }}- return storage.mount, nil-}--func (s *Service) ensureBackupVolumeInternal(ctx context.Context) error {slog.DebugContext(ctx, "volume service: ensure backup volume", "backupVolume", s.deps.BackupVolumeName)- dockerClient, err := s.deps.Docker.GetClient(ctx)+ ensureClient, err := s.deps.Docker.GetClient(ctx)if err != nil {- return err+ return mount.Mount{}, err}-- _, err = dockerClient.VolumeInspect(ctx, s.deps.BackupVolumeName, client.VolumeInspectOptions{})- if err != nil {- _, err = dockerClient.VolumeCreate(ctx, client.VolumeCreateOptions{- Name: s.deps.BackupVolumeName,- })- if err != nil {- return fmt.Errorf("failed to create backup volume: %w", err)+ if _, inspectErr := ensureClient.VolumeInspect(ctx, s.deps.BackupVolumeName, client.VolumeInspectOptions{}); inspectErr != nil {+ if _, createErr := ensureClient.VolumeCreate(ctx, client.VolumeCreateOptions{Name: s.deps.BackupVolumeName}); createErr != nil {+ return mount.Mount{}, fmt.Errorf("failed to create backup volume: %w", createErr)}}- return nil+ return mount.Mount{Type: mount.TypeVolume, Source: s.deps.BackupVolumeName, Target: target, ReadOnly: readOnly}, nil}-func (- s *Service,-) stopRunningContainersForBackupInternal(- ctx context.Context,- dockerClient *client.Client,- volumeName string,- user usertypes.Actor,- refuseArcaneWriters bool,-) (- []container.Summary,- error,-) {+func (s *Service) stopRunningContainersForBackup(ctx context.Context, dockerClient *client.Client, volumeName string, user usertypes.Actor, refuseArcaneWriters bool) ([]container.Summary, error) {if s.deps.StopContainer == nil || s.deps.StartContainer == nil {return nil, errors.New("container service is unavailable")}@@ -195,10 +115,8 @@ func (arcaneOwned = append(arcaneOwned, candidate)continue}- // Arcane's own helper containers mount the volume as well, but they are- // auto-removed when stopped and recreated on demand. Stopping one here- // would leave nothing for the restart pass to find, failing an- // otherwise successful backup or restore.+ // Arcane's helper containers are auto-removed when stopped, so stopping+ // one would leave nothing for the restart pass and fail the operation.if strings.EqualFold(candidate.Labels[libarcane.InternalResourceLabel], "true") {continue}@@ -212,16 +130,14 @@ func (}}containerIDs := docker.FilterContainersUsingVolume(eligible, volumeName)- containersByID := make(map[string]container.Summary, len(eligible))- for _, candidate := range eligible {- containersByID[candidate.ID] = candidate- }stopped := make([]container.Summary, 0, len(containerIDs))- for _, containerID := range containerIDs {- candidate := containersByID[containerID]- if stopContainerErr := s.deps.StopContainer(ctx, containerID, user); stopContainerErr != nil {- stillStopped, restartErr := s.startContainersAfterBackupInternal(context.WithoutCancel(ctx), dockerClient, stopped, user)- return stillStopped, errors.Join(fmt.Errorf("failed to stop container %s before volume backup: %w", containerID, stopContainerErr), restartErr)+ for _, candidate := range eligible {+ if !slices.Contains(containerIDs, candidate.ID) {+ continue+ }+ if stopContainerErr := s.deps.StopContainer(ctx, candidate.ID, user); stopContainerErr != nil {+ stillStopped, restartErr := s.startContainersAfterBackup(context.WithoutCancel(ctx), dockerClient, stopped, user)+ return stillStopped, errors.Join(fmt.Errorf("failed to stop container %s before volume backup: %w", candidate.ID, stopContainerErr), restartErr)}stopped = append(stopped, candidate)}@@ -229,7 +145,7 @@ func (}//nolint:gocognit // recovery retries must reconcile IDs, names, and Compose identities in one bounded loop-func (s *Service) startContainersAfterBackupInternal(ctx context.Context, dockerClient *client.Client, stoppedContainers []container.Summary, user usertypes.Actor) ([]container.Summary, error) {+func (s *Service) startContainersAfterBackup(ctx context.Context, dockerClient *client.Client, stoppedContainers []container.Summary, user usertypes.Actor) ([]container.Summary, error) {recoveryCtx, cancel := context.WithTimeout(ctx, volumeBackupContainerRecoveryTimeout)defer cancel()@@ -305,113 +221,80 @@ func (s *Service) startContainersAfterBackupInternal(ctx context.Context, dockerreturn nil, nil}-func sanitizeBackupPathInternal(input string) (string, error) {- trimmed := strings.TrimSpace(input)- if trimmed == "" {- return "", errors.New("invalid path: empty")- }- cleaned := path.Clean(trimmed)- if cleaned == "." || cleaned == "/" {- return "", fmt.Errorf("invalid path: %s", input)- }- if path.IsAbs(cleaned) {- cleaned = strings.TrimPrefix(cleaned, "/")- }- if cleaned == "" || cleaned == "." || cleaned == "/" || strings.HasPrefix(cleaned, "..") || strings.Contains(cleaned, "/../") {- return "", fmt.Errorf("invalid path: %s", input)- }- return cleaned, nil-}--func sanitizeBackupIDInternal(backupID string) (string, error) {- cleaned, err := sanitizeBackupPathInternal(backupID)+func backupArchiveFilename(backupID string) (string, error) {+ name, err := kit.ValidateFileName(backupID)if err != nil {return "", fmt.Errorf("invalid backup id: %w", err)}- if strings.Contains(cleaned, "/") {- return "", errors.New("invalid backup id: path separators not allowed")- }- return cleaned, nil+ return name + ".tar.gz", nil}-func backupArchiveFilenameInternal(backupID string) (string, error) {- sanitizedBackupID, err := sanitizeBackupIDInternal(backupID)- if err != nil {- return "", err- }- return sanitizedBackupID + ".tar.gz", nil-}--// volumeBackupPasswordInternal returns the recovery key once one is stored, re-keying the given repositories off the legacy derivation first.-func (s *Service) volumeBackupPasswordInternal(ctx context.Context, dockerClient *client.Client, repositories ...backup.Repository) (string, error) {+// volumeBackupPassword returns the recovery key once one is stored, re-keying the given repositories off the legacy derivation first.+func (s *Service) volumeBackupPassword(ctx context.Context, dockerClient *client.Client, repositories ...backup.Repository) (string, error) {+ legacyPassword := kit.SHA256Hex(legacyVolumePasswordSalt + s.deps.EncryptionKey)if s.deps.RecoveryKeys == nil {- return kit.SHA256Hex(legacyVolumePasswordSaltInternal + s.deps.EncryptionKey), nil+ return legacyPassword, nil}key, err := s.deps.RecoveryKeys.Get(ctx)if errors.Is(err, backup.ErrRecoveryKeyNotConfigured) {- return kit.SHA256Hex(legacyVolumePasswordSaltInternal + s.deps.EncryptionKey), nil+ return legacyPassword, nil}if err != nil {return "", err}- for _, repository := range repositories {- s.rekeyRepositoryInternal(ctx, dockerClient, repository, key)- }- return key, nil-}--// rekeyRepositoryInternal re-keys a legacy repository once per process; one that opens with neither password is left for the caller's operation to create or report.-func (s *Service) rekeyRepositoryInternal(ctx context.Context, dockerClient *client.Client, repository backup.Repository, recoveryKey string) {if s.deps.Engine == nil {- return+ return key, nil}- if done, _ := s.rekeyed.Load(repository.ID); done == recoveryKey {- return- }- if _, err := s.deps.Engine.ListSnapshots(ctx, dockerClient, repository, recoveryKey); err == nil {- s.rekeyed.Store(repository.ID, recoveryKey)- return- }- if repository.ID == localVolumeRepositoryID {- writable, err := s.localRusticRepositoryInternal(ctx, dockerClient, false)- if err != nil {- slog.WarnContext(ctx, "could not open the local volume backup repository for re-keying", "error", err.Error())- return+ // Re-key each legacy repository once per process; one that opens with neither password is left for the caller to create or report.+ for _, repository := range repositories {+ if done, _ := s.rekeyed.Load(repository.ID); done == key {+ continue}- repository = writable- }- if err := s.deps.Engine.ChangeRepositoryPassword(ctx, dockerClient, repository, kit.SHA256Hex(legacyVolumePasswordSaltInternal+s.deps.EncryptionKey), recoveryKey); err != nil {- slog.DebugContext(ctx, "volume backup repository was not re-keyed", "repository", repository.ID, "error", err.Error())- return+ if _, listErr := s.deps.Engine.ListSnapshots(ctx, dockerClient, repository, key); listErr == nil {+ s.rekeyed.Store(repository.ID, key)+ continue+ }+ if repository.ID == localVolumeRepositoryID {+ writable, localErr := s.localRusticRepository(ctx, dockerClient, false)+ if localErr != nil {+ slog.WarnContext(ctx, "could not open the local volume backup repository for re-keying", "error", localErr.Error())+ continue+ }+ repository = writable+ }+ if changeErr := s.deps.Engine.ChangeRepositoryPassword(ctx, dockerClient, repository, legacyPassword, key); changeErr != nil {+ slog.DebugContext(ctx, "volume backup repository was not re-keyed", "repository", repository.ID, "error", changeErr.Error())+ continue+ }+ slog.InfoContext(ctx, "Re-keyed volume backup repository to the recovery key", "repository", repository.ID)+ s.rekeyed.Store(repository.ID, key)}- slog.InfoContext(ctx, "Re-keyed volume backup repository to the recovery key", "repository", repository.ID)- s.rekeyed.Store(repository.ID, recoveryKey)+ return key, nil}-func (s *Service) localRusticRepositoryInternal(ctx context.Context, dockerClient *client.Client, readOnly bool) (backup.Repository, error) {- storage, err := s.resolveUsableBackupStorageMountInternal(ctx, dockerClient, "/repository", readOnly)+func (s *Service) localRusticRepository(ctx context.Context, dockerClient *client.Client, readOnly bool) (backup.Repository, error) {+ storage, err := s.StorageMount(ctx, dockerClient, "/repository", readOnly)if err != nil {return backup.Repository{}, err}return backup.Repository{ID: localVolumeRepositoryID,Environment: []string{"RUSTIC_REPOSITORY=" + volumeRusticRepositoryPath},- Mounts: []mount.Mount{storage.mount},+ Mounts: []mount.Mount{storage},}, nil}-func (s *Service) remoteRusticRepositoryInternal(ctx context.Context, destinationID string) (backup.Repository, error) {- return s.remoteRusticRepositoryForInstanceInternal(ctx, destinationID, "")+// remoteInstanceID resolves the instance whose S3 root holds a backup; empty means this instance.+func (s *Service) remoteInstanceID(instanceID string) string {+ return cmp.Or(strings.TrimSpace(instanceID), strings.TrimSpace(s.deps.Settings.GetSettingsConfig().InstanceID.Value))}-// remoteRusticRepositoryForInstanceInternal addresses one instance's root on the destination; empty means this instance.-func (s *Service) remoteRusticRepositoryForInstanceInternal(ctx context.Context, destinationID, instanceID string) (backup.Repository, error) {+// remoteRusticRepository addresses one instance's root on the destination; empty instanceID means this instance.+func (s *Service) remoteRusticRepository(ctx context.Context, destinationID, instanceID string) (backup.Repository, error) {if s.deps.S3Destinations == nil {- return backup.Repository{}, errors.New("S3 backup service is unavailable")- }- if strings.TrimSpace(instanceID) == "" {- instanceID = strings.TrimSpace(s.deps.Settings.GetSettingsConfig().InstanceID.Value)+ return backup.Repository{}, errors.New("S3 backup destinations are unavailable")}+ instanceID = s.remoteInstanceID(instanceID)if instanceID == "" {return backup.Repository{}, errors.New("arcane instance ID is unavailable")}@@ -421,94 +304,74 @@ func (s *Service) remoteRusticRepositoryForInstanceInternal(ctx context.Context,}return backup.Repository{ID: "volumes:s3:" + destinationID + ":" + instanceID,- Environment: configuration.RusticEnvironment("arcane-volume-backups", instanceID),+ Environment: configuration.RusticEnvironment(backup.VolumeRoot, instanceID),}, nil}-func (s *Service) rusticRepositoryForBackupInternal(ctx context.Context, dockerClient *client.Client, entry *volume.Backup) (backup.Repository, string, error) {- if entry.LocalSnapshotID != "" {- repository, err := s.localRusticRepositoryInternal(ctx, dockerClient, true)- return repository, entry.LocalSnapshotID, err+// acquireVolumeRun takes the volume's run lease, failing with AlreadyRunning while another backup operation holds it.+func (s *Service) acquireVolumeRun(ctx context.Context, volumeName string) (*runs.Lease, error) {+ lease, admitted, err := s.deps.Engine.TryAcquireRun(ctx, backup.VolumeAdmissionScope, volumeName)+ if err != nil {+ return nil, err}- if entry.RemoteSnapshotID != "" {- repository, err := s.remoteRusticRepositoryForInstanceInternal(ctx, entry.S3DestinationID, entry.RemoteInstanceID)- return repository, entry.RemoteSnapshotID, err+ if !admitted {+ return nil, s.deps.AlreadyRunning}- return backup.Repository{}, "", errors.New("volume backup has no Rustic snapshot")+ return lease, nil}-func (s *Service) createBackupTempContainerWithMountInternal(ctx context.Context, dockerClient *client.Client, helperImage string, backupMount mount.Mount) (string, func(), error) {- var err error- if dockerClient == nil {- dockerClient, err = s.deps.Docker.GetClient(ctx)- if err != nil {- return "", nil, err- }+// backupDestination names the destination that holds a backup's local and remote copies.+func backupDestination(local, remote bool) volume.BackupDestination {+ switch {+ case local && remote:+ return volume.BackupDestinationLocalS3+ case remote:+ return volume.BackupDestinationS3+ default:+ return volume.BackupDestinationLocal}+}- if strings.TrimSpace(helperImage) == "" {- helperImage, err = s.deps.HelperImage(ctx, dockerClient)- if err != nil {- return "", nil, err- }+// logBackupEvent records a volume backup event; a nil user logs as the system user.+func (s *Service) logBackupEvent(ctx context.Context, eventType event.EventType, volumeName string, user *usertypes.Actor, metadata database.JSON) {+ actor := cmp.Or(user, &usertypes.SystemUser)+ if err := s.deps.Events.LogVolumeEvent(ctx, eventType, volumeName, volumeName, actor.ID, actor.Username, "0", metadata); err != nil {+ slog.WarnContext(ctx, "could not log volume backup event", "volume", volumeName, "action", metadata["action"], "error", err)}+}- config := &container.Config{- Image: helperImage,- Cmd: []string{"sleep", "infinity"},- NetworkDisabled: true,- Labels: volumehelper.Labels(),+func (s *Service) createBackupTempContainerWithMount(ctx context.Context, dockerClient *client.Client, backupMount mount.Mount) (string, func(), error) {+ helperImage, err := s.deps.HelperImage(ctx, dockerClient)+ if err != nil {+ return "", nil, err}-- hostConfig := volumehelper.HostConfig(helperImage, nil, []mount.Mount{backupMount})-resp, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{- Config: config,- HostConfig: hostConfig,+ Config: &container.Config{Image: helperImage, Cmd: []string{"sleep", "infinity"}, NetworkDisabled: true, Labels: volumehelper.Labels()},+ HostConfig: volumehelper.HostConfig(helperImage, nil, []mount.Mount{backupMount}),})if err != nil {return "", nil, fmt.Errorf("failed to create backup temp container: %w", err)}-if _, containerStartErr := dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); containerStartErr != nil {_, _ = dockerClient.ContainerRemove(ctx, resp.ID, volumehelper.RemoveOptions())return "", nil, fmt.Errorf("failed to start backup temp container: %w", containerStartErr)}-- cleanup := func() {+ return resp.ID, func() {_, _ = dockerClient.ContainerRemove(context.WithoutCancel(ctx), resp.ID, volumehelper.RemoveOptions())- }-- return resp.ID, cleanup, nil+ }, nil}-func (s *Service) createBackupTempContainerInternal(ctx context.Context, dockerClient *client.Client, target string, readOnly bool) (string, func(), error) {+func (s *Service) createBackupTempContainer(ctx context.Context, dockerClient *client.Client, target string, readOnly bool) (string, func(), error) {slog.DebugContext(ctx, "volume service: create backup temp container", "target", target, "readOnly", readOnly)var err errorif dockerClient == nil {- dockerClient, err = s.deps.Docker.GetClient(ctx)- if err != nil {+ if dockerClient, err = s.deps.Docker.GetClient(ctx); err != nil {return "", nil, err}}-- backupStorage, err := s.resolveUsableBackupStorageMountInternal(ctx, dockerClient, target, readOnly)+ backupMount, err := s.StorageMount(ctx, dockerClient, target, readOnly)if err != nil {return "", nil, err}-- return s.createBackupTempContainerWithMountInternal(ctx, dockerClient, "", backupStorage.mount)-}--func volumeSourceMountInternal(volumeName string) mount.Mount {- return mount.Mount{Type: mount.TypeVolume, Source: volumeName, Target: "/volume", ReadOnly: true}-}--func backupDestinationAttemptedInternal(previous scheduler.Run, backupID, destination string) bool {- for _, evidence := range previous.Outcome.Targets {- if evidence.ID == backupID+":"+destination {- return true- }- }- return false+ return s.createBackupTempContainerWithMount(ctx, dockerClient, backupMount)}diff --git a/backend/internal/volume/children/backup/helpers_test.go b/backend/internal/volume/children/backup/helpers_test.goindex 2ab24dbd5..200f8e645 100644--- a/backend/internal/volume/children/backup/helpers_test.go+++ b/backend/internal/volume/children/backup/helpers_test.go@@ -24,19 +24,17 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/event")-func TestResolveBackupStorageMountFromMountsInternal(t *testing.T) {+func TestResolveBackupStorageMountFromMounts(t *testing.T) {tests := []struct {name stringmounts []container.MountPointtarget stringreadOnly boolwantResolved bool- wantMode backupStorageModewantType mount.TypewantSource stringwantTarget stringwantReadOnly bool- wantEnsure bool}{{name: "mirrors bind mount",@@ -46,7 +44,6 @@ func TestResolveBackupStorageMountFromMountsInternal(t *testing.T) {target: "/volume",readOnly: true,wantResolved: true,- wantMode: backupStorageModeArcaneMount,wantType: mount.TypeBind,wantSource: "/host/backups",wantTarget: "/volume",@@ -60,7 +57,6 @@ func TestResolveBackupStorageMountFromMountsInternal(t *testing.T) {target: "/volume",readOnly: false,wantResolved: true,- wantMode: backupStorageModeArcaneMount,wantType: mount.TypeBind,wantSource: "/host/backups",wantTarget: "/volume",@@ -74,7 +70,6 @@ func TestResolveBackupStorageMountFromMountsInternal(t *testing.T) {target: "/backups",readOnly: false,wantResolved: true,- wantMode: backupStorageModeArcaneMount,wantType: mount.TypeVolume,wantSource: "arcane-backups",wantTarget: "/backups",@@ -99,40 +94,21 @@ func TestResolveBackupStorageMountFromMountsInternal(t *testing.T) {for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {- got, ok := resolveBackupStorageMountFromMountsInternal(t.Context(), tt.mounts, tt.target, tt.readOnly).Get()- require.Equal(t, tt.wantResolved, ok)+ got := resolveBackupStorageMountFromMounts(t.Context(), tt.mounts, tt.target, tt.readOnly)+ require.Equal(t, tt.wantResolved, got != nil)if !tt.wantResolved {return}- require.Equal(t, tt.wantMode, got.mode)- require.Equal(t, tt.wantType, got.mount.Type)- require.Equal(t, tt.wantSource, got.mount.Source)- require.Equal(t, tt.wantTarget, got.mount.Target)- require.Equal(t, tt.wantReadOnly, got.mount.ReadOnly)- require.Equal(t, tt.wantEnsure, got.requiresEnsure)+ require.Equal(t, tt.wantType, got.Type)+ require.Equal(t, tt.wantSource, got.Source)+ require.Equal(t, tt.wantTarget, got.Target)+ require.Equal(t, tt.wantReadOnly, got.ReadOnly)})}}-func TestResolveBackupStorageMountInternalFallsBackToNamedVolume(t *testing.T) {- svc := NewService(Dependencies{BackupVolumeName: "arcane-backups"})-- got := svc.resolveBackupStorageMountInternal(t.Context(), nil, "/backups", true)- require.Equal(t, backupStorageModeNamedVolumeFallback, got.mode)- require.Equal(t, mount.TypeVolume, got.mount.Type)- require.Equal(t, "arcane-backups", got.mount.Source)- require.Equal(t, "/backups", got.mount.Target)- require.True(t, got.mount.ReadOnly)- require.True(t, got.requiresEnsure)-}--func TestBackupMountWarningForStorageInternal(t *testing.T) {- require.Empty(t, backupMountWarningForStorageInternal(backupStorageMountInternal{mode: backupStorageModeArcaneMount}))- require.Equal(t, backupMountMissingWarning, backupMountWarningForStorageInternal(backupStorageMountInternal{mode: backupStorageModeNamedVolumeFallback}))-}--func TestBackupMountWarningFromArcaneMountsInternal(t *testing.T) {+func TestBackupMountWarningFromArcaneMounts(t *testing.T) {tests := []struct {name stringmounts []container.MountPoint@@ -184,12 +160,12 @@ func TestBackupMountWarningFromArcaneMountsInternal(t *testing.T) {for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {- require.Equal(t, tt.want, backupMountWarningFromArcaneMountsInternal(t.Context(), tt.mounts))+ require.Equal(t, tt.want, backupMountWarningFromArcaneMounts(tt.mounts))})}}-func setupVolumeBackupLifecycleTestInternal(t *testing.T, handler http.Handler) (*Service, *client.Client) {+func setupVolumeBackupLifecycleTest(t *testing.T, handler http.Handler) (*Service, *client.Client) {t.Helper()server := httptest.NewServer(handler)@@ -235,14 +211,14 @@ func TestVolumeBackupContainerLifecycleStopsAndRestartsOnlyRunningContainersUsin}})- service, dockerClient := setupVolumeBackupLifecycleTestInternal(t, serverHandler)+ service, dockerClient := setupVolumeBackupLifecycleTest(t, serverHandler)actor := user.Actor{ID: "user-1", Username: "tester"}- stopped, err := service.stopRunningContainersForBackupInternal(t.Context(), dockerClient, "app-data", actor, false)+ stopped, err := service.stopRunningContainersForBackup(t.Context(), dockerClient, "app-data", actor, false)require.NoError(t, err)require.Len(t, stopped, 1)require.Equal(t, "uses-volume", stopped[0].ID)- remaining, err := service.startContainersAfterBackupInternal(t.Context(), dockerClient, stopped, actor)+ remaining, err := service.startContainersAfterBackup(t.Context(), dockerClient, stopped, actor)require.NoError(t, err)require.Empty(t, remaining)require.Equal(t, []string{"stop:uses-volume", "start:uses-volume"}, operations)@@ -279,9 +255,9 @@ func TestVolumeBackupContainerLifecycleRollsBackStoppedContainersOnStopFailure(t}})- service, dockerClient := setupVolumeBackupLifecycleTestInternal(t, serverHandler)+ service, dockerClient := setupVolumeBackupLifecycleTest(t, serverHandler)actor := user.Actor{ID: "user-1", Username: "tester"}- stillStopped, err := service.stopRunningContainersForBackupInternal(t.Context(), dockerClient, "app-data", actor, false)+ stillStopped, err := service.stopRunningContainersForBackup(t.Context(), dockerClient, "app-data", actor, false)require.ErrorContains(t, err, "failed to stop container second")require.Empty(t, stillStopped)require.Equal(t, []string{"stop:first", "stop:second", "start:first"}, operations)@@ -339,14 +315,14 @@ func TestVolumeBackupContainerLifecycleWaitsForRunningComposeReplacement(t *test}})- service, dockerClient := setupVolumeBackupLifecycleTestInternal(t, serverHandler)+ service, dockerClient := setupVolumeBackupLifecycleTest(t, serverHandler)actor := user.Actor{ID: "user-1", Username: "tester"}- stopped, err := service.stopRunningContainersForBackupInternal(t.Context(), dockerClient, "app-data", actor, false)+ stopped, err := service.stopRunningContainersForBackup(t.Context(), dockerClient, "app-data", actor, false)require.NoError(t, err)require.Len(t, stopped, 1)require.Equal(t, "old-id", stopped[0].ID)- remaining, err := service.startContainersAfterBackupInternal(t.Context(), dockerClient, stopped, actor)+ remaining, err := service.startContainersAfterBackup(t.Context(), dockerClient, stopped, actor)require.NoError(t, err)require.Empty(t, remaining)require.Equal(t, []string{"stop:old-id"}, operations)diff --git a/backend/internal/volume/children/backup/service.go b/backend/internal/volume/children/backup/service.goindex 4f080b64e..e567099b4 100644--- a/backend/internal/volume/children/backup/service.go+++ b/backend/internal/volume/children/backup/service.go@@ -9,6 +9,7 @@ import ("io""log/slog""path"+ "slices""strings""sync""time"@@ -19,18 +20,19 @@ import ("github.com/getarcaneapp/arcane/types/v2/scheduler"usertypes "github.com/getarcaneapp/arcane/types/v2/user""github.com/getarcaneapp/arcane/types/v2/volume"+ "github.com/italypaleale/francis/actor"+ "github.com/italypaleale/francis/builtin/workflow""github.com/moby/moby/api/types/container""github.com/moby/moby/api/types/mount""github.com/moby/moby/client"+ kit "go.getarcane.app/kit/pkg""gorm.io/gorm"- "github.com/getarcaneapp/arcane/backend/v2/internal/activity""github.com/getarcaneapp/arcane/backend/v2/internal/backup""github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/docker""github.com/getarcaneapp/arcane/backend/v2/internal/event"- "github.com/getarcaneapp/arcane/backend/v2/internal/middleware""github.com/getarcaneapp/arcane/backend/v2/internal/s3""github.com/getarcaneapp/arcane/backend/v2/internal/settings"userdomain "github.com/getarcaneapp/arcane/backend/v2/internal/user"@@ -41,6 +43,7 @@ import (activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/volumehelper""github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs""github.com/getarcaneapp/arcane/backend/v2/pkg/utils"@@ -66,7 +69,6 @@ type Dependencies struct {DB *database.DBDocker *docker.DockerClientServiceEvents *event.EventService- Activity *activity.ActivityServiceSettings *settings.SettingsServiceEngine *backup.EngineS3Destinations *s3.S3DestinationService@@ -89,39 +91,81 @@ type Service struct {policies *policies.Servicebrowser *browser.Servicerestore *restore.Service+ flow *flow.Engine+ // backupWorkflow runs manual backups that StartBackup accepted.+ backupWorkflow *flow.Workflow}-// lockHeldInternal marks a context whose caller already holds the volume's workspace lock.-type lockHeldInternal struct {+// backupPlan is one run's resolved destinations; it doubles as the run's recovery checkpoint.+type backupPlan struct {+ BackupID string `json:"backupId"`+ LocalEnabled bool `json:"localEnabled"`+ S3Enabled bool `json:"s3Enabled"`+ S3DestinationID string `json:"s3DestinationId"`+ Policy *policies.VolumeBackupPolicy `json:"policy,omitempty"`+}++// manualBackupInput is a manual backup's workflow payload; the row and plan are frozen before it starts.+type manualBackupInput struct {+ Checkpoint backupPlan `json:"checkpoint"`+ VolumeName string `json:"volumeName"`+ Requester backuptypes.Requester `json:"requester"`+}++// lockHeld marks a context whose caller already holds the volume's workspace lock.+type lockHeld struct {service *ServicevolumeName string}-type lockHeldKeyInternal struct{}+type lockHeldKey struct{}++type remoteRepositoryKey struct {+ destinationID string+ instanceID string+}func NewService(deps Dependencies) *Service {s := &Service{deps: deps}+ repository := func(ctx context.Context, dockerClient *client.Client, entry *volume.Backup) (backup.Repository, string, error) {+ if entry.LocalSnapshotID != "" {+ local, err := s.localRusticRepository(ctx, dockerClient, true)+ return local, entry.LocalSnapshotID, err+ }+ if entry.RemoteSnapshotID != "" {+ remote, err := s.remoteRusticRepository(ctx, entry.S3DestinationID, entry.RemoteInstanceID)+ return remote, entry.RemoteSnapshotID, err+ }+ return backup.Repository{}, "", errors.New("volume backup has no Rustic snapshot")+ }s.policies = policies.NewService(policies.Dependencies{DB: deps.DB,S3Destinations: deps.S3Destinations,Settings: deps.Settings,- Activity: deps.Activity,AlreadyRunning: deps.AlreadyRunning,- LatestRun: s.latestRunInternal,+ LatestRun: func(ctx context.Context, policyID string) (*volume.BackupEntry, error) {+ entry, err := s.deps.Store.Latest(ctx, policyID)+ if err != nil || entry == nil {+ return nil, err+ }+ return new(entry.Entry()), nil+ },CreateBackup: func(ctx context.Context, volumeName, policyID string) (*volume.Backup, error) {return s.CreateBackup(ctx, volumeName, usertypes.SystemUser, backuptypes.VolumeBackupTriggerScheduled, volume.CreateBackupRequest{PolicyID: policyID})},Reconcile: s.ReconcileBackup,})s.browser = browser.NewService(browser.Dependencies{- Docker: deps.Docker,- Engine: deps.Engine,- Run: deps.Store.Run,- Repository: s.rusticRepositoryForBackupInternal,- Password: s.volumeBackupPasswordInternal,- SanitizePath: sanitizeBackupPathInternal,- ArchiveFilename: backupArchiveFilenameInternal,- TempContainer: s.createBackupTempContainerInternal,+ Docker: deps.Docker,+ Engine: deps.Engine,+ Run: deps.Store.Run,+ Repository: repository,+ Password: s.volumeBackupPassword,+ SanitizePath: func(input string) (string, error) {+ return kit.NormalizeRelativePath(strings.TrimLeft(strings.TrimSpace(input), "/"))+ },+ ArchiveFilename: backupArchiveFilename,+ TempContainer: s.createBackupTempContainer,Exec: deps.Exec,})s.restore = restore.NewService(restore.Dependencies{@@ -130,24 +174,50 @@ func NewService(deps Dependencies) *Service {Events: deps.Events,Locks: deps.Locks,Run: deps.Store.Run,- Repository: s.rusticRepositoryForBackupInternal,- Password: s.volumeBackupPasswordInternal,- StopContainers: s.stopRunningContainersForBackupInternal,- StartContainers: s.startContainersAfterBackupInternal,- SafetyBackup: s.safetyBackupInternal,+ Repository: repository,+ Password: s.volumeBackupPassword,+ StopContainers: s.stopRunningContainersForBackup,+ StartContainers: s.startContainersAfterBackup,+ // The safety backup runs under the restore's workspace lock, so executeBackup must not take it again.+ SafetyBackup: func(ctx context.Context, volumeName string, user usertypes.Actor) (string, error) {+ ctx = context.WithValue(ctx, lockHeldKey{}, lockHeld{service: s, volumeName: volumeName})+ request := volume.CreateBackupRequest{Destination: volume.BackupDestinationLocal}+ entry, err := s.CreateBackup(ctx, volumeName, user, backuptypes.VolumeBackupTriggerSafety, request)+ if err != nil {+ return "", err+ }+ return entry.ID, nil+ },ArchivePaths: s.browser.ArchivePaths,- ArchiveFilename: backupArchiveFilenameInternal,+ ArchiveFilename: backupArchiveFilename,HelperImage: deps.HelperImage,StorageMount: s.StorageMount,AcquireHelper: deps.AcquireHelper,Exec: deps.Exec,})- if deps.Engine != nil {- deps.Engine.RegisterRunKind("volume", s.executeDurableBackupInternal, s.failDurableBackupInternal)- }return s}+// RegisterWorkflows defines the manual and scheduled backup workflows while the host is still unstarted.+func (s *Service) RegisterWorkflows(engine *flow.Engine) error {+ var err error+ s.flow = engine+ s.backupWorkflow, err = engine.Define(flow.Definition{+ Name: "volume-backup",+ Version: 1,+ Fingerprint: "8bcb88f11ff66a2b5825101d5bd5f9914f9b76bc56e5d131a82499b819bf9c4a",+ Concurrency: 4,+ Timeout: 24 * time.Hour,+ Activity: activitylib.StartRequest{Type: activitytypes.TypeResourceAction},+ Labels: map[string]string{"backup": "Creating backup"},+ Steps: []workflow.StepSpec{workflow.Step("backup", engine.Handler(s.runManualBackup), workflow.WithMaxAttempts(1))},+ })+ if err != nil {+ return err+ }+ return s.policies.RegisterWorkflows(engine)+}+// SetScheduler injects the dynamic scheduler and admission gate for per-policy// backup jobs. Agent mode passes them too: agents run their own volume backups.func (s *Service) SetScheduler(ctx context.Context, dynamicScheduler scheduler.DynamicScheduler, admissionGate *runs.Admission) error {@@ -173,193 +243,93 @@ func (s *Service) RenamePolicies(tx *gorm.DB, oldName, newName string) error {return s.policies.Rename(tx, oldName, newName)}-func entryInternal(entry *volume.Backup) volume.BackupEntry {- return volume.BackupEntry{- RemoteAvailable: entry.RemoteAvailable,- ActivityID: entry.ActivityID,- ID: entry.ID,- VolumeName: entry.VolumeName,- Size: entry.Size,- CreatedAt: entry.CreatedAt.Format(time.RFC3339),- Status: entry.Status,- Trigger: entry.Trigger,- Destination: entry.Destination,- Format: entry.Format,- LocalSnapshotID: entry.LocalSnapshotID,- RemoteSnapshotID: entry.RemoteSnapshotID,- S3DestinationID: entry.S3DestinationID,- S3DestinationName: entry.S3DestinationName,- PolicyID: entry.PolicyID,- Error: entry.Error,- Type: entry.Type,- }-}--func (s *Service) latestRunInternal(ctx context.Context, policyID string) (*volume.BackupEntry, error) {- entry, err := s.deps.Store.Latest(ctx, policyID)- if err != nil || entry == nil {- return nil, err- }- return new(entryInternal(entry)), nil-}-+// backupEntry converts a backup row to its API entry for policy last runs and accepted manual starts.func (s *Service) List(ctx context.Context, volumeName string, params pagination.QueryParams) ([]volume.Backup, pagination.Response, error) {- slog.DebugContext(- ctx,- "volume service: list backups paginated",- "volume",- volumeName,- "search",- params.Search,- "sort",- params.Sort,- "order",- params.Order,- "start",- params.Start,- "limit",- params.Limit,- )+ slog.DebugContext(ctx, "volume service: list backups paginated", "volume", volumeName, "search", params.Search, "sort", params.Sort,+ "order", params.Order, "start", params.Start, "limit", params.Limit)backups, totalItems, err := s.deps.Store.List(ctx, volumeName, params)if err != nil {return nil, pagination.Response{}, err}- hasS3Destination := false- for i := range backups {- if strings.TrimSpace(backups[i].S3DestinationID) != "" {- hasS3Destination = true- break+ if slices.ContainsFunc(backups, func(entry volume.Backup) bool { return strings.TrimSpace(entry.S3DestinationID) != "" }) {+ destinations := s.deps.S3Destinations.DestinationsByID(ctx)+ for i := range backups {+ backups[i].S3DestinationName = destinations[backups[i].S3DestinationID].Name}}- if s.deps.S3Destinations != nil && hasS3Destination {- // Destination names are decoration; never fail the listing over them.- destinations, destinationErr := s.deps.S3Destinations.ListS3DestinationsByID(ctx)- if destinationErr != nil {- slog.WarnContext(ctx, "could not resolve volume backup S3 destination names", "volume", volumeName, "error", destinationErr)- } else {- for i := range backups {- backups[i].S3DestinationName = destinations[backups[i].S3DestinationID].Name- }- }- }- root := ""- if s.deps.Settings != nil {- root = "arcane-volume-backups/" + s.deps.Settings.GetSettingsConfig().InstanceID.Value- }- remoteAvailable := backup.RemoteSnapshotChecker(ctx, s.deps.S3Destinations, root)+ remoteAvailable := backup.RemoteSnapshotChecker(ctx, s.deps.S3Destinations)for i := range backups {- backups[i].RemoteAvailable = remoteAvailable(backups[i].S3DestinationID, backups[i].RemoteSnapshotID)+ // Discovered backups live under the instance that wrote them, not necessarily this one.+ root := ""+ if s.deps.Settings != nil {+ root = path.Join(backup.VolumeRoot, s.remoteInstanceID(backups[i].RemoteInstanceID))+ }+ backups[i].RemoteAvailable = remoteAvailable(backups[i].S3DestinationID, root, backups[i].RemoteSnapshotID)}return backups, pagination.BuildResponse(totalItems, totalItems, params), nil}-// backupPlanInternal is the resolved destination selection for one backup run.-type backupPlanInternal struct {- policy *policies.VolumeBackupPolicy- localEnabled bool- s3Enabled bool- s3DestinationID string- destination volume.BackupDestination-}--func (- s *Service,-) resolveBackupPlanInternal(- ctx context.Context,- volumeName string,- trigger string,- request volume.CreateBackupRequest,- suppliedPolicy *policies.VolumeBackupPolicy,-) (- backupPlanInternal,- error,-) {+func (s *Service) resolveBackupPlan(ctx context.Context, volumeName, trigger string, request volume.CreateBackupRequest, policy *policies.VolumeBackupPolicy) (backupPlan, error) {destination := request.Destination- if destination != "" && destination != volume.BackupDestinationLocal && destination != volume.BackupDestinationS3 && destination != volume.BackupDestinationLocalS3 {- return backupPlanInternal{}, errors.New("invalid volume backup destination")+ validDestinations := []volume.BackupDestination{volume.BackupDestinationLocal, volume.BackupDestinationS3, volume.BackupDestinationLocalS3}+ if destination != "" && !slices.Contains(validDestinations, destination) {+ return backupPlan{}, errors.New("invalid volume backup destination")}- policy := suppliedPolicyif trigger != backuptypes.VolumeBackupTriggerSafety && policy == nil {var err error- policy, err = s.policies.Policy(ctx, volumeName, request.PolicyID)- if err != nil {- return backupPlanInternal{}, err+ if policy, err = s.policies.Policy(ctx, volumeName, request.PolicyID); err != nil {+ return backupPlan{}, err}if request.PolicyID != "" && policy == nil {- return backupPlanInternal{}, errors.New("volume backup policy not found")+ return backupPlan{}, errors.New("volume backup policy not found")}}- localEnabled, s3Enabled, s3DestinationID := true, false, strings.TrimSpace(request.S3DestinationID)+ plan := backupPlan{LocalEnabled: true, S3DestinationID: strings.TrimSpace(request.S3DestinationID), Policy: policy}if policy != nil {- localEnabled, s3Enabled, s3DestinationID = policy.LocalEnabled, policy.S3Enabled, policy.S3DestinationID+ plan.LocalEnabled, plan.S3Enabled, plan.S3DestinationID = policy.LocalEnabled, policy.S3Enabled, policy.S3DestinationID}if trigger == backuptypes.VolumeBackupTriggerSafety {- localEnabled, s3Enabled = true, false+ plan.LocalEnabled, plan.S3Enabled = true, false} else if destination != "" {- localEnabled = destination != volume.BackupDestinationS3- s3Enabled = destination != volume.BackupDestinationLocal+ plan.LocalEnabled = destination != volume.BackupDestinationS3+ plan.S3Enabled = destination != volume.BackupDestinationLocal}- if !localEnabled && !s3Enabled {- return backupPlanInternal{}, errors.New("select at least one volume backup destination")+ if !plan.LocalEnabled && !plan.S3Enabled {+ return backupPlan{}, errors.New("select at least one volume backup destination")}- if s3Enabled && strings.TrimSpace(s3DestinationID) == "" {- return backupPlanInternal{}, errors.New("select an S3 destination for the volume backup")+ if !plan.S3Enabled {+ plan.S3DestinationID = ""+ return plan, nil}- if s3Enabled {- if s.deps.S3Destinations == nil {- return backupPlanInternal{}, errors.New("S3 backup destinations are unavailable")- }- if _, err := s.deps.S3Destinations.Configuration(ctx, s3DestinationID); err != nil {- return backupPlanInternal{}, errors.New("select a valid S3 destination for the volume backup")- }- } else {- s3DestinationID = ""+ if strings.TrimSpace(plan.S3DestinationID) == "" {+ return backupPlan{}, errors.New("select an S3 destination for the volume backup")+ }+ if s.deps.S3Destinations == nil {+ return backupPlan{}, errors.New("S3 backup destinations are unavailable")}- switch {- case localEnabled && s3Enabled:- destination = volume.BackupDestinationLocalS3- case s3Enabled:- destination = volume.BackupDestinationS3- default:- destination = volume.BackupDestinationLocal+ if _, err := s.deps.S3Destinations.Configuration(ctx, plan.S3DestinationID); err != nil {+ return backupPlan{}, errors.New("select a valid S3 destination for the volume backup")}- return backupPlanInternal{policy: policy, localEnabled: localEnabled, s3Enabled: s3Enabled, s3DestinationID: s3DestinationID, destination: destination}, nil+ return plan, nil}-func (s *Service) CreateBackup(ctx context.Context, volumeName string, user usertypes.Actor, trigger string, request volume.CreateBackupRequest) (_ *volume.Backup, err error) {+func (s *Service) CreateBackup(ctx context.Context, volumeName string, user usertypes.Actor, trigger string, request volume.CreateBackupRequest) (*volume.Backup, error) {trigger = cmp.Or(trigger, backuptypes.VolumeBackupTriggerManual)- plan, err := s.resolveBackupPlanInternal(ctx, volumeName, trigger, request, nil)+ plan, err := s.resolveBackupPlan(ctx, volumeName, trigger, request, nil)if err != nil {return nil, err}- return s.createBackupInternal(ctx, volumeName, user, trigger, request.PolicyID, plan)-}--// safetyBackupInternal takes the local pre-restore backup for a caller that already holds the volume's workspace lock.-func (s *Service) safetyBackupInternal(ctx context.Context, volumeName string, user usertypes.Actor) (string, error) {- ctx = context.WithValue(ctx, lockHeldKeyInternal{}, lockHeldInternal{service: s, volumeName: volumeName})- entry, err := s.CreateBackup(ctx, volumeName, user, backuptypes.VolumeBackupTriggerSafety, volume.CreateBackupRequest{Destination: volume.BackupDestinationLocal})+ entry, lease, err := s.prepareBackup(ctx, volumeName, trigger, request.PolicyID, &plan)if err != nil {- return "", err+ return nil, err}- return entry.ID, nil+ defer lease.Release(ctx)+ return entry, s.completeBackup(ctx, entry, s.executeBackup(ctx, entry, user, plan))}// CreateSystemManagedBackup runs the existing volume backup workflow with a transient centralized policy.-func (- s *Service,-) CreateSystemManagedBackup(- ctx context.Context,- volumeName string,- user usertypes.Actor,- trigger string,- policyID string,- policy backuptypes.UpdateBackupPolicy,-) (- *volume.Backup,- error,-) {+func (s *Service) CreateSystemManagedBackup(ctx context.Context, volumeName string, user usertypes.Actor, trigger, policyID string, policy backuptypes.UpdateBackupPolicy) (*volume.Backup, error) {if !strings.HasPrefix(policyID, backuptypes.SystemVolumePolicyPrefix) {return nil, errors.New("invalid system-managed volume backup policy id")}@@ -370,44 +340,28 @@ func (S3DestinationID: policy.S3DestinationID,}transient.ID = policyID- plan, err := s.resolveBackupPlanInternal(ctx, volumeName, trigger, volume.CreateBackupRequest{}, transient)+ plan, err := s.resolveBackupPlan(ctx, volumeName, trigger, volume.CreateBackupRequest{}, transient)if err != nil {return nil, err}- return s.createBackupInternal(ctx, volumeName, user, trigger, policyID, plan)-}--func (s *Service) createBackupInternal(ctx context.Context, volumeName string, user usertypes.Actor, trigger, policyID string, plan backupPlanInternal) (*volume.Backup, error) {- entry, lease, err := s.prepareBackupInternal(ctx, volumeName, trigger, policyID, plan)+ entry, lease, err := s.prepareBackup(ctx, volumeName, trigger, policyID, &plan)if err != nil {return nil, err}defer lease.Release(ctx)- err = s.executeBackupInternal(ctx, entry, user, plan)- err = s.completeBackupInternal(ctx, entry, err)- return entry, err+ return entry, s.completeBackup(ctx, entry, s.executeBackup(ctx, entry, user, plan))}-type volumeBackupRecoveryInternal struct {- BackupID string `json:"backupId"`- LocalEnabled bool `json:"localEnabled"`- S3Enabled bool `json:"s3Enabled"`- S3DestinationID string `json:"s3DestinationId"`- Policy *policies.VolumeBackupPolicy `json:"policy,omitempty"`-}--func (s *Service) prepareBackupInternal(ctx context.Context, volumeName, trigger, policyID string, plan backupPlanInternal) (*volume.Backup, *runs.Lease, error) {- lease, admitted, err := s.deps.Engine.TryAcquireRun(ctx, backup.VolumeAdmissionScope, volumeName)+// prepareBackup admits the run, persists its row, and records plan (now carrying the row ID) as the checkpoint.+func (s *Service) prepareBackup(ctx context.Context, volumeName, trigger, policyID string, plan *backupPlan) (*volume.Backup, *runs.Lease, error) {+ lease, err := s.acquireVolumeRun(ctx, volumeName)if err != nil {return nil, nil, err}- if !admitted {- return nil, nil, s.deps.AlreadyRunning- }entry := &volume.Backup{VolumeName: volumeName, CreatedAt: time.Now(), Status: backuptypes.VolumeBackupStatusRunning,- Trigger: trigger, Destination: plan.destination, Format: volume.BackupFormatRustic,- S3DestinationID: plan.s3DestinationID, PolicyID: policyID,+ Trigger: trigger, Destination: backupDestination(plan.LocalEnabled, plan.S3Enabled), Format: volume.BackupFormatRustic,+ S3DestinationID: plan.S3DestinationID, PolicyID: policyID,Type: backuptypes.ManagementTypeForPolicy(policyID),}entry.ID = fmt.Sprintf("%s-%d-%s", volumeName, time.Now().UnixNano(), uuid.New().String()[:8])@@ -415,15 +369,8 @@ func (s *Service) prepareBackupInternal(ctx context.Context, volumeName, triggerlease.Release(ctx)return nil, nil, createBackupErr}- checkpoint, err := json.Marshal(- volumeBackupRecoveryInternal{- BackupID: entry.ID,- LocalEnabled: plan.localEnabled,- S3Enabled: plan.s3Enabled,- S3DestinationID: plan.s3DestinationID,- Policy: plan.policy,- },- )+ plan.BackupID = entry.ID+ checkpoint, err := json.Marshal(*plan)if err == nil {err = jobcontext.Progress(ctx, scheduler.TargetOutcome{ResourceType: "volume_backup", ID: volumeName, Status: scheduler.Running, RecoveryData: checkpoint})}@@ -434,7 +381,12 @@ func (s *Service) prepareBackupInternal(ctx context.Context, volumeName, triggerreturn entry, lease, nil}-func (s *Service) completeBackupInternal(ctx context.Context, entry *volume.Backup, err error) error {+func (s *Service) completeBackup(ctx context.Context, entry *volume.Backup, err error) error {+ // A workflow task stopped by shutdown, a plain cancel, is handed back, so its row stays running for the resumed+ // task to finish; a user cancel, a workflow that ended or timed out, or work outside a workflow records the failure.+ if _, resumable := jobcontext.Run(ctx); resumable && errors.Is(context.Cause(ctx), context.Canceled) {+ return ctx.Err()+ }if err == nil {err = ctx.Err()}@@ -452,28 +404,26 @@ func (s *Service) completeBackupInternal(ctx context.Context, entry *volume.Backreturn err}-func (s *Service) executeBackupInternal(ctx context.Context, entry *volume.Backup, user usertypes.Actor, plan backupPlanInternal) (err error) {+func (s *Service) executeBackup(ctx context.Context, entry *volume.Backup, user usertypes.Actor, plan backupPlan) (err error) {defer utils.RecoverToError(&err, "volume backup")- volumeName, trigger := entry.VolumeName, entry.Trigger- lockHeld, _ := ctx.Value(lockHeldKeyInternal{}).(lockHeldInternal)- if lockHeld.service != s || lockHeld.volumeName != volumeName {+ volumeName := entry.VolumeName+ if held, _ := ctx.Value(lockHeldKey{}).(lockHeld); held.service != s || held.volumeName != volumeName {defer s.deps.Locks.Lock(volumeName)()}- if cancellationErr := ctx.Err(); cancellationErr != nil {- return cancellationErr+ if cancelErr := ctx.Err(); cancelErr != nil {+ return cancelErr}dockerClient, err := s.deps.Docker.GetClient(ctx)if err != nil {return err}+ // stopped holds the containers still awaiting a restart.var stopped []container.Summary- containersStopped := false- if plan.policy != nil && plan.policy.StopContainers {- stopped, err = s.stopRunningContainersForBackupInternal(ctx, dockerClient, volumeName, user, false)- containersStopped = len(stopped) > 0+ if plan.Policy != nil && plan.Policy.StopContainers {+ stopped, err = s.stopRunningContainersForBackup(ctx, dockerClient, volumeName, user, false)defer func() {- if containersStopped {- _, restartErr := s.startContainersAfterBackupInternal(context.WithoutCancel(ctx), dockerClient, stopped, user)+ if len(stopped) > 0 {+ _, restartErr := s.startContainersAfterBackup(context.WithoutCancel(ctx), dockerClient, stopped, user)err = errors.Join(err, restartErr)}}()@@ -481,146 +431,94 @@ func (s *Service) executeBackupInternal(ctx context.Context, entry *volume.Backureturn err}}- if plan.localEnabled && entry.LocalSnapshotID == "" {- if createLocalBackupSnapshotErr := s.createLocalBackupSnapshotInternal(ctx, dockerClient, entry); createLocalBackupSnapshotErr != nil {- return createLocalBackupSnapshotErr- }- }- if plan.s3Enabled && entry.RemoteSnapshotID == "" {- if createRemoteBackupSnapshotErr := s.createRemoteBackupSnapshotInternal(ctx, dockerClient, entry, plan); createRemoteBackupSnapshotErr != nil {- return createRemoteBackupSnapshotErr- }+ if snapshotErr := s.createSnapshots(ctx, dockerClient, entry, plan); snapshotErr != nil {+ return snapshotErr}- if containersStopped {- stopped, err = s.startContainersAfterBackupInternal(context.WithoutCancel(ctx), dockerClient, stopped, user)- containersStopped = len(stopped) > 0- if err != nil {+ if len(stopped) > 0 {+ if stopped, err = s.startContainersAfterBackup(context.WithoutCancel(ctx), dockerClient, stopped, user); err != nil {return err}}- // Retention failures must not fail the run: the backup itself succeeded,- // and the deferred status-save would otherwise mark it failed and hide it- // from ExpiredRunIDs forever.- if trigger != backuptypes.VolumeBackupTriggerSafety && plan.policy != nil && plan.policy.RetentionCount > 0 {- if retentionErr := s.applyVolumeBackupRetentionInternal(ctx, plan.policy.ID, plan.policy.RetentionCount, plan.s3Enabled); retentionErr != nil {- slog.ErrorContext(ctx, "Volume backup retention failed", "volume", volumeName, "policy", plan.policy.ID, "error", retentionErr)+ // Retention failures must not fail the run: the status save would mark the+ // succeeded backup failed and hide it from ExpiredRunIDs forever.+ if entry.Trigger != backuptypes.VolumeBackupTriggerSafety && plan.Policy != nil && plan.Policy.RetentionCount > 0 {+ expired, retentionErr := backup.ExpiredRunIDs(ctx, s.deps.DB, "volume_backups", plan.Policy.ID, plan.Policy.RetentionCount)+ if retentionErr == nil && len(expired) > 0 {+ var expiredEntries []*volume.Backup+ if expiredEntries, retentionErr = s.deps.Store.Runs(ctx, expired); retentionErr == nil {+ retentionErr = s.deleteRusticBackups(ctx, expiredEntries, nil, plan.S3Enabled)+ }+ }+ if retentionErr != nil {+ slog.ErrorContext(ctx, "Volume backup retention failed", "volume", volumeName, "policy", plan.Policy.ID, "error", retentionErr)}}metadata := database.JSON{"action": "backup_create", "backup_id": entry.ID, "size": entry.Size, "destination": entry.Destination}- if logErr := s.deps.Events.LogVolumeEvent(ctx, event.EventTypeVolumeBackupCreate, volumeName, volumeName, user.ID, user.Username, "0", metadata); logErr != nil {- slog.WarnContext(ctx, "could not log volume backup create event", "volume", volumeName, "error", logErr)- }+ s.logBackupEvent(ctx, event.EventTypeVolumeBackupCreate, volumeName, &user, metadata)return ctx.Err()}-func (s *Service) createLocalBackupSnapshotInternal(ctx context.Context, dockerClient *client.Client, entry *volume.Backup) error {- repository, repoErr := s.localRusticRepositoryInternal(ctx, dockerClient, false)- if repoErr != nil {- return repoErr+// createSnapshots writes the plan's missing snapshots; S3 replicates the local snapshot when both are enabled.+func (s *Service) createSnapshots(ctx context.Context, dockerClient *client.Client, entry *volume.Backup, plan backupPlan) error {+ source := mount.Mount{Type: mount.TypeVolume, Source: entry.VolumeName, Target: "/volume", ReadOnly: true}+ input := backup.RootSnapshotInput(source, backup.RunSnapshotTag(entry.ID))+ if plan.LocalEnabled && entry.LocalSnapshotID == "" {+ repository, err := s.localRusticRepository(ctx, dockerClient, false)+ if err != nil {+ return err+ }+ password, err := s.volumeBackupPassword(ctx, dockerClient, repository)+ if err != nil {+ return err+ }+ if progressErr := jobcontext.Progress(ctx, scheduler.TargetOutcome{ResourceType: "backup_destination", ID: entry.ID + ":local", Status: scheduler.Running}); progressErr != nil {+ return progressErr+ }+ snapshot, err := s.deps.Engine.CreateSnapshot(ctx, dockerClient, repository, password, entry.VolumeName, input)+ if err != nil {+ return fmt.Errorf("failed to create local Rustic snapshot: %w", err)+ }+ entry.LocalSnapshotID, entry.Size = snapshot.ID, snapshot.Size+ if saveErr := s.deps.Store.Save(ctx, entry); saveErr != nil {+ return saveErr+ }+ succeeded := scheduler.TargetOutcome{ResourceType: "backup_destination", ID: entry.ID + ":local", Status: scheduler.Succeeded, Message: snapshot.ID}+ if progressErr := jobcontext.Progress(ctx, succeeded); progressErr != nil {+ return progressErr+ }}- password, passwordErr := s.volumeBackupPasswordInternal(ctx, dockerClient, repository)- if passwordErr != nil {- return passwordErr+ if !plan.S3Enabled || entry.RemoteSnapshotID != "" {+ return nil}- if err := jobcontext.Progress(ctx, scheduler.TargetOutcome{ResourceType: "backup_destination", ID: entry.ID + ":local", Status: scheduler.Running}); err != nil {+ remoteRepository, err := s.remoteRusticRepository(ctx, plan.S3DestinationID, "")+ if err != nil {return err}- localSnapshot, err := s.deps.Engine.CreateSnapshot(- ctx,- dockerClient,- repository,- password,- entry.VolumeName,- backup.RootSnapshotInput(- volumeSourceMountInternal(- entry.VolumeName,- ),- backup.RunSnapshotTag(- entry.ID,- ),- ),- )+ password, err := s.volumeBackupPassword(ctx, dockerClient, remoteRepository)if err != nil {- return fmt.Errorf("failed to create local Rustic snapshot: %w", err)- }- entry.LocalSnapshotID = localSnapshot.ID- entry.Size = localSnapshot.Size- if saveLocalBackupErr := s.deps.Store.Save(ctx, entry); saveLocalBackupErr != nil {- return saveLocalBackupErr- }- if progressErr := jobcontext.Progress(- ctx,- scheduler.TargetOutcome{- ResourceType: "backup_destination",- ID: entry.ID + ":local",- Status: scheduler.Succeeded,- Message: localSnapshot.ID,- },- ); progressErr != nil {- return progressErr- }- return nil-}--func (s *Service) createRemoteBackupSnapshotInternal(ctx context.Context, dockerClient *client.Client, entry *volume.Backup, plan backupPlanInternal) error {- var err error- remoteRepository, repoErr := s.remoteRusticRepositoryInternal(ctx, plan.s3DestinationID)- if repoErr != nil {- return repoErr- }- password, passwordErr := s.volumeBackupPasswordInternal(ctx, dockerClient, remoteRepository)- if passwordErr != nil {- return passwordErr+ return err}if progressErr := jobcontext.Progress(ctx, scheduler.TargetOutcome{ResourceType: "backup_destination", ID: entry.ID + ":remote", Status: scheduler.Running}); progressErr != nil {return progressErr}- var remoteSnapshot backup.Snapshot- if plan.localEnabled {- localRepository, localErr := s.localRusticRepositoryInternal(ctx, dockerClient, true)+ var snapshot backup.Snapshot+ if plan.LocalEnabled {+ localRepository, localErr := s.localRusticRepository(ctx, dockerClient, true)if localErr != nil {return localErr}- remoteSnapshot, err = s.deps.Engine.Replicate(ctx, dockerClient, localRepository, entry.LocalSnapshotID, remoteRepository, password, entry.VolumeName, backup.RunSnapshotTag(entry.ID))+ snapshot, err = s.deps.Engine.Replicate(ctx, dockerClient, localRepository, entry.LocalSnapshotID, remoteRepository, password, entry.VolumeName, backup.RunSnapshotTag(entry.ID))} else {- remoteSnapshot, err = s.deps.Engine.CreateSnapshot(- ctx,- dockerClient,- remoteRepository,- password,- entry.VolumeName,- backup.RootSnapshotInput(- volumeSourceMountInternal(- entry.VolumeName,- ),- backup.RunSnapshotTag(- entry.ID,- ),- ),- )+ snapshot, err = s.deps.Engine.CreateSnapshot(ctx, dockerClient, remoteRepository, password, entry.VolumeName, input)}if err != nil {return fmt.Errorf("failed to create S3 Rustic snapshot: %w", err)}- entry.RemoteSnapshotID = remoteSnapshot.ID- if entry.Size == 0 {- entry.Size = remoteSnapshot.Size+ entry.RemoteSnapshotID, entry.Size = snapshot.ID, cmp.Or(entry.Size, snapshot.Size)+ if saveErr := s.deps.Store.Save(ctx, entry); saveErr != nil {+ return saveErr}- if saveRemoteBackupErr := s.deps.Store.Save(ctx, entry); saveRemoteBackupErr != nil {- return saveRemoteBackupErr- }- if remoteDestinationProgressErr := jobcontext.Progress(- ctx,- scheduler.TargetOutcome{- ResourceType: "backup_destination",- ID: entry.ID + ":remote",- Status: scheduler.Succeeded,- Message: remoteSnapshot.ID,- },- ); remoteDestinationProgressErr != nil {- return remoteDestinationProgressErr- }- return nil+ return jobcontext.Progress(ctx, scheduler.TargetOutcome{ResourceType: "backup_destination", ID: entry.ID + ":remote", Status: scheduler.Succeeded, Message: snapshot.ID})}func (s *Service) UploadBackup(ctx context.Context, backupID, s3DestinationID string) (*volume.Backup, error) {@@ -628,17 +526,13 @@ func (s *Service) UploadBackup(ctx context.Context, backupID, s3DestinationID stif err != nil {return nil, err}- lease, admitted, err := s.deps.Engine.TryAcquireRun(ctx, backup.VolumeAdmissionScope, entry.VolumeName)+ lease, err := s.acquireVolumeRun(ctx, entry.VolumeName)if err != nil {return nil, err}- if !admitted {- return nil, s.deps.AlreadyRunning- }defer lease.Release(ctx)// Reload under the lease: a delete may have raced the first read.- entry, err = s.deps.Store.Run(ctx, backupID)- if err != nil {+ if entry, err = s.deps.Store.Run(ctx, backupID); err != nil {return nil, err}if entry.Status != backuptypes.VolumeBackupStatusSucceeded || entry.LocalSnapshotID == "" {@@ -650,25 +544,19 @@ func (s *Service) UploadBackup(ctx context.Context, backupID, s3DestinationID stif strings.TrimSpace(s3DestinationID) == "" {return nil, errors.New("select an S3 destination for the upload")}- if s.deps.S3Destinations == nil {- return nil, errors.New("S3 backup destinations are unavailable")- }- if _, configurationErr := s.deps.S3Destinations.Configuration(ctx, s3DestinationID); configurationErr != nil {- return nil, errors.New("select a valid S3 destination for the upload")- }- dockerClient, err := s.deps.Docker.GetClient(ctx)+ remoteRepository, err := s.remoteRusticRepository(ctx, s3DestinationID, "")if err != nil {return nil, err}- localRepository, err := s.localRusticRepositoryInternal(ctx, dockerClient, true)+ dockerClient, err := s.deps.Docker.GetClient(ctx)if err != nil {return nil, err}- remoteRepository, err := s.remoteRusticRepositoryInternal(ctx, s3DestinationID)+ localRepository, err := s.localRusticRepository(ctx, dockerClient, true)if err != nil {return nil, err}- password, err := s.volumeBackupPasswordInternal(ctx, dockerClient, localRepository, remoteRepository)+ password, err := s.volumeBackupPassword(ctx, dockerClient, localRepository, remoteRepository)if err != nil {return nil, err}@@ -676,14 +564,11 @@ func (s *Service) UploadBackup(ctx context.Context, backupID, s3DestinationID stif err != nil {return nil, fmt.Errorf("failed to upload Rustic snapshot to S3: %w", err)}- entry.RemoteSnapshotID = snapshot.ID- entry.S3DestinationID = s3DestinationID- entry.Destination = volume.BackupDestinationLocalS3+ entry.RemoteSnapshotID, entry.S3DestinationID, entry.Destination = snapshot.ID, s3DestinationID, volume.BackupDestinationLocalS3if saveUploadedBackupErr := s.deps.Store.Save(ctx, entry); saveUploadedBackupErr != nil {return nil, fmt.Errorf("failed to save uploaded volume backup: %w", saveUploadedBackupErr)}- return entry,- nil+ return entry, nil}func (s *Service) DeleteBackup(ctx context.Context, backupID string, user *usertypes.Actor) error {@@ -691,56 +576,93 @@ func (s *Service) DeleteBackup(ctx context.Context, backupID string, user *usertif err != nil {return err}- // Deletes contend with create/restore/upload on the volume's run lease so- // a slow operation cannot resurrect or double-forget snapshots. Retention- // deletes run under CreateBackup's lease and call the internal path.- lease, admitted, err := s.deps.Engine.TryAcquireRun(ctx, backup.VolumeAdmissionScope, entry.VolumeName)+ // Deletes share the volume's run lease with create/restore/upload so a slow+ // operation cannot resurrect or double-forget snapshots.+ lease, err := s.acquireVolumeRun(ctx, entry.VolumeName)if err != nil {return err}- if !admitted {- return s.deps.AlreadyRunning- }defer lease.Release(ctx)- return s.deleteBackupInternal(ctx, backupID, user)-}--func (s *Service) deleteBackupInternal(ctx context.Context, backupID string, user *usertypes.Actor) error {- entry, err := s.deps.Store.Run(ctx, backupID)- if err != nil {+ if entry, err = s.deps.Store.Run(ctx, backupID); err != nil {return err}- if entry.Format == volume.BackupFormatArchive {- return s.deleteArchiveBackupInternal(ctx, entry, user)+ if entry.Format != volume.BackupFormatArchive {+ return s.deleteRusticBackups(ctx, []*volume.Backup{entry}, user, true)}- return s.deleteRusticBackupsInternal(ctx, []*volume.Backup{entry}, user, true)-}--type remoteRepositoryKeyInternal struct {- destinationID string- instanceID string+ // Delete the row first: a failed file removal then only leaves an orphan file.+ if deleteErr := s.deps.Store.Delete(ctx, backupID); deleteErr != nil {+ return deleteErr+ }+ containerID, cleanup, err := s.createBackupTempContainer(ctx, nil, "/volume", false)+ if err != nil {+ slog.WarnContext(ctx, "failed to create container for backup file cleanup", "backupId", backupID, "error", err.Error())+ } else {+ defer cleanup()+ filename, filenameErr := backupArchiveFilename(backupID)+ if filenameErr != nil {+ slog.WarnContext(ctx, "failed to sanitize backup id for file cleanup", "backupId", backupID, "error", filenameErr.Error())+ } else if _, _, err = s.deps.Exec(ctx, containerID, "", []string{"rm", "-f", path.Join("/volume", filename)}); err != nil {+ slog.WarnContext(ctx, "failed to delete backup file (orphan file may remain)", "backupId", backupID, "error", err.Error())+ }+ }+ s.logBackupEvent(ctx, event.EventTypeVolumeBackupDelete, entry.VolumeName, user, database.JSON{"action": "backup_delete", "backup_id": backupID})+ return nil}-// deleteRusticBackupsInternal forgets snapshots grouped by repository so each repository is pruned once.-func (s *Service) deleteRusticBackupsInternal(ctx context.Context, entries []*volume.Backup, user *usertypes.Actor, includeRemote bool) error {+// deleteRusticBackups forgets snapshots grouped by repository so each repository is pruned once.+func (s *Service) deleteRusticBackups(ctx context.Context, entries []*volume.Backup, user *usertypes.Actor, includeRemote bool) error {dockerClient, err := s.deps.Docker.GetClient(ctx)if err != nil {return err}- var localEntries []*volume.Backup- remoteGroups := make(map[remoteRepositoryKeyInternal][]*volume.Backup)+ var localSnapshotIDs []string+ remoteGroups := make(map[remoteRepositoryKey][]*volume.Backup)for _, entry := range entries {if entry.LocalSnapshotID != "" {- localEntries = append(localEntries, entry)+ localSnapshotIDs = append(localSnapshotIDs, entry.LocalSnapshotID)}if includeRemote && entry.RemoteSnapshotID != "" {- key := remoteRepositoryKeyInternal{destinationID: entry.S3DestinationID, instanceID: entry.RemoteInstanceID}+ key := remoteRepositoryKey{destinationID: entry.S3DestinationID, instanceID: s.remoteInstanceID(entry.RemoteInstanceID)}remoteGroups[key] = append(remoteGroups[key], entry)}}- deleteErr := s.forgetLocalSnapshotsInternal(ctx, dockerClient, localEntries)+ var deleteErr error+ if len(localSnapshotIDs) > 0 {+ repository, repoErr := s.localRusticRepository(ctx, dockerClient, false)+ if repoErr == nil {+ repoErr = s.forgetSnapshots(ctx, dockerClient, repository, localSnapshotIDs)+ }+ if repoErr != nil {+ deleteErr = fmt.Errorf("failed to delete local Rustic snapshots: %w", repoErr)+ } else {+ for _, entry := range entries {+ entry.LocalSnapshotID = ""+ }+ }+ }for key, group := range remoteGroups {- deleteErr = errors.Join(deleteErr, s.forgetRemoteSnapshotsInternal(ctx, dockerClient, key.destinationID, key.instanceID, group))+ snapshotIDs := make([]string, len(group))+ for index, entry := range group {+ snapshotIDs[index] = entry.RemoteSnapshotID+ }+ repository, repoErr := s.remoteRusticRepository(ctx, key.destinationID, key.instanceID)+ var observation backuptypes.RepositoryObservation+ if repoErr == nil {+ observation, repoErr = backup.CheckRemoteRepository(ctx, s.deps.S3Destinations, key.destinationID, path.Join(backup.VolumeRoot, key.instanceID))+ }+ if repoErr == nil && observation.Available {+ repoErr = s.forgetSnapshots(ctx, dockerClient, repository, snapshotIDs)+ } else if repoErr == nil {+ // Nothing remains to forget; drop the references so the entry can be removed.+ slog.WarnContext(ctx, "S3 repository is missing; dropping remote snapshot references", "destinationId", key.destinationID, "snapshots", len(snapshotIDs))+ }+ if repoErr != nil {+ deleteErr = errors.Join(deleteErr, fmt.Errorf("failed to delete S3 Rustic snapshots: %w", repoErr))+ continue+ }+ for _, entry := range group {+ entry.RemoteSnapshotID, entry.S3DestinationID = "", ""+ }}for _, entry := range entries {if entry.LocalSnapshotID == "" && entry.RemoteSnapshotID == "" {@@ -748,17 +670,10 @@ func (s *Service) deleteRusticBackupsInternal(ctx context.Context, entries []*vodeleteErr = errors.Join(deleteErr, fmt.Errorf("failed to delete volume backup record: %w", deleteBackupErr))continue}- s.logBackupDeleteEventInternal(ctx, entry.VolumeName, entry.ID, user)+ s.logBackupEvent(ctx, event.EventTypeVolumeBackupDelete, entry.VolumeName, user, database.JSON{"action": "backup_delete", "backup_id": entry.ID})continue}- switch {- case entry.LocalSnapshotID != "" && entry.RemoteSnapshotID != "":- entry.Destination = volume.BackupDestinationLocalS3- case entry.LocalSnapshotID != "":- entry.Destination = volume.BackupDestinationLocal- default:- entry.Destination = volume.BackupDestinationS3- }+ entry.Destination = backupDestination(entry.LocalSnapshotID != "", entry.RemoteSnapshotID != "")if saveErr := s.deps.Store.Save(ctx, entry); saveErr != nil {deleteErr = errors.Join(deleteErr, saveErr)}@@ -772,235 +687,97 @@ func (s *Service) PruneLocalRepository(ctx context.Context) error {if err != nil {return err}- repository, err := s.localRusticRepositoryInternal(ctx, dockerClient, false)+ repository, err := s.localRusticRepository(ctx, dockerClient, false)if err != nil {return err}- return s.forgetSnapshotsInternal(ctx, dockerClient, repository, nil)-}--func (s *Service) forgetLocalSnapshotsInternal(ctx context.Context, dockerClient *client.Client, entries []*volume.Backup) error {- if len(entries) == 0 {- return nil- }- snapshotIDs := make([]string, len(entries))- for index, entry := range entries {- snapshotIDs[index] = entry.LocalSnapshotID- }- repository, repoErr := s.localRusticRepositoryInternal(ctx, dockerClient, false)- if repoErr == nil {- repoErr = s.forgetSnapshotsInternal(ctx, dockerClient, repository, snapshotIDs)- }- if repoErr != nil {- return fmt.Errorf("failed to delete local Rustic snapshots: %w", repoErr)- }- for _, entry := range entries {- entry.LocalSnapshotID = ""- }- return nil-}--func (s *Service) forgetRemoteSnapshotsInternal(ctx context.Context, dockerClient *client.Client, destinationID, instanceID string, entries []*volume.Backup) error {- snapshotIDs := make([]string, len(entries))- for index, entry := range entries {- snapshotIDs[index] = entry.RemoteSnapshotID- }- repository, repoErr := s.remoteRusticRepositoryForInstanceInternal(ctx, destinationID, instanceID)- if repoErr == nil {- // Mirror remoteRusticRepositoryForInstanceInternal so the check targets the Rustic root.- root := instanceID- if strings.TrimSpace(root) == "" {- root = strings.TrimSpace(s.deps.Settings.GetSettingsConfig().InstanceID.Value)- }- var observation backuptypes.RepositoryObservation- observation, repoErr = backup.CheckRemoteRepository(ctx, s.deps.S3Destinations, destinationID, path.Join("arcane-volume-backups", root))- switch {- case repoErr != nil:- case !observation.Available:- // Nothing remains to forget; drop the references so the entry can be removed.- slog.WarnContext(ctx, "S3 repository is missing; dropping remote snapshot references", "destinationId", destinationID, "snapshots", len(snapshotIDs))- default:- repoErr = s.forgetSnapshotsInternal(ctx, dockerClient, repository, snapshotIDs)- }- }- if repoErr != nil {- return fmt.Errorf("failed to delete S3 Rustic snapshots: %w", repoErr)- }- for _, entry := range entries {- entry.RemoteSnapshotID = ""- entry.S3DestinationID = ""- }- return nil+ return s.forgetSnapshots(ctx, dockerClient, repository, nil)}-func (s *Service) forgetSnapshotsInternal(ctx context.Context, dockerClient *client.Client, repository backup.Repository, snapshotIDs []string) error {- password, err := s.volumeBackupPasswordInternal(ctx, dockerClient, repository)+func (s *Service) forgetSnapshots(ctx context.Context, dockerClient *client.Client, repository backup.Repository, snapshotIDs []string) error {+ password, err := s.volumeBackupPassword(ctx, dockerClient, repository)if err != nil {return err}return s.deps.Engine.ForgetSnapshots(ctx, dockerClient, repository, password, snapshotIDs)}-func (s *Service) logBackupDeleteEventInternal(ctx context.Context, volumeName, backupID string, user *usertypes.Actor) {- actingUser := user- if actingUser == nil {- actingUser = &usertypes.SystemUser- }- if logErr := s.deps.Events.LogVolumeEvent(- ctx,- event.EventTypeVolumeBackupDelete,- volumeName,- volumeName,- actingUser.ID,- actingUser.Username,- "0",- database.JSON{- "action": "backup_delete",- "backup_id": backupID,- },- ); logErr != nil {- slog.WarnContext(ctx, "could not log volume backup delete event", "volume", volumeName, "error", logErr)- }-}--// DownloadBackup streams a backup as a tar.gz archive. Legacy archive rows-// stream their stored file; local Rustic rows are restored into a scratch-// volume and packaged on the fly.+// DownloadBackup streams a backup as a tar.gz archive: legacy archives stream their stored+// file, and local Rustic snapshots are restored into a scratch volume and packaged on the fly.func (s *Service) DownloadBackup(ctx context.Context, backupID string, user *usertypes.Actor) (io.ReadCloser, int64, error) {entry, err := s.deps.Store.Run(ctx, backupID)if err != nil {return nil, 0, err}- var reader io.ReadCloser- var size int64- if entry.Format == volume.BackupFormatArchive {- reader, size, err = s.downloadArchiveBackupInternal(ctx, backupID)- } else {- reader, size, err = s.downloadRusticBackupInternal(ctx, entry)- }- if err != nil {- return nil, 0, err- }- actingUser := user- if actingUser == nil {- actingUser = &usertypes.SystemUser- }- metadata := database.JSON{"action": "backup_download", "backup_id": backupID, "size": size}- if logErr := s.deps.Events.LogVolumeEvent(ctx, event.EventTypeVolumeBackupDownload, entry.VolumeName, entry.VolumeName, actingUser.ID, actingUser.Username, "0", metadata); logErr != nil {- slog.WarnContext(ctx, "could not log volume backup download event", "volume", entry.VolumeName, "error", logErr)- }- return reader, size, nil-}--func (s *Service) downloadRusticBackupInternal(ctx context.Context, entry *volume.Backup) (io.ReadCloser, int64, error) {- if entry.LocalSnapshotID == "" {+ if entry.Format != volume.BackupFormatArchive && entry.LocalSnapshotID == "" {return nil, 0, errors.New("only local volume backups can be downloaded")}dockerClient, err := s.deps.Docker.GetClient(ctx)if err != nil {return nil, 0, err}- repository, err := s.localRusticRepositoryInternal(ctx, dockerClient, true)- if err != nil {- return nil, 0, err- }- password, err := s.volumeBackupPasswordInternal(ctx, dockerClient, repository)- if err != nil {- return nil, 0, err- }- scratchVolume := "arcane-rustic-download-" + uuid.New().String()- if _, volumeCreateErr := dockerClient.VolumeCreate(ctx, client.VolumeCreateOptions{Name: scratchVolume, Labels: volumehelper.Labels()}); volumeCreateErr != nil {- return nil, 0, fmt.Errorf("failed to create download scratch volume: %w", volumeCreateErr)- }- removeScratch := func() {- _, _ = dockerClient.VolumeRemove(context.WithoutCancel(ctx), scratchVolume, client.VolumeRemoveOptions{Force: true})- }- if restoreSnapshotErr := s.deps.Engine.RestoreSnapshot(- ctx,- dockerClient,- repository,- password,- entry.LocalSnapshotID,- mount.Mount{- Type: mount.TypeVolume,- Source: scratchVolume,- Target: "/volume",- },- backup.RestoreOptions{- DeleteExtra: true,- },- ); restoreSnapshotErr != nil {- removeScratch()- return nil, 0, fmt.Errorf("failed to restore Rustic snapshot for download: %w", restoreSnapshotErr)- }- helperImage, err := s.deps.HelperImage(ctx, dockerClient)- if err != nil {- removeScratch()- return nil, 0, err- }- archiveMount := mount.Mount{Type: mount.TypeVolume, Source: scratchVolume, Target: "/volume", ReadOnly: false}- containerID, removeContainer, err := s.createBackupTempContainerWithMountInternal(ctx, dockerClient, helperImage, archiveMount)- if err != nil {- removeScratch()- return nil, 0, err- }- cleanup := func() {- removeContainer()- removeScratch()- }- archivePath := "/tmp/" + entry.ID + ".tar.gz"- if _, _, execInContainerErr := s.deps.Exec(ctx, containerID, "", []string{"tar", "-czf", archivePath, "-C", "/volume", "."}); execInContainerErr != nil {- cleanup()- return nil, 0, fmt.Errorf("failed to package Rustic snapshot for download: %w", execInContainerErr)- }- return volumehelper.DownloadFileFromContainer(ctx, dockerClient, containerID, archivePath, cleanup)-}--func (s *Service) downloadArchiveBackupInternal(ctx context.Context, backupID string) (io.ReadCloser, int64, error) {- filename, err := backupArchiveFilenameInternal(backupID)- if err != nil {- return nil, 0, err+ var containerID, archivePath string+ var cleanup func()+ if entry.Format == volume.BackupFormatArchive {+ filename, filenameErr := backupArchiveFilename(backupID)+ if filenameErr != nil {+ return nil, 0, filenameErr+ }+ archivePath = path.Join("/volume", filename)+ containerID, cleanup, err = s.createBackupTempContainer(ctx, dockerClient, "/volume", true)+ } else {+ // Restore the local snapshot into a scratch volume and tar it inside a helper container.+ archivePath = "/tmp/" + entry.ID + ".tar.gz"+ repository, repoErr := s.localRusticRepository(ctx, dockerClient, true)+ if repoErr != nil {+ return nil, 0, repoErr+ }+ password, passwordErr := s.volumeBackupPassword(ctx, dockerClient, repository)+ if passwordErr != nil {+ return nil, 0, passwordErr+ }+ scratchVolume := "arcane-rustic-download-" + uuid.New().String()+ if _, volumeCreateErr := dockerClient.VolumeCreate(ctx, client.VolumeCreateOptions{Name: scratchVolume, Labels: volumehelper.Labels()}); volumeCreateErr != nil {+ return nil, 0, fmt.Errorf("failed to create download scratch volume: %w", volumeCreateErr)+ }+ removeScratch := func() {+ _, _ = dockerClient.VolumeRemove(context.WithoutCancel(ctx), scratchVolume, client.VolumeRemoveOptions{Force: true})+ }+ scratchMount := mount.Mount{Type: mount.TypeVolume, Source: scratchVolume, Target: "/volume"}+ restoreOptions := backup.RestoreOptions{DeleteExtra: true}+ if restoreErr := s.deps.Engine.RestoreSnapshot(ctx, dockerClient, repository, password, entry.LocalSnapshotID, scratchMount, restoreOptions); restoreErr != nil {+ removeScratch()+ return nil, 0, fmt.Errorf("failed to restore Rustic snapshot for download: %w", restoreErr)+ }+ var removeContainer func()+ if containerID, removeContainer, err = s.createBackupTempContainerWithMount(ctx, dockerClient, scratchMount); err != nil {+ removeScratch()+ return nil, 0, err+ }+ cleanup = func() {+ removeContainer()+ removeScratch()+ }+ if _, _, err = s.deps.Exec(ctx, containerID, "", []string{"tar", "-czf", archivePath, "-C", "/volume", "."}); err != nil {+ cleanup()+ err = fmt.Errorf("failed to package Rustic snapshot for download: %w", err)+ }}- dockerClient, err := s.deps.Docker.GetClient(ctx)if err != nil {return nil, 0, err}- containerID, cleanup, err := s.createBackupTempContainerInternal(ctx, dockerClient, "/volume", true)+ reader, size, err := volumehelper.DownloadFileFromContainer(ctx, dockerClient, containerID, archivePath, cleanup)if err != nil {return nil, 0, err}- return volumehelper.DownloadFileFromContainer(ctx, dockerClient, containerID, path.Join("/volume", filename), cleanup)-}--func (s *Service) deleteArchiveBackupInternal(ctx context.Context, entry *volume.Backup, user *usertypes.Actor) error {- // Delete from DB first - if this fails, no changes are made. If file- // deletion fails afterward we just have an orphan file, easier to clean up- // than an orphan DB record pointing at a missing file.- volumeName := entry.VolumeName- backupID := entry.ID- if err := s.deps.Store.Delete(ctx, backupID); err != nil {- return err- }- containerID, cleanup, err := s.createBackupTempContainerInternal(ctx, nil, "/volume", false)- if err != nil {- slog.WarnContext(ctx, "failed to create container for backup file cleanup", "backupId", backupID, "error", err.Error())- } else {- defer cleanup()- filename, filenameErr := backupArchiveFilenameInternal(backupID)- if filenameErr != nil {- slog.WarnContext(ctx, "failed to sanitize backup id for file cleanup", "backupId", backupID, "error", filenameErr.Error())- } else if _, _, err = s.deps.Exec(ctx, containerID, "", []string{"rm", "-f", path.Join("/volume", filename)}); err != nil {- slog.WarnContext(ctx, "failed to delete backup file (orphan file may remain)", "backupId", backupID, "error", err.Error())- }- }- s.logBackupDeleteEventInternal(ctx, volumeName, backupID, user)- return nil+ s.logBackupEvent(ctx, event.EventTypeVolumeBackupDownload, entry.VolumeName, user, database.JSON{"action": "backup_download", "backup_id": backupID, "size": size})+ return reader, size, nil}// ArchiveFileTarget resolves a legacy archive backup of volumeName and the// sanitized member path for a single-file workspace restore.func (s *Service) ArchiveFileTarget(ctx context.Context, volumeName, backupID, rel string) (string, string, error) {- filename, err := backupArchiveFilenameInternal(backupID)+ filename, err := backupArchiveFilename(backupID)if err != nil {return "", "", common.Classify(common.ErrVolumeWorkspaceNotFound, err)}@@ -1011,7 +788,7 @@ func (s *Service) ArchiveFileTarget(ctx context.Context, volumeName, backupID, rif entry.VolumeName != volumeName {return "", "", common.Classify(common.ErrVolumeWorkspaceForbidden, errors.New("backup does not belong to volume"))}- cleaned, err := sanitizeBackupPathInternal(rel)+ cleaned, err := kit.NormalizeRelativePath(strings.TrimLeft(strings.TrimSpace(rel), "/"))if err != nil {return "", "", common.Classify(common.ErrVolumeWorkspaceBadRequest, err)}@@ -1023,18 +800,6 @@ func (s *Service) RestoreArchiveMembers(ctx context.Context, containerID, filenareturn s.restore.RestoreArchiveMembers(ctx, containerID, filename, cleanedPaths)}-func (s *Service) applyVolumeBackupRetentionInternal(ctx context.Context, policyID string, retentionCount int, includeRemote bool) error {- expired, err := backup.ExpiredRunIDs(ctx, s.deps.DB, "volume_backups", policyID, retentionCount)- if err != nil || len(expired) == 0 {- return err- }- entries, err := s.deps.Store.Runs(ctx, expired)- if err != nil {- return err- }- return s.deleteRusticBackupsInternal(ctx, entries, nil, includeRemote)-}-// DiscoverRemoteBackups imports snapshots from every instance root on the destination; per-root failures are returned without blocking the others.func (s *Service) DiscoverRemoteBackups(ctx context.Context, destinationID string) (int, []string, error) {if s.deps.Engine == nil || s.deps.S3Destinations == nil || s.deps.RecoveryKeys == nil {@@ -1051,7 +816,7 @@ func (s *Service) DiscoverRemoteBackups(ctx context.Context, destinationID strinif err != nil {return 0, nil, errors.New("the selected S3 backup destination is not configured")}- roots, err := s3utils.ListRepositoryRoots(ctx, configuration, "arcane-volume-backups")+ roots, err := s3utils.ListRepositoryRoots(ctx, configuration, backup.VolumeRoot)if err != nil {return 0, nil, fmt.Errorf("failed to list volume backup repositories: %w", err)}@@ -1070,7 +835,7 @@ func (s *Service) DiscoverRemoteBackups(ctx context.Context, destinationID strincreated := 0var failures []stringfor _, root := range roots {- repository, repoErr := s.remoteRusticRepositoryForInstanceInternal(ctx, destinationID, root)+ repository, repoErr := s.remoteRusticRepository(ctx, destinationID, root)if repoErr != nil {failures = append(failures, fmt.Sprintf("instance %s: %v", root, repoErr))continue@@ -1084,7 +849,7 @@ func (s *Service) DiscoverRemoteBackups(ctx context.Context, destinationID strinif _, exists := known[snapshot.ID]; exists {continue}- entry := discoveredVolumeBackupInternal(destinationID, root, snapshot)+ entry := discoveredVolumeBackup(destinationID, root, snapshot)if entry == nil {continue}@@ -1101,8 +866,8 @@ func (s *Service) DiscoverRemoteBackups(ctx context.Context, destinationID strinreturn created, failures, nil}-// discoveredVolumeBackupInternal maps a snapshot to a backup record via its volume label; unlabeled and system snapshots map to nil.-func discoveredVolumeBackupInternal(destinationID, root string, snapshot backup.DiscoveredSnapshot) *volume.Backup {+// discoveredVolumeBackup maps a snapshot to a backup record via its volume label; unlabeled and system snapshots map to nil.+func discoveredVolumeBackup(destinationID, root string, snapshot backup.DiscoveredSnapshot) *volume.Backup {volumeName := strings.TrimSpace(snapshot.Label)if volumeName == "" || volumeName == systemRecoverySnapshotLabel {return nil@@ -1125,7 +890,7 @@ func (s *Service) MigrateRepositoryPasswords(ctx context.Context) error {if s.deps.RecoveryKeys == nil || s.deps.Engine == nil {return nil}- key, err := s.deps.RecoveryKeys.Get(ctx)+ _, err := s.deps.RecoveryKeys.Get(ctx)if errors.Is(err, backup.ErrRecoveryKeyNotConfigured) {return nil}@@ -1136,88 +901,89 @@ func (s *Service) MigrateRepositoryPasswords(ctx context.Context) error {if err != nil {return err}- local, err := s.localRusticRepositoryInternal(ctx, dockerClient, true)+ local, err := s.localRusticRepository(ctx, dockerClient, true)if err != nil {return err}repositories := []backup.Repository{local}if s.deps.S3Destinations != nil {- destinations, listS3DestinationsByIDErr := s.deps.S3Destinations.ListS3DestinationsByID(ctx)- if listS3DestinationsByIDErr != nil {- return fmt.Errorf("failed to list S3 destinations for re-key: %w", listS3DestinationsByIDErr)+ destinations, listErr := s.deps.S3Destinations.ListAllS3Destinations(ctx)+ if listErr != nil {+ return fmt.Errorf("failed to list S3 destinations for re-key: %w", listErr)}- for destinationID := range destinations {- remote, remoteRusticRepositoryErr := s.remoteRusticRepositoryInternal(ctx, destinationID)+ for _, destination := range destinations {+ remote, remoteRusticRepositoryErr := s.remoteRusticRepository(ctx, destination.ID, "")if remoteRusticRepositoryErr != nil {return remoteRusticRepositoryErr}repositories = append(repositories, remote)}}- for _, repository := range repositories {- s.rekeyRepositoryInternal(ctx, dockerClient, repository, key)- }- return nil+ _, err = s.volumeBackupPassword(ctx, dockerClient, repositories...)+ return err}-// StartBackup persists a running backup and submits application-owned work.+// StartBackup persists a running backup and submits its workflow.func (s *Service) StartBackup(ctx context.Context, environmentID, volumeName string, user usertypes.Actor, request volume.CreateBackupRequest) (volume.BackupEntry, error) {- plan, err := s.resolveBackupPlanInternal(ctx, volumeName, backuptypes.VolumeBackupTriggerManual, request, nil)+ plan, err := s.resolveBackupPlan(ctx, volumeName, backuptypes.VolumeBackupTriggerManual, request, nil)if err != nil {return volume.BackupEntry{}, err}- entry, lease, err := s.prepareBackupInternal(ctx, volumeName, backuptypes.VolumeBackupTriggerManual, request.PolicyID, plan)+ requester := backup.NewRequester(ctx, user, environmentID, authz.PermVolumesBackup)+ if authorizeErr := s.deps.Engine.Authorize(ctx, requester); authorizeErr != nil {+ return volume.BackupEntry{}, authorizeErr+ }+ entry, lease, err := s.prepareBackup(ctx, volumeName, backuptypes.VolumeBackupTriggerManual, request.PolicyID, &plan)if err != nil {return volume.BackupEntry{}, err}- activityID, workCtx := activitylib.StartHandlerActivity(ctx, s.deps.Activity, environmentID, activitytypes.TypeResourceAction,- "volume", volumeName, volumeName, &user, "Creating backup", "Creating volume backup",- database.JSON{"action": "create_volume_backup", "backupId": entry.ID, "destination": entry.Destination, "policyId": request.PolicyID, "s3DestinationId": entry.S3DestinationID}, false)- if activityID == "" {- defer lease.Release(ctx)- return volume.BackupEntry{}, s.completeBackupInternal(ctx, entry, errors.New("failed to start backup activity"))- }+ release := s.deps.Engine.Hold(backup.VolumeAdmissionScope, volumeName, entry.ID, lease)+ input := manualBackupInput{Checkpoint: plan, VolumeName: volumeName, Requester: requester}+ activityID, err := s.flow.Submit(ctx, s.backupWorkflow, input, activitylib.StartRequest{+ EnvironmentID: environmentID, ResourceType: new("volume"), ResourceID: new(volumeName), ResourceName: new(volumeName), StartedBy: &user,+ Step: "Creating backup", LatestMessage: "Creating volume backup",+ Metadata: database.JSON{"action": "create_volume_backup", "backupId": entry.ID, "destination": entry.Destination, "policyId": request.PolicyID, "s3DestinationId": entry.S3DestinationID},+ }, backup.AcceptedTarget(entry.ID))+ if err != nil {+ release(ctx)+ return volume.BackupEntry{}, s.completeBackup(ctx, entry, err)+ }+ // A backup canceled before its task runs never takes the parked lease.+ go func() {+ background := context.WithoutCancel(ctx)+ s.watchBackup(background, entry.ID, activityID)+ release(background)+ }()entry.ActivityID = &activityID- accepted := entryInternal(entry)- keyID, _ := ctx.Value(middleware.ContextKeyApiKeyID).(string)- payload, err := json.Marshal(- manualVolumeBackupInternal{- Checkpoint: volumeBackupRecoveryInternal{- BackupID: entry.ID,- LocalEnabled: plan.localEnabled,- S3Enabled: plan.s3Enabled,- S3DestinationID: plan.s3DestinationID,- Policy: plan.policy,- },- VolumeName: volumeName,- UserID: user.ID,- ActivityID: activityID,- },- )- if err == nil {- err = s.deps.Engine.SubmitDurableRun(- workCtx,- backuptypes.DurableRunCommand{- Kind: "volume",- RunID: entry.ID,- ActivityID: activityID,- Payload: payload,- UserID: user.ID,- EnvironmentID: environmentID,- Permission: authz.PermVolumesBackup,- RequestedWithKey: keyID,- },- lease,- )- }+ return entry.Entry(), nil+}++// WatchAcceptedBackups restores, at startup, the watch over manual backups whose saved workflows resume.+func (s *Service) WatchAcceptedBackups(ctx context.Context) error {+ standalone, err := s.flow.Standalone(ctx)if err != nil {- lease.Release(ctx)- err = s.completeBackupInternal(workCtx, entry, err)- activitylib.CompleteHandlerActivity(workCtx, s.deps.Activity, activityID, "Volume backup created successfully", err)- return volume.BackupEntry{}, err+ return err+ }+ for _, run := range standalone {+ if backupID := backup.AcceptedBackupID(run); backupID != "" && run.JobID == s.backupWorkflow.Francis().Name() {+ go s.watchBackup(context.WithoutCancel(ctx), backupID, run.ID)+ }}+ return nil+}- return accepted, nil+// watchBackup fails the backup's row when its workflow ends without the task completing it, such as a cancel+// before the task started.+func (s *Service) watchBackup(ctx context.Context, backupID, instanceID string) {+ outcome, err := s.flow.Wait(ctx, s.backupWorkflow, instanceID)+ if err != nil || outcome.Status == scheduler.Succeeded {+ return+ }+ if current, loadErr := s.deps.Store.Run(ctx, backupID); loadErr == nil && current.Status == backuptypes.VolumeBackupStatusRunning {+ if failErr := s.deps.Store.Fail(ctx, backupID, cmp.Or(outcome.Message, "Backup did not run")); failErr != nil {+ slog.WarnContext(ctx, "Failed to close a volume backup whose workflow ended early", "backupId", backupID, "error", failErr)+ }+ }}func (s *Service) ReconcileBackup(ctx context.Context, previous scheduler.Run, volumeName string) (scheduler.Outcome, error) {@@ -1225,247 +991,189 @@ func (s *Service) ReconcileBackup(ctx context.Context, previous scheduler.Run, vif outcome.Status == scheduler.Succeeded {return outcome, nil}- for _, target := range previous.Outcome.Targets {- if target.ID != volumeName || len(target.RecoveryData) == 0 {- continue- }- var checkpoint volumeBackupRecoveryInternal- if err := json.Unmarshal(target.RecoveryData, &checkpoint); err != nil {- return outcome, err- }- entry, loadErr := s.deps.Store.Run(ctx, checkpoint.BackupID)- if loadErr != nil {- return outcome, loadErr+ index := slices.IndexFunc(previous.Outcome.Targets, func(target scheduler.TargetOutcome) bool {+ return target.ID == volumeName && len(target.RecoveryData) > 0+ })+ if index < 0 {+ return outcome, nil+ }+ target := previous.Outcome.Targets[index]+ var checkpoint backupPlan+ if err := json.Unmarshal(target.RecoveryData, &checkpoint); err != nil {+ return outcome, err+ }+ entry, err := s.deps.Store.Run(ctx, checkpoint.BackupID)+ if err != nil {+ return outcome, err+ }+ // Only an interrupted backup is still running; a failed one was settled by its attempt, even if its volume+ // was renamed since, and the caller starts a new backup under the current name.+ if entry.Status == backuptypes.VolumeBackupStatusFailed {+ return outcome, backup.ErrBackupSettled+ }+ if entry.VolumeName != volumeName {+ return outcome, gorm.ErrRecordNotFound+ }+ if entry.Status != backuptypes.VolumeBackupStatusSucceeded {+ // Resuming would copy data its stopped containers may since have changed, so the row fails and the+ // caller starts a new backup that stops them again.+ if checkpoint.Policy != nil && checkpoint.Policy.StopContainers {+ if failErr := s.deps.Store.Fail(ctx, entry.ID, "Backup was interrupted while its containers were stopped"); failErr != nil {+ return outcome, failErr+ }+ return outcome, backup.ErrBackupSettled}- if entry.VolumeName != volumeName {- return outcome, gorm.ErrRecordNotFound+ lease, admitted, acquireErr := s.deps.Engine.AcquireRun(ctx, backup.VolumeAdmissionScope, volumeName, "")+ if acquireErr != nil {+ return outcome, acquireErr}- if entry.Status != backuptypes.VolumeBackupStatusSucceeded {- if checkpoint.Policy != nil && checkpoint.Policy.StopContainers {- return outcome, nil- }- lease, admitted, err := s.deps.Engine.AcquireDurableRun(ctx, previous.ID, backup.VolumeAdmissionScope, volumeName)- if err != nil {- return outcome, err- }- if !admitted {- return outcome, nil- }- defer lease.Release(ctx) //nolint:gocritic // The matching recovery target always returns before the loop advances.- if resumeBackupErr := s.resumeBackupInternal(ctx, previous, entry, checkpoint); resumeBackupErr != nil {- return outcome, resumeBackupErr- }+ // Another backup of this volume is running; the task is handed back, without spending its one attempt,+ // so a later delivery resumes this one rather than leaving it running.+ if !admitted {+ return outcome, fmt.Errorf("%w: %w", actor.ErrJobRejected, s.deps.AlreadyRunning)}- target.Status = scheduler.Succeeded- if err := jobcontext.Progress(ctx, target); err != nil {- return outcome, err+ defer lease.Release(ctx)+ if resumeErr := s.resumeBackup(ctx, previous, entry, checkpoint); resumeErr != nil {+ return outcome, resumeErr}- return scheduler.Outcome{Status: scheduler.Succeeded, Targets: []scheduler.TargetOutcome{target}}, nil}- return outcome, nil+ target.Status = scheduler.Succeeded+ if progressErr := jobcontext.Progress(ctx, target); progressErr != nil {+ return outcome, progressErr+ }+ return scheduler.Outcome{Status: scheduler.Succeeded, Targets: []scheduler.TargetOutcome{target}}, nil}-func (- s *Service,-) recoverSnapshotInternal(- ctx context.Context,- dockerClient *client.Client,- repository backup.Repository,- entry *volume.Backup,- snapshotID string,-) (- backup.Snapshot,- bool,- error,-) {- password, err := s.volumeBackupPasswordInternal(ctx, dockerClient, repository)+// resumeBackup adopts snapshots an interrupted run already wrote, then finishes the backup.+func (s *Service) resumeBackup(ctx context.Context, previous scheduler.Run, entry *volume.Backup, checkpoint backupPlan) error {+ dockerClient, err := s.deps.Docker.GetClient(ctx)if err != nil {- return backup.Snapshot{}, false, err+ return err}- return s.deps.Engine.FindRunSnapshot(ctx, dockerClient, repository, password, entry.ID, snapshotID)-}--func (- s *Service,-) observeBackupDestinationsInternal(- ctx context.Context,- dockerClient *client.Client,- previous scheduler.Run,- entry *volume.Backup,- checkpoint volumeBackupRecoveryInternal,-) (- backup.Repository,- backup.Repository,- error,-) {var local, remote backup.Repository- var err error+ var repositories []backup.Repositoryif checkpoint.LocalEnabled {- local, err = s.localRusticRepositoryInternal(ctx, dockerClient, false)- if err != nil {- return local, remote, err- }- if entry.LocalSnapshotID != "" || backupDestinationAttemptedInternal(previous, entry.ID, "local") {- snapshot, found, recoverSnapshotErr := s.recoverSnapshotInternal(ctx, dockerClient, local, entry, entry.LocalSnapshotID)- if recoverSnapshotErr != nil {- return local, remote, recoverSnapshotErr- }- if found {- entry.LocalSnapshotID, entry.Size = snapshot.ID, snapshot.Size- }+ if local, err = s.localRusticRepository(ctx, dockerClient, false); err != nil {+ return err}+ repositories = append(repositories, local)}if checkpoint.S3Enabled {- remote, err = s.remoteRusticRepositoryInternal(ctx, checkpoint.S3DestinationID)- if err != nil {- return local, remote, err- }- if entry.RemoteSnapshotID != "" || backupDestinationAttemptedInternal(previous, entry.ID, "remote") {- snapshot, found, recoverSnapshotErr2 := s.recoverSnapshotInternal(ctx, dockerClient, remote, entry, entry.RemoteSnapshotID)- if recoverSnapshotErr2 != nil {- return local, remote, recoverSnapshotErr2- }- if found {- entry.RemoteSnapshotID = snapshot.ID- if entry.Size == 0 {- entry.Size = snapshot.Size- }- }+ if remote, err = s.remoteRusticRepository(ctx, checkpoint.S3DestinationID, ""); err != nil {+ return err}+ repositories = append(repositories, remote)}- return local, remote, nil-}--func (s *Service) resumeBackupInternal(ctx context.Context, previous scheduler.Run, entry *volume.Backup, checkpoint volumeBackupRecoveryInternal) error {- dockerClient, err := s.deps.Docker.GetClient(ctx)+ password, err := s.volumeBackupPassword(ctx, dockerClient, repositories...)if err != nil {return err}- local, remote, err := s.observeBackupDestinationsInternal(ctx, dockerClient, previous, entry, checkpoint)- if err != nil {- return err+ attempted := func(destination string) bool {+ return slices.ContainsFunc(previous.Outcome.Targets, func(target scheduler.TargetOutcome) bool { return target.ID == entry.ID+":"+destination })}- if checkpoint.LocalEnabled && entry.LocalSnapshotID == "" && entry.RemoteSnapshotID != "" {- password, volumeBackupPasswordErr := s.volumeBackupPasswordInternal(ctx, dockerClient, remote)- if volumeBackupPasswordErr != nil {- return volumeBackupPasswordErr+ if checkpoint.LocalEnabled && (entry.LocalSnapshotID != "" || attempted("local")) {+ snapshot, found, findErr := s.deps.Engine.FindRunSnapshot(ctx, dockerClient, local, password, entry.ID, entry.LocalSnapshotID)+ if findErr != nil {+ return findErr}- snapshot, volumeBackupPasswordErr := s.deps.Engine.Replicate(ctx, dockerClient, remote, entry.RemoteSnapshotID, local, password, entry.VolumeName, backup.RunSnapshotTag(entry.ID))- if volumeBackupPasswordErr != nil {- return volumeBackupPasswordErr+ if found {+ entry.LocalSnapshotID, entry.Size = snapshot.ID, snapshot.Size}- entry.LocalSnapshotID = snapshot.ID}- if operationErr := s.deps.Store.Save(ctx, entry); operationErr != nil {- return operationErr+ if checkpoint.S3Enabled && (entry.RemoteSnapshotID != "" || attempted("remote")) {+ snapshot, found, findErr := s.deps.Engine.FindRunSnapshot(ctx, dockerClient, remote, password, entry.ID, entry.RemoteSnapshotID)+ if findErr != nil {+ return findErr+ }+ if found {+ entry.RemoteSnapshotID, entry.Size = snapshot.ID, cmp.Or(entry.Size, snapshot.Size)+ }+ }+ if checkpoint.LocalEnabled && entry.LocalSnapshotID == "" && entry.RemoteSnapshotID != "" {+ snapshot, replicateErr := s.deps.Engine.Replicate(ctx, dockerClient, remote, entry.RemoteSnapshotID, local, password, entry.VolumeName, backup.RunSnapshotTag(entry.ID))+ if replicateErr != nil {+ return replicateErr+ }+ entry.LocalSnapshotID = snapshot.ID}- plan := backupPlanInternal{- localEnabled: checkpoint.LocalEnabled,- s3Enabled: checkpoint.S3Enabled,- s3DestinationID: checkpoint.S3DestinationID,- policy: checkpoint.Policy,- destination: entry.Destination,+ if saveErr := s.deps.Store.Save(ctx, entry); saveErr != nil {+ return saveErr}- err = s.executeBackupInternal(ctx, entry, usertypes.SystemUser, plan)- return s.completeBackupInternal(ctx, entry, err)-}--type manualVolumeBackupInternal struct {- Checkpoint volumeBackupRecoveryInternal `json:"checkpoint"`- VolumeName string `json:"volumeName"`- UserID string `json:"userId"`- ActivityID string `json:"activityId"`+ return s.completeBackup(ctx, entry, s.executeBackup(ctx, entry, usertypes.SystemUser, checkpoint))}-func (s *Service) executeDurableBackupInternal(ctx context.Context, runID string, payload []byte, interrupted bool) (err error) {- var command manualVolumeBackupInternal- if decodeErr := json.Unmarshal(payload, &command); decodeErr != nil {- return decodeErr+// runManualBackup is the volume-backup step. A delivery that finds the checkpoint it recorded+// resumes that backup; otherwise it takes over the admission StartBackup won and runs it.+func (s *Service) runManualBackup(ctx context.Context, t flow.Task) (_ any, err error) {+ var input manualBackupInput+ if decodeErr := t.Payload(&input); decodeErr != nil {+ return nil, decodeErr}defer func() {- if ctx.Err() == nil {- if err != nil {- saveErr := s.deps.Store.Fail(ctx, command.Checkpoint.BackupID, err.Error())- err = errors.Join(err, saveErr)- }- activitylib.CompleteHandlerActivity(ctx, s.deps.Activity, command.ActivityID, "Volume backup created successfully", err)+ if err != nil && ctx.Err() == nil && !errors.Is(err, actor.ErrJobRejected) && !errors.Is(err, backup.ErrBackupSettled) {+ err = errors.Join(err, s.deps.Store.Fail(ctx, input.Checkpoint.BackupID, err.Error()))}}()- entry, loadBackupErr := s.deps.Store.Run(ctx, command.Checkpoint.BackupID)- if loadBackupErr != nil {- return loadBackupErr+ previous, _ := jobcontext.Run(ctx)+ resuming := slices.ContainsFunc(previous.Outcome.Targets, func(target scheduler.TargetOutcome) bool {+ return target.ID == input.VolumeName && len(target.RecoveryData) > 0+ })+ if !resuming {+ lease, admitted, acquireErr := s.deps.Engine.AcquireRun(ctx, backup.VolumeAdmissionScope, input.VolumeName, input.Checkpoint.BackupID)+ if acquireErr != nil {+ return nil, acquireErr+ }+ if !admitted {+ return nil, s.deps.AlreadyRunning+ }+ defer lease.Release(ctx)}-- if entry.Status == backuptypes.VolumeBackupStatusSucceeded {- return nil+ if authorizeErr := s.deps.Engine.Authorize(ctx, input.Requester); authorizeErr != nil {+ return nil, authorizeErr}- if interrupted {- previous, _ := jobcontext.Run(ctx)- outcome, recoveryErr := s.ReconcileBackup(ctx, previous, command.VolumeName)- if recoveryErr != nil {- return recoveryErr+ succeeded := scheduler.Outcome{Status: scheduler.Succeeded, Message: "Volume backup created successfully"}+ if resuming {+ outcome, reconcileErr := s.ReconcileBackup(ctx, previous, input.VolumeName)+ // A settled row already records why it failed, so the run reports that rather than the sentinel.+ if errors.Is(reconcileErr, backup.ErrBackupSettled) {+ if entry, loadErr := s.deps.Store.Run(ctx, input.Checkpoint.BackupID); loadErr == nil && entry.Error != "" {+ return nil, fmt.Errorf("%w: %s", backup.ErrBackupSettled, entry.Error)+ }+ }+ if reconcileErr != nil {+ return nil, reconcileErr}if outcome.Status != scheduler.Succeeded {- return errors.New(outcome.Message)+ return nil, errors.New(outcome.Message)}- return nil+ return succeeded, nil}- lease, admitted, err := s.deps.Engine.AcquireDurableRun(ctx, runID, backup.VolumeAdmissionScope, command.VolumeName)+ entry, err := s.deps.Store.Run(ctx, input.Checkpoint.BackupID)if err != nil {- return err- }- if !admitted {- return s.deps.AlreadyRunning+ return nil, err}- defer lease.Release(ctx)- actor := usertypes.SystemUser- if command.UserID != "" && command.UserID != "agent" {+ requestedBy := usertypes.SystemUser+ if input.Requester.UserID != "" && input.Requester.UserID != "agent" {var localUser userdomain.User- if loadUserErr := s.deps.DB.WithContext(ctx).Where("id = ?", command.UserID).First(&localUser).Error; loadUserErr != nil {- return loadUserErr+ if loadUserErr := s.deps.DB.WithContext(ctx).Where("id = ?", input.Requester.UserID).First(&localUser).Error; loadUserErr != nil {+ return nil, loadUserErr}- actor = *localUser.Actor()+ requestedBy = *localUser.Actor()}- checkpoint, err := json.Marshal(command.Checkpoint)+ checkpoint, err := json.Marshal(input.Checkpoint)if err != nil {- return err- }- if progressErr := jobcontext.Progress(- ctx,- scheduler.TargetOutcome{- ResourceType: "volume_backup",- ID: command.VolumeName,- Status: scheduler.Running,- RecoveryData: checkpoint,- ActivityID: command.ActivityID,- },- ); progressErr != nil {- return progressErr+ return nil, err}- plan := backupPlanInternal{- localEnabled: command.Checkpoint.LocalEnabled,- s3Enabled: command.Checkpoint.S3Enabled,- s3DestinationID: command.Checkpoint.S3DestinationID,- policy: command.Checkpoint.Policy,- destination: entry.Destination,+ target := scheduler.TargetOutcome{ResourceType: "volume_backup", ID: input.VolumeName, Status: scheduler.Running, RecoveryData: checkpoint, ActivityID: t.ActivityID()}+ if progressErr := jobcontext.Progress(ctx, target); progressErr != nil {+ return nil, progressErr}- err = s.executeBackupInternal(ctx, entry, actor, plan)+ err = s.executeBackup(ctx, entry, requestedBy, input.Checkpoint)if ctx.Err() != nil {- return ctx.Err()- }- return s.completeBackupInternal(ctx, entry, err)-}--func (s *Service) failDurableBackupInternal(ctx context.Context, _ string, payload []byte, runErr error) error {- var command manualVolumeBackupInternal- if err := json.Unmarshal(payload, &command); err != nil {- return err+ return nil, ctx.Err()}- entry, err := s.deps.Store.Run(ctx, command.Checkpoint.BackupID)- if err != nil {- return err+ if completeErr := s.completeBackup(ctx, entry, err); completeErr != nil {+ return nil, completeErr}- err = s.deps.Store.Fail(ctx, entry.ID, runErr.Error())- activitylib.CompleteHandlerActivity(ctx, s.deps.Activity, command.ActivityID, "Volume backup created successfully", runErr)- return err+ return succeeded, nil}diff --git a/backend/internal/volume/children/backup/service_test.go b/backend/internal/volume/children/backup/service_test.goindex 530c2afbc..3c70af66f 100644--- a/backend/internal/volume/children/backup/service_test.go+++ b/backend/internal/volume/children/backup/service_test.go@@ -16,7 +16,7 @@ import ("github.com/getarcaneapp/arcane/types/v2/volume""github.com/libtnb/sqlite""github.com/stretchr/testify/require"- "go.getarcane.app/kit/pkg"+ kit "go.getarcane.app/kit/pkg""go.getarcane.app/sys/crypto""gorm.io/gorm"@@ -24,11 +24,11 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/backup""github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/volume/children/backup/children/policies"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow/flowtest""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"- francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing")-func applyVolumeBackupMigrationsInternal(t *testing.T, gormDB *gorm.DB) {+func applyVolumeBackupMigrations(t *testing.T, gormDB *gorm.DB) {t.Helper()for _, name := range []string{"032_add_volume_backups.sql", "073_add_backup_support.sql", "087_add_volume_backup_remote_instance.sql"} {migration, err := os.ReadFile("../../../../resources/migrations/sqlite/" + name)@@ -37,41 +37,32 @@ func applyVolumeBackupMigrationsInternal(t *testing.T, gormDB *gorm.DB) {}}-func newVolumeAdmissionForTestInternal(t testing.TB) *runs.Admission {- t.Helper()- runtime := francistest.New(t)- gate := runs.NewAdmission(runtime.Service(), t.Name())- require.NoError(t, gate.Register(runtime))- francistest.Start(t, runtime)- return gate-}--type volumeBackupPolicySchedulerInternal struct {+type volumeBackupPolicyScheduler struct {jobs map[string]scheduler.Job}-func (s *volumeBackupPolicySchedulerInternal) AddJob(_ context.Context, job scheduler.Job) error {+func (s *volumeBackupPolicyScheduler) AddJob(_ context.Context, job scheduler.Job) error {s.jobs[job.Name()] = jobreturn nil}-func (s *volumeBackupPolicySchedulerInternal) RemoveJob(_ context.Context, name string) {+func (s *volumeBackupPolicyScheduler) RemoveJob(_ context.Context, name string) {delete(s.jobs, name)}-func (s *volumeBackupPolicySchedulerInternal) HasJob(name string) bool {+func (s *volumeBackupPolicyScheduler) HasJob(name string) bool {_, ok := s.jobs[name]return ok}-func (s *volumeBackupPolicySchedulerInternal) Submit(_ context.Context, request scheduler.Request) (scheduler.Run, error) {+func (s *volumeBackupPolicyScheduler) Submit(_ context.Context, request scheduler.Request) (scheduler.Run, error) {return scheduler.Run{ID: request.RunID, JobID: request.JobID, EnvironmentID: request.EnvironmentID, Status: scheduler.Queued}, nil}func TestVolumeBackupPolicy_RetentionIgnoresFailedRuns(t *testing.T) {gormDB, err := gorm.Open(sqlite.Open("file:volume-backup-retention-failed?mode=memory&cache=shared"), &gorm.Config{})require.NoError(t, err)- applyVolumeBackupMigrationsInternal(t, gormDB)+ applyVolumeBackupMigrations(t, gormDB)policyID := "policy-1"require.NoError(t, gormDB.Exec(@@ -83,20 +74,15 @@ func TestVolumeBackupPolicy_RetentionIgnoresFailedRuns(t *testing.T) {"failed-run", "app-data", 0, time.Now(), policyID, backuptypes.VolumeBackupStatusFailed,).Error)- service := NewService(Dependencies{DB: &database.DB{DB: gormDB}})- require.NoError(t, service.applyVolumeBackupRetentionInternal(t.Context(), policyID, 1, true))-- var backups []volume.Backup- require.NoError(t, gormDB.Table("volume_backups").Order("created_at ASC").Find(&backups).Error)- require.Len(t, backups, 2)- require.Equal(t, "snapshot-1", backups[0].LocalSnapshotID)- require.Equal(t, backuptypes.VolumeBackupStatusFailed, backups[1].Status)+ expired, err := backup.ExpiredRunIDs(t.Context(), &database.DB{DB: gormDB}, "volume_backups", policyID, 1)+ require.NoError(t, err)+ require.Empty(t, expired)}func TestVolumeBackupPolicy_ScheduledRunCreatesActivity(t *testing.T) {gormDB, err := gorm.Open(sqlite.Open("file:volume-backup-scheduled-activity?mode=memory&cache=shared"), &gorm.Config{})require.NoError(t, err)- require.NoError(t, gormDB.AutoMigrate(&policies.VolumeBackupPolicy{}, &activity.Activity{}))+ require.NoError(t, gormDB.AutoMigrate(&policies.VolumeBackupPolicy{}, &activity.Activity{}, &activity.ActivityMessage{}))db := &database.DB{DB: gormDB}policy := &policies.VolumeBackupPolicy{VolumeName: "app-data",@@ -106,23 +92,21 @@ func TestVolumeBackupPolicy_ScheduledRunCreatesActivity(t *testing.T) {LocalEnabled: true,}require.NoError(t, gormDB.Create(policy).Error)- gate := newVolumeAdmissionForTestInternal(t)- engine := backup.NewEngine(t.Context(), gate, nil)+ harness := flowtest.New(t, activity.NewActivityService(db, nil))+ gate := runs.NewAdmission(harness.Runtime.Service(), t.Name())+ require.NoError(t, gate.Register(harness.Runtime))+ engine := backup.NewEngine(gate, nil)t.Cleanup(func() { require.NoError(t, engine.Stop(context.WithoutCancel(t.Context()))) })- errAlreadyRunning := errors.New("a backup is already running for this volume")- service := NewService(Dependencies{- DB: db,- Activity: activity.NewActivityService(db, nil),- Engine: engine,- AlreadyRunning: errAlreadyRunning,- })- jobScheduler := &volumeBackupPolicySchedulerInternal{jobs: make(map[string]scheduler.Job)}+ service := NewService(Dependencies{DB: db, Engine: engine, AlreadyRunning: errors.New("a backup is already running for this volume")})+ require.NoError(t, service.RegisterWorkflows(harness.Engine))+ harness.Start(t)+ flowtest.AssertDefinitions(t, harness)+ jobScheduler := &volumeBackupPolicyScheduler{jobs: make(map[string]scheduler.Job)}require.NoError(t, service.SetScheduler(t.Context(), jobScheduler, gate))service.RegisterJobsOnStartup(t.Context())job, registered := jobScheduler.jobs["volume-backup:"+policy.ID]require.True(t, registered)- // Holding the volume's admission lease makes the scheduled run fail with- // "already running", which still must record a failed activity.+ // Holding the volume's admission lease skips the scheduled run, which still completes its activity.lease, admitted, err := engine.TryAcquireRun(t.Context(), backup.VolumeAdmissionScope, policy.VolumeName)require.NoError(t, err)require.True(t, admitted)@@ -134,7 +118,7 @@ func TestVolumeBackupPolicy_ScheduledRunCreatesActivity(t *testing.T) {var backupActivity activity.Activityrequire.NoError(t, gormDB.Where("resource_type = ?", "volume_backup").First(&backupActivity).Error)- require.Equal(t, activitytypes.StatusFailed, backupActivity.Status)+ require.Equal(t, activitytypes.StatusSuccess, backupActivity.Status)require.Equal(t, "scheduled_volume_backup", backupActivity.Metadata["action"])require.Equal(t, policy.Schedule, backupActivity.Metadata["schedule"])}@@ -171,13 +155,13 @@ func TestVolumeBackupPasswordPrefersRecoveryKey(t *testing.T) {RecoveryKeys: recoveryKeys,})- password, err := service.volumeBackupPasswordInternal(t.Context(), nil)+ password, err := service.volumeBackupPassword(t.Context(), nil)require.NoError(t, err)- require.Equal(t, kit.SHA256Hex(legacyVolumePasswordSaltInternal+service.deps.EncryptionKey), password)+ require.Equal(t, kit.SHA256Hex(legacyVolumePasswordSalt+service.deps.EncryptionKey), password)recoveryKey := "QWERTY-ABCDEF-234567-GHIJKL-MNOPQR-STUVWX-YZ2345-ZXCVBN"require.NoError(t, recoveryKeys.Set(t.Context(), recoveryKey))- password, err = service.volumeBackupPasswordInternal(t.Context(), nil)+ password, err = service.volumeBackupPassword(t.Context(), nil)require.NoError(t, err)require.Equal(t, recoveryKey, password)}@@ -186,7 +170,7 @@ func TestDiscoveredVolumeBackupMapping(t *testing.T) {snapshot := backup.DiscoveredSnapshot{ID: "abc123", Label: "app-data", Time: time.Unix(1700000000, 0).UTC()}snapshot.Summary.TotalBytesProcessed = 2048- entry := discoveredVolumeBackupInternal("destination-1", "instance-a", snapshot)+ entry := discoveredVolumeBackup("destination-1", "instance-a", snapshot)require.NotNil(t, entry)require.Equal(t, "app-data", entry.VolumeName)require.Equal(t, "remote-destination-1-instance-a-abc123", entry.ID)@@ -201,11 +185,11 @@ func TestDiscoveredVolumeBackupMapping(t *testing.T) {// Snapshots without a volume label, and system recovery snapshots, never// map to a volume backup.- require.Nil(t, discoveredVolumeBackupInternal("destination-1", "instance-a", backup.DiscoveredSnapshot{ID: "x"}))- require.Nil(t, discoveredVolumeBackupInternal("destination-1", "instance-a", backup.DiscoveredSnapshot{ID: "x", Label: "arcane-system-recovery"}))+ require.Nil(t, discoveredVolumeBackup("destination-1", "instance-a", backup.DiscoveredSnapshot{ID: "x"}))+ require.Nil(t, discoveredVolumeBackup("destination-1", "instance-a", backup.DiscoveredSnapshot{ID: "x", Label: "arcane-system-recovery"}))// A missing snapshot timestamp falls back to now.- fallback := discoveredVolumeBackupInternal("d", "r", backup.DiscoveredSnapshot{ID: "x", Label: "vol"})+ fallback := discoveredVolumeBackup("d", "r", backup.DiscoveredSnapshot{ID: "x", Label: "vol"})require.NotNil(t, fallback)require.False(t, fallback.CreatedAt.IsZero())}diff --git a/backend/internal/volume/handler.go b/backend/internal/volume/handler.goindex b6ec321d9..39eb24532 100644--- a/backend/internal/volume/handler.go+++ b/backend/internal/volume/handler.go@@ -307,15 +307,7 @@ func (h *VolumeHandler) RemoveVolume(ctx context.Context, input *RemoveVolumeInpreturn nil, huma.Error500InternalServerError("Failed to delete volume: " + err.Error())}- return &handlerutil.Out[base.MessageResponse]{- Body: base.ApiResponse[base.MessageResponse]{- Success: true,- Data: base.MessageResponse{- Message: "Volume removed successfully",- ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer(),- },- },- }, nil+ return handlerutil.MessageOutput("Volume removed successfully", activityID), nil}// PruneVolumes removes all unused Docker volumes.diff --git a/backend/internal/volume/service.go b/backend/internal/volume/service.goindex bc41c12e4..731ad0741 100644--- a/backend/internal/volume/service.go+++ b/backend/internal/volume/service.go@@ -22,11 +22,10 @@ import ("github.com/moby/moby/client""github.com/samber/mo""go.getarcane.app/docker"- "go.getarcane.app/kit/pkg"+ kit "go.getarcane.app/kit/pkg""golang.org/x/sync/singleflight""gorm.io/gorm"- "github.com/getarcaneapp/arcane/backend/v2/internal/activity""github.com/getarcaneapp/arcane/backend/v2/internal/backup""github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/internal/config"@@ -43,6 +42,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/volumes""github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs""github.com/getarcaneapp/arcane/backend/v2/pkg/utils"workspacepkg "github.com/getarcaneapp/arcane/backend/v2/pkg/workspace"@@ -75,7 +75,6 @@ func NewVolumeService(db *database.DB,dockerService *dockerInternal.DockerClientService,eventService *event.EventService,- activityService *activity.ActivityService,settingsService *settings.SettingsService,containerService *containerdomain.ContainerService,imageService *image.ImageService,@@ -113,7 +112,6 @@ func NewVolumeService(DB: db,Docker: dockerService,Events: eventService,- Activity: activityService,Settings: settingsService,Engine: engine,S3Destinations: s3Destinations,@@ -194,12 +192,22 @@ func (s *VolumeService) ReconcileBackup(ctx context.Context, previous scheduler.}// ReconcileInterruptedBackups runs before this process can accept backup work.+// RegisterWorkflows defines the backup workflows while the host is still unstarted.+func (s *VolumeService) RegisterWorkflows(engine *flow.Engine) error {+ return s.backup.RegisterWorkflows(engine)+}++// ReconcileInterruptedBackups fails backups left running by the previous process and watches the manual+// backups that saved workflows resume.func (s *VolumeService) ReconcileInterruptedBackups(ctx context.Context, protectedIDs ...string) error {query := s.db.WithContext(ctx).Model(&VolumeBackup{}).Where("status = ?", VolumeBackupStatusRunning)if len(protectedIDs) > 0 {query = query.Where("id NOT IN ?", protectedIDs)}- return query.Updates(map[string]any{"status": VolumeBackupStatusFailed, "error": "Backup interrupted by Arcane restart"}).Error+ if err := query.Updates(map[string]any{"status": VolumeBackupStatusFailed, "error": "Backup interrupted by Arcane restart"}).Error; err != nil {+ return err+ }+ return s.backup.WatchAcceptedBackups(ctx)}// PruneLocalBackupRepository frees the space of deleted local volume backups.diff --git a/backend/internal/volume/service_test.go b/backend/internal/volume/service_test.goindex 2e0248f52..63d88ab34 100644--- a/backend/internal/volume/service_test.go+++ b/backend/internal/volume/service_test.go@@ -124,7 +124,7 @@ func TestRenameVolumeMetadataInternalPreservesPoliciesAndHistory(t *testing.T) {require.NoError(t, gormDB.Exec("INSERT INTO volume_backup_policies (id, volume_name, schedule) VALUES (?, ?, ?)", "policy-1", "source-data", "0 0 2 * * *").Error)require.NoError(t, gormDB.Create(&VolumeBackup{VolumeName: "source-data", PolicyID: "policy-1"}).Error)- service := NewVolumeService(&database.DB{DB: gormDB}, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)+ service := NewVolumeService(&database.DB{DB: gormDB}, nil, nil, nil, nil, nil, nil, nil, nil, nil)require.NoError(t, service.renameVolumeMetadataInternal(t.Context(), "source-data", "renamed-data"))var renamedPolicy string@@ -271,7 +271,7 @@ func TestVolumeWorkspaceReadsWaitForMutationLock(t *testing.T) {t.Cleanup(server.Close)dockerService := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newVolumeWorkspaceTestDockerClientInternal(t, server))- service := NewVolumeService(nil, dockerService, nil, nil, nil, nil, nil, nil, nil, nil, nil)+ service := NewVolumeService(nil, dockerService, nil, nil, nil, nil, nil, nil, nil, nil)unlockMutation := service.workspaceLocks.Lock("workspace-volume")readStarted := make(chan struct{})readDone := make(chan error, 1)@@ -351,7 +351,7 @@ func TestDownloadVolumeWorkspaceFileHoldsReadLockUntilClosed(t *testing.T) {t.Cleanup(server.Close)dockerService := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newVolumeWorkspaceTestDockerClientInternal(t, server))- service := NewVolumeService(nil, dockerService, nil, nil, nil, nil, nil, nil, nil, nil, nil)+ service := NewVolumeService(nil, dockerService, nil, nil, nil, nil, nil, nil, nil, nil)service.helperByVolume["workspace-volume"] = &volumeHelper{id: "helper", lastUsedAt: time.Now(), protocol: types.ProtocolVersion}reader, size, err := service.workspace.DownloadVolumeWorkspaceFile(t.Context(), "workspace-volume", "file.txt")require.NoError(t, err)@@ -417,7 +417,7 @@ func TestUpdateVolumeWorkspaceRejectsStaleRevisionBeforeStaging(t *testing.T) {t.Cleanup(server.Close)dockerClient := newVolumeWorkspaceTestDockerClientInternal(t, server)- service := NewVolumeService(nil, docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(dockerClient), nil, nil, nil, nil, nil, nil, nil, nil, nil)+ service := NewVolumeService(nil, docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(dockerClient), nil, nil, nil, nil, nil, nil, nil, nil)uploadIndex := 0workspace, err := service.workspace.UpdateVolumeWorkspace(t.Context(), "workspace-volume", volumetypes.WorkspaceUpdateManifest{FileTreeRevision: "stale",@@ -457,7 +457,7 @@ func TestVolumeBackup_ListResolvesDestinationName(t *testing.T) {S3DestinationID: destination.ID,}).Error)- service := NewVolumeService(db, nil, nil, nil, nil, nil, nil, nil, s3.NewS3DestinationService(db, nil), nil, nil)+ service := NewVolumeService(db, nil, nil, nil, nil, nil, nil, s3.NewS3DestinationService(db, nil), nil, nil)backups, _, err := service.backup.List(t.Context(), "app-data", pagination.QueryParams{})require.NoError(t, err)require.Len(t, backups, 1)@@ -472,7 +472,7 @@ func TestVolumeBackupPolicy_GetReturnsLastRunForEachPolicyFromStore(t *testing.Trequire.NoError(t, gormDB.Create(&VolumeBackup{VolumeName: "app-data", PolicyID: "first-policy", Status: VolumeBackupStatusSucceeded}).Error)require.NoError(t, gormDB.Create(&VolumeBackup{VolumeName: "app-data", PolicyID: "second-policy", Status: VolumeBackupStatusFailed}).Error)- store := NewVolumeService(&database.DB{DB: gormDB}, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil).backupStoreInternal()+ store := NewVolumeService(&database.DB{DB: gormDB}, nil, nil, nil, nil, nil, nil, nil, nil, nil).backupStoreInternal()first, err := store.Latest(t.Context(), "first-policy")require.NoError(t, err)require.Equal(t, "succeeded", first.Status)diff --git a/backend/internal/vulnerability/children/scan/children/trivy/service.go b/backend/internal/vulnerability/children/scan/children/trivy/service.goindex 6436e27a2..8c169bfd2 100644--- a/backend/internal/vulnerability/children/scan/children/trivy/service.go+++ b/backend/internal/vulnerability/children/scan/children/trivy/service.go@@ -429,6 +429,10 @@ func (return nil, fmt.Errorf("trivy scan failed: %s", errMsg)}+ // The name can move to another image after it was selected; never store that image's findings under imageID.+ if scanned := trivyReport.Metadata.ImageID; scanned != "" && strings.TrimPrefix(scanned, "sha256:") != strings.TrimPrefix(imageID, "sha256:") {+ return nil, fmt.Errorf("%s now refers to image %s, not the selected image %s", imageName, scanned, imageID)+ }result := vulnerability.ConvertTrivyReportToScanResult(trivyReport, imageID, time.Now(), 0)result.ScannerVersion = scannerVersionresult.HasReport = s.persistRawTrivyReportInternal(ctx, imageID, trivyOutput, stdoutFile)diff --git a/backend/internal/vulnerability/children/scan/service.go b/backend/internal/vulnerability/children/scan/service.goindex 11bbe7194..dda53a942 100644--- a/backend/internal/vulnerability/children/scan/service.go+++ b/backend/internal/vulnerability/children/scan/service.go@@ -6,20 +6,19 @@ import ("errors""fmt""log/slog"+ "slices""strings""sync"- "sync/atomic""time"activitytypes "github.com/getarcaneapp/arcane/types/v2/activity""github.com/getarcaneapp/arcane/types/v2/features"usertypes "github.com/getarcaneapp/arcane/types/v2/user""github.com/getarcaneapp/arcane/types/v2/vulnerability"- "github.com/moby/moby/api/types/image""github.com/moby/moby/api/types/registry"+ "github.com/moby/moby/client""github.com/samber/mo"- "go.getarcane.app/kit/pkg"- "golang.org/x/sync/errgroup"+ kit "go.getarcane.app/kit/pkg""github.com/getarcaneapp/arcane/backend/v2/internal/activity""github.com/getarcaneapp/arcane/backend/v2/internal/common"@@ -32,14 +31,23 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability/children/scan/children/trivy"activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts""github.com/getarcaneapp/arcane/backend/v2/pkg/utils")const (- scansTableInternal = "vulnerability_scans"- scanStaleTimeout = 30 * time.Minute+ scansTable = "vulnerability_scans"+ scanStaleTimeout = 30 * time.Minute+ scheduledScanMaxAge = 7 * 24 * time.Hour+ // scanPoolIdle removes shared scan containers left behind by a scan that never finalized.+ scanPoolIdle = 2 * time.Minute+ // scanPoolSetupTimeout bounds creating the shared scan containers, including pulling the scanner image.+ scanPoolSetupTimeout = 10 * time.Minute)+// scheduledScanUser is the actor recorded for scheduled scans.+var scheduledScanUser = usertypes.Actor{Username: "System"}+// Records updates scan records, whose model the vulnerability domain owns.type Records struct {MarkScanning func(ctx context.Context, imageID string) error@@ -78,10 +86,39 @@ type Service struct {saveResult func(ctx context.Context, result *vulnerability.ScanResult) errorensureSummary func(result *vulnerability.ScanResult)filterIgnored func(ctx context.Context, envID, imageID string, vulns []vulnerability.Vulnerability) ([]vulnerability.Vulnerability, error)- // scanPhases tracks live scan phases in-memory for in-progress scans.- scanPhases sync.Map // map[string]vulnerability.ScanPhase+ scanPhases sync.Map // map[string]vulnerability.ScanPhase+ poolMu sync.Mutex+ pool *scanPool}+type (+ // Target is one image a scheduled scan covers.+ Target struct {+ ImageID string `json:"imageId"`+ ImageName string `json:"imageName"`+ ScannerVersion string `json:"scannerVersion,omitempty"`+ }++ // Fixable summarizes one image's fixable findings for the scan notification.+ Fixable struct {+ Total int `json:"total,omitempty"`+ Severity map[string]int `json:"severity,omitempty"`+ CVEs []string `json:"cves,omitempty"`+ }++ // scanPool is the set of long-running Trivy containers scheduled scans exec+ // into. Scan tasks share it across deliveries until finalize or idleness removes it.+ scanPool struct {+ containers chan string+ files map[string][]byte+ cleanup func()+ users int+ idle *time.Timer+ // retired closes when a container could not be stopped; waiting scans then retry on a fresh pool.+ retired chan struct{}+ }+)+func NewService(deps Dependencies) *Service {s := &Service{db: deps.DB,@@ -95,12 +132,12 @@ func NewService(deps Dependencies) *Service {ensureSummary: deps.EnsureSummary,filterIgnored: deps.FilterIgnored,}- s.trivy = trivy.NewService(deps.Config, deps.Docker, deps.Settings, deps.KV, deps.RegistryAuthConfigs, deps.SaveReport, s.setScanPhaseInternal)+ s.trivy = trivy.NewService(deps.Config, deps.Docker, deps.Settings, deps.KV, deps.RegistryAuthConfigs, deps.SaveReport, s.setScanPhase)return s}-// scanStateRowInternal is the freshness projection of a scan record.-type scanStateRowInternal struct {+// scanStateRow is the freshness projection of a scan record.+type scanStateRow struct {ID string `gorm:"column:id"`ImageName string `gorm:"column:image_name"`Status string `gorm:"column:status"`@@ -109,150 +146,149 @@ type scanStateRowInternal struct {ScanPolicy string `gorm:"column:scan_policy"`}-func (s *Service) setScanPhaseInternal(imageID string, phase vulnerability.ScanPhase) {+func (s *Service) setScanPhase(imageID string, phase vulnerability.ScanPhase) {if imageID == "" || phase == "" {return}s.scanPhases.Store(imageID, phase)}-func (s *Service) clearScanPhaseInternal(imageID string) {- if imageID == "" {- return- }- s.scanPhases.Delete(imageID)-}--func (s *Service) getScanPhaseInternal(imageID string) vulnerability.ScanPhase {- if imageID == "" {- return ""- }-- phase, ok := s.scanPhases.Load(imageID)- if !ok {- return ""- }-- if typed, localOk := phase.(vulnerability.ScanPhase); localOk {- return typed- }-- if str, localOk2 := phase.(string); localOk2 {- return vulnerability.ScanPhase(str)- }-- return ""-}--func (s *Service) resolveScanPhaseInternal(imageID string, status vulnerability.ScanStatus) vulnerability.ScanPhase {+// resolveScanPhase reports the live phase of an in-progress scan, falling back to one implied by its status.+func (s *Service) resolveScanPhase(imageID string, status vulnerability.ScanStatus) vulnerability.ScanPhase {if status != vulnerability.ScanStatusPending && status != vulnerability.ScanStatusScanning {return ""}-- if phase := s.getScanPhaseInternal(imageID); phase != "" {- return phase- }-- if status == vulnerability.ScanStatusPending {- return vulnerability.ScanPhaseCreatingContainer- }- return vulnerability.ScanPhaseScanningImage-}--func (s *Service) applyScanPhaseToSummaryInternal(summary *vulnerability.ScanSummary) {- if summary == nil {- return- }-- summary.ScanPhase = s.resolveScanPhaseInternal(summary.ImageID, summary.Status)-}--func (s *Service) applyScanPhaseToResultInternal(result *vulnerability.ScanResult) {- if result == nil {- return+ if phase, ok := s.scanPhases.Load(imageID); ok {+ if typed, isPhase := phase.(vulnerability.ScanPhase); isPhase && typed != "" {+ return typed+ }}-- result.ScanPhase = s.resolveScanPhaseInternal(result.ImageID, result.Status)+ return kit.Ternary(status == vulnerability.ScanStatusPending, vulnerability.ScanPhaseCreatingContainer, vulnerability.ScanPhaseScanningImage)}-// ScanImage scans an image for vulnerabilities using Trivy+// ScanImage queues a Trivy scan of an image and runs it in the background.func (s *Service) ScanImage(ctx context.Context, envID, imageID string, user usertypes.Actor) (*vulnerability.ScanResult, error) {if err := s.settingsService.RequireFeature(ctx, features.VulnerabilityManagement); err != nil {return nil, err}scanCtx := utils.ActivityRuntimeContext(ctx, nil)- // Get Docker client to inspect the imagedockerClient, err := s.dockerService.GetClient(ctx)if err != nil {return nil, fmt.Errorf("failed to connect to Docker: %w", err)}-- // Inspect the image to get detailsimageInspect, err := dockerClient.ImageInspect(scanCtx, imageID)if err != nil {return nil, fmt.Errorf("failed to inspect image: %w", err)}-- // Determine image name to scan- imageName := imageID- if len(imageInspect.RepoTags) > 0 {- imageName = imageInspect.RepoTags[0]- } else if len(imageInspect.RepoDigests) > 0 {- imageName = imageInspect.RepoDigests[0]- }+ imageName := slices.Concat(imageInspect.RepoTags, imageInspect.RepoDigests, []string{imageID})[0]slotID, releaseSlot, err := s.trivy.AcquireSlot(scanCtx, imageID, false)if err != nil {return nil, err}status := kit.Ternary(releaseSlot != nil, vulnerability.ScanStatusScanning, vulnerability.ScanStatusPending)-- slog.InfoContext(scanCtx,- "vulnerability scan queued",- "environmentId", envID,- "imageId", imageID,- "imageName", imageName,- )- activityID := s.startVulnerabilityScanActivityInternal(scanCtx, envID, imageID, imageName, &user)- // Make the scan cancelable: cancelling the activity cancels scanCtx, which- // unblocks the Trivy container wait and triggers its (cancel-safe) teardown.+ slog.InfoContext(scanCtx, "vulnerability scan queued", "environmentId", envID, "imageId", imageID, "imageName", imageName)++ activityID := ""+ if s.activityService != nil {+ scanActivity, startErr := s.activityService.StartActivity(scanCtx, activity.StartActivityRequest{+ EnvironmentID: envID,+ Type: activitytypes.TypeVulnerabilityScan,+ ResourceType: new("image"),+ ResourceID: &imageID,+ ResourceName: &imageName,+ StartedBy: &user,+ Progress: new(0),+ Step: string(vulnerability.ScanPhaseCreatingContainer),+ LatestMessage: "Vulnerability scan queued",+ })+ if startErr != nil {+ slog.WarnContext(scanCtx, "failed to create vulnerability scan activity", "error", startErr, "imageId", imageID)+ } else {+ activityID = scanActivity.ID+ }+ }+ activityRef := mo.EmptyableToOption(activityID).ToPointer()+ // Cancelling the activity cancels scanCtx, which stops the Trivy container wait and tears it down.scanCtx = s.activityService.Track(scanCtx, activityID)- s.setScanPhaseInternal(imageID, vulnerability.ScanPhaseCreatingContainer)+ s.setScanPhase(imageID, vulnerability.ScanPhaseCreatingContainer)- // Create pending scan record- pendingResult := &vulnerability.ScanResult{- ImageID: imageID,- ImageName: imageName,- ScanTime: time.Now(),- Status: status,- ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer(),- }- s.applyScanPhaseToResultInternal(pendingResult)+ pendingResult := &vulnerability.ScanResult{ImageID: imageID, ImageName: imageName, ScanTime: time.Now(), Status: status, ActivityID: activityRef}+ pendingResult.ScanPhase = s.resolveScanPhase(imageID, status)if saveErr := s.saveResult(scanCtx, pendingResult); saveErr != nil {- slog.WarnContext(scanCtx,- "failed to save pending scan result",- "error", saveErr,- "imageId", imageID,- "imageName", imageName,- )- } else {- slog.DebugContext(scanCtx,- "saved pending vulnerability scan record",- "imageId", imageID,- "imageName", imageName,- "scanTime", pendingResult.ScanTime,- )- }-- go s.scanImageInBackgroundInternal(scanCtx, envID, imageID, imageName, user, activityID, slotID, releaseSlot)+ slog.WarnContext(scanCtx, "failed to save pending scan result", "error", saveErr, "imageId", imageID, "imageName", imageName)+ }++ go func() {+ var runErr error+ if releaseSlot == nil {+ slotID, releaseSlot, runErr = s.trivy.AcquireSlot(scanCtx, imageID, true)+ }+ if releaseSlot != nil {+ defer releaseSlot()+ if updateErr := s.records.MarkScanning(scanCtx, imageID); updateErr != nil {+ slog.WarnContext(scanCtx, "failed to mark queued scan as running", "imageId", imageID, "error", updateErr)+ }+ }+ defer s.scanPhases.Delete(imageID)+ // fail records a failed scan; a scan that never started carries no duration.+ fail := func(message string, duration int64) {+ s.setScanPhase(imageID, vulnerability.ScanPhaseStoringResults)+ s.updateVulnerabilityScanActivity(scanCtx, activityID, vulnerability.ScanPhaseStoringResults, 90, "Storing failed scan result")+ failed := &vulnerability.ScanResult{+ ImageID: imageID, ImageName: imageName, ScanTime: time.Now(), Status: vulnerability.ScanStatusFailed,+ ActivityID: activityRef, Error: message, Duration: duration,+ }+ if saveErr := s.saveResult(scanCtx, failed); saveErr != nil {+ slog.WarnContext(scanCtx, "failed to save failed scan result", "error", saveErr)+ }+ slog.WarnContext(scanCtx, "background vulnerability scan failed", "environmentId", envID, "imageId", imageID, "imageName", imageName, "error", message)+ s.completeVulnerabilityScanActivity(scanCtx, activityID, false, message)+ }++ s.setScanPhase(imageID, vulnerability.ScanPhaseCreatingContainer)+ s.updateVulnerabilityScanActivity(scanCtx, activityID, vulnerability.ScanPhaseCreatingContainer, 10, "Preparing vulnerability scanner")+ var scannerImage string+ if runErr == nil {+ scannerImage, runErr = s.trivy.EnsureImage(scanCtx)+ }+ if runErr != nil {+ fail(kit.Ternary(errors.Is(runErr, common.ErrFeatureDisabled), runErr.Error(), "Trivy scanner is not available: "+runErr.Error()), 0)+ return+ }++ s.setScanPhase(imageID, vulnerability.ScanPhaseScanningImage)+ s.updateVulnerabilityScanActivity(scanCtx, activityID, vulnerability.ScanPhaseScanningImage, 45, "Scanning image for vulnerabilities")+ startTime := time.Now()+ result, runErr := s.trivy.Scan(scanCtx, scannerImage, imageName, imageID, slotID)+ duration := time.Since(startTime).Milliseconds()+ if runErr != nil {+ s.logScanEvent(scanCtx, event.EventTypeImageScan, "vulnerability_scan", envID, imageID, imageName, user, runErr.Error())+ fail(runErr.Error(), duration)+ return+ }++ result.Duration = duration+ result.ActivityID = activityRef+ s.ensureSummary(result)+ s.setScanPhase(imageID, vulnerability.ScanPhaseStoringResults)+ s.updateVulnerabilityScanActivity(scanCtx, activityID, vulnerability.ScanPhaseStoringResults, 90, "Storing vulnerability results")+ if saveErr := s.saveResult(scanCtx, result); saveErr != nil {+ slog.WarnContext(scanCtx, "failed to save scan result", "error", saveErr)+ }+ slog.InfoContext(scanCtx, "background vulnerability scan completed",+ "environmentId", envID, "imageId", imageID, "imageName", imageName, "durationMs", duration, "vulnerabilities", result.Summary.Total)+ s.logScanEvent(scanCtx, event.EventTypeImageScan, "vulnerability_scan", envID, imageID, imageName, user, "")+ s.completeVulnerabilityScanActivity(scanCtx, activityID, true, "")+ }()return pendingResult, nil}-// markStaleInternal fails a pending or scanning scan that outlived-// scanStaleTimeout and returns the failure message and duration.-func (s *Service) markStaleInternal(ctx context.Context, imageID string, status vulnerability.ScanStatus, scanTime time.Time) (string, int64, bool) {+// markStale fails a pending or scanning scan that outlived scanStaleTimeout+// and returns the failure message and duration.+func (s *Service) markStale(ctx context.Context, imageID string, status vulnerability.ScanStatus, scanTime time.Time) (string, int64, bool) {if status != vulnerability.ScanStatusScanning && status != vulnerability.ScanStatusPending {return "", 0, false}@@ -262,12 +298,10 @@ func (s *Service) markStaleInternal(ctx context.Context, imageID string, statuserrMsg := "Scan timed out or was interrupted. Please retry."duration := time.Since(scanTime).Milliseconds()- s.clearScanPhaseInternal(imageID)-+ s.scanPhases.Delete(imageID)if s.records.MarkFailed == nil {return errMsg, duration, true}-if err := s.records.MarkFailed(ctx, imageID, errMsg, duration); err != nil {slog.WarnContext(ctx, "failed to mark stale scan as failed", "error", err, "scanId", imageID)}@@ -279,11 +313,11 @@ func (s *Service) RefreshSummary(ctx context.Context, summary *vulnerability.Scaif summary == nil {return}- if errMsg, _, stale := s.markStaleInternal(ctx, summary.ImageID, summary.Status, summary.ScanTime); stale {+ if errMsg, _, stale := s.markStale(ctx, summary.ImageID, summary.Status, summary.ScanTime); stale {summary.Status = vulnerability.ScanStatusFailedsummary.Error = errMsg}- s.applyScanPhaseToSummaryInternal(summary)+ summary.ScanPhase = s.resolveScanPhase(summary.ImageID, summary.Status)}// RefreshResult fails stale in-progress scans and attaches the live scan phase.@@ -291,153 +325,18 @@ func (s *Service) RefreshResult(ctx context.Context, result *vulnerability.ScanRif result == nil {return}- if errMsg, duration, stale := s.markStaleInternal(ctx, result.ImageID, result.Status, result.ScanTime); stale {+ if errMsg, duration, stale := s.markStale(ctx, result.ImageID, result.Status, result.ScanTime); stale {result.Status = vulnerability.ScanStatusFailedresult.Error = errMsgresult.Duration = duration}- s.applyScanPhaseToResultInternal(result)-}--func (s *Service) scanImageInBackgroundInternal(ctx context.Context, envID, imageID, imageName string, user usertypes.Actor, activityID string, slotID int, releaseSlot func()) {- var err error- if releaseSlot == nil {- slotID, releaseSlot, err = s.trivy.AcquireSlot(ctx, imageID, true)- }- if releaseSlot != nil {- defer releaseSlot()- if updateErr := s.records.MarkScanning(ctx, imageID); updateErr != nil {- slog.WarnContext(ctx, "failed to mark queued scan as running", "imageId", imageID, "error", updateErr)- }- }- defer s.clearScanPhaseInternal(imageID)- s.setScanPhaseInternal(imageID, vulnerability.ScanPhaseCreatingContainer)- s.updateVulnerabilityScanActivityInternal(ctx, activityID, vulnerability.ScanPhaseCreatingContainer, 10, "Preparing vulnerability scanner")-- slog.DebugContext(ctx,- "starting background vulnerability scan",- "environmentId", envID,- "imageId", imageID,- "imageName", imageName,- )-- var scannerImage string- if err == nil {- scannerImage, err = s.trivy.EnsureImage(ctx)- }- if err != nil {- s.setScanPhaseInternal(imageID, vulnerability.ScanPhaseStoringResults)- s.updateVulnerabilityScanActivityInternal(ctx, activityID, vulnerability.ScanPhaseStoringResults, 90, "Unable to prepare vulnerability scanner")- errorMessage := "Trivy scanner is not available: " + err.Error()- if errors.Is(err, common.ErrFeatureDisabled) {- errorMessage = err.Error()- }- result := &vulnerability.ScanResult{- ImageID: imageID,- ImageName: imageName,- ScanTime: time.Now(),- Status: vulnerability.ScanStatusFailed,- ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer(),- Error: errorMessage,- }- if saveErr := s.saveResult(ctx, result); saveErr != nil {- slog.WarnContext(ctx, "failed to save scan result", "error", saveErr)- }- slog.WarnContext(ctx,- "background vulnerability scan failed before execution",- "environmentId", envID,- "imageId", imageID,- "imageName", imageName,- "error", result.Error,- )- s.completeVulnerabilityScanActivityInternal(ctx, activityID, false, result.Error)- return- }-- s.setScanPhaseInternal(imageID, vulnerability.ScanPhaseScanningImage)- s.updateVulnerabilityScanActivityInternal(ctx, activityID, vulnerability.ScanPhaseScanningImage, 45, "Scanning image for vulnerabilities")- startTime := time.Now()- result, err := s.trivy.Scan(ctx, scannerImage, imageName, imageID, slotID)- duration := time.Since(startTime).Milliseconds()-- if err != nil {- s.setScanPhaseInternal(imageID, vulnerability.ScanPhaseStoringResults)- s.updateVulnerabilityScanActivityInternal(ctx, activityID, vulnerability.ScanPhaseStoringResults, 90, "Storing failed scan result")- failedResult := &vulnerability.ScanResult{- ImageID: imageID,- ImageName: imageName,- ScanTime: time.Now(),- Status: vulnerability.ScanStatusFailed,- ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer(),- Error: err.Error(),- Duration: duration,- }- if saveErr := s.saveResult(ctx, failedResult); saveErr != nil {- slog.WarnContext(ctx, "failed to save failed scan result", "error", saveErr)- }- slog.WarnContext(ctx,- "background vulnerability scan failed",- "environmentId", envID,- "imageId", imageID,- "imageName", imageName,- "durationMs", duration,- "error", err,- )- s.logScanEvent(ctx, envID, imageID, imageName, user, false, err.Error())- s.completeVulnerabilityScanActivityInternal(ctx, activityID, false, err.Error())- return- }-- result.Duration = duration- result.ActivityID = mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()- s.ensureSummary(result)- s.setScanPhaseInternal(imageID, vulnerability.ScanPhaseStoringResults)- s.updateVulnerabilityScanActivityInternal(ctx, activityID, vulnerability.ScanPhaseStoringResults, 90, "Storing vulnerability results")- if saveErr := s.saveResult(ctx, result); saveErr != nil {- slog.WarnContext(ctx, "failed to save scan result", "error", saveErr)- }-- slog.InfoContext(ctx,- "background vulnerability scan completed",- "environmentId", envID,- "imageId", imageID,- "imageName", imageName,- "durationMs", duration,- "vulnerabilities", result.Summary.Total,- )-- s.logScanEvent(ctx, envID, imageID, imageName, user, true, "")- s.completeVulnerabilityScanActivityInternal(ctx, activityID, true, "")-}--func (s *Service) startVulnerabilityScanActivityInternal(ctx context.Context, envID, imageID, imageName string, user *usertypes.Actor) string {- if s.activityService == nil {- return ""- }-- scanActivity, err := s.activityService.StartActivity(ctx, activity.StartActivityRequest{- EnvironmentID: envID,- Type: activitytypes.TypeVulnerabilityScan,- ResourceType: new("image"),- ResourceID: &imageID,- ResourceName: &imageName,- StartedBy: user,- Progress: new(0),- Step: string(vulnerability.ScanPhaseCreatingContainer),- LatestMessage: "Vulnerability scan queued",- })- if err != nil {- slog.WarnContext(ctx, "failed to create vulnerability scan activity", "error", err, "imageId", imageID)- return ""- }- return scanActivity.ID+ result.ScanPhase = s.resolveScanPhase(result.ImageID, result.Status)}-func (s *Service) updateVulnerabilityScanActivityInternal(ctx context.Context, activityID string, phase vulnerability.ScanPhase, progress int, message string) {+func (s *Service) updateVulnerabilityScanActivity(ctx context.Context, activityID string, phase vulnerability.ScanPhase, progress int, message string) {if s.activityService == nil || activityID == "" {return}-if _, err := s.activityService.AppendMessage(ctx, activityID, activity.AppendActivityMessageRequest{Level: activitytypes.MessageLevelInfo,Message: message,@@ -448,34 +347,36 @@ func (s *Service) updateVulnerabilityScanActivityInternal(ctx context.Context, a}}-func (s *Service) completeVulnerabilityScanActivityInternal(ctx context.Context, activityID string, success bool, errMessage string) {+func (s *Service) completeVulnerabilityScanActivity(ctx context.Context, activityID string, success bool, errMessage string) {if s.activityService == nil || activityID == "" {return}-status := activitytypes.StatusSuccessmessage := "Vulnerability scan completed"var errorPtr *string- if !success {- if activitylib.CancelledByContext(ctx) {- status = activitytypes.StatusCancelled- message = "Vulnerability scan cancelled"- } else {- status = activitytypes.StatusFailed- message = "Vulnerability scan failed"- if strings.TrimSpace(errMessage) != "" {- errorPtr = &errMessage- message = errMessage- }- }+ switch {+ case success:+ case activitylib.CancelledByContext(ctx):+ status, message = activitytypes.StatusCancelled, "Vulnerability scan cancelled"+ default:+ status, message = activitytypes.StatusFailed, cmp.Or(errMessage, "Vulnerability scan failed")+ errorPtr = mo.EmptyableToOption(errMessage).ToPointer()}-if _, err := s.activityService.CompleteActivity(utils.ActivityRuntimeContext(ctx, nil), activityID, status, message, errorPtr); err != nil {slog.DebugContext(ctx, "failed to complete vulnerability scan activity", "activityId", activityID, "error", err)}}-const scheduledScanMaxAge = 7 * 24 * time.Hour+// logScanEvent records a manual or scheduled scan event; an empty errMsg means it succeeded.+func (s *Service) logScanEvent(ctx context.Context, eventType event.EventType, action, envID, imageID, imageName string, user usertypes.Actor, errMsg string) {+ metadata := database.JSON{"action": action, "imageId": imageID, "imageName": imageName, "success": errMsg == ""}+ if errMsg != "" {+ metadata["error"] = errMsg+ }+ if err := s.eventService.LogImageEvent(ctx, eventType, imageID, imageName, user.ID, user.Username, cmp.Or(envID, "0"), metadata); err != nil {+ slog.WarnContext(ctx, "failed to log vulnerability scan event", "action", action, "error", err)+ }+}// Policy describes the scan settings a stored result was produced with.func (s *Service) Policy() string {@@ -483,414 +384,254 @@ func (s *Service) Policy() string {return fmt.Sprintf("ignoreUnfixed=%t;server=%s", s.trivy.IgnoreUnfixed(), serverURL)}-type scheduledScanTargetInternal struct {- imageID string- imageName string-}--type scheduledScanNotificationSummary struct {- mu sync.Mutex- totalFixable int- imagesWithFixable int- severityCounts map[string]int- sampleCVEs []string- seenCVEs map[string]struct{}-}--func newScheduledScanNotificationSummary() *scheduledScanNotificationSummary {- return &scheduledScanNotificationSummary{- severityCounts: map[string]int{- "CRITICAL": 0,- "HIGH": 0,- "MEDIUM": 0,- "LOW": 0,- "UNKNOWN": 0,- },- sampleCVEs: make([]string, 0, 5),- seenCVEs: make(map[string]struct{}),- }-}--func (s *Service) runScheduledImageScanInternal(- ctx context.Context,- containerPool chan string,- imageName, imageID, scannerVersion string,- files map[string][]byte,-) (*vulnerability.ScanResult, int64, error) {- startTime := time.Now()- slotID, releaseSlot, slotErr := s.trivy.AcquireSlot(ctx, "", true)- if slotErr != nil {- return nil, 0, slotErr- }- defer releaseSlot()-- var containerID string- select {- case containerID = <-containerPool:- case <-ctx.Done():- return nil, 0, ctx.Err()- }- defer func() { containerPool <- containerID }()-+// Targets lists the images due for a scheduled scan, skipping completed results+// from the last week with the same scanner version and policy.+func (s *Service) Targets(ctx context.Context) ([]Target, error) {if err := s.settingsService.RequireFeature(ctx, features.VulnerabilityManagement); err != nil {- return nil, 0, err- }- result, scanErr := s.trivy.ExecScan(ctx, containerID, imageName, imageID, scannerVersion, slotID, files)-- return result, time.Since(startTime).Milliseconds(), scanErr-}--func (s *Service) saveScheduledFailedScanInternal(- ctx context.Context,- envID, imageID, imageName string,- user usertypes.Actor,- duration int64,- scanErr error,-) {- failedResult := &vulnerability.ScanResult{- ImageID: imageID,- ImageName: imageName,- ScanTime: time.Now(),- Status: vulnerability.ScanStatusFailed,- Error: scanErr.Error(),- Duration: duration,- }- if saveErr := s.saveResult(ctx, failedResult); saveErr != nil {- slog.WarnContext(ctx, "failed to save failed scan result", "error", saveErr)- }- s.logScheduledScanEvent(ctx, envID, imageID, imageName, user, false, scanErr.Error())-}--func (s *Service) saveScheduledSuccessfulScanInternal(- ctx context.Context,- envID, imageID, imageName string,- user usertypes.Actor,- result *vulnerability.ScanResult,- duration int64,-) {- result.Duration = duration- s.ensureSummary(result)- if saveErr := s.saveResult(ctx, result); saveErr != nil {- slog.WarnContext(ctx, "failed to save scan result", "error", saveErr)- }- s.logScheduledScanEvent(ctx, envID, imageID, imageName, user, true, "")-}--func (s *Service) collectScheduledFixableSummaryInternal(- ctx context.Context,- envID string,- result *vulnerability.ScanResult,- summary *scheduledScanNotificationSummary,-) {- fixableVulns := s.fixedVulnerabilitiesForNotifications(ctx, envID, result)- if len(fixableVulns) == 0 {- return- }-- summary.mu.Lock()- defer summary.mu.Unlock()- summary.imagesWithFixable++- for i := range fixableVulns {- v := fixableVulns[i]- summary.totalFixable++-- sev := strings.ToUpper(strings.TrimSpace(string(v.Severity)))- if _, ok := summary.severityCounts[sev]; !ok {- sev = "UNKNOWN"- }- summary.severityCounts[sev]++-- cveID := strings.TrimSpace(v.VulnerabilityID)- if cveID == "" {- continue- }- if _, alreadyAdded := summary.seenCVEs[cveID]; alreadyAdded {- continue- }- summary.seenCVEs[cveID] = struct{}{}- if len(summary.sampleCVEs) < 5 {- summary.sampleCVEs = append(summary.sampleCVEs, cveID)- }- }-}--func (s *Service) sendScheduledVulnerabilitySummaryInternal(- ctx context.Context,- scanned int,- summary *scheduledScanNotificationSummary,-) {- if !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) || summary.totalFixable == 0 || s.notificationService == nil {- return- }-- summaryDate := time.Now().UTC().Format("2006-01-02")- payload := notification.VulnerabilityNotificationPayload{- CVEID: "Daily Summary - " + summaryDate,- Severity: fmt.Sprintf(- "Critical:%d High:%d Medium:%d Low:%d Unknown:%d",- summary.severityCounts["CRITICAL"],- summary.severityCounts["HIGH"],- summary.severityCounts["MEDIUM"],- summary.severityCounts["LOW"],- summary.severityCounts["UNKNOWN"],- ),- ImageName: fmt.Sprintf("%d image(s) scanned, %d with fixable vulnerabilities", scanned, summary.imagesWithFixable),- FixedVersion: fmt.Sprintf("%d fixable vulnerability record(s)", summary.totalFixable),- }- if len(summary.sampleCVEs) > 0 {- payload.PkgName = strings.Join(summary.sampleCVEs, ", ")- }-- if err := s.notificationService.SendVulnerabilityNotification(ctx, payload); err != nil {- slog.WarnContext(ctx, "failed to send daily vulnerability summary notification", "error", err)- }-}--// ScanAllImages scans all Docker images for vulnerabilities. It is intended-// for use by the scheduled vulnerability scan job. A single long-running Trivy-// container is created and reused for every image via docker exec, which avoids-// the overhead of creating/destroying a container per scan. The caller-supplied-// user is recorded in the event log.-func (s *Service) ScanAllImages(ctx context.Context, envID string, user usertypes.Actor, force bool) (scanned, failed int, err error) {- if requireFeatureErr := s.settingsService.RequireFeature(ctx, features.VulnerabilityManagement); requireFeatureErr != nil {- return 0, 0, requireFeatureErr+ return nil, err}images, err := s.dockerService.ListImages(ctx)if err != nil {- return 0, 0, fmt.Errorf("failed to list images: %w", err)- }-- targets := make([]scheduledScanTargetInternal, 0, len(images))- for _, img := range images {- imageID, imageName, shouldScan := scheduledScanImageTargetInternal(img)- if shouldScan {- targets = append(targets, scheduledScanTargetInternal{imageID: imageID, imageName: imageName})- }+ return nil, fmt.Errorf("failed to list images: %w", err)}-scannerVersion := s.trivy.Version(ctx)- if !force {- targets, err = s.skipRecentlyScannedTargetsInternal(ctx, targets, scannerVersion)- if err != nil {- return 0, 0, err+ targets := make([]Target, 0, len(images))+ ids := make([]string, 0, len(images))+ for _, img := range images {+ imageName := slices.Concat(img.RepoTags, img.RepoDigests, []string{img.ID})[0]+ // Skip intermediate and dangling images with no usable name.+ if imageName != "<none>:<none>" && imageName != img.ID {+ targets = append(targets, Target{ImageID: img.ID, ImageName: imageName, ScannerVersion: scannerVersion})+ ids = append(ids, img.ID)}}- if len(targets) == 0 {- return 0, 0, nil- }-- concurrency := s.trivy.ConcurrentScanContainers()- containerIDs, files, cleanup, err := s.trivy.SetupBatchContainers(ctx, concurrency)- if err != nil {- return 0, 0, err- }- defer cleanup()-- containerPool := make(chan string, len(containerIDs))- for _, containerID := range containerIDs {- containerPool <- containerID- }-- notificationSummary := newScheduledScanNotificationSummary()- var scannedCount, failedCount atomic.Int64- var featureStopped atomic.Bool-- g, gctx := errgroup.WithContext(ctx)- g.SetLimit(concurrency)- for _, target := range targets {- g.Go(func() (workerErr error) {- defer utils.RecoverToError(&workerErr, "scheduled vulnerability scan worker", "imageId", target.imageID)- if gctx.Err() != nil {- return gctx.Err()- }- if !s.settingsService.IsFeatureEnabled(gctx, features.VulnerabilityManagement) {- featureStopped.Store(true)- return nil- }- switch s.scanSingleImage(gctx, containerPool, target.imageID, target.imageName, scannerVersion, envID, user, notificationSummary, files) {- case scanStatusSuccess:- scannedCount.Add(1)- case scanStatusFailed:- failedCount.Add(1)- case scanStatusSkipped:- featureStopped.Store(true)- }- return nil- })- }-- err = g.Wait()- scanned, failed = int(scannedCount.Load()), int(failedCount.Load())- if err != nil {- return scanned, failed, err- }-- if featureStopped.Load() {- return scanned, failed, fmt.Errorf("remaining scans stopped because feature %s was disabled: %w", features.VulnerabilityManagement, common.ErrFeatureDisabled)- }- s.sendScheduledVulnerabilitySummaryInternal(ctx, scanned, notificationSummary)-- return scanned, failed, nil-}--func (s *Service) skipRecentlyScannedTargetsInternal(ctx context.Context, targets []scheduledScanTargetInternal, scannerVersion string) ([]scheduledScanTargetInternal, error) {if len(targets) == 0 {return targets, nil}- ids := make([]string, 0, len(targets))- for _, target := range targets {- ids = append(ids, target.imageID)- }-- var records []scanStateRowInternal- if err := s.db.WithContext(ctx).- Table(scansTableInternal).+ var records []scanStateRow+ if queryErr := s.db.WithContext(ctx).+ Table(scansTable).Select("id", "image_name", "status", "scan_time", "scanner_version", "scan_policy").Where("id IN ?", ids).- Find(&records).Error; err != nil {- return nil, fmt.Errorf("failed to load existing scan records: %w", err)+ Find(&records).Error; queryErr != nil {+ return nil, fmt.Errorf("failed to load existing scan records: %w", queryErr)}-cutoff := time.Now().Add(-scheduledScanMaxAge)policy := s.Policy()- recent := make(map[string]scanStateRowInternal, len(records))+ recent := make(map[string]scanStateRow, len(records))for _, record := range records {if record.Status == string(vulnerability.ScanStatusCompleted) && record.ScannerVersion == scannerVersion && record.ScanPolicy == policy && record.ScanTime.After(cutoff) {recent[record.ID] = record}}-- remaining := make([]scheduledScanTargetInternal, 0, len(targets))- for _, target := range targets {- record, ok := recent[target.imageID]- if !ok {- remaining = append(remaining, target)- continue- }- if record.ImageName != target.imageName {- if err := s.records.SetImageName(ctx, target.imageID, target.imageName); err != nil {- slog.WarnContext(ctx, "failed to refresh scan record image name", "imageId", target.imageID, "error", err)+ targets = slices.DeleteFunc(targets, func(target Target) bool {+ record, ok := recent[target.ImageID]+ if ok && record.ImageName != target.ImageName {+ if renameErr := s.records.SetImageName(ctx, target.ImageID, target.ImageName); renameErr != nil {+ slog.WarnContext(ctx, "failed to refresh scan record image name", "imageId", target.ImageID, "error", renameErr)}}+ return ok+ })+ if len(recent) > 0 {+ slog.InfoContext(ctx, "scheduled vulnerability scan: skipping recently scanned images", "skipped", len(recent), "maxAge", scheduledScanMaxAge.String())}+ return targets, nil+}- if skipped := len(targets) - len(remaining); skipped > 0 {- slog.InfoContext(ctx, "scheduled vulnerability scan: skipping recently scanned images", "skipped", skipped, "maxAge", scheduledScanMaxAge.String())+// joinPool returns the shared scan containers, creating them on first use; leave must be called when the scan ends.+func (s *Service) joinPool(ctx context.Context) (*scanPool, func(), error) {+ s.poolMu.Lock()+ if s.pool == nil {+ // Setup stays cancelable and bounded, since later scans wait on poolMu; removing the containers detaches on its own.+ setupCtx, cancelSetup := context.WithTimeout(ctx, scanPoolSetupTimeout)+ containerIDs, files, cleanup, err := s.trivy.SetupBatchContainers(setupCtx, s.trivy.ConcurrentScanContainers())+ cancelSetup()+ if err != nil {+ s.poolMu.Unlock()+ return nil, nil, common.Classify(common.ErrUnavailable, fmt.Errorf("prepare scan containers: %w", err))+ }+ s.pool = &scanPool{containers: make(chan string, len(containerIDs)), files: files, cleanup: cleanup, retired: make(chan struct{})}+ for _, containerID := range containerIDs {+ s.pool.containers <- containerID+ }}-- return remaining, nil+ pool := s.pool+ if pool.idle != nil {+ pool.idle.Stop()+ pool.idle = nil+ }+ pool.users+++ s.poolMu.Unlock()+ return pool, func() {+ s.poolMu.Lock()+ pool.users--+ idle, retired := pool.users == 0 && s.pool == pool, pool.users == 0 && s.pool != pool+ // An interrupted last scan, such as at shutdown, removes the pool now; the idle timer would not outlive the process.+ if idle && ctx.Err() != nil {+ s.pool, idle, retired = nil, false, true+ }+ if idle {+ pool.idle = time.AfterFunc(scanPoolIdle, s.ReleaseScanPool)+ }+ s.poolMu.Unlock()+ // The last scan out of a retired pool removes its containers, the unstoppable one included.+ if retired {+ pool.cleanup()+ }+ }, nil}-func (s *Service) fixedVulnerabilitiesForNotifications(- ctx context.Context,- envID string,- result *vulnerability.ScanResult,-) []vulnerability.Vulnerability {- if result == nil || result.Status != vulnerability.ScanStatusCompleted || len(result.Vulnerabilities) == 0 {- return nil+// ScanTarget scans one image in the shared scan containers and saves its result.+// It returns common.ErrFeatureDisabled, without saving, once the feature is off.+func (s *Service) ScanTarget(ctx context.Context, envID string, target Target) (Fixable, error) {+ pool, leave, err := s.joinPool(ctx)+ if err != nil {+ return Fixable{}, err}+ defer leave()- vulnerabilities := result.Vulnerabilities- filtered, err := s.filterIgnored(ctx, envID, result.ImageID, vulnerabilities)+ slog.InfoContext(ctx, "scheduled vulnerability scan: scanning image", "image", target.ImageName, "imageId", target.ImageID)+ startTime := time.Now()+ slotID, releaseSlot, err := s.trivy.AcquireSlot(ctx, "", true)if err != nil {- slog.WarnContext(ctx, "failed to filter ignored vulnerabilities for notifications", "error", err)- } else {- vulnerabilities = filtered+ return Fixable{}, err}+ defer releaseSlot()+ var containerID string+ select {+ case containerID = <-pool.containers:+ case <-pool.retired:+ return Fixable{}, common.Classify(common.ErrUnavailable, errors.New("scan containers were replaced; retrying"))+ case <-ctx.Done():+ return Fixable{}, ctx.Err()+ }+ defer func() {+ // A cancel only detaches from the exec, so the container is restarted to stop Trivy before another scan reuses it.+ if ctx.Err() != nil {+ restartCtx, cancelRestart := context.WithTimeout(context.WithoutCancel(ctx), timeouts.GetDuration(0, timeouts.DefaultDockerAPI))+ defer cancelRestart()+ dockerClient, clientErr := s.dockerService.GetClient(restartCtx)+ if clientErr == nil {+ _, clientErr = dockerClient.ContainerRestart(restartCtx, containerID, client.ContainerRestartOptions{})+ }+ if clientErr != nil {+ // Without this container the pool cannot serve scans, so later scans start a fresh one.+ slog.WarnContext(restartCtx, "scheduled vulnerability scan: failed to stop a canceled scan; retiring its scan containers", "containerId", containerID, "error", clientErr)+ s.poolMu.Lock()+ select {+ case <-pool.retired:+ default:+ close(pool.retired)+ }+ if s.pool == pool {+ s.pool = nil+ }+ s.poolMu.Unlock()+ return+ }+ }+ pool.containers <- containerID+ }()+ if featureErr := s.settingsService.RequireFeature(ctx, features.VulnerabilityManagement); featureErr != nil {+ return Fixable{}, featureErr+ }+ result, err := s.trivy.ExecScan(ctx, containerID, target.ImageName, target.ImageID, target.ScannerVersion, slotID, pool.files)+ duration := time.Since(startTime).Milliseconds()+ if err != nil {+ failed := &vulnerability.ScanResult{+ ImageID: target.ImageID, ImageName: target.ImageName, ScanTime: time.Now(),+ Status: vulnerability.ScanStatusFailed, Error: err.Error(), Duration: duration,+ }+ if saveErr := s.saveResult(ctx, failed); saveErr != nil {+ slog.WarnContext(ctx, "failed to save failed scan result", "error", saveErr)+ }+ s.logScanEvent(ctx, event.EventTypeImageVulnerabilityScan, "scheduled_vulnerability_scan", envID, target.ImageID, target.ImageName, scheduledScanUser, err.Error())+ return Fixable{}, err+ }+ result.Duration = duration+ s.ensureSummary(result)+ if saveErr := s.saveResult(ctx, result); saveErr != nil {+ slog.WarnContext(ctx, "failed to save scan result", "error", saveErr)+ }+ s.logScanEvent(ctx, event.EventTypeImageVulnerabilityScan, "scheduled_vulnerability_scan", envID, target.ImageID, target.ImageName, scheduledScanUser, "")- fixable := make([]vulnerability.Vulnerability, 0, len(vulnerabilities))- for i := range vulnerabilities {- v := vulnerabilities[i]+ fixable := Fixable{Severity: map[string]int{}}+ if result.Status != vulnerability.ScanStatusCompleted || len(result.Vulnerabilities) == 0 {+ return fixable, nil+ }+ vulnerabilities, err := s.filterIgnored(ctx, envID, result.ImageID, result.Vulnerabilities)+ if err != nil {+ slog.WarnContext(ctx, "failed to filter ignored vulnerabilities for notifications", "error", err)+ vulnerabilities = result.Vulnerabilities+ }+ for _, v := range vulnerabilities {if strings.TrimSpace(v.FixedVersion) == "" {continue}- fixable = append(fixable, v)+ fixable.Total+++ fixable.Severity[strings.ToUpper(strings.TrimSpace(string(v.Severity)))]+++ if cveID := strings.TrimSpace(v.VulnerabilityID); cveID != "" && len(fixable.CVEs) < 5 && !slices.Contains(fixable.CVEs, cveID) {+ fixable.CVEs = append(fixable.CVEs, cveID)+ }}-- return fixable+ return fixable, nil}-// logScanEvent logs a vulnerability scan event-func (s *Service) logScanEvent(ctx context.Context, envID, imageID, imageName string, user usertypes.Actor, success bool, errMsg string) {- metadata := database.JSON{- "action": "vulnerability_scan",- "imageId": imageID,- "imageName": imageName,- "success": success,- }- if errMsg != "" {- metadata["error"] = errMsg+// ReleaseScanPool removes the shared scan containers once no scan is using them.+func (s *Service) ReleaseScanPool() {+ s.poolMu.Lock()+ pool := s.pool+ if pool == nil || pool.users > 0 {+ s.poolMu.Unlock()+ return}-- environmentID := cmp.Or(envID, "0")-- eventType := event.EventTypeImageScan- if err := s.eventService.LogImageEvent(ctx, eventType, imageID, imageName, user.ID, user.Username, environmentID, metadata); err != nil {- slog.WarnContext(ctx, "failed to log vulnerability scan event", "error", err)+ s.pool = nil+ if pool.idle != nil {+ pool.idle.Stop()}+ s.poolMu.Unlock()+ pool.cleanup()}-// logScheduledScanEvent logs a vulnerability scan event using the dedicated scheduled scan event type.-func (s *Service) logScheduledScanEvent(ctx context.Context, envID, imageID, imageName string, user usertypes.Actor, success bool, errMsg string) {- metadata := database.JSON{- "action": "scheduled_vulnerability_scan",- "imageId": imageID,- "imageName": imageName,- "success": success,+// NotifySummary sends the scheduled scan's daily summary of fixable findings.+func (s *Service) NotifySummary(ctx context.Context, scanned int, fixables []Fixable) {+ if !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) || s.notificationService == nil {+ return}- if errMsg != "" {- metadata["error"] = errMsg+ severity := map[string]int{"CRITICAL": 0, "HIGH": 0, "MEDIUM": 0, "LOW": 0, "UNKNOWN": 0}+ var total, images int+ var sampleCVEs []string+ for _, fixable := range fixables {+ if fixable.Total == 0 {+ continue+ }+ images+++ total += fixable.Total+ for sev, count := range fixable.Severity {+ if _, known := severity[sev]; !known {+ sev = "UNKNOWN"+ }+ severity[sev] += count+ }+ for _, cveID := range fixable.CVEs {+ if len(sampleCVEs) < 5 && !slices.Contains(sampleCVEs, cveID) {+ sampleCVEs = append(sampleCVEs, cveID)+ }+ }}-- environmentID := cmp.Or(envID, "0")-- if err := s.eventService.LogImageEvent(ctx, event.EventTypeImageVulnerabilityScan, imageID, imageName, user.ID, user.Username, environmentID, metadata); err != nil {- slog.WarnContext(ctx, "failed to log scheduled vulnerability scan event", "error", err)+ if total == 0 {+ return}-}--type scanResultStatus int--const (- scanStatusSuccess scanResultStatus = iota- scanStatusFailed- scanStatusSkipped-)--func (s *Service) scanSingleImage(- ctx context.Context,- containerPool chan string,- imageID, imageName, scannerVersion, envID string,- user usertypes.Actor,- notificationSummary *scheduledScanNotificationSummary,- files map[string][]byte,-) scanResultStatus {- slog.InfoContext(ctx, "scheduled vulnerability scan: scanning image", "image", imageName, "imageId", imageID)-- result, duration, scanErr := s.runScheduledImageScanInternal(ctx, containerPool, imageName, imageID, scannerVersion, files)- if scanErr != nil {- if errors.Is(scanErr, common.ErrFeatureDisabled) {- return scanStatusSkipped- }- s.saveScheduledFailedScanInternal(ctx, envID, imageID, imageName, user, duration, scanErr)- return scanStatusFailed+ payload := notification.VulnerabilityNotificationPayload{+ CVEID: "Daily Summary - " + time.Now().UTC().Format("2006-01-02"),+ Severity: fmt.Sprintf("Critical:%d High:%d Medium:%d Low:%d Unknown:%d", severity["CRITICAL"], severity["HIGH"], severity["MEDIUM"], severity["LOW"], severity["UNKNOWN"]),+ ImageName: fmt.Sprintf("%d image(s) scanned, %d with fixable vulnerabilities", scanned, images),+ FixedVersion: fmt.Sprintf("%d fixable vulnerability record(s)", total),+ PkgName: strings.Join(sampleCVEs, ", "),}-- s.saveScheduledSuccessfulScanInternal(ctx, envID, imageID, imageName, user, result, duration)- s.collectScheduledFixableSummaryInternal(ctx, envID, result, notificationSummary)- return scanStatusSuccess-}--func scheduledScanImageTargetInternal(img image.Summary) (imageID, imageName string, shouldScan bool) {- imageID = img.ID- imageName = imageID- if len(img.RepoTags) > 0 {- imageName = img.RepoTags[0]- } else if len(img.RepoDigests) > 0 {- imageName = img.RepoDigests[0]+ if err := s.notificationService.SendVulnerabilityNotification(ctx, payload); err != nil {+ slog.WarnContext(ctx, "failed to send daily vulnerability summary notification", "error", err)}-- // Skip intermediate / dangling images with no usable name.- return imageID, imageName, imageName != "<none>:<none>" && imageName != imageID}diff --git a/backend/internal/vulnerability/children/scan/service_test.go b/backend/internal/vulnerability/children/scan/service_test.goindex ad2bdc126..80ffde583 100644--- a/backend/internal/vulnerability/children/scan/service_test.go+++ b/backend/internal/vulnerability/children/scan/service_test.go@@ -6,18 +6,18 @@ import ("time""github.com/getarcaneapp/arcane/types/v2/features"- "github.com/getarcaneapp/arcane/types/v2/user""github.com/getarcaneapp/arcane/types/v2/vulnerability""github.com/libtnb/sqlite""github.com/stretchr/testify/require""gorm.io/gorm"+ "github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/notification""github.com/getarcaneapp/arcane/backend/v2/internal/settings")-func featureSettingsInternal(t *testing.T) (*settings.SettingsService, *database.DB) {+func featureSettings(t *testing.T) (*settings.SettingsService, *database.DB) {t.Helper()gdb, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})require.NoError(t, err)@@ -32,9 +32,9 @@ func featureSettingsInternal(t *testing.T) (*settings.SettingsService, *database}func TestDisabledScheduledItemPreservesPreviousResult(t *testing.T) {- settingsSvc, db := featureSettingsInternal(t)- require.NoError(t, db.Table("vulnerability_scans").AutoMigrate(&scanStateRowInternal{}))- require.NoError(t, db.Table("vulnerability_scans").Create(&scanStateRowInternal{+ settingsSvc, db := featureSettings(t)+ require.NoError(t, db.Table("vulnerability_scans").AutoMigrate(&scanStateRow{}))+ require.NoError(t, db.Table("vulnerability_scans").Create(&scanStateRow{ID: "test-image", Status: string(vulnerability.ScanStatusCompleted), ScanTime: time.Now(),}).Error)svc := NewService(Dependencies{@@ -45,21 +45,19 @@ func TestDisabledScheduledItemPreservesPreviousResult(t *testing.T) {},})require.NoError(t, settingsSvc.SetBoolSetting(t.Context(), features.VulnerabilityManagementSettingKey, false))- status := svc.scanSingleImage(t.Context(), nil, "test-image", "test:latest", "", "0", user.Actor{}, newScheduledScanNotificationSummary(), nil)+ _, err := svc.ScanTarget(t.Context(), "0", Target{ImageID: "test-image", ImageName: "test:latest"})- require.Equal(t, scanStatusSkipped, status)- var record scanStateRowInternal+ require.ErrorIs(t, err, common.ErrFeatureDisabled)+ var record scanStateRowrequire.NoError(t, db.Table("vulnerability_scans").First(&record, "id = ?", "test-image").Error)require.Equal(t, string(vulnerability.ScanStatusCompleted), record.Status)}func TestDisabledFeatureSuppressesScheduledSummaryNotification(t *testing.T) {- settingsSvc, _ := featureSettingsInternal(t)+ settingsSvc, _ := featureSettings(t)require.NoError(t, settingsSvc.SetBoolSetting(t.Context(), features.VulnerabilityManagementSettingKey, false))svc := &Service{settingsService: settingsSvc, notificationService: ¬ification.NotificationService{}}- summary := newScheduledScanNotificationSummary()- summary.totalFixable = 1require.NotPanics(t, func() {- svc.sendScheduledVulnerabilitySummaryInternal(t.Context(), 1, summary)+ svc.NotifySummary(t.Context(), 1, []Fixable{{Total: 1}})})}diff --git a/backend/internal/vulnerability/model.go b/backend/internal/vulnerability/model.goindex 729a0ede4..d86a8e88d 100644--- a/backend/internal/vulnerability/model.go+++ b/backend/internal/vulnerability/model.go@@ -110,7 +110,7 @@ func (VulnerabilityThreatIntel) TableName() string {return "vulnerability_threat_intel"}-type vulnerabilityScanItemDetailsInternal struct {+type vulnerabilityScanItemDetails struct {Description string `json:"description,omitempty"`References []string `json:"references,omitempty"`CVSS *vulnerability.CVSSInfo `json:"cvss,omitempty"`diff --git a/backend/internal/vulnerability/service.go b/backend/internal/vulnerability/service.goindex 10d83d071..fc66dfe25 100644--- a/backend/internal/vulnerability/service.go+++ b/backend/internal/vulnerability/service.go@@ -9,13 +9,20 @@ import ("net/http""time"+ "github.com/getarcaneapp/arcane/types/v2"+ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"+ "github.com/getarcaneapp/arcane/types/v2/features"+ "github.com/getarcaneapp/arcane/types/v2/scheduler"usertypes "github.com/getarcaneapp/arcane/types/v2/user""github.com/getarcaneapp/arcane/types/v2/vulnerability"+ "github.com/italypaleale/francis/builtin/workflow"dockerregistry "github.com/moby/moby/api/types/registry"+ kit "go.getarcane.app/kit/pkg""gorm.io/gorm""gorm.io/gorm/clause""github.com/getarcaneapp/arcane/backend/v2/internal/activity"+ "github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/docker"@@ -28,20 +35,36 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability/children/query""github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability/children/scan""github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability/children/store"+ activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/vuln"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/dbutil")+// ignoreIdentityQuery matches the ignore of one finding in one image and environment.+const ignoreIdentityQuery = "environment_id = ? AND image_id = ? AND vulnerability_id = ? AND pkg_name = ? AND installed_version = ?"++// scanParallelism caps concurrent scan tasks; Trivy slots apply the configured limit below it.+const scanParallelism = 16++// scanSlot is one scan task's output; failures travel as task errors.+type scanSlot struct {+ Skipped bool `json:"skipped,omitempty"`+ Fixable scan.Fixable `json:"fixable"`+}+// VulnerabilityService coordinates vulnerability scanning, findings, ignores,// and risk, and owns the scan record and raw report persistence.type VulnerabilityService struct {db *database.DBsettingsService *settings.SettingsService- scan *scan.Service- query *query.Service- store *store.Service- ignores *ignores.Service+ scan *scan.Service+ query *query.Service+ // scanWorkflow runs scheduled scans; see RegisterWorkflows.+ scanWorkflow *flow.Workflow+ store *store.Service+ ignores *ignores.Service}// NewVulnerabilityService creates a new VulnerabilityService instance@@ -65,46 +88,183 @@ func NewVulnerabilityService(}s.store = store.NewService(db, store.Records{- Save: s.saveRecordInternal,- SaveStatus: s.saveRecordStatusInternal,- Delete: s.deleteRecordsInternal,- DeleteReports: s.deleteReportsInternal,- IDsExcept: s.recordIDsExceptInternal,- DeleteExcept: s.deleteRecordsExceptInternal,- EachMissingItems: s.eachRecordMissingItemsInternal,- FixableCounts: s.fixableCountsInternal,- SaveReport: s.saveReportInternal,- SetFixableCount: s.setFixableCountInternal,- DeleteItems: deleteItemsInternal,- SaveItems: saveItemsInternal,- CountKnownExploited: s.countKnownExploitedInternal,- ReplaceKnownExploited: s.replaceKnownExploitedInternal,- StaleEPSSCVEs: s.staleEPSSCVEsInternal,- SaveEPSS: s.saveEPSSInternal,- PruneThreatIntel: s.pruneThreatIntelInternal,+ Save: s.saveRecord,+ SaveStatus: func(ctx context.Context, result *vulnerability.ScanResult, summary *vulnerability.SeveritySummary, errMessage string) error {+ updates := map[string]any{+ "image_name": result.ImageName,+ "status": string(result.Status),+ "scan_time": result.ScanTime,+ "duration": result.Duration,+ "critical_count": summary.Critical,+ "high_count": summary.High,+ "medium_count": summary.Medium,+ "low_count": summary.Low,+ "unknown_count": summary.Unknown,+ "total_count": summary.Total,+ "scanner_version": result.ScannerVersion,+ "error": nil,+ }+ if errMessage != "" {+ updates["error"] = errMessage+ }+ return s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{}).Where("id = ?", result.ImageID).Updates(updates).Error+ },+ Delete: func(ctx context.Context, imageIDs []string) error {+ if err := s.db.WithContext(ctx).Where("id IN ?", imageIDs).Delete(&VulnerabilityScanRecord{}).Error; err != nil {+ return err+ }+ s.query.InvalidateRiskOverview()+ return nil+ },+ DeleteReports: func(ctx context.Context, imageIDs []string) error {+ return s.db.WithContext(ctx).Where("image_id IN ?", imageIDs).Delete(&VulnerabilityReportRecord{}).Error+ },+ IDsExcept: func(ctx context.Context, keepIDs []string) ([]string, error) {+ scope := s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{})+ if len(keepIDs) > 0 {+ scope = scope.Where("id NOT IN ?", keepIDs)+ }+ var ids []string+ err := scope.Pluck("id", &ids).Error+ return ids, err+ },+ DeleteExcept: func(ctx context.Context, keepIDs []string) (int64, error) {+ scope := s.db.WithContext(ctx).Where("1 = 1")+ if len(keepIDs) > 0 {+ scope = s.db.WithContext(ctx).Where("id NOT IN ?", keepIDs)+ }+ result := scope.Delete(&VulnerabilityScanRecord{})+ if result.Error != nil {+ return 0, result.Error+ }+ return result.RowsAffected, nil+ },+ EachMissingItems: s.eachRecordMissingItems,+ FixableCounts: func(ctx context.Context) (map[string]*int, error) {+ var scans []VulnerabilityScanRecord+ if err := s.db.WithContext(ctx).Select("id", "fixable_count").Find(&scans).Error; err != nil {+ return nil, err+ }+ counts := make(map[string]*int, len(scans))+ for i := range scans {+ counts[scans[i].ID] = scans[i].FixableCount+ }+ return counts, nil+ },+ SaveReport: s.saveReport,+ SetFixableCount: func(ctx context.Context, imageID string, count int) error {+ return s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{}).Where("id = ?", imageID).Update("fixable_count", count).Error+ },+ DeleteItems: func(tx *gorm.DB, imageIDs []string) error {+ return tx.Where("image_id IN ?", imageIDs).Delete(&VulnerabilityScanItem{}).Error+ },+ SaveItems: saveItems,+ CountKnownExploited: func(ctx context.Context) (int64, error) {+ var count int64+ err := s.db.WithContext(ctx).Model(&VulnerabilityThreatIntel{}).Where("known_exploited = ?", true).Count(&count).Error+ return count, err+ },+ ReplaceKnownExploited: s.replaceKnownExploited,+ StaleEPSSCVEs: func(ctx context.Context, checkedBefore time.Time, limit int) ([]string, error) {+ var cveIDs []string+ err := s.db.WithContext(ctx).+ Model(&VulnerabilityScanItem{}).+ Distinct("vulnerability_scan_items.vulnerability_id").+ Joins("LEFT JOIN vulnerability_threat_intel ON vulnerability_threat_intel.cve_id = vulnerability_scan_items.vulnerability_id").+ Where("vulnerability_scan_items.vulnerability_id LIKE ?", "CVE-%").+ Where("vulnerability_threat_intel.epss_checked_at IS NULL OR vulnerability_threat_intel.epss_checked_at < ?", checkedBefore).+ Limit(limit).+ Pluck("vulnerability_scan_items.vulnerability_id", &cveIDs).Error+ return cveIDs, err+ },+ SaveEPSS: func(ctx context.Context, entries []vulnerability.ThreatIntelEntry) error {+ batch := make([]VulnerabilityThreatIntel, 0, len(entries))+ for i := range entries {+ batch = append(batch, VulnerabilityThreatIntel(entries[i]))+ }+ return s.db.WithContext(ctx).Clauses(clause.OnConflict{+ Columns: []clause.Column{{Name: "cve_id"}},+ DoUpdates: clause.AssignmentColumns([]string{"epss_score", "epss_percentile", "epss_checked_at", "updated_at"}),+ }).Create(&batch).Error+ },+ PruneThreatIntel: func(ctx context.Context) error {+ return s.db.WithContext(ctx).+ Where("known_exploited = ? AND NOT EXISTS (SELECT 1 FROM vulnerability_scan_items WHERE vulnerability_scan_items.vulnerability_id = vulnerability_threat_intel.cve_id)", false).+ Delete(&VulnerabilityThreatIntel{}).Error+ },}, dockerService, settingsService, kvService, httpClient)- s.query = query.NewService(db, cfg, dockerService, settingsService, s.loadResultInternal, s.refreshSummaryInternal, s.store.ThreatIntelStatus, s.filterIgnoredInternal)+ s.query = query.NewService(db, cfg, dockerService, settingsService, s.loadResult, func(ctx context.Context, summary *vulnerability.ScanSummary) {+ s.scan.RefreshSummary(ctx, summary)+ }, s.store.ThreatIntelStatus, func(ctx context.Context, envID, imageID string, vulns []vulnerability.Vulnerability) ([]vulnerability.Vulnerability, error) {+ return s.ignores.FilterForImage(ctx, envID, imageID, vulns)+ })s.ignores = ignores.NewService(db, ignores.Records{- ForImage: s.ignoresForImageInternal,- Exists: s.ignoreExistsInternal,- Get: s.getIgnoreInternal,- Create: createIgnoresInternal,- DeleteIdentity: s.deleteIgnoreIdentityInternal,- DeleteIDs: deleteIgnoreIDsInternal,- Page: s.ignorePageInternal,- Changed: s.query.InvalidateRiskOverview,+ ForImage: func(ctx context.Context, envID, imageID string) ([]vulnerability.IgnoredVulnerability, error) {+ var ignoreRecords []VulnerabilityIgnore+ if err := s.db.WithContext(ctx).Where("environment_id = ? AND image_id = ?", envID, imageID).Find(&ignoreRecords).Error; err != nil {+ return nil, err+ }+ return ignoredVulnerabilities(ignoreRecords), nil+ },+ Exists: func(ctx context.Context, ignore *vulnerability.IgnoredVulnerability) (bool, error) {+ var existing VulnerabilityIgnore+ err := s.db.WithContext(ctx).Where(+ ignoreIdentityQuery,+ ignore.EnvironmentID, ignore.ImageID, ignore.VulnerabilityID, ignore.PkgName, ignore.InstalledVersion,+ ).First(&existing).Error+ if errors.Is(err, gorm.ErrRecordNotFound) {+ return false, nil+ }+ return err == nil, err+ },+ Get: s.getIgnore,+ Create: func(tx *gorm.DB, entries []vulnerability.IgnoredVulnerability, batchSize int) ([]vulnerability.IgnoredVulnerability, error) {+ records := make([]VulnerabilityIgnore, len(entries))+ for i := range entries {+ records[i] = VulnerabilityIgnore(entries[i])+ }+ if err := tx.CreateInBatches(&records, batchSize).Error; err != nil {+ return nil, err+ }+ return ignoredVulnerabilities(records), nil+ },+ DeleteIdentity: func(ctx context.Context, ignore *vulnerability.IgnoredVulnerability) error {+ return s.db.WithContext(ctx).Where(+ ignoreIdentityQuery,+ ignore.EnvironmentID, ignore.ImageID, ignore.VulnerabilityID, ignore.PkgName, ignore.InstalledVersion,+ ).Delete(&VulnerabilityIgnore{}).Error+ },+ DeleteIDs: func(tx *gorm.DB, envID string, ignoreIDs []string) (int64, error) {+ result := tx.Where("environment_id = ? AND id IN ?", envID, ignoreIDs).Delete(&VulnerabilityIgnore{})+ return result.RowsAffected, result.Error+ },+ Page: s.ignorePage,+ Changed: s.query.InvalidateRiskOverview,}, s.query.ActiveItemsQuery)s.scan = scan.NewService(scan.Dependencies{DB: db,Records: scan.Records{- MarkScanning: s.markScanningInternal,- MarkFailed: s.markFailedInternal,- SetImageName: s.setImageNameInternal,+ MarkScanning: func(ctx context.Context, imageID string) error {+ return s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{}).Where("id = ?", imageID).Update("status", ScanStatusScanning).Error+ },+ MarkFailed: func(ctx context.Context, imageID, message string, duration int64) error {+ if s.db == nil {+ return nil+ }+ return s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{}).Where("id = ?", imageID).Updates(map[string]any{+ "status": ScanStatusFailed,+ "error": message,+ "duration": duration,+ }).Error+ },+ SetImageName: func(ctx context.Context, imageID, imageName string) error {+ return s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{}).Where("id = ?", imageID).Update("image_name", imageName).Error+ },},Config: cfg,KV: kvService,RegistryAuthConfigs: registryAuthConfigs,- SaveReport: s.saveReportInternal,+ SaveReport: s.saveReport,Docker: dockerService,Settings: settingsService,Events: eventService,@@ -129,11 +289,115 @@ func (s *VulnerabilityService) ScanImage(ctx context.Context, envID, imageID strreturn s.scan.ScanImage(ctx, envID, imageID, user)}-// ScanAllImages runs the scheduled scan of every image.-func (s *VulnerabilityService) ScanAllImages(ctx context.Context, envID string, user usertypes.Actor, force bool) (scanned, failed int, err error) {- return s.scan.ScanAllImages(ctx, envID, user, force)+// RegisterWorkflows defines the scheduled scan: discover due images, scan each+// as its own retryable task, then notify. Call it before the host starts.+func (s *VulnerabilityService) RegisterWorkflows(engine *flow.Engine) error {+ scanWorkflow, err := engine.Define(flow.Definition{+ Name: "vulnerability-scan",+ Version: 1,+ Fingerprint: "d221e3e86aded6a3afaed48abc267783425ff861c89accd834abf14148a754d3",+ Concurrency: scanParallelism,+ // Batches of large images can take hours; Trivy bounds each scan itself.+ Timeout: 12 * time.Hour,+ Activity: activitylib.StartRequest{+ Type: activitytypes.TypeVulnerabilityScan,+ ResourceType: new("images"),+ ResourceName: new("Scheduled vulnerability scan"),+ Step: "Discovering images",+ LatestMessage: "Scheduled vulnerability scan started",+ },+ Labels: map[string]string{+ "discover": "Discovering images",+ "scan": "Scanning images",+ "finalize": "Finishing vulnerability scan",+ },+ Steps: []workflow.StepSpec{+ workflow.Step("discover", engine.Handler(func(ctx context.Context, _ flow.Task) (any, error) {+ if !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) {+ return []scan.Target{}, nil+ }+ if deleted, err := s.CleanupOrphanedScanRecords(ctx); err != nil {+ slog.WarnContext(ctx, "orphaned vulnerability scan cleanup failed", "error", err)+ } else if deleted > 0 {+ slog.InfoContext(ctx, "cleaned up orphaned vulnerability scan records", "deleted", deleted)+ }+ // A feature disabled since the check above fails this attempt; the retry returns no targets.+ targets, err := s.scan.Targets(ctx)+ if err != nil {+ return nil, common.Classify(common.ErrUnavailable, err)+ }+ return targets, nil+ })),+ // Slots and the shared container pool bound real parallelism to the configured limit.+ workflow.ForEach("scan", engine.Handler(func(ctx context.Context, t flow.Task) (any, error) {+ var target scan.Target+ if err := t.DecodeItem(&target); err != nil {+ return nil, err+ }+ fixable, err := s.scan.ScanTarget(ctx, types.LocalDockerEnvironmentID, target)+ if errors.Is(err, common.ErrFeatureDisabled) {+ return scanSlot{Skipped: true}, nil+ }+ if err != nil {+ return nil, err+ }+ return scanSlot{Fixable: fixable}, nil+ }),+ workflow.WithItemsFrom("discover"),+ workflow.WithMaxParallel(scanParallelism),+ workflow.WithFailurePolicy(workflow.TolerateFailures),+ workflow.WithMaxAttempts(3),+ workflow.WithRetryBackoff(30*time.Second, 2*time.Minute)),+ workflow.Step("finalize", engine.Handler(func(ctx context.Context, t flow.Task) (any, error) {+ s.scan.ReleaseScanPool()+ var targets []scan.Target+ if err := t.DecodeOutput("discover", &targets); err != nil {+ return nil, err+ }+ slots, err := flow.Results[scanSlot](t, "scan")+ if err != nil {+ return nil, err+ }+ outcome := scheduler.Outcome{Status: scheduler.Succeeded}+ scanned, skipped := 0, 0+ fixables := make([]scan.Fixable, 0, len(slots))+ for index, slot := range slots {+ switch {+ case slot.Err != "":+ outcome.Targets = append(outcome.Targets, scheduler.TargetOutcome{ResourceType: "image", ID: targets[index].ImageName, Status: scheduler.Failed, Message: slot.Err})+ case slot.Value.Skipped:+ skipped+++ default:+ scanned+++ fixables = append(fixables, slot.Value.Fixable)+ }+ }+ outcome.Message = fmt.Sprintf("Vulnerability scan completed: %d scanned, %d failed", scanned, len(outcome.Targets))+ switch {+ case skipped > 0 || (len(targets) == 0 && !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement)):+ outcome.Status = kit.Ternary(scanned > 0, scheduler.Partial, scheduler.Skipped)+ outcome.Message = fmt.Sprintf("Remaining scans stopped because feature %s was disabled", features.VulnerabilityManagement)+ case len(outcome.Targets) > 0:+ outcome.Status = scheduler.Partial+ }+ if skipped == 0 {+ s.scan.NotifySummary(ctx, scanned, fixables)+ }+ slog.InfoContext(ctx, "scheduled vulnerability scan completed", "scanned", scanned, "failed", len(outcome.Targets), "skipped", skipped)+ return outcome, nil+ }), workflow.WithInputFrom("discover")),+ },+ })+ if err != nil {+ return err+ }+ s.scanWorkflow = scanWorkflow+ return nil}+// ScanWorkflow is the scheduled vulnerability scan workflow.+func (s *VulnerabilityService) ScanWorkflow() *flow.Workflow { return s.scanWorkflow }+// GetScanSummariesByImageIDs retrieves scan summaries for multiple images.func (s *VulnerabilityService) GetScanSummariesByImageIDs(ctx context.Context, imageIDs []string) (map[string]*vulnerability.ScanSummary, error) {return s.query.GetScanSummariesByImageIDs(ctx, imageIDs)@@ -179,16 +443,8 @@ func (s *VulnerabilityService) ImportLegacyReportFiles(ctx context.Context) {s.store.ImportLegacyReportFiles(ctx)}-func (s *VulnerabilityService) refreshSummaryInternal(ctx context.Context, summary *vulnerability.ScanSummary) {- s.scan.RefreshSummary(ctx, summary)-}--func (s *VulnerabilityService) filterIgnoredInternal(ctx context.Context, envID, imageID string, vulns []vulnerability.Vulnerability) ([]vulnerability.Vulnerability, error) {- return s.ignores.FilterForImage(ctx, envID, imageID, vulns)-}--// loadResultInternal reads the stored scan result for an image without insights.-func (s *VulnerabilityService) loadResultInternal(ctx context.Context, imageID string) (*vulnerability.ScanResult, error) {+// loadResult reads the stored scan result for an image without insights.+func (s *VulnerabilityService) loadResult(ctx context.Context, imageID string) (*vulnerability.ScanResult, error) {var record VulnerabilityScanRecorderr := s.db.WithContext(ctx).Where("id = ?", imageID).First(&record).Errorif err != nil {@@ -198,17 +454,6 @@ func (s *VulnerabilityService) loadResultInternal(ctx context.Context, imageID sreturn nil, fmt.Errorf("failed to get scan result: %w", err)}- result := convertRecordToResultInternal(ctx, &record)- s.scan.RefreshResult(ctx, result)- var reportCount int64- if countReportsErr := s.db.WithContext(ctx).Model(&VulnerabilityReportRecord{}).Where("image_id = ?", record.ID).Count(&reportCount).Error; countReportsErr == nil {- result.HasReport = reportCount > 0- }- return result, nil-}--// convertRecordToResultInternal converts a database record to a ScanResult.-func convertRecordToResultInternal(ctx context.Context, record *VulnerabilityScanRecord) *vulnerability.ScanResult {result := &vulnerability.ScanResult{ImageID: record.ID,ImageName: record.ImageName,@@ -232,19 +477,24 @@ func convertRecordToResultInternal(ctx context.Context, record *VulnerabilityScaif len(record.Vulnerabilities) > 0 && record.Vulnerabilities[0] != "" {var vulns []vulnerability.Vulnerability- if err := json.Unmarshal([]byte(record.Vulnerabilities[0]), &vulns); err != nil {- slog.WarnContext(ctx, "failed to unmarshal vulnerabilities", "error", err)+ if decodeErr := json.Unmarshal([]byte(record.Vulnerabilities[0]), &vulns); decodeErr != nil {+ slog.WarnContext(ctx, "failed to unmarshal vulnerabilities", "error", decodeErr)} else {result.Vulnerabilities = vulns}}store.EnsureSummary(result)- return result+ s.scan.RefreshResult(ctx, result)+ var reportCount int64+ if countReportsErr := s.db.WithContext(ctx).Model(&VulnerabilityReportRecord{}).Where("image_id = ?", record.ID).Count(&reportCount).Error; countReportsErr == nil {+ result.HasReport = reportCount > 0+ }+ return result, nil}-// saveRecordInternal upserts the scan record of result inside tx.-func (s *VulnerabilityService) saveRecordInternal(tx *gorm.DB, result *vulnerability.ScanResult, fixable int) error {+// saveRecord upserts the scan record of result inside tx.+func (s *VulnerabilityService) saveRecord(tx *gorm.DB, result *vulnerability.ScanResult, fixable int) error {var vulnJSON database.StringSliceif len(result.Vulnerabilities) > 0 {vulnBytes, err := json.Marshal(result.Vulnerabilities)@@ -288,66 +538,8 @@ func (s *VulnerabilityService) saveRecordInternal(tx *gorm.DB, result *vulnerabi}).Create(&record).Error}-// saveRecordStatusInternal writes only the status, counts, and error of result.-func (s *VulnerabilityService) saveRecordStatusInternal(ctx context.Context, result *vulnerability.ScanResult, summary *vulnerability.SeveritySummary, errMessage string) error {- updates := map[string]any{- "image_name": result.ImageName,- "status": string(result.Status),- "scan_time": result.ScanTime,- "duration": result.Duration,- "critical_count": summary.Critical,- "high_count": summary.High,- "medium_count": summary.Medium,- "low_count": summary.Low,- "unknown_count": summary.Unknown,- "total_count": summary.Total,- "scanner_version": result.ScannerVersion,- "error": nil,- }- if errMessage != "" {- updates["error"] = errMessage- }- return s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{}).Where("id = ?", result.ImageID).Updates(updates).Error-}--func (s *VulnerabilityService) deleteRecordsInternal(ctx context.Context, imageIDs []string) error {- if err := s.db.WithContext(ctx).Where("id IN ?", imageIDs).Delete(&VulnerabilityScanRecord{}).Error; err != nil {- return err- }- s.query.InvalidateRiskOverview()- return nil-}--func (s *VulnerabilityService) deleteReportsInternal(ctx context.Context, imageIDs []string) error {- return s.db.WithContext(ctx).Where("image_id IN ?", imageIDs).Delete(&VulnerabilityReportRecord{}).Error-}--// recordIDsExceptInternal lists scan record IDs outside keepIDs; an empty keepIDs lists every record.-func (s *VulnerabilityService) recordIDsExceptInternal(ctx context.Context, keepIDs []string) ([]string, error) {- scope := s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{})- if len(keepIDs) > 0 {- scope = scope.Where("id NOT IN ?", keepIDs)- }- var ids []string- err := scope.Pluck("id", &ids).Error- return ids, err-}--// deleteRecordsExceptInternal deletes scan records outside keepIDs; an empty keepIDs deletes every record.-func (s *VulnerabilityService) deleteRecordsExceptInternal(ctx context.Context, keepIDs []string) (int64, error) {- scope := s.db.WithContext(ctx).Where("1 = 1")- if len(keepIDs) > 0 {- scope = s.db.WithContext(ctx).Where("id NOT IN ?", keepIDs)- }- result := scope.Delete(&VulnerabilityScanRecord{})- if result.Error != nil {- return 0, result.Error- }- return result.RowsAffected, nil-}--// eachRecordMissingItemsInternal visits completed scans with findings but no finding rows.-func (s *VulnerabilityService) eachRecordMissingItemsInternal(ctx context.Context, batchSize int, fn func(imageID, vulnerabilitiesJSON string)) error {+// eachRecordMissingItems visits completed scans with findings but no finding rows.+func (s *VulnerabilityService) eachRecordMissingItems(ctx context.Context, batchSize int, fn func(imageID, vulnerabilitiesJSON string)) error {var batch []VulnerabilityScanRecordreturn s.db.WithContext(ctx).Select("id", "vulnerabilities").@@ -365,20 +557,8 @@ func (s *VulnerabilityService) eachRecordMissingItemsInternal(ctx context.Contex}).Error}-func (s *VulnerabilityService) fixableCountsInternal(ctx context.Context) (map[string]*int, error) {- var scans []VulnerabilityScanRecord- if err := s.db.WithContext(ctx).Select("id", "fixable_count").Find(&scans).Error; err != nil {- return nil, err- }- counts := make(map[string]*int, len(scans))- for i := range scans {- counts[scans[i].ID] = scans[i].FixableCount- }- return counts, nil-}--// saveReportInternal stores the raw Trivy report of an image, replacing any earlier one.-func (s *VulnerabilityService) saveReportInternal(ctx context.Context, imageID, data string) error {+// saveReport stores the raw Trivy report of an image, replacing any earlier one.+func (s *VulnerabilityService) saveReport(ctx context.Context, imageID, data string) error {if s.db == nil {return errors.New("database unavailable")}@@ -389,50 +569,12 @@ func (s *VulnerabilityService) saveReportInternal(ctx context.Context, imageID,}).Create(&record).Error}-func (s *VulnerabilityService) setFixableCountInternal(ctx context.Context, imageID string, count int) error {- return s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{}).Where("id = ?", imageID).Update("fixable_count", count).Error-}--func (s *VulnerabilityService) markScanningInternal(ctx context.Context, imageID string) error {- return s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{}).Where("id = ?", imageID).Update("status", ScanStatusScanning).Error-}--func (s *VulnerabilityService) markFailedInternal(ctx context.Context, imageID, message string, duration int64) error {- if s.db == nil {- return nil- }- return s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{}).Where("id = ?", imageID).Updates(map[string]any{- "status": ScanStatusFailed,- "error": message,- "duration": duration,- }).Error-}--func (s *VulnerabilityService) setImageNameInternal(ctx context.Context, imageID, imageName string) error {- return s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{}).Where("id = ?", imageID).Update("image_name", imageName).Error-}-const (- scanItemInsertBatchSizeInternal = 500- threatIntelUpsertBatchInternal = 500+ scanItemInsertBatchSize = 500+ threatIntelUpsertBatch = 500)-func deleteItemsInternal(tx *gorm.DB, imageIDs []string) error {- return tx.Where("image_id IN ?", imageIDs).Delete(&VulnerabilityScanItem{}).Error-}--func saveItemsInternal(tx *gorm.DB, imageID string, vulns []vulnerability.Vulnerability) error {- items, err := buildScanItemsInternal(imageID, vulns)- if err != nil {- return err- }- if storeItemsErr := tx.CreateInBatches(items, scanItemInsertBatchSizeInternal).Error; storeItemsErr != nil {- return fmt.Errorf("failed to store scan items: %w", storeItemsErr)- }- return nil-}--func buildScanItemsInternal(imageID string, vulns []vulnerability.Vulnerability) ([]VulnerabilityScanItem, error) {+func saveItems(tx *gorm.DB, imageID string, vulns []vulnerability.Vulnerability) error {items := make([]VulnerabilityScanItem, 0, len(vulns))for i := range vulns {v := &vulns[i]@@ -452,7 +594,7 @@ func buildScanItemsInternal(imageID string, vulns []vulnerability.Vulnerability)item.CVSSScore = new(score)}if v.Description != "" || len(v.References) > 0 || v.CVSS != nil || v.PublishedDate != nil || v.LastModifiedDate != nil {- raw, err := json.Marshal(vulnerabilityScanItemDetailsInternal{+ raw, err := json.Marshal(vulnerabilityScanItemDetails{Description: v.Description,References: v.References,CVSS: v.CVSS,@@ -460,23 +602,20 @@ func buildScanItemsInternal(imageID string, vulns []vulnerability.Vulnerability)LastModifiedDate: v.LastModifiedDate,})if err != nil {- return nil, fmt.Errorf("failed to marshal vulnerability details: %w", err)+ return fmt.Errorf("failed to marshal vulnerability details: %w", err)}item.Details = new(string(raw))}items = append(items, item)}- return items, nil-}--func (s *VulnerabilityService) countKnownExploitedInternal(ctx context.Context) (int64, error) {- var count int64- err := s.db.WithContext(ctx).Model(&VulnerabilityThreatIntel{}).Where("known_exploited = ?", true).Count(&count).Error- return count, err+ if storeItemsErr := tx.CreateInBatches(items, scanItemInsertBatchSize).Error; storeItemsErr != nil {+ return fmt.Errorf("failed to store scan items: %w", storeItemsErr)+ }+ return nil}-// replaceKnownExploitedInternal clears every stored KEV flag and upserts entries in one transaction.-func (s *VulnerabilityService) replaceKnownExploitedInternal(ctx context.Context, entries []vulnerability.ThreatIntelEntry, now time.Time) error {+// replaceKnownExploited clears every stored KEV flag and upserts entries in one transaction.+func (s *VulnerabilityService) replaceKnownExploited(ctx context.Context, entries []vulnerability.ThreatIntelEntry, now time.Time) error {rows := make([]VulnerabilityThreatIntel, 0, len(entries))for i := range entries {rows = append(rows, VulnerabilityThreatIntel(entries[i]))@@ -490,63 +629,11 @@ func (s *VulnerabilityService) replaceKnownExploitedInternal(ctx context.Contextreturn tx.Clauses(clause.OnConflict{Columns: []clause.Column{{Name: "cve_id"}},DoUpdates: clause.AssignmentColumns([]string{"known_exploited", "kev_date_added", "kev_due_date", "kev_ransomware", "updated_at"}),- }).CreateInBatches(rows, threatIntelUpsertBatchInternal).Error+ }).CreateInBatches(rows, threatIntelUpsertBatch).Error})}-func (s *VulnerabilityService) staleEPSSCVEsInternal(ctx context.Context, checkedBefore time.Time, limit int) ([]string, error) {- var cveIDs []string- err := s.db.WithContext(ctx).- Model(&VulnerabilityScanItem{}).- Distinct("vulnerability_scan_items.vulnerability_id").- Joins("LEFT JOIN vulnerability_threat_intel ON vulnerability_threat_intel.cve_id = vulnerability_scan_items.vulnerability_id").- Where("vulnerability_scan_items.vulnerability_id LIKE ?", "CVE-%").- Where("vulnerability_threat_intel.epss_checked_at IS NULL OR vulnerability_threat_intel.epss_checked_at < ?", checkedBefore).- Limit(limit).- Pluck("vulnerability_scan_items.vulnerability_id", &cveIDs).Error- return cveIDs, err-}--// saveEPSSInternal upserts EPSS scores without touching the KEV columns.-func (s *VulnerabilityService) saveEPSSInternal(ctx context.Context, entries []vulnerability.ThreatIntelEntry) error {- batch := make([]VulnerabilityThreatIntel, 0, len(entries))- for i := range entries {- batch = append(batch, VulnerabilityThreatIntel(entries[i]))- }- return s.db.WithContext(ctx).Clauses(clause.OnConflict{- Columns: []clause.Column{{Name: "cve_id"}},- DoUpdates: clause.AssignmentColumns([]string{"epss_score", "epss_percentile", "epss_checked_at", "updated_at"}),- }).Create(&batch).Error-}--// pruneThreatIntelInternal drops cached CVEs that are neither KEV-listed nor found by any scan.-func (s *VulnerabilityService) pruneThreatIntelInternal(ctx context.Context) error {- return s.db.WithContext(ctx).- Where("known_exploited = ? AND NOT EXISTS (SELECT 1 FROM vulnerability_scan_items WHERE vulnerability_scan_items.vulnerability_id = vulnerability_threat_intel.cve_id)", false).- Delete(&VulnerabilityThreatIntel{}).Error-}--func (s *VulnerabilityService) ignoresForImageInternal(ctx context.Context, envID, imageID string) ([]vulnerability.IgnoredVulnerability, error) {- var ignoreRecords []VulnerabilityIgnore- if err := s.db.WithContext(ctx).Where("environment_id = ? AND image_id = ?", envID, imageID).Find(&ignoreRecords).Error; err != nil {- return nil, err- }- return ignoredVulnerabilitiesInternal(ignoreRecords), nil-}--func (s *VulnerabilityService) ignoreExistsInternal(ctx context.Context, ignore *vulnerability.IgnoredVulnerability) (bool, error) {- var existing VulnerabilityIgnore- err := s.db.WithContext(ctx).Where(- "environment_id = ? AND image_id = ? AND vulnerability_id = ? AND pkg_name = ? AND installed_version = ?",- ignore.EnvironmentID, ignore.ImageID, ignore.VulnerabilityID, ignore.PkgName, ignore.InstalledVersion,- ).First(&existing).Error- if errors.Is(err, gorm.ErrRecordNotFound) {- return false, nil- }- return err == nil, err-}--func (s *VulnerabilityService) getIgnoreInternal(ctx context.Context, envID, ignoreID string) (*vulnerability.IgnoredVulnerability, bool, error) {+func (s *VulnerabilityService) getIgnore(ctx context.Context, envID, ignoreID string) (*vulnerability.IgnoredVulnerability, bool, error) {var record VulnerabilityIgnoreif err := s.db.WithContext(ctx).Where("id = ? AND environment_id = ?", ignoreID, envID).First(&record).Error; err != nil {if errors.Is(err, gorm.ErrRecordNotFound) {@@ -554,45 +641,11 @@ func (s *VulnerabilityService) getIgnoreInternal(ctx context.Context, envID, ign}return nil, false, err}- ignore := ignoredVulnerabilityInternal(&record)+ ignore := vulnerability.IgnoredVulnerability(record)return &ignore, true, nil}-func createIgnoresInternal(tx *gorm.DB, entries []vulnerability.IgnoredVulnerability, batchSize int) ([]vulnerability.IgnoredVulnerability, error) {- records := make([]VulnerabilityIgnore, 0, len(entries))- for i := range entries {- ignore := &entries[i]- records = append(records, VulnerabilityIgnore{- ID: ignore.ID,- EnvironmentID: ignore.EnvironmentID,- ImageID: ignore.ImageID,- VulnerabilityID: ignore.VulnerabilityID,- PkgName: ignore.PkgName,- InstalledVersion: ignore.InstalledVersion,- Reason: ignore.Reason,- CreatedBy: ignore.CreatedBy,- CreatedAt: ignore.CreatedAt,- })- }- if err := tx.CreateInBatches(&records, batchSize).Error; err != nil {- return nil, err- }- return ignoredVulnerabilitiesInternal(records), nil-}--func (s *VulnerabilityService) deleteIgnoreIdentityInternal(ctx context.Context, ignore *vulnerability.IgnoredVulnerability) error {- return s.db.WithContext(ctx).Where(- "environment_id = ? AND image_id = ? AND vulnerability_id = ? AND pkg_name = ? AND installed_version = ?",- ignore.EnvironmentID, ignore.ImageID, ignore.VulnerabilityID, ignore.PkgName, ignore.InstalledVersion,- ).Delete(&VulnerabilityIgnore{}).Error-}--func deleteIgnoreIDsInternal(tx *gorm.DB, envID string, ignoreIDs []string) (int64, error) {- result := tx.Where("environment_id = ? AND id IN ?", envID, ignoreIDs).Delete(&VulnerabilityIgnore{})- return result.RowsAffected, result.Error-}--func (s *VulnerabilityService) ignorePageInternal(+func (s *VulnerabilityService) ignorePage(ctx context.Context,envID string,order func(*gorm.DB) *gorm.DB,@@ -609,27 +662,13 @@ func (s *VulnerabilityService) ignorePageInternal(if err := scope.Offset(offset).Limit(limit).Find(&ignoreRecords).Error; err != nil {return nil, 0, fmt.Errorf("failed to list ignored vulnerabilities: %w", err)}- return ignoredVulnerabilitiesInternal(ignoreRecords), total, nil-}--func ignoredVulnerabilityInternal(record *VulnerabilityIgnore) vulnerability.IgnoredVulnerability {- return vulnerability.IgnoredVulnerability{- ID: record.ID,- EnvironmentID: record.EnvironmentID,- ImageID: record.ImageID,- VulnerabilityID: record.VulnerabilityID,- PkgName: record.PkgName,- InstalledVersion: record.InstalledVersion,- Reason: record.Reason,- CreatedBy: record.CreatedBy,- CreatedAt: record.CreatedAt,- }+ return ignoredVulnerabilities(ignoreRecords), total, nil}-func ignoredVulnerabilitiesInternal(records []VulnerabilityIgnore) []vulnerability.IgnoredVulnerability {+func ignoredVulnerabilities(records []VulnerabilityIgnore) []vulnerability.IgnoredVulnerability {result := make([]vulnerability.IgnoredVulnerability, len(records))for i := range records {- result[i] = ignoredVulnerabilityInternal(&records[i])+ result[i] = vulnerability.IgnoredVulnerability(records[i])}return result}diff --git a/backend/internal/vulnerability/service_test.go b/backend/internal/vulnerability/service_test.goindex 3326d5ba6..5817aaabc 100644--- a/backend/internal/vulnerability/service_test.go+++ b/backend/internal/vulnerability/service_test.go@@ -13,6 +13,7 @@ import ("time""github.com/getarcaneapp/arcane/types/v2/features"+ "github.com/getarcaneapp/arcane/types/v2/scheduler""github.com/getarcaneapp/arcane/types/v2/user""github.com/getarcaneapp/arcane/types/v2/vulnerability""github.com/libtnb/sqlite"@@ -21,13 +22,16 @@ import ("go.getarcane.app/kit/pkg""gorm.io/gorm"+ "github.com/getarcaneapp/arcane/backend/v2/internal/activity""github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/kv""github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability/children/query"+ activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/vuln""github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow/flowtest")func setupVulnerabilityScanTestDB(t *testing.T) *database.DB {@@ -42,7 +46,7 @@ func setupVulnerabilityScanTestDB(t *testing.T) *database.DB {func TestSaveScanResult_FixableCountsOSPackagesOnly(t *testing.T) {ctx := t.Context()db := setupVulnerabilityScanTestDB(t)- svc := newTestServiceInternal(db, nil)+ svc := newTestService(db, nil)result := &vulnerability.ScanResult{ImageID: "sha256:img-fixable",@@ -65,7 +69,7 @@ func TestSaveScanResult_FixableCountsOSPackagesOnly(t *testing.T) {func TestVulnerabilityService_ListVulnerabilities_NoScanReturnsNotFound(t *testing.T) {db := setupVulnerabilityScanTestDB(t)- svc := newTestServiceInternal(db, nil)+ svc := newTestService(db, nil)_, _, err := svc.query.ListVulnerabilities(t.Context(), "sha256:unknown", pagination.QueryParams{Limit: 20})@@ -75,7 +79,7 @@ func TestVulnerabilityService_ListVulnerabilities_NoScanReturnsNotFound(t *testifunc TestVulnerabilityService_DeleteScanResultsByImageIDs(t *testing.T) {ctx := t.Context()db := setupVulnerabilityScanTestDB(t)- svc := newTestServiceInternal(db, nil)+ svc := newTestService(db, nil)records := []VulnerabilityScanRecord{{ID: "sha256:img1", ImageName: "nginx:latest", Status: ScanStatusCompleted, ScanTime: time.Now()},@@ -97,7 +101,7 @@ func TestVulnerabilityService_DeleteScanResultsByImageIDs(t *testing.T) {func TestVulnerabilityService_DeleteScanResultsByImageIDs_EmptyInput(t *testing.T) {ctx := t.Context()db := setupVulnerabilityScanTestDB(t)- svc := newTestServiceInternal(db, nil)+ svc := newTestService(db, nil)require.NoError(t, svc.DeleteScanResultsByImageIDs(ctx, nil))require.NoError(t, svc.DeleteScanResultsByImageIDs(ctx, []string{}))@@ -106,7 +110,7 @@ func TestVulnerabilityService_DeleteScanResultsByImageIDs_EmptyInput(t *testing.func TestVulnerabilityService_ListAllVulnerabilities_FiltersIgnoredInline(t *testing.T) {ctx := t.Context()db := setupVulnerabilityScanTestDB(t)- svc := newTestServiceInternal(db, nil)+ svc := newTestService(db, nil)vulns := []vulnerability.Vulnerability{{@@ -154,7 +158,7 @@ func TestVulnerabilityService_ListAllVulnerabilities_FiltersIgnoredInline(t *tesfunc TestVulnerabilityService_ListAllVulnerabilityImageOptions_SeverityFilter(t *testing.T) {ctx := t.Context()db := setupVulnerabilityScanTestDB(t)- svc := newTestServiceInternal(db, nil)+ svc := newTestService(db, nil)now := time.Now()records := []VulnerabilityScanRecord{@@ -178,7 +182,7 @@ func TestVulnerabilityService_ListAllVulnerabilityImageOptions_SeverityFilter(tfunc TestVulnerabilityService_ListAllVulnerabilityImageOptions_InvalidSeverityAndNoFilter(t *testing.T) {ctx := t.Context()db := setupVulnerabilityScanTestDB(t)- svc := newTestServiceInternal(db, nil)+ svc := newTestService(db, nil)now := time.Now()records := []VulnerabilityScanRecord{@@ -206,7 +210,7 @@ func TestVulnerabilityService_ListAllVulnerabilityImageOptions_InvalidSeverityAnfunc TestVulnerabilityService_ActionableCountExcludingIgnored(t *testing.T) {ctx := t.Context()db := setupVulnerabilityScanTestDB(t)- svc := newTestServiceInternal(db, nil)+ svc := newTestService(db, nil)now := time.Now()@@ -297,7 +301,7 @@ func TestVulnerabilityService_ActionableCountExcludingIgnored(t *testing.T) {func TestVulnerabilityService_ListAllVulnerabilities_PagesAndCountsWithInlineFilters(t *testing.T) {ctx := t.Context()db := setupVulnerabilityScanTestDB(t)- svc := newTestServiceInternal(db, nil)+ svc := newTestService(db, nil)vulnsA := []vulnerability.Vulnerability{{VulnerabilityID: "CVE-A1", PkgName: "openssl", InstalledVersion: "1.0.0", Severity: vulnerability.SeverityCritical},@@ -356,9 +360,9 @@ func TestVulnerabilityService_ListAllVulnerabilities_PagesAndCountsWithInlineFilrequire.EqualValues(t, 3, page.GrandTotalItems)}-func riskTestServiceInternal(t *testing.T, httpClient *http.Client) (*VulnerabilityService, *settings.SettingsService, *database.DB) {+func riskTestService(t *testing.T, httpClient *http.Client) (*VulnerabilityService, *settings.SettingsService, *database.DB) {t.Helper()- settingsSvc, db := featureSettingsInternal(t)+ settingsSvc, db := featureSettings(t)sqlDB, err := db.DB.DB()require.NoError(t, err)sqlDB.SetMaxOpenConns(1)@@ -367,7 +371,7 @@ func riskTestServiceInternal(t *testing.T, httpClient *http.Client) (*Vulnerabilreturn svc, settingsSvc, db}-func seedRiskScansInternal(t *testing.T, svc *VulnerabilityService) {+func seedRiskScans(t *testing.T, svc *VulnerabilityService) {t.Helper()scans := []*vulnerability.ScanResult{{@@ -394,8 +398,8 @@ func seedRiskScansInternal(t *testing.T, svc *VulnerabilityService) {}func TestVulnerabilityService_GetRiskOverview(t *testing.T) {- svc, _, db := riskTestServiceInternal(t, nil)- seedRiskScansInternal(t, svc)+ svc, _, db := riskTestService(t, nil)+ seedRiskScans(t, svc)require.NoError(t, db.Create(&VulnerabilityThreatIntel{CVEID: "CVE-2024-0002", KnownExploited: true, KEVRansomware: true, KEVDueDate: "2024-06-01"}).Error)require.NoError(t, db.Create(&VulnerabilityIgnore{ID: "ignore-1", EnvironmentID: "0", ImageID: "sha256:web", VulnerabilityID: "CVE-2024-0003", PkgName: "zlib", InstalledVersion: "1.2.0"}).Error)@@ -460,8 +464,8 @@ func TestVulnerabilityService_GetRiskOverview(t *testing.T) {}func TestVulnerabilityService_GetRiskOverview_ImageScoreMatchesInsights(t *testing.T) {- svc, _, db := riskTestServiceInternal(t, nil)- seedRiskScansInternal(t, svc)+ svc, _, db := riskTestService(t, nil)+ seedRiskScans(t, svc)require.NoError(t, db.Create(&VulnerabilityThreatIntel{CVEID: "CVE-2024-0001", EPSSScore: new(0.25)}).Error)overview, err := svc.query.GetRiskOverview(t.Context(), "0")@@ -482,8 +486,8 @@ func TestVulnerabilityService_GetRiskOverview_ImageScoreMatchesInsights(t *testi}func TestVulnerabilityService_RecordRiskSnapshot(t *testing.T) {- svc, _, db := riskTestServiceInternal(t, nil)- seedRiskScansInternal(t, svc)+ svc, _, db := riskTestService(t, nil)+ seedRiskScans(t, svc)now := time.Now().UTC()require.NoError(t, db.Create(&[]query.VulnerabilityRiskSnapshot{{SnapshotDate: now.AddDate(0, 0, -500).Format(time.DateOnly), ScoringVersion: vuln.ScoringVersion, RiskScore: 10, RiskBand: "low"},@@ -513,7 +517,7 @@ func TestVulnerabilityService_RecordRiskSnapshot(t *testing.T) {require.Equal(t, 3, overview.Drivers7dAgo.KnownExploited)}-// seedBulkIgnoreFixtureInternal stores nginx with 45 findings (more than one page) and redis with three findings covering every filter dimension.+// seedBulkIgnoreFixture stores nginx with 45 findings (more than one page) and redis with three findings covering every filter dimension.type roundTripFunc func(*http.Request) (*http.Response, error)func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) {@@ -568,7 +572,7 @@ func TestVulnerabilityService_SyncThreatIntel(t *testing.T) {return srv.Client().Transport.RoundTrip(r)})}- svc, settingsSvc, db := riskTestServiceInternal(t, httpClient)+ svc, settingsSvc, db := riskTestService(t, httpClient)vulns := make([]vulnerability.Vulnerability, 0, 151)for i := range 150 {@@ -619,7 +623,7 @@ func TestVulnerabilityService_SyncThreatIntel(t *testing.T) {require.EqualValues(t, 1, kevRequests.Load(), "disabled intel makes no requests")}-func seedBulkIgnoreFixtureInternal(t *testing.T, db *database.DB, svc *VulnerabilityService) {+func seedBulkIgnoreFixture(t *testing.T, db *database.DB, svc *VulnerabilityService) {t.Helper()nginx := make([]vulnerability.Vulnerability, 0, 45)for i := range 45 {@@ -665,7 +669,7 @@ func seedBulkIgnoreFixtureInternal(t *testing.T, db *database.DB, svc *Vulnerabisvc.BackfillScanItems(t.Context())}-func activeCountInternal(t *testing.T, svc *VulnerabilityService, envID string) int64 {+func activeCount(t *testing.T, svc *VulnerabilityService, envID string) int64 {t.Helper()_, page, err := svc.query.ListAllVulnerabilities(t.Context(), envID, pagination.QueryParams{Start: 0, Limit: 20})require.NoError(t, err)@@ -689,8 +693,8 @@ func TestBulkIgnoreVulnerabilities_AllMatchingHonorsFilters(t *testing.T) {} {t.Run(tc.name, func(t *testing.T) {db := setupVulnerabilityScanTestDB(t)- svc := newTestServiceInternal(db, nil)- seedBulkIgnoreFixtureInternal(t, db, svc)+ svc := newTestService(db, nil)+ seedBulkIgnoreFixture(t, db, svc)before, err := svc.ActionableCountExcludingIgnored(ctx)require.NoError(t, err)@@ -699,8 +703,8 @@ func TestBulkIgnoreVulnerabilities_AllMatchingHonorsFilters(t *testing.T) {affected, err := svc.ignores.BulkIgnoreVulnerabilities(ctx, "env-1", &tc.filters, "bob")require.NoError(t, err)require.Equal(t, tc.expected, affected)- require.EqualValues(t, 48-tc.expected, activeCountInternal(t, svc, "env-1"))- require.EqualValues(t, 48, activeCountInternal(t, svc, "env-2"), "other environments keep their own view")+ require.EqualValues(t, 48-tc.expected, activeCount(t, svc, "env-1"))+ require.EqualValues(t, 48, activeCount(t, svc, "env-2"), "other environments keep their own view")var actionableIgnored int64require.NoError(t, db.Model(&VulnerabilityIgnore{}).@@ -723,7 +727,7 @@ func TestBulkIgnoreVulnerabilities_AllMatchingHonorsFilters(t *testing.T) {restored, err := svc.ignores.BulkUnignoreVulnerabilities(ctx, "env-1", &tc.filters)require.NoError(t, err)require.Equal(t, tc.expected, restored)- require.EqualValues(t, 48, activeCountInternal(t, svc, "env-1"))+ require.EqualValues(t, 48, activeCount(t, svc, "env-1"))after, err = svc.ActionableCountExcludingIgnored(ctx)require.NoError(t, err)require.Equal(t, before, after)@@ -731,7 +735,7 @@ func TestBulkIgnoreVulnerabilities_AllMatchingHonorsFilters(t *testing.T) {}}-func featureSettingsInternal(t *testing.T) (*settings.SettingsService, *database.DB) {+func featureSettings(t *testing.T) (*settings.SettingsService, *database.DB) {t.Helper()gdb, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})require.NoError(t, err)@@ -746,15 +750,15 @@ func featureSettingsInternal(t *testing.T) (*settings.SettingsService, *database}func TestVulnerabilityFeatureDisabledRejectsScansAndRetainsReports(t *testing.T) {- settingsSvc, db := featureSettingsInternal(t)- svc := newTestServiceInternal(db, settingsSvc)+ settingsSvc, db := featureSettings(t)+ svc := newTestService(db, settingsSvc)require.NoError(t, db.Create(&VulnerabilityScanRecord{ID: "test-image", ImageName: "test:latest", Status: ScanStatusCompleted, ScanTime: time.Now()}).Error)require.NoError(t, db.Create(&VulnerabilityReportRecord{ImageID: "test-image", Data: "stored report"}).Error)require.NoError(t, settingsSvc.SetBoolSetting(t.Context(), features.VulnerabilityManagementSettingKey, false))_, err := svc.ScanImage(t.Context(), "0", "test-image", user.Actor{})require.ErrorIs(t, err, common.ErrFeatureDisabled)- _, _, err = svc.ScanAllImages(t.Context(), "0", user.Actor{}, false)+ _, err = svc.scan.Targets(t.Context())require.ErrorIs(t, err, common.ErrFeatureDisabled)summaries, err := svc.GetScanSummariesByImageIDs(t.Context(), []string{"test-image"})require.NoError(t, err)@@ -772,7 +776,23 @@ func TestVulnerabilityFeatureDisabledRejectsScansAndRetainsReports(t *testing.T)require.Equal(t, "stored report", report.Data)}-// newTestServiceInternal wires a VulnerabilityService and its children over db.-func newTestServiceInternal(db *database.DB, settingsService *settings.SettingsService) *VulnerabilityService {+// newTestService wires a VulnerabilityService and its children over db.+func newTestService(db *database.DB, settingsService *settings.SettingsService) *VulnerabilityService {return NewVulnerabilityService(db, nil, nil, nil, settingsService, nil, nil, nil, nil, nil)}++func TestScanWorkflowSkipsWhenFeatureDisabled(t *testing.T) {+ settingsSvc, db := featureSettings(t)+ require.NoError(t, db.AutoMigrate(&activity.Activity{}, &activity.ActivityMessage{}))+ harness := flowtest.New(t, activity.NewActivityService(db, nil))+ svc := newTestService(db, settingsSvc)+ require.NoError(t, svc.RegisterWorkflows(harness.Engine))+ harness.Start(t)+ flowtest.AssertDefinitions(t, harness)+ require.NoError(t, settingsSvc.SetBoolSetting(t.Context(), features.VulnerabilityManagementSettingKey, false))++ outcome, err := harness.Engine.Run(t.Context(), svc.ScanWorkflow(), nil, activitylib.StartRequest{}, nil)+ require.NoError(t, err)+ require.Equal(t, scheduler.Skipped, outcome.Status)+ require.Len(t, outcome.Steps, 3)+}diff --git a/backend/pkg/libarcane/activity/requests.go b/backend/pkg/libarcane/activity/requests.goindex 6fc11eeff..a50f0c481 100644--- a/backend/pkg/libarcane/activity/requests.go+++ b/backend/pkg/libarcane/activity/requests.go@@ -40,6 +40,8 @@ type SlotWaiter interface {}type StartRequest struct {+ // ID is an optional caller-chosen ID; starting an existing ID returns that row.+ ID stringEnvironmentID string// BatchID groups activities spawned by one logical user action. When nil,// the batch ID attached to the request context (if any) is used instead.diff --git a/backend/pkg/scheduler/analytics_job.go b/backend/pkg/scheduler/analytics_job.goindex b7107d7ea..46916f152 100644--- a/backend/pkg/scheduler/analytics_job.go+++ b/backend/pkg/scheduler/analytics_job.go@@ -26,9 +26,13 @@ const (AnalyticsJobName = "analytics-heartbeat"defaultHeartbeatEndpoint = "https://checkin.getarcane.app/heartbeat"devHeartbeatEndpoint = "http://localhost:8080/heartbeat"+ analyticsHeartbeatNextAttemptKey = "analytics.heartbeat.next_attempt_at"+ // analyticsHeartbeatLastAttemptKey is the window older releases kept; it is carried forward once.analyticsHeartbeatLastAttemptKey = "analytics.heartbeat.last_attempt_at"analyticsHeartbeatDedupeWindow = 24 * time.Hour- analyticsHeartbeatCheckSchedule = "0 0 * * * *"+ // analyticsHeartbeatRetryDelay spaces retries of a failed send, which the hourly check then picks up.+ analyticsHeartbeatRetryDelay = time.Hour+ analyticsHeartbeatCheckSchedule = "0 0 * * * *")type AnalyticsJob struct {@@ -79,7 +83,7 @@ func (j *AnalyticsJob) Run(ctx context.Context) (schedulertypes.Outcome, error)return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil}- allowed, err := j.claimHeartbeatAttemptWindowInternal(ctx)+ allowed, err := j.claimHeartbeat(ctx)if err != nil {slog.ErrorContext(ctx, "failed to acquire analytics heartbeat send window", "error", err)return schedulertypes.Outcome{}, err@@ -106,22 +110,8 @@ func (j *AnalyticsJob) Run(ctx context.Context) (schedulertypes.Outcome, error)return schedulertypes.Outcome{}, err}- slog.InfoContext(- ctx,- "sending analytics heartbeat",- "jobName",- AnalyticsJobName,- "version",- payload.Version,- "instanceId",- payload.InstanceID,- "serverType",- payload.ServerType,- "heartbeatUrl",- j.heartbeatURL,- "env",- j.cfg.Environment,- )+ slog.InfoContext(ctx, "sending analytics heartbeat", "version", payload.Version, "instanceId", payload.InstanceID,+ "serverType", payload.ServerType, "heartbeatUrl", j.heartbeatURL, "env", j.cfg.Environment)_, err = backoff.Retry(ctx,@@ -158,7 +148,12 @@ func (j *AnalyticsJob) Run(ctx context.Context) (schedulertypes.Outcome, error)if bodyText != "" {details = append(details, "response="+bodyText)}- return struct{}{}, fmt.Errorf("analytics heartbeat request failed: %s", strings.Join(details, "; "))+ failure := fmt.Errorf("analytics heartbeat request failed: %s", strings.Join(details, "; "))+ // Retrying a rejection within seconds only spends more of the endpoint's limit.+ if resp.StatusCode < http.StatusInternalServerError {+ return struct{}{}, backoff.Permanent(failure)+ }+ return struct{}{}, failure}return struct{}{}, nil},@@ -166,26 +161,15 @@ func (j *AnalyticsJob) Run(ctx context.Context) (schedulertypes.Outcome, error)backoff.WithMaxTries(3),)if err != nil {- slog.ErrorContext(ctx, "analytics heartbeat failed", "error", err)+ slog.ErrorContext(ctx, "analytics heartbeat failed; retrying after the retry delay", "error", err, "retryDelay", analyticsHeartbeatRetryDelay)+ retryAt := j.now().UTC().Add(analyticsHeartbeatRetryDelay).Format(time.RFC3339Nano)+ if retryErr := j.kvService.Set(ctx, analyticsHeartbeatNextAttemptKey, retryAt); retryErr != nil {+ slog.WarnContext(ctx, "failed to shorten analytics heartbeat window after a failed send", "error", retryErr)+ }return schedulertypes.Outcome{}, err}- slog.InfoContext(- ctx,- "analytics heartbeat sent successfully",- "jobName",- AnalyticsJobName,- "version",- payload.Version,- "instanceId",- payload.InstanceID,- "serverType",- payload.ServerType,- "heartbeatUrl",- j.heartbeatURL,- "env",- j.cfg.Environment,- )+ slog.InfoContext(ctx, "analytics heartbeat sent successfully", "version", payload.Version, "instanceId", payload.InstanceID, "serverType", payload.ServerType)return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil}@@ -194,58 +178,41 @@ func (j *AnalyticsJob) Reschedule(ctx context.Context) error {return nil}-func (j *AnalyticsJob) claimHeartbeatAttemptWindowInternal(ctx context.Context) (bool, error) {+// claimHeartbeat reports whether a heartbeat is due and, if so, schedules the next one a day out. Claiming before+// sending keeps restarts from sending twice; a failed send moves the next attempt up to the retry delay.+func (j *AnalyticsJob) claimHeartbeat(ctx context.Context) (bool, error) {if j.kvService == nil {return false, errors.New("analytics heartbeat kv service is not configured")}-j.runMu.Lock()defer j.runMu.Unlock()now := j.now().UTC()- rawLastAttemptAt, ok, err := j.kvService.Get(ctx, analyticsHeartbeatLastAttemptKey)+ rawNextAttemptAt, ok, err := j.kvService.Get(ctx, analyticsHeartbeatNextAttemptKey)if err != nil {return false, fmt.Errorf("failed to load analytics heartbeat attempt state: %w", err)}-- if ok {- lastAttemptAt, parseErr := time.Parse(time.RFC3339Nano, rawLastAttemptAt)- if parseErr != nil {- slog.WarnContext(- ctx,- "invalid analytics heartbeat attempt timestamp; resetting dedupe window",- "key",- analyticsHeartbeatLastAttemptKey,- "value",- rawLastAttemptAt,- "error",- parseErr,- )- } else {- nextEligibleAt := lastAttemptAt.Add(analyticsHeartbeatDedupeWindow)- if now.Before(nextEligibleAt) {- slog.InfoContext(- ctx,- "skipping analytics heartbeat; already attempted within dedupe window",- "jobName",- AnalyticsJobName,- "lastAttemptAt",- lastAttemptAt,- "nextEligibleAt",- nextEligibleAt,- )- return false, nil- }+ nextAttemptAt, parseErr := time.Parse(time.RFC3339Nano, rawNextAttemptAt)+ if !ok {+ // An upgrade from a release that stored the last attempt keeps that window instead of sending again.+ rawLastAttemptAt, hadLast, lastErr := j.kvService.Get(ctx, analyticsHeartbeatLastAttemptKey)+ if lastErr != nil {+ return false, fmt.Errorf("failed to load analytics heartbeat attempt state: %w", lastErr)+ }+ if lastAttemptAt, lastParseErr := time.Parse(time.RFC3339Nano, rawLastAttemptAt); hadLast && lastParseErr == nil {+ ok, parseErr, nextAttemptAt = true, nil, lastAttemptAt.Add(analyticsHeartbeatDedupeWindow)}}-- // Persist the attempt window before sending on purpose: the product behavior is- // best-effort at-most-once-per-24h check-ins, which avoids duplicate heartbeats- // after restarts or partially completed outbound requests.- if setErr := j.kvService.Set(ctx, analyticsHeartbeatLastAttemptKey, now.Format(time.RFC3339Nano)); setErr != nil {+ if ok && parseErr != nil {+ slog.WarnContext(ctx, "invalid analytics heartbeat attempt timestamp; resetting window", "value", rawNextAttemptAt, "error", parseErr)+ }+ if ok && parseErr == nil && now.Before(nextAttemptAt) {+ slog.InfoContext(ctx, "skipping analytics heartbeat; next one is not due yet", "nextAttemptAt", nextAttemptAt)+ return false, nil+ }+ if setErr := j.kvService.Set(ctx, analyticsHeartbeatNextAttemptKey, now.Add(analyticsHeartbeatDedupeWindow).Format(time.RFC3339Nano)); setErr != nil {return false, fmt.Errorf("failed to persist analytics heartbeat attempt state: %w", setErr)}-return true, nil}diff --git a/backend/pkg/scheduler/auto_heal_job.go b/backend/pkg/scheduler/auto_heal_job.goindex 1ee282ad5..8af0ac596 100644--- a/backend/pkg/scheduler/auto_heal_job.go+++ b/backend/pkg/scheduler/auto_heal_job.go@@ -1,7 +1,6 @@package schedulerimport (- "cmp""context""encoding/json/v2""errors"@@ -104,15 +103,7 @@ func (j *AutoHealJob) ShouldSchedule(ctx context.Context) bool {}func (j *AutoHealJob) Schedule(ctx context.Context) string {- schedule := cmp.Or(j.settingsService.GetStringSetting(ctx, "autoHealInterval", autoHealDefaultSchedule), autoHealDefaultSchedule)-- parser := scheduleutil.Parser()- if _, err := parser.Parse(schedule); err != nil {- slog.WarnContext(ctx, "Invalid cron expression for auto-heal, using default", "invalidSchedule", schedule, "error", err)- return autoHealDefaultSchedule- }-- return schedule+ return scheduleutil.Or(ctx, j.settingsService.GetStringSetting(ctx, "autoHealInterval", autoHealDefaultSchedule), autoHealDefaultSchedule, "auto-heal")}func (j *AutoHealJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {diff --git a/backend/pkg/scheduler/auto_patch_job.go b/backend/pkg/scheduler/auto_patch_job.goindex fa3500e5e..2f8b54f75 100644--- a/backend/pkg/scheduler/auto_patch_job.go+++ b/backend/pkg/scheduler/auto_patch_job.go@@ -1,90 +1,33 @@package schedulerimport (- "cmp""context"- "errors"- "log/slog"- "github.com/getarcaneapp/arcane/types/v2""github.com/getarcaneapp/arcane/types/v2/features"- schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"- "github.com/getarcaneapp/arcane/types/v2/user"- "go.getarcane.app/kit/pkg"- "github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/internal/image""github.com/getarcaneapp/arcane/backend/v2/internal/settings"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow"scheduleutil "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/schedule")-const AutoPatchJobName = "auto-patch"--var autoPatchSystemUser = user.Actor{- Username: "System",-}--// AutoPatchJob periodically patches images whose latest vulnerability scan-// found fixable OS package vulnerabilities. It is opt-in via the-// "imageAutoPatchEnabled" setting.-type AutoPatchJob struct {- imageService *image.ImageService- settingsService *settings.SettingsService-}--// NewAutoPatchJob creates a new AutoPatchJob.-func NewAutoPatchJob(imageService *image.ImageService, settingsService *settings.SettingsService) *AutoPatchJob {- return &AutoPatchJob{- imageService: imageService,- settingsService: settingsService,- }-}--func (j *AutoPatchJob) Name() string {- return AutoPatchJobName-}--func (j *AutoPatchJob) ShouldSchedule(ctx context.Context) bool {- return j.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) && j.settingsService.GetBoolSetting(ctx, "imageAutoPatchEnabled", false)-}--// Schedule returns the cron expression for the job. Defaults to daily at 3 AM.-func (j *AutoPatchJob) Schedule(ctx context.Context) string {- schedule := cmp.Or(j.settingsService.GetStringSetting(ctx, "imageAutoPatchInterval", "0 0 3 * * *"), "0 0 3 * * *")-- parser := scheduleutil.Parser()- if _, err := parser.Parse(schedule); err != nil {- slog.WarnContext(ctx, "Invalid cron expression for auto-patch, using default", "invalidSchedule", schedule, "error", err)- return "0 0 3 * * *"- }-- return schedule-}--func (j *AutoPatchJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {- if !j.ShouldSchedule(ctx) {- slog.DebugContext(ctx, "scheduled image patching disabled; skipping run")- return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil- }-- slog.InfoContext(ctx, "scheduled image patching started")+const (+ AutoPatchJobName = "auto-patch"+ autoPatchDefaultCron = "0 0 3 * * *"+)- patched, skipped, err := j.imageService.PatchFlaggedImages(ctx, types.LocalDockerEnvironmentID, autoPatchSystemUser)- if errors.Is(err, common.ErrFeatureDisabled) {- status := kit.Ternary(patched > 0, schedulertypes.Partial, schedulertypes.Skipped)- return schedulertypes.Outcome{Status: status, Message: err.Error()}, nil+// NewAutoPatchJob runs the durable auto-patch workflow. It is opt-in via the+// "imageAutoPatchEnabled" setting and defaults to daily at 03:00.+func NewAutoPatchJob(engine *flow.Engine, imageService *image.ImageService, settingsService *settings.SettingsService) *flow.Job {+ return &flow.Job{+ Engine: engine,+ Workflow: imageService.PatchWorkflow(),+ JobName: AutoPatchJobName,+ ScheduleFn: func(ctx context.Context) string {+ return scheduleutil.Or(ctx, settingsService.GetStringSetting(ctx, "imageAutoPatchInterval", autoPatchDefaultCron), autoPatchDefaultCron, AutoPatchJobName)+ },+ ShouldRunFn: func(ctx context.Context) bool {+ return settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) && settingsService.GetBoolSetting(ctx, "imageAutoPatchEnabled", false)+ },}- if err != nil {- slog.ErrorContext(ctx, "scheduled image patching failed", "error", err)- if patched > 0 {- return schedulertypes.Outcome{Status: schedulertypes.Partial}, err- }- return schedulertypes.Outcome{}, err- }-- slog.InfoContext(ctx, "scheduled image patching completed",- "patched", patched,- "skipped", skipped,- )- return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil}diff --git a/backend/pkg/scheduler/auto_update_job.go b/backend/pkg/scheduler/auto_update_job.goindex e77612ba2..732a104fc 100644--- a/backend/pkg/scheduler/auto_update_job.go+++ b/backend/pkg/scheduler/auto_update_job.go@@ -1,254 +1,32 @@package schedulerimport (- "cmp""context"- "errors"- "log/slog"- "strings"- schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"- updatertypes "github.com/getarcaneapp/arcane/types/v2/updater"kit "go.getarcane.app/kit/pkg""github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/backend/v2/internal/updater"- "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"- "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow")-const autoUpdateAdmissionScopeInternal = "auto-update"--// pendingUpdateApplierInternal is the slice of UpdaterService the job needs,-// kept as an interface so the overlap guard is testable with a fake.-type pendingUpdateApplierInternal interface {- ApplyPending(ctx context.Context, options updatertypes.Options) (*updatertypes.Result, error)-}--type AutoUpdateJob struct {- updaterService pendingUpdateApplierInternal- settingsService *settings.SettingsService- admissionGate *runs.Admission-}--func NewAutoUpdateJob(updaterService *updater.UpdaterService, settingsService *settings.SettingsService, admissionGate *runs.Admission) (*AutoUpdateJob, error) {- if admissionGate == nil {- return nil, errors.New("auto-update admission gate unavailable")- }- return &AutoUpdateJob{- updaterService: updaterService,- settingsService: settingsService,- admissionGate: admissionGate,- }, nil-}--func (j *AutoUpdateJob) Name() string {- return "auto-update"-}--func (j *AutoUpdateJob) ShouldSchedule(ctx context.Context) bool {- enabled := j.settingsService.GetBoolSetting(ctx, "autoUpdate", false)- pollingEnabled := j.settingsService.GetBoolSetting(ctx, "pollingEnabled", true)- return enabled && pollingEnabled-}--func (j *AutoUpdateJob) Schedule(ctx context.Context) string {- s := j.settingsService.GetStringSetting(ctx, "autoUpdateInterval", "0 0 0 * * *")- return kit.Ternary(s == "", "0 0 0 * * *", s)-}--func (j *AutoUpdateJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {- options := updatertypes.Options{}- unresolvedTargets := false- if previous, ok := jobcontext.Run(ctx); ok && (previous.AttemptCount > 1 || len(previous.Outcome.Targets) > 0) {- var priorOutcome schedulertypes.Outcome- options, priorOutcome, unresolvedTargets = autoUpdateRetryOptionsInternal(previous)- if priorOutcome.Status != "" {- priorOutcome.ActivityID = cmp.Or(priorOutcome.ActivityID, previous.Outcome.ActivityID)- if priorOutcome.Status == schedulertypes.Failed && previous.Outcome.Message != "" {- priorOutcome.Message += ": " + previous.Outcome.Message- }- return priorOutcome, nil- }- }-- enabled := j.settingsService.GetBoolSetting(ctx, "autoUpdate", false)- pollingEnabled := j.settingsService.GetBoolSetting(ctx, "pollingEnabled", true)- if !enabled || !pollingEnabled {- if len(options.ResourceIds) > 0 {- return schedulertypes.Outcome{Status: schedulertypes.Failed, Message: "Auto-update is disabled; the remaining targets were not updated"}, nil- }- slog.DebugContext(ctx, "auto-update disabled or polling disabled; skipping run",- "autoUpdate", enabled, "pollingEnabled", pollingEnabled)- return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil- }-- lease, admitted, err := j.admissionGate.TryAcquire(ctx, schedulertypes.AdmissionKey{Scope: autoUpdateAdmissionScopeInternal})- if err != nil {- slog.ErrorContext(ctx, "auto-update admission failed", "error", err)- return schedulertypes.Outcome{}, err- }- if !admitted {- if len(options.ResourceIds) > 0 {- return schedulertypes.Outcome{Status: schedulertypes.Failed, Message: "Another update is active; the remaining targets were not updated"}, nil- }- slog.WarnContext(ctx, "auto-update run still in progress; skipping overlapping run")- return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil- }- defer lease.Release(ctx)-- slog.InfoContext(ctx, "auto-update run started")-- result, err := j.updaterService.ApplyPending(ctx, options)- if err != nil {- slog.ErrorContext(ctx, "auto-update run failed", "err", err)- if result == nil {- return schedulertypes.Outcome{}, err- }- }- if result == nil {- return schedulertypes.Outcome{Status: schedulertypes.Failed, Message: "Updater returned no operation result"}, nil- }-- return autoUpdateOutcomeInternal(ctx, result, unresolvedTargets, err)-}--func (j *AutoUpdateJob) Reschedule(ctx context.Context) error {- slog.InfoContext(ctx, "rescheduling auto-update job in new scheduler; currently requires restart")- return nil-}--func (j *AutoUpdateJob) Reconcile(ctx context.Context, previous schedulertypes.Run) (schedulertypes.Outcome, error) {- if service, ok := j.updaterService.(*updater.UpdaterService); ok {- return service.ReconcilePending(ctx, previous)- }- confirmed := confirmedAutoUpdateInternal(previous)- if confirmed.Status != schedulertypes.Succeeded {- confirmed.Message = "Interrupted auto-update has no confirmed full-batch completion"- if previous.Outcome.Message != "" {- confirmed.Message += ": " + previous.Outcome.Message- }- }- return confirmed, nil-}--func autoUpdateRetryOptionsInternal(previous schedulertypes.Run) (updatertypes.Options, schedulertypes.Outcome, bool) {- options := updatertypes.Options{}- unresolvedTargets := true- for _, target := range previous.Outcome.Targets {- if target.ID == "auto-update" && target.ResourceType == "update-batch" && (target.Status == schedulertypes.Succeeded || target.Status == schedulertypes.Partial) {- unresolvedTargets = false- }- }- confirmed := confirmedAutoUpdateInternal(previous)- if confirmed.Status == schedulertypes.Succeeded {- return options, confirmed, false- }- hasResourceTargets := false- for _, target := range previous.Outcome.Targets {- if target.ID == "auto-update" {- continue- }- hasResourceTargets = true- if target.Status == schedulertypes.Succeeded || target.Status == schedulertypes.Skipped {- continue- }- if target.ResourceType != "container" || target.Status != schedulertypes.Failed || strings.TrimSpace(target.ID) == "" {- unresolvedTargets = true- continue- }- options.ResourceIds = append(options.ResourceIds, target.ID)- }- if hasResourceTargets && len(options.ResourceIds) == 0 {- if unresolvedTargets {- return options, schedulertypes.Outcome{Status: schedulertypes.Failed, Message: "Remaining targets have no confirmed completion or safe retry", Targets: previous.Outcome.Targets}, true- }- return options,- schedulertypes.Outcome{- Status: schedulertypes.Failed,- Message: "Target results do not confirm full-batch completion",- ActivityID: previous.Outcome.ActivityID,- Targets: previous.Outcome.Targets,- },- true- }- if !hasResourceTargets {- return options,- schedulertypes.Outcome{- Status: schedulertypes.Failed,- Message: "The previous update has no confirmed retry targets",- ActivityID: previous.Outcome.ActivityID,- Targets: previous.Outcome.Targets,- },- true- }- if len(options.ResourceIds) > 0 {- options.Type = "container"- }- return options, schedulertypes.Outcome{}, unresolvedTargets-}--// ValidateRetry restricts replay to explicitly failed container updates.-func (j *AutoUpdateJob) ValidateRetry(_ context.Context, previous schedulertypes.Run) error {- options, outcome, _ := autoUpdateRetryOptionsInternal(previous)- if outcome.Status != "" || len(options.ResourceIds) == 0 {- return errors.New("auto-update has no safely retryable failed containers")- }- return nil-}--func autoUpdateOutcomeInternal(ctx context.Context, result *updatertypes.Result, unresolvedTargets bool, err error) (schedulertypes.Outcome, error) {- slog.InfoContext(ctx, "auto-update run completed",- "checked", result.Checked,- "updated", result.Updated,- "restarted", result.Restarted,- "skipped", result.Skipped,- "failed", result.Failed,- )- outcome := schedulertypes.Outcome{Status: schedulertypes.Succeeded}- if result.ActivityID != nil {- outcome.ActivityID = *result.ActivityID- }- for _, item := range result.Items {- status := kit.Ternary(item.Status == "failed", schedulertypes.Failed, schedulertypes.Succeeded)- if item.Status == "skipped" {- status = schedulertypes.Skipped- }- outcome.Targets = append(outcome.Targets, schedulertypes.TargetOutcome{ID: item.ResourceID, ResourceType: item.ResourceType, Status: status, Message: item.Error, ActivityID: outcome.ActivityID})- }- if result.Failed > 0 || err != nil || unresolvedTargets {- outcome.Status = schedulertypes.Partial- outcome.Message = "Some updates failed"- if err != nil {- outcome.Message += ": " + err.Error()- }- }- if unresolvedTargets {- outcome.Status = schedulertypes.Failed- outcome.Message = "Some target outcomes could not be confirmed"- }- if outcomeErr, ok := errors.AsType[*schedulertypes.OutcomeError](err); ok {- outcome.Status = outcomeErr.Outcome.Status- outcome.Message = outcomeErr.Outcome.Message- outcome.Targets = nil- }- return outcome, err-}--func confirmedAutoUpdateInternal(previous schedulertypes.Run) schedulertypes.Outcome {- confirmed := jobcontext.ConfirmedTarget(previous, "auto-update")- for _, target := range previous.Outcome.Targets {- if target.ID == "auto-update" && target.ResourceType != "update-batch" {- confirmed.Status = schedulertypes.Failed- }- }- if confirmed.Status == schedulertypes.Succeeded {- for _, target := range previous.Outcome.Targets {- if target.Status != schedulertypes.Succeeded && target.Status != schedulertypes.Skipped {- confirmed.Status = schedulertypes.Failed- break- }- }+const autoUpdateDefaultSchedule = "0 0 0 * * *"++// NewAutoUpdateJob runs the durable auto-update workflow on the configured schedule.+// Interrupted runs that cannot resume fall back to frozen-plan recovery.+func NewAutoUpdateJob(engine *flow.Engine, updaterService *updater.UpdaterService, settingsService *settings.SettingsService) *flow.Job {+ return &flow.Job{+ Engine: engine,+ Workflow: updaterService.AutoUpdateWorkflow(),+ JobName: "auto-update",+ ScheduleFn: func(ctx context.Context) string {+ schedule := settingsService.GetStringSetting(ctx, "autoUpdateInterval", autoUpdateDefaultSchedule)+ return kit.Ternary(schedule == "", autoUpdateDefaultSchedule, schedule)+ },+ ShouldRunFn: func(ctx context.Context) bool {+ return settingsService.GetBoolSetting(ctx, "autoUpdate", false) && settingsService.GetBoolSetting(ctx, "pollingEnabled", true)+ },+ FallbackFn: updaterService.ReconcilePending,+ ValidateRetryFn: updaterService.ValidateAutoUpdateRetry,}- return confirmed}diff --git a/backend/pkg/scheduler/auto_update_job_test.go b/backend/pkg/scheduler/auto_update_job_test.goindex 7fae0cc42..b0392d247 100644--- a/backend/pkg/scheduler/auto_update_job_test.go+++ b/backend/pkg/scheduler/auto_update_job_test.go@@ -1,23 +1,17 @@package schedulerimport (- "context"- "sync/atomic""testing"- "time"- schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"- "github.com/getarcaneapp/arcane/types/v2/updater""github.com/stretchr/testify/require"- "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"+ "github.com/getarcaneapp/arcane/backend/v2/internal/updater")func TestAutoUpdateJob_ShouldSchedule_RequiresAutoUpdateAndPolling(t *testing.T) {ctx := t.Context()_, settingsSvc, _ := setupAnalyticsStateServicesInternal(t)- job, err := NewAutoUpdateJob(nil, settingsSvc, newTestAdmissionGateInternal(t))- require.NoError(t, err)+ job := NewAutoUpdateJob(nil, &updater.UpdaterService{}, settingsSvc)require.False(t, job.ShouldSchedule(ctx))@@ -30,181 +24,3 @@ func TestAutoUpdateJob_ShouldSchedule_RequiresAutoUpdateAndPolling(t *testing.T)require.NoError(t, settingsSvc.SetBoolSetting(ctx, "autoUpdate", false))require.False(t, job.ShouldSchedule(ctx))}--type blockingApplierFakeInternal struct {- calls atomic.Int32- started chan struct{}- release chan struct{}-}--func (f *blockingApplierFakeInternal) ApplyPending(context.Context, updater.Options) (*updater.Result, error) {- f.calls.Add(1)- select {- case f.started <- struct{}{}:- default:- }- <-f.release- return &updater.Result{}, nil-}--func TestAutoUpdateJob_OverlappingRunIsSkippedInternal(t *testing.T) {- ctx := t.Context()- _, settingsSvc, _ := setupAnalyticsStateServicesInternal(t)- require.NoError(t, settingsSvc.SetBoolSetting(ctx, "autoUpdate", true))-- applier := &blockingApplierFakeInternal{- started: make(chan struct{}, 1),- release: make(chan struct{}),- }- job := &AutoUpdateJob{updaterService: applier, settingsService: settingsSvc, admissionGate: newTestAdmissionGateInternal(t)}-- firstDone := make(chan struct{})- go func() {- _, runErr1 := job.Run(ctx)- if runErr1 != nil {- t.Errorf("first auto update job run failed: %v", runErr1)- close(firstDone)- return- }- close(firstDone)- }()- <-applier.started-- // Overlapping tick returns immediately without a second ApplyPending.- _, runErr2 := job.Run(ctx)- require.NoError(t, runErr2)- require.Equal(t, int32(1), applier.calls.Load())-- close(applier.release)- select {- case <-firstDone:- case <-time.After(time.Second):- require.FailNow(t, "first auto-update run did not finish")- }-- // The guard resets once the run finishes.- _, runErr3 := job.Run(ctx)- require.NoError(t, runErr3)- require.Equal(t, int32(2), applier.calls.Load())- for _, tc := range []struct {- name string- targets []schedulertypes.TargetOutcome- retryIDs []string- status schedulertypes.RunStatus- unresolved bool- }{- {name: "missing progress", unresolved: true, status: schedulertypes.Failed},- {- name: "image pull alone",- unresolved: true,- targets: []schedulertypes.TargetOutcome{{- ID: "image",- ResourceType: "image",- Status: schedulertypes.Succeeded,- }},- status: schedulertypes.Failed,- },- {- name: "failed containers only",- unresolved: true,- targets: []schedulertypes.TargetOutcome{- {- ID: "done",- ResourceType: "container",- Status: schedulertypes.Succeeded,- },- {- ID: "failed",- ResourceType: "container",- Status: schedulertypes.Failed,- },- {- ID: "image",- ResourceType: "image",- Status: schedulertypes.Failed,- },- },- retryIDs: []string{"failed"},- },- {- name: "completed failed batch retry",- targets: []schedulertypes.TargetOutcome{- {- ID: "auto-update",- ResourceType: "update-batch",- Status: schedulertypes.Partial,- },- {- ID: "failed",- ResourceType: "container",- Status: schedulertypes.Failed,- },- },- retryIDs: []string{"failed"},- },- {- name: "scoped retry cannot confirm original batch",- targets: []schedulertypes.TargetOutcome{- {- ID: "auto-update",- ResourceType: "update-retry",- Status: schedulertypes.Succeeded,- },- {- ID: "done",- ResourceType: "container",- Status: schedulertypes.Succeeded,- },- },- status: schedulertypes.Failed,- unresolved: true,- },- {- name: "completed batch",- targets: []schedulertypes.TargetOutcome{{- ID: "auto-update",- ResourceType: "update-batch",- Status: schedulertypes.Succeeded,- }},- status: schedulertypes.Succeeded,- },- } {- t.Run(tc.name, func(t *testing.T) {- previous := schedulertypes.Run{AttemptCount: 2, Outcome: schedulertypes.Outcome{Targets: tc.targets}}- options, outcome, unresolved := autoUpdateRetryOptionsInternal(previous)- require.Equal(t, tc.unresolved, unresolved)- require.Equal(t, tc.retryIDs, options.ResourceIds)- require.Equal(t, tc.status, outcome.Status)- if len(tc.retryIDs) == 0 {- require.Error(t, job.ValidateRetry(ctx, previous))- } else {- require.NoError(t, job.ValidateRetry(ctx, previous))- }- reconciled, err := job.Reconcile(ctx, previous)- require.NoError(t, err)- if tc.status == schedulertypes.Succeeded {- require.Equal(t, schedulertypes.Succeeded, reconciled.Status)- } else {- require.Equal(t, schedulertypes.Failed, reconciled.Status)- }- })- }-- require.NoError(t, settingsSvc.SetBoolSetting(ctx, "autoUpdate", false))- retryCtx := jobcontext.WithExecution(- ctx,- schedulertypes.Run{- AttemptCount: 2,- Outcome: schedulertypes.Outcome{Targets: []schedulertypes.TargetOutcome{{- ID: "failed",- ResourceType: "container",- Status: schedulertypes.Failed,- }}},- },- nil,- )- outcome, err := job.Run(retryCtx)- require.NoError(t, err)- require.Equal(t, schedulertypes.Failed, outcome.Status)- require.Equal(t, int32(2), applier.calls.Load())-}diff --git a/backend/pkg/scheduler/docker_client_refresh_job.go b/backend/pkg/scheduler/docker_client_refresh_job.goindex b8aac5532..70649eefa 100644--- a/backend/pkg/scheduler/docker_client_refresh_job.go+++ b/backend/pkg/scheduler/docker_client_refresh_job.go@@ -1,7 +1,6 @@package schedulerimport (- "cmp""context""log/slog"@@ -37,15 +36,7 @@ func (j *DockerClientRefreshJob) Name() string {}func (j *DockerClientRefreshJob) Schedule(ctx context.Context) string {- schedule := cmp.Or(j.settingsService.GetStringSetting(ctx, "dockerClientRefreshInterval", dockerClientRefreshDefaultSchedule), dockerClientRefreshDefaultSchedule)-- parser := scheduleutil.Parser()- if _, err := parser.Parse(schedule); err != nil {- slog.WarnContext(ctx, "Invalid cron expression for Docker client refresh, using default", "invalidSchedule", schedule, "error", err)- return dockerClientRefreshDefaultSchedule- }-- return schedule+ return scheduleutil.Or(ctx, j.settingsService.GetStringSetting(ctx, "dockerClientRefreshInterval", dockerClientRefreshDefaultSchedule), dockerClientRefreshDefaultSchedule, "docker-client-refresh")}func (j *DockerClientRefreshJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {diff --git a/backend/pkg/scheduler/entityjobs/entityjobs.go b/backend/pkg/scheduler/entityjobs/entityjobs.goindex 68366150c..928b2f02a 100644--- a/backend/pkg/scheduler/entityjobs/entityjobs.go+++ b/backend/pkg/scheduler/entityjobs/entityjobs.go@@ -1,7 +1,4 @@-// Package entityjobs holds the per-entity dynamic-job registry shared by the-// services that schedule one job per database row (GitOps syncs, environment-// health checks). It lives beside the scheduler rather than inside it because-// package scheduler imports the services that would use this registry.+// Package entityjobs registers one dynamic scheduler job per database row, such as a GitOps sync or a backup policy.package entityjobsimport (@@ -17,12 +14,8 @@ import (// GitOpsSyncJobPrefix is shared by the GitOps registry and environment cleanup.const GitOpsSyncJobPrefix = "gitops-sync:"-// Registry owns one dynamic scheduler job per entity ID, plus the admission-// gate that keeps a single run of any one entity in flight at a time.-//-// The scheduler and the app lifecycle context arrive post-construction via-// SetScheduler: the manager wires them during bootstrap, and agent mode leaves-// them nil so every registration becomes a no-op.+// Registry owns one job per entity ID and admits one run per entity at a time.+// Without SetScheduler, as in agent mode, every registration is a no-op.type Registry struct {jobPrefix stringadmissionScope string@@ -38,15 +31,10 @@ func New(jobPrefix, admissionScope string) *Registry {return &Registry{jobPrefix: jobPrefix, admissionScope: admissionScope}}-// SetScheduler injects the job scheduler, the admission gate and the app-// lifecycle context. The lifecycle context is what scheduled runs execute on,-// so they outlive the request or bootstrap goroutine that registered them.+// SetScheduler injects the scheduler, the admission gate, and the app lifecycle context scheduled runs execute on.////nolint:contextcheck // scheduled runs must capture the app lifecycle context, not request contexts-func (r *Registry) SetScheduler(ctx context.Context,- scheduler schedulertypes.DynamicScheduler,- admissionGate *runs.Admission,-) error {+func (r *Registry) SetScheduler(ctx context.Context, scheduler schedulertypes.DynamicScheduler, admissionGate *runs.Admission) error {if scheduler == nil || admissionGate == nil {return fmt.Errorf("%s scheduler dependencies unavailable", r.admissionScope)}@@ -62,12 +50,10 @@ func (r *Registry) SetScheduler(ctx context.Context,// Enabled reports whether a scheduler has been injected.func (r *Registry) Enabled() bool { return r.scheduler != nil }-// Scheduler exposes the injected scheduler for the rare caller that must touch-// a job this registry does not own (nil when none was injected).+// Scheduler returns the injected scheduler, or nil.func (r *Registry) Scheduler() schedulertypes.DynamicScheduler { return r.scheduler }-// Context returns the context scheduled work should run on: the app lifecycle-// context when one was injected, otherwise a cancel-detached copy of ctx.+// Context returns the app lifecycle context, or a cancel-detached copy of ctx before one is injected.func (r *Registry) Context(ctx context.Context) context.Context {if r.lifecycleCtx != nil {return r.lifecycleCtx@@ -82,29 +68,24 @@ func (r *Registry) Context(ctx context.Context) context.Context {func (r *Registry) JobName(entityID string) string { return r.jobPrefix + entityID }// Register adds (or replaces) the dynamic job for entityID.-func (r *Registry) Register(ctx context.Context,- entityID string,- schedule func(context.Context) string,- run func(context.Context) (schedulertypes.Outcome,- error),- reconcile ...func(context.Context,- schedulertypes.Run) (schedulertypes.Outcome,- error),+func (r *Registry) Register(ctx context.Context, entityID string, schedule func(context.Context) string,+ run func(context.Context) (schedulertypes.Outcome, error), reconcile ...func(context.Context, schedulertypes.Run) (schedulertypes.Outcome, error),) {+ job := &schedulertypes.GenericJob{JobName: r.JobName(entityID), ScheduleFn: schedule, RunFn: run}+ if len(reconcile) > 0 {+ job.ReconcileFn = reconcile[0]+ }+ r.Add(ctx, job)+}++// Add adds (or replaces) a prebuilt job, such as a flow.Job named with JobName.+func (r *Registry) Add(ctx context.Context, job schedulertypes.Job) {if r.scheduler == nil {return}schedulerCtx := r.Context(ctx)- job := &schedulertypes.GenericJob{- JobName: r.JobName(entityID),- ScheduleFn: schedule,- RunFn: run,- }- if len(reconcile) > 0 {- job.ReconcileFn = reconcile[0]- }if err := r.scheduler.AddJob(schedulerCtx, job); err != nil {- slog.ErrorContext(schedulerCtx, "failed to register scheduled job", "job", job.JobName, "error", err)+ slog.ErrorContext(schedulerCtx, "failed to register scheduled job", "job", job.Name(), "error", err)}}@@ -116,8 +97,7 @@ func (r *Registry) Unregister(ctx context.Context, entityID string) {r.scheduler.RemoveJob(r.Context(ctx), r.JobName(entityID))}-// TryAcquire admits at most one in-flight run per entity ID. It refuses-// immediately when a run is already active.+// TryAcquire admits one in-flight run per entity ID and refuses at once while one is active.func (r *Registry) TryAcquire(ctx context.Context, entityID string) (*runs.Lease, bool, error) {return r.admissionGate.TryAcquire(ctx, schedulertypes.AdmissionKey{Scope: r.admissionScope, ID: entityID})}diff --git a/backend/pkg/scheduler/flow/flow.go b/backend/pkg/scheduler/flow/flow.gonew file mode 100644index 000000000..f9c46e845--- /dev/null+++ b/backend/pkg/scheduler/flow/flow.go@@ -0,0 +1,630 @@+// Package flow runs durable multi-step jobs as Francis workflows inside+// coordinator runs. The run stays the source of truth for status, the Jobs UI,+// and Activity; the workflow instance is its execution graph.+//+// Migrating a job:+// 1. Write each step as a func(ctx context.Context, t flow.Task) (any, error), inline in the definition+// unless it is shared or too complex to inline.+// Return common.Classify(common.ErrUnavailable or common.ErrTimeout, err) to retry; other errors fail the task.+// Tasks run at least once, so confirm external effects before repeating them.+// 2. Keep inputs and outputs small: keys, not lists of data, since Francis ships the instance input with+// every task. Keep data in domain tables and never put secrets in a payload.+// 3. Define the workflow with Engine.Define in a RegisterWorkflows method called from the service's DI+// provider. The last step returns scheduler.Outcome.+// 4. Register a flow.Job in place of the old job.+// 5. Cover the definition with flowtest.AssertDefinitions. On any graph or policy change, bump Version and+// set Fingerprint to the value the test reports.+package flow++import (+ "cmp"+ "context"+ "encoding/json"+ "errors"+ "fmt"+ "log/slog"+ "slices"+ "strconv"+ "strings"+ "sync"+ "time"+ "uuid"++ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"+ "github.com/getarcaneapp/arcane/types/v2/scheduler"+ "github.com/italypaleale/francis/actor"+ "github.com/italypaleale/francis/builtin/workflow"++ activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis"+)++const (+ defaultMaxPayloadSize = 256 << 10+ defaultMaxJournalSize = 4 << 20+ // stepWriteInterval bounds how often step progress rewrites the run record.+ stepWriteInterval = 500 * time.Millisecond+ // evidenceActorType stores the target progress of standalone instances, which have no run to hold it.+ evidenceActorType = "workflow-evidence"+)++var (+ // ErrNotResumable reports that a run has no workflow instance this host can serve.+ ErrNotResumable = errors.New("workflow instance cannot be resumed")+ // errInstanceFinished releases a root's cancellation once the instance ends.+ errInstanceFinished = errors.New("workflow instance finished")+)++type (+ // Definition declares a workflow. Bump Version and update Fingerprint on any+ // graph or policy change; flowtest.AssertDefinitions reports the new value.+ Definition struct {+ Name string+ Version int+ Fingerprint string+ Concurrency int+ Timeout time.Duration+ // Activity is the template for the activity each root instance owns; an empty Type means none.+ Activity activitylib.StartRequest+ // Labels name steps on the activity; steps without one show their name.+ Labels map[string]string+ // Steps use Francis step builders with handlers wrapped by Engine.Handler.+ // The last step returns scheduler.Outcome.+ Steps []workflow.StepSpec+ Options []workflow.Option+ }++ // Workflow is a defined workflow registered on the engine's host.+ Workflow struct {+ def Definition+ wf *workflow.Workflow+ svc *workflow.WorkflowService+ }++ // Activities is the activity service surface the engine drives.+ Activities interface {+ activitylib.Service+ activitylib.MessageAppender+ activitylib.Tracker+ UpdateActivity(ctx context.Context, activityID string, req activitylib.UpdateRequest) (*activitytypes.Activity, error)+ AwaitActivitySlotBounded(ctx context.Context, activityID, environmentID string) error+ }++ // Engine defines workflows, runs them inside coordinator runs, and owns their activities.+ Engine struct {+ runtime *francis.Runtime+ coordinator *runs.Coordinator+ activities Activities+ ctx context.Context+ cancel context.CancelFunc+ mu sync.Mutex+ workflows []*Workflow+ monitors map[string]*instanceMonitor+ roots map[string]rootCancellation+ evidence utils.KeyedMutex+ // slots serializes reacquiring a root activity's slot, so its sibling tasks never wait on the slot it holds.+ slots utils.KeyedMutex+ // tracked holds each starting activity's cancel registration until its monitor takes it over.+ tracked map[string]context.Context+ // admitted holds the runs whose executor authorized them and waits on them; other runs' tasks wait for reconcile.+ admitted map[string]struct{}+ // ready closes once startup recovery has protected standalone work; until then standalone tasks wait.+ ready chan struct{}+ // submitted holds the standalone instances this process started; tasks of others are recovered.+ submitted map[string]struct{}+ readyOnce sync.Once+ }++ // rootCancellation interrupts every running task of one root instance.+ rootCancellation struct {+ ctx context.Context+ cancel context.CancelCauseFunc+ }+)++// New builds an engine whose monitors live until appCtx ends or Stop is called.+func New(appCtx context.Context, runtime *francis.Runtime, coordinator *runs.Coordinator, activities Activities) (*Engine, error) {+ if err := runtime.RegisterActor(evidenceActorType, func(string, *actor.Service) actor.Actor { return struct{}{} }); err != nil {+ return nil, err+ }+ ctx, cancel := context.WithCancel(appCtx)+ return &Engine{+ runtime: runtime,+ coordinator: coordinator,+ activities: activities,+ ctx: ctx,+ cancel: cancel,+ monitors: map[string]*instanceMonitor{},+ roots: map[string]rootCancellation{},+ tracked: map[string]context.Context{},+ admitted: map[string]struct{}{},+ ready: make(chan struct{}),+ submitted: map[string]struct{}{},+ }, nil+}++// Define validates and registers a workflow. It must run before the host starts.+func (e *Engine) Define(def Definition) (*Workflow, error) {+ switch {+ case def.Version < 1:+ return nil, fmt.Errorf("workflow %q requires a version", def.Name)+ case def.Concurrency < 1:+ return nil, fmt.Errorf("workflow %q requires a concurrency", def.Name)+ case def.Timeout <= 0:+ return nil, fmt.Errorf("workflow %q requires a timeout", def.Name)+ case len(def.Steps) == 0:+ return nil, fmt.Errorf("workflow %q requires steps", def.Name)+ }+ options := append([]workflow.Option{+ workflow.WithVersion(def.Version),+ workflow.WithConcurrency(def.Concurrency),+ workflow.WithTimeout(def.Timeout),+ workflow.WithSteps(def.Steps...),+ workflow.WithUnknownVersionPolicy(workflow.FailUnknownVersion),+ workflow.WithMaxInputSize(defaultMaxPayloadSize),+ workflow.WithMaxOutputSize(defaultMaxPayloadSize),+ workflow.WithMaxJournalSize(defaultMaxJournalSize),+ workflow.WithLogger(slog.Default().With("scope", "workflow")),+ }, def.Options...)+ wf, err := workflow.New(def.Name, options...)+ if err != nil {+ return nil, fmt.Errorf("define workflow %q: %w", def.Name, err)+ }+ if registerErr := e.runtime.RegisterWorkflow(wf); registerErr != nil {+ return nil, registerErr+ }+ defined := &Workflow{def: def, wf: wf, svc: wf.Service(e.runtime.Service())}+ e.mu.Lock()+ e.workflows = append(e.workflows, defined)+ e.mu.Unlock()+ return defined, nil+}++// Fingerprint is the graph hash the definition declares for its version.+func (w *Workflow) Fingerprint() string { return w.def.Fingerprint }++// Francis exposes the underlying definition for use with workflow.WithChild.+func (w *Workflow) Francis() *workflow.Workflow { return w.wf }++// Workflows lists every defined workflow.+func (e *Engine) Workflows() []*Workflow {+ e.mu.Lock()+ defer e.mu.Unlock()+ return slices.Clone(e.workflows)+}++// Start monitors every unfinished root instance so activities complete without a caller.+func (e *Engine) Start(ctx context.Context) error {+ return e.eachRoot(ctx, func(wf *Workflow, instanceID string) error {+ e.monitor(wf, instanceID)+ return nil+ })+}++// Ready lets standalone tasks run once startup recovery has protected the work Standalone reported.+func (e *Engine) Ready() {+ e.readyOnce.Do(func() { close(e.ready) })+}++// Standalone lists unfinished instances started by Submit or an unbound Run as runs of the workflow's+// name, carrying the targets their tasks recorded, so startup recovery can protect their work.+func (e *Engine) Standalone(ctx context.Context) ([]scheduler.Run, error) {+ var standalone []scheduler.Run+ err := e.eachRoot(ctx, func(wf *Workflow, instanceID string) error {+ if isInstanceOf(instanceID, wf.def.Name) {+ return nil+ }+ run := scheduler.Run{ID: instanceID, JobID: wf.def.Name, ActivityID: wf.activityFor(instanceID), Status: scheduler.Running}+ if err := e.runtime.Service().GetState(ctx, evidenceActorType, instanceID, &run.Outcome.Targets); err != nil && !errors.Is(err, actor.ErrStateNotFound) {+ return fmt.Errorf("load %s progress: %w", instanceID, err)+ }+ standalone = append(standalone, run)+ return nil+ })+ return standalone, err+}++// eachRoot visits every unfinished root instance of every defined workflow.+func (e *Engine) eachRoot(ctx context.Context, visit func(wf *Workflow, instanceID string) error) error {+ for _, wf := range e.Workflows() {+ for _, status := range []workflow.Status{workflow.StatusPending, workflow.StatusRunning, workflow.StatusSuspended, workflow.StatusCompensating} {+ after := ""+ for {+ page, err := wf.svc.List(ctx, &workflow.ListOptions{Status: status, After: after, Limit: 100})+ if err != nil {+ return fmt.Errorf("list %s workflows: %w", wf.def.Name, err)+ }+ for _, instance := range page.Instances {+ if instance.Parent != nil {+ continue+ }+ if visitErr := visit(wf, instance.InstanceID); visitErr != nil {+ return visitErr+ }+ }+ if after = page.AfterID(); after == "" {+ break+ }+ }+ }+ }+ return nil+}++// Stop ends monitoring; unfinished instances resume on the next Start.+func (e *Engine) Stop(context.Context) error {+ e.cancel()+ return nil+}++// Run starts the workflow, or attaches to the run's active instance, and waits for its outcome. A run in+// ctx (jobcontext.Run) binds the instance to that run; set fields of activity override the definition's+// activity template; a non-nil output receives the last step's data alongside its scheduler.Outcome fields.+func (e *Engine) Run(ctx context.Context, wf *Workflow, payload any, activity activitylib.StartRequest, output any) (scheduler.Outcome, error) {+ run, bound := jobcontext.Run(ctx)+ instanceID := ""+ if bound {+ defer e.admit(run.ID)()+ if target, ok := activeTarget(run, wf.def.Name, false); ok {+ status, err := wf.svc.GetStatus(ctx, target.ID)+ if err != nil && !errors.Is(err, workflow.ErrInstanceNotFound) {+ return scheduler.Outcome{}, fmt.Errorf("read %s instance: %w", wf.def.Name, err)+ }+ if err == nil && status.Version == wf.def.Version {+ instanceID = target.ID+ } else {+ e.supersede(ctx, wf, target, "Superseded by a newer workflow instance")+ }+ }+ }+ if instanceID == "" {+ var err error+ if instanceID, err = e.startInstance(ctx, wf, run, bound, payload, activity, nil); err != nil {+ return scheduler.Outcome{Status: scheduler.Failed, Message: err.Error()}, err+ }+ }+ outcome, err := e.await(ctx, wf, run, bound, instanceID)+ if err != nil || output == nil {+ return outcome, err+ }+ status, err := wf.svc.GetStatus(ctx, instanceID)+ if err != nil {+ return outcome, fmt.Errorf("read %s output: %w", wf.def.Name, err)+ }+ return outcome, status.DecodeOutput(output)+}++// Submit starts a standalone instance without waiting for it and returns its ID,+// which is also its activity's ID. Its tasks record target progress like a run's;+// evidence is recorded before the instance starts, such as the record it owns.+func (e *Engine) Submit(ctx context.Context, wf *Workflow, payload any, activity activitylib.StartRequest, evidence ...scheduler.TargetOutcome) (string, error) {+ instanceID, err := e.startInstance(ctx, wf, scheduler.Run{}, false, payload, activity, evidence)+ if err != nil {+ return "", err+ }+ e.monitor(wf, instanceID)+ return instanceID, nil+}++// Wait blocks until a submitted instance finishes, ctx ends, or the engine stops.+func (e *Engine) Wait(ctx context.Context, wf *Workflow, instanceID string) (scheduler.Outcome, error) {+ ctx, cancel := context.WithCancel(ctx)+ defer cancel()+ defer context.AfterFunc(e.ctx, cancel)()+ return e.await(ctx, wf, scheduler.Run{}, false, instanceID)+}++// Resume attaches to the run's active instance, or reads the result of one that finished before the run saved it.+// It returns ErrNotResumable when the instance is gone or belongs to a version this host no longer serves.+func (e *Engine) Resume(ctx context.Context, wf *Workflow, run scheduler.Run) (scheduler.Outcome, error) {+ target, ok := activeTarget(run, wf.def.Name, true)+ if !ok {+ return scheduler.Outcome{}, ErrNotResumable+ }+ defer e.admit(run.ID)()+ status, err := wf.svc.GetStatus(ctx, target.ID)+ if err != nil && !errors.Is(err, workflow.ErrInstanceNotFound) {+ return scheduler.Outcome{}, fmt.Errorf("read %s instance: %w", wf.def.Name, err)+ }+ if err != nil || status.Version != wf.def.Version {+ if !target.Status.Terminal() {+ e.supersede(ctx, wf, target, "Workflow instance could not be resumed")+ }+ return scheduler.Outcome{}, ErrNotResumable+ }+ return e.await(ctx, wf, run, true, target.ID)+}++// Retire cancels the run's unfinished instance without resuming it, such as for a job that was disabled. An instance+// that finished before the run saved its result reports that result instead, with finished set.+func (e *Engine) Retire(ctx context.Context, wf *Workflow, run scheduler.Run, reason string) (outcome scheduler.Outcome, finished bool, err error) {+ if target, ok := activeTarget(run, wf.def.Name, true); ok && target.Status.Terminal() {+ if outcome, err = e.Resume(ctx, wf, run); !errors.Is(err, ErrNotResumable) {+ return outcome, true, err+ }+ }+ if target, ok := activeTarget(run, wf.def.Name, false); ok {+ e.supersede(ctx, wf, target, reason)+ }+ return scheduler.Outcome{}, false, nil+}++func (e *Engine) startInstance(ctx context.Context, wf *Workflow, run scheduler.Run, bound bool, payload any, activity activitylib.StartRequest, evidence []scheduler.TargetOutcome) (string, error) {+ instanceID := wf.def.Name + "-" + uuid.NewV7().String()+ binding := Binding{RootID: instanceID}+ if bound {+ instanceID = fmt.Sprintf("%s-%s-%d", wf.def.Name, run.ID, run.AttemptCount)+ binding = Binding{EnvironmentID: run.EnvironmentID, JobID: run.JobID, RunID: run.ID, RootID: instanceID}+ }+ encoded, err := json.Marshal(payload)+ if err != nil {+ return "", fmt.Errorf("encode %s input: %w", wf.def.Name, err)+ }+ activityID := wf.activityFor(instanceID)+ if bound {+ // Record the instance first so its tasks pass the run fence.+ target := scheduler.TargetOutcome{ResourceType: scheduler.WorkflowTarget, ID: instanceID, Status: scheduler.Running, ActivityID: activityID}+ if progressErr := jobcontext.Progress(ctx, target); progressErr != nil {+ return "", fmt.Errorf("record %s instance: %w", wf.def.Name, progressErr)+ }+ }+ if activityID != "" {+ if activityErr := e.startActivity(ctx, wf, activityID, cmp.Or(run.EnvironmentID, "0"), activity); activityErr != nil {+ return "", activityErr+ }+ }+ if !bound {+ e.mu.Lock()+ e.submitted[instanceID] = struct{}{}+ e.mu.Unlock()+ }+ startErr := error(nil)+ if len(evidence) > 0 {+ startErr = e.runtime.Service().SetState(ctx, evidenceActorType, instanceID, evidence, nil)+ }+ if startErr == nil {+ _, _, startErr = wf.svc.Start(ctx, envelope{Binding: binding, Payload: encoded}, workflow.WithInstanceID(instanceID))+ }+ if startErr != nil {+ e.releaseRoot(instanceID)+ if len(evidence) > 0 {+ _ = e.runtime.Service().DeleteState(context.WithoutCancel(ctx), evidenceActorType, instanceID)+ }+ if activityID != "" {+ e.untrack(activityID)+ message := startErr.Error()+ _, _ = e.activities.CompleteActivity(ctx, activityID, activitytypes.StatusFailed, "Failed to start workflow", &message)+ }+ return "", fmt.Errorf("start %s: %w", wf.def.Name, startErr)+ }+ return instanceID, nil+}++func (e *Engine) startActivity(ctx context.Context, wf *Workflow, activityID, environmentID string, details activitylib.StartRequest) error {+ request := wf.def.Activity+ request.ID = activityID+ request.EnvironmentID = cmp.Or(details.EnvironmentID, request.EnvironmentID, environmentID)+ request.ResourceType = cmp.Or(details.ResourceType, request.ResourceType)+ request.ResourceID = cmp.Or(details.ResourceID, request.ResourceID)+ request.ResourceName = cmp.Or(details.ResourceName, request.ResourceName)+ request.StartedBy = cmp.Or(details.StartedBy, request.StartedBy)+ request.Step = cmp.Or(details.Step, request.Step)+ request.LatestMessage = cmp.Or(details.LatestMessage, request.LatestMessage)+ if details.Metadata != nil {+ request.Metadata = details.Metadata+ }+ // The registration a user cancel reaches exists before the activity does, so no cancel is missed; the+ // monitor takes it over once the instance runs.+ tracked := e.activities.Track(e.ctx, activityID)+ e.mu.Lock()+ e.tracked[activityID] = tracked+ e.mu.Unlock()+ if _, err := e.activities.StartActivity(ctx, request); err != nil {+ e.untrack(activityID)+ // Completing drops the service's cancel registration and settles a row the failed start may have left.+ message := err.Error()+ _, _ = e.activities.CompleteActivity(context.WithoutCancel(ctx), activityID, activitytypes.StatusFailed, "Failed to start workflow", &message)+ return fmt.Errorf("start %s activity: %w", wf.def.Name, err)+ }+ // A deferred slot is awaited by the task that does the work, so a submit never blocks on it.+ if request.Queue && !request.DeferSlot {+ waitCtx, cancel := context.WithCancel(ctx)+ defer cancel()+ defer context.AfterFunc(tracked, cancel)()+ if err := e.activities.AwaitActivitySlotBounded(waitCtx, activityID, request.EnvironmentID); err != nil && !errors.Is(context.Cause(tracked), activitylib.ErrCanceled) {+ e.untrack(activityID)+ message := err.Error()+ _, _ = e.activities.CompleteActivity(context.WithoutCancel(ctx), activityID, activitytypes.StatusFailed, "Failed to start workflow", &message)+ return err+ }+ }+ if errors.Is(context.Cause(tracked), activitylib.ErrCanceled) {+ e.untrack(activityID)+ _, _ = e.activities.CompleteActivity(context.WithoutCancel(ctx), activityID, activitytypes.StatusCancelled, activitylib.ErrCanceled.Error(), nil)+ return fmt.Errorf("start %s: %w", wf.def.Name, activitylib.ErrCanceled)+ }+ return nil+}++// untrack drops a starting activity's registration; the engine only holds it until the monitor starts.+func (e *Engine) untrack(activityID string) {+ e.mu.Lock()+ delete(e.tracked, activityID)+ e.mu.Unlock()+}++// await waits for the instance's outcome, projecting steps onto a bound run at+// most every stepWriteInterval.+func (e *Engine) await(ctx context.Context, wf *Workflow, run scheduler.Run, bound bool, instanceID string) (scheduler.Outcome, error) {+ monitor := e.monitor(wf, instanceID)+ var recorded []scheduler.StepOutcome+ var lastWrite time.Time+ for {+ steps, outcome, changed := monitor.snapshot()+ var flush <-chan time.Time+ if bound && !stepsEqual(recorded, steps) {+ if wait := stepWriteInterval - time.Since(lastWrite); outcome == nil && wait > 0 {+ flush = time.After(wait)+ } else if err := e.coordinator.RecordSteps(ctx, run, instanceID, steps); err != nil && ctx.Err() == nil {+ slog.DebugContext(ctx, "Failed to record workflow steps", "instanceId", instanceID, "error", err)+ } else {+ recorded, lastWrite = steps, time.Now()+ }+ }+ if outcome != nil {+ if bound {+ // Bound to the executor: a write lost to shutdown is redone when reconcile re-attaches.+ target := scheduler.TargetOutcome{ResourceType: scheduler.WorkflowTarget, ID: instanceID, Status: outcome.Status, ActivityID: wf.activityFor(instanceID)}+ // Run never attaches a finished instance, so a retry starts a new one.+ if !target.Status.Terminal() {+ target.Status = scheduler.Failed+ }+ if err := jobcontext.Progress(ctx, target); err != nil {+ slog.WarnContext(ctx, "Failed to record workflow completion", "instanceId", instanceID, "error", err)+ }+ }+ return *outcome, nil+ }+ select {+ case <-ctx.Done():+ return scheduler.Outcome{}, ctx.Err()+ case <-changed:+ case <-flush:+ }+ }+}++// supersede retires an instance that a run no longer follows.+func (e *Engine) supersede(ctx context.Context, wf *Workflow, target scheduler.TargetOutcome, reason string) {+ target.Status = scheduler.Failed+ target.Message = reason+ if err := jobcontext.Progress(ctx, target); err != nil {+ slog.WarnContext(ctx, "Failed to retire workflow instance", "instanceId", target.ID, "error", err)+ }+ if err := wf.svc.Cancel(ctx, target.ID, reason); err != nil && !errors.Is(err, workflow.ErrInstanceNotFound) && !errors.Is(err, workflow.ErrInstanceTerminated) {+ slog.DebugContext(ctx, "Failed to cancel superseded workflow instance", "instanceId", target.ID, "error", err)+ }+ e.mu.Lock()+ monitor := e.monitors[target.ID]+ delete(e.monitors, target.ID)+ e.mu.Unlock()+ if monitor != nil {+ monitor.stop()+ }+ e.releaseRoot(target.ID)+ if target.ActivityID != "" {+ _, _ = e.activities.CompleteActivity(ctx, target.ActivityID, activitytypes.StatusFailed, reason, new(reason))+ }+}++// admit lets the run's tasks execute until the returned func is called.+func (e *Engine) admit(runID string) func() {+ e.mu.Lock()+ e.admitted[runID] = struct{}{}+ e.mu.Unlock()+ return func() {+ e.mu.Lock()+ delete(e.admitted, runID)+ e.mu.Unlock()+ }+}++// rootContext is canceled, with the user's cause, when a root instance is canceled.+func (e *Engine) rootContext(rootID string) context.Context {+ e.mu.Lock()+ defer e.mu.Unlock()+ root, ok := e.roots[rootID]+ if !ok {+ ctx, cancel := context.WithCancelCause(e.ctx)+ root = rootCancellation{ctx: ctx, cancel: cancel}+ e.roots[rootID] = root+ }+ return root.ctx+}++// cancelRoot interrupts the running tasks of a root instance and its children.+func (e *Engine) cancelRoot(rootID string, cause error) {+ e.mu.Lock()+ root, ok := e.roots[rootID]+ e.mu.Unlock()+ if ok {+ root.cancel(cause)+ }+}++func (e *Engine) releaseRoot(rootID string) {+ e.mu.Lock()+ root, ok := e.roots[rootID]+ delete(e.roots, rootID)+ delete(e.submitted, rootID)+ e.mu.Unlock()+ if ok {+ root.cancel(errInstanceFinished)+ }+}++// activeTarget returns the run's latest instance of a workflow and whether the run may attach to it: an unfinished+// instance, or when recovering, a finished one whose attempt ended before the run saved its result.+func activeTarget(run scheduler.Run, name string, recovering bool) (scheduler.TargetOutcome, bool) {+ for _, target := range slices.Backward(run.Outcome.Targets) {+ if target.ResourceType != scheduler.WorkflowTarget || !isInstanceOf(target.ID, name) {+ continue+ }+ if !recovering || !target.Status.Terminal() {+ return target, !target.Status.Terminal()+ }+ attempt, err := strconv.Atoi(target.ID[strings.LastIndexByte(target.ID, '-')+1:])+ if err != nil {+ return target, false+ }+ // Redeliveries repeat an attempt's number; only its latest entry records whether the result was saved.+ for _, previous := range slices.Backward(run.Attempts) {+ if previous.Number == attempt {+ return target, previous.FinishedAt == nil+ }+ }+ return target, false+ }+ return scheduler.TargetOutcome{}, false+}++// isInstanceOf matches run-bound instance IDs of the form <name>-<runID>-<attempt>.+func isInstanceOf(instanceID, name string) bool {+ rest, ok := strings.CutPrefix(instanceID, name+"-")+ if !ok || len(rest) < 38 || rest[36] != '-' {+ return false+ }+ _, err := uuid.Parse(rest[:36])+ return err == nil+}++// rootWorkflow returns the workflow a root instance ID belongs to: <name>-<uuid> standalone, or <name>-<runID>-<attempt>.+func (e *Engine) rootWorkflow(rootID string) *Workflow {+ for _, wf := range e.Workflows() {+ rest, ok := strings.CutPrefix(rootID, wf.def.Name+"-")+ if !ok {+ continue+ }+ if _, err := uuid.Parse(rest); err == nil || isInstanceOf(rootID, wf.def.Name) {+ return wf+ }+ }+ return nil+}++// activityFor returns the activity an instance owns, or "" when the workflow has+// none. The instance ID doubles as the activity ID, so redelivery finds the same row.+func (w *Workflow) activityFor(instanceID string) string {+ if w.def.Activity.Type == "" {+ return ""+ }+ return instanceID+}++// stepLabel names a step for the activity.+func (w *Workflow) stepLabel(name string) string {+ return cmp.Or(w.def.Labels[name], name)+}diff --git a/backend/pkg/scheduler/flow/flow_test.go b/backend/pkg/scheduler/flow/flow_test.gonew file mode 100644index 000000000..7feacdbf8--- /dev/null+++ b/backend/pkg/scheduler/flow/flow_test.go@@ -0,0 +1,442 @@+package flow_test++import (+ "context"+ "errors"+ "fmt"+ "net"+ "net/url"+ "os"+ "path/filepath"+ "sync"+ "sync/atomic"+ "testing"+ "time"++ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"+ "github.com/getarcaneapp/arcane/types/v2/scheduler"+ "github.com/italypaleale/francis/builtin/workflow"+ "github.com/stretchr/testify/require"++ "github.com/getarcaneapp/arcane/backend/v2/internal/common"+ activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow/flowtest"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"+)++type activitiesFake struct {+ mu sync.Mutex+ statuses map[string]activitytypes.Status+ completed map[string]int+ tracked map[string]context.CancelCauseFunc+}++type hostFixture struct {+ harness *flowtest.Harness+ job *flow.Job+ cancel context.CancelFunc+}++func TestRestartBetweenStepsReattachesSQLite(t *testing.T) {+ testRestartBetweenStepsInternal(t, "file:"+filepath.ToSlash(filepath.Join(t.TempDir(), "actors.db")))+}++func TestRestartBetweenStepsReattachesPostgres(t *testing.T) {+ databaseURL := os.Getenv("ARCANE_TEST_POSTGRES_DSN")+ if databaseURL == "" {+ t.Skip("ARCANE_TEST_POSTGRES_DSN is not set")+ }+ testRestartBetweenStepsInternal(t, databaseURL)+}++func testRestartBetweenStepsInternal(t *testing.T, databaseURL string) {+ jobID := fmt.Sprintf("flow-restart-%d", time.Now().UnixNano())+ activities := newActivitiesFake()+ var firstRuns atomic.Int32+ blocked := make(chan struct{})+ first := startHostInternal(t, databaseURL, jobID, 1, activities, &firstRuns, func(ctx context.Context, _ flow.Task) (any, error) {+ close(blocked)+ <-ctx.Done()+ return nil, ctx.Err()+ }, nil)+ run, err := first.harness.Coordinator.Submit(t.Context(), scheduler.Request{JobID: jobID, EnvironmentID: "0", Trigger: "manual"})+ require.NoError(t, err)+ select {+ case <-blocked:+ case <-time.After(20 * time.Second):+ t.Fatal("second step never started")+ }+ started, err := first.harness.Coordinator.Get(t.Context(), "0", jobID, run.ID)+ require.NoError(t, err)+ instanceID := workflowTargetInternal(t, started).ID+ first.stop(t)++ second := startHostInternal(t, databaseURL, jobID, 1, activities, &firstRuns, func(context.Context, flow.Task) (any, error) {+ return nil, nil+ }, nil)+ finished := waitTerminalInternal(t, second, jobID, run.ID)++ require.Equal(t, scheduler.Succeeded, finished.Status)+ require.Equal(t, int32(1), firstRuns.Load(), "completed steps must not run again")+ target := workflowTargetInternal(t, finished)+ require.Equal(t, instanceID, target.ID, "reconcile must re-attach to the same instance")+ require.Equal(t, scheduler.Succeeded, target.Status)+ require.Len(t, finished.Outcome.Steps, 3)+ for _, step := range finished.Outcome.Steps {+ require.Equal(t, scheduler.Succeeded, step.Status, step.Name)+ }+ require.Eventually(t, func() bool { return activities.status(instanceID) == activitytypes.StatusSuccess }, 10*time.Second, 50*time.Millisecond)+}++func TestVersionBumpFallsBack(t *testing.T) {+ databaseURL := "file:" + filepath.ToSlash(filepath.Join(t.TempDir(), "actors.db"))+ jobID := fmt.Sprintf("flow-version-%d", time.Now().UnixNano())+ activities := newActivitiesFake()+ var firstRuns atomic.Int32+ blocked := make(chan struct{})+ first := startHostInternal(t, databaseURL, jobID, 1, activities, &firstRuns, func(ctx context.Context, _ flow.Task) (any, error) {+ close(blocked)+ <-ctx.Done()+ return nil, ctx.Err()+ }, nil)+ run, err := first.harness.Coordinator.Submit(t.Context(), scheduler.Request{JobID: jobID, EnvironmentID: "0", Trigger: "manual"})+ require.NoError(t, err)+ select {+ case <-blocked:+ case <-time.After(20 * time.Second):+ t.Fatal("second step never started")+ }+ first.stop(t)++ var fallbacks atomic.Int32+ second := startHostInternal(t, databaseURL, jobID, 2, activities, &firstRuns, func(context.Context, flow.Task) (any, error) {+ return nil, nil+ }, func(context.Context, scheduler.Run) (scheduler.Outcome, error) {+ fallbacks.Add(1)+ return scheduler.Outcome{Status: scheduler.Succeeded, Message: "recovered"}, nil+ })+ finished := waitTerminalInternal(t, second, jobID, run.ID)++ require.Equal(t, scheduler.Succeeded, finished.Status)+ require.Equal(t, "recovered", finished.Outcome.Message)+ require.Equal(t, int32(1), fallbacks.Load())+ target := workflowTargetInternal(t, finished)+ require.Equal(t, scheduler.Failed, target.Status, "the unserved instance must be retired")+}++func TestChildInheritsRunBinding(t *testing.T) {+ jobID := fmt.Sprintf("flow-child-%d", time.Now().UnixNano())+ harness := flowtest.New(t, newActivitiesFake())+ engine := harness.Engine+ childRuns := make(chan string, 1)+ child, err := engine.Define(flow.Definition{+ Name: "flow-child", Version: 1, Concurrency: 1, Timeout: time.Minute,+ Steps: []workflow.StepSpec{+ workflow.Step("work", engine.Handler(func(ctx context.Context, t flow.Task) (any, error) {+ var label string+ if err := t.Payload(&label); err != nil {+ return nil, err+ }+ run, _ := jobcontext.Run(ctx)+ childRuns <- run.ID+ return scheduler.Outcome{Status: scheduler.Partial, Message: "child " + label}, nil+ })),+ },+ })+ require.NoError(t, err)+ parent, err := engine.Define(flow.Definition{+ Name: "flow-parent", Version: 1, Concurrency: 1, Timeout: time.Minute,+ Steps: []workflow.StepSpec{+ workflow.Step("prepare", engine.Handler(func(_ context.Context, t flow.Task) (any, error) {+ return t.ChildInput("payload")+ })),+ workflow.Child("child", workflow.WithDefinition(child.Francis())),+ workflow.Step("finalize", engine.Handler(func(_ context.Context, t flow.Task) (any, error) {+ var childOutcome scheduler.Outcome+ if decodeErr := t.DecodeOutput("child", &childOutcome); decodeErr != nil {+ return nil, decodeErr+ }+ return scheduler.Outcome{Status: scheduler.Succeeded, Message: childOutcome.Message}, nil+ })),+ },+ })+ require.NoError(t, err)+ fixture := startJobInternal(t, harness, &flow.Job{Engine: engine, Workflow: parent, JobName: jobID, ScheduleFn: func(context.Context) string { return "" }})++ run, err := harness.Coordinator.Submit(t.Context(), scheduler.Request{JobID: jobID, EnvironmentID: "0", Trigger: "manual"})+ require.NoError(t, err)+ finished := waitTerminalInternal(t, fixture, jobID, run.ID)++ require.Equal(t, scheduler.Succeeded, finished.Status)+ require.Equal(t, "child payload", finished.Outcome.Message)+ require.Equal(t, run.ID, <-childRuns, "child tasks must run inside the parent's run")+}++func TestActivityCancelInterruptsRunningStep(t *testing.T) {+ activities := newActivitiesFake()+ harness := flowtest.New(t, activities)+ engine := harness.Engine+ started := make(chan string, 1)+ causes := make(chan error, 1)+ wf, err := engine.Define(flow.Definition{+ Name: "flow-cancel", Version: 1, Concurrency: 1, Timeout: time.Minute,+ Activity: activitylib.StartRequest{Type: activitytypes.TypeAutoUpdate},+ Steps: []workflow.StepSpec{+ workflow.Step("block", engine.Handler(func(ctx context.Context, t flow.Task) (any, error) {+ started <- t.ActivityID()+ <-ctx.Done()+ causes <- context.Cause(ctx)+ return nil, ctx.Err()+ })),+ },+ })+ require.NoError(t, err)+ harness.Start(t)++ outcomes := make(chan scheduler.Outcome, 1)+ go func() {+ outcome, _ := engine.Run(t.Context(), wf, nil, activitylib.StartRequest{}, nil)+ outcomes <- outcome+ }()+ activityID := <-started+ activities.requestCancel(activityID)++ select {+ case cause := <-causes:+ require.ErrorIs(t, cause, activitylib.ErrCanceled, "the running step must stop when the user cancels")+ case <-time.After(10 * time.Second):+ t.Fatal("the running step was not interrupted")+ }+ select {+ case outcome := <-outcomes:+ require.Equal(t, scheduler.Canceled, outcome.Status)+ case <-time.After(20 * time.Second):+ t.Fatal("the workflow did not finish after cancel")+ }+ require.Equal(t, activitytypes.StatusCancelled, activities.status(activityID))+}++func TestStartFinishesSubmittedInstanceWithItsProgress(t *testing.T) {+ databaseURL := "file:" + filepath.ToSlash(filepath.Join(t.TempDir(), "actors.db"))+ activities := newActivitiesFake()+ started := make(chan struct{})+ define := func(harness *flowtest.Harness, work func(context.Context, flow.Task) (any, error)) *flow.Workflow {+ wf, err := harness.Engine.Define(flow.Definition{+ Name: "flow-orphan", Version: 1, Concurrency: 1, Timeout: time.Minute,+ Activity: activitylib.StartRequest{Type: activitytypes.TypeAutoUpdate},+ Steps: []workflow.StepSpec{workflow.Step("work", harness.Engine.Handler(work))},+ })+ require.NoError(t, err)+ return wf+ }++ first := flowtest.New(t, activities, databaseURL)+ wf := define(first, func(ctx context.Context, _ flow.Task) (any, error) {+ if err := jobcontext.Progress(ctx, scheduler.TargetOutcome{ID: "backup", Status: scheduler.Running, RecoveryData: []byte(`{"backupId":"frozen"}`)}); err != nil {+ return nil, err+ }+ close(started)+ <-ctx.Done()+ return nil, ctx.Err()+ })+ first.Start(t)+ activityID, err := first.Engine.Submit(t.Context(), wf, nil, activitylib.StartRequest{})+ require.NoError(t, err)+ <-started+ standalone, err := first.Engine.Standalone(t.Context())+ require.NoError(t, err)+ require.Len(t, standalone, 1)+ require.Equal(t, activityID, standalone[0].ActivityID)+ require.JSONEq(t, `{"backupId":"frozen"}`, string(standalone[0].Outcome.Targets[0].RecoveryData))+ (&hostFixture{harness: first, cancel: func() {}}).stop(t)++ // No caller awaits the instance on the new host; Start alone must finish it, and the+ // redelivered task must see the progress the interrupted one recorded.+ second := flowtest.New(t, activities, databaseURL)+ define(second, func(ctx context.Context, _ flow.Task) (any, error) {+ previous, _ := jobcontext.Run(ctx)+ if len(previous.Outcome.Targets) != 1 || string(previous.Outcome.Targets[0].RecoveryData) != `{"backupId":"frozen"}` {+ return nil, errors.New("interrupted progress was lost")+ }+ return scheduler.Outcome{Status: scheduler.Succeeded}, nil+ })+ second.Start(t)+ require.Eventually(t, func() bool { return activities.status(activityID) == activitytypes.StatusSuccess }, 20*time.Second, 50*time.Millisecond)+ standalone, err = second.Engine.Standalone(t.Context())+ require.NoError(t, err)+ require.Empty(t, standalone)+}++func TestRetryable(t *testing.T) {+ var netErr net.Error = &net.DNSError{IsTimeout: true}+ cases := map[string]struct {+ err error+ want bool+ }{+ "unavailable": {common.Classify(common.ErrUnavailable, errors.New("503")), true},+ "timeout": {common.Classify(common.ErrTimeout, errors.New("slow")), true},+ "deadline": {fmt.Errorf("lookup: %w", context.DeadlineExceeded), true},+ "network": {netErr, true},+ "tls": {&url.Error{Op: "Get", URL: "https://registry", Err: errors.New("x509: certificate signed by unknown authority")}, false},+ "unauthorized": {common.Classify(common.ErrUnauthorized, errors.New("denied")), false},+ "plain": {errors.New("boom"), false},+ }+ for name, tc := range cases {+ require.Equal(t, tc.want, flow.Retryable(tc.err), name)+ }+}++func startHostInternal(+ t *testing.T,+ databaseURL, jobID string,+ version int,+ activities *activitiesFake,+ firstRuns *atomic.Int32,+ second func(context.Context, flow.Task) (any, error),+ fallback func(context.Context, scheduler.Run) (scheduler.Outcome, error),+) *hostFixture {+ t.Helper()+ harness := flowtest.New(t, activities, databaseURL)+ engine := harness.Engine+ steps := []workflow.StepSpec{+ workflow.Step("first", engine.Handler(func(context.Context, flow.Task) (any, error) {+ firstRuns.Add(1)+ return nil, nil+ })),+ workflow.Step("second", engine.Handler(second), workflow.WithMaxAttempts(1)),+ workflow.Step("finalize", engine.Handler(func(context.Context, flow.Task) (any, error) {+ return scheduler.Outcome{Status: scheduler.Succeeded, Message: "done"}, nil+ })),+ }+ if version > 1 {+ steps = append(steps[:2], workflow.Step("extra", engine.Handler(func(context.Context, flow.Task) (any, error) { return nil, nil })), steps[2])+ }+ wf, err := engine.Define(flow.Definition{+ Name: "flow-test",+ Version: version,+ Concurrency: 2,+ Timeout: time.Minute,+ Activity: activitylib.StartRequest{Type: activitytypes.TypeAutoUpdate},+ Steps: steps,+ })+ require.NoError(t, err)+ job := &flow.Job{Engine: engine, Workflow: wf, JobName: jobID, ScheduleFn: func(context.Context) string { return "" }, FallbackFn: fallback}+ return startJobInternal(t, harness, job)+}++// startJobInternal starts the host and a coordinator that executes job.+func startJobInternal(t *testing.T, harness *flowtest.Harness, job *flow.Job) *hostFixture {+ t.Helper()+ coordinator := harness.Coordinator+ coordinator.SetExecutor(+ func(ctx context.Context, run scheduler.Run) (scheduler.Outcome, error) {+ return job.Run(coordinator.ExecutionContext(ctx, run, ""))+ },+ func(ctx context.Context, run scheduler.Run) (scheduler.Outcome, error) {+ return job.Reconcile(coordinator.ExecutionContext(ctx, run, ""), run)+ },+ )+ harness.Start(t)+ appCtx, cancel := context.WithCancel(t.Context())+ require.NoError(t, coordinator.Start(t.Context(), appCtx))+ coordinator.Activate()+ fixture := &hostFixture{harness: harness, job: job, cancel: cancel}+ t.Cleanup(func() { fixture.stop(t) })+ return fixture+}++// stop simulates a process exit between steps.+func (f *hostFixture) stop(t *testing.T) {+ t.Helper()+ if f.cancel == nil {+ return+ }+ f.cancel()+ f.cancel = nil+ ctx, cancel := context.WithTimeout(context.WithoutCancel(t.Context()), 15*time.Second)+ defer cancel()+ _ = f.harness.Coordinator.Stop(ctx)+ _ = f.harness.Engine.Stop(ctx)+ require.NoError(t, f.harness.Runtime.Stop(ctx))+}++func waitTerminalInternal(t *testing.T, fixture *hostFixture, jobID, runID string) scheduler.Run {+ t.Helper()+ var run scheduler.Run+ require.Eventually(t, func() bool {+ var err error+ run, err = fixture.harness.Coordinator.Get(t.Context(), "0", jobID, runID)+ return err == nil && run.Status.Terminal()+ }, 45*time.Second, 100*time.Millisecond)+ return run+}++func workflowTargetInternal(t *testing.T, run scheduler.Run) scheduler.TargetOutcome {+ t.Helper()+ var found []scheduler.TargetOutcome+ for _, target := range run.Outcome.Targets {+ if target.ResourceType == scheduler.WorkflowTarget {+ found = append(found, target)+ }+ }+ require.Len(t, found, 1)+ return found[0]+}++func newActivitiesFake() *activitiesFake {+ return &activitiesFake{statuses: map[string]activitytypes.Status{}, completed: map[string]int{}, tracked: map[string]context.CancelCauseFunc{}}+}++func (f *activitiesFake) status(id string) activitytypes.Status {+ f.mu.Lock()+ defer f.mu.Unlock()+ return f.statuses[id]+}++func (f *activitiesFake) StartActivity(_ context.Context, req activitylib.StartRequest) (*activitytypes.Activity, error) {+ f.mu.Lock()+ defer f.mu.Unlock()+ if _, ok := f.statuses[req.ID]; !ok {+ f.statuses[req.ID] = activitytypes.StatusRunning+ }+ return &activitytypes.Activity{ID: req.ID, Status: f.statuses[req.ID]}, nil+}++func (f *activitiesFake) CompleteActivity(_ context.Context, activityID string, status activitytypes.Status, _ string, _ *string, _ ...string) (*activitytypes.Activity, error) {+ f.mu.Lock()+ defer f.mu.Unlock()+ f.statuses[activityID] = status+ f.completed[activityID]+++ return &activitytypes.Activity{ID: activityID, Status: status}, nil+}++func (f *activitiesFake) AppendMessage(context.Context, string, activitylib.AppendMessageRequest) (*activitytypes.Message, error) {+ return &activitytypes.Message{}, nil+}++func (f *activitiesFake) AppendMessages(context.Context, string, []activitylib.AppendMessageRequest) ([]activitytypes.Message, error) {+ return nil, nil+}++func (f *activitiesFake) Track(ctx context.Context, activityID string) context.Context {+ f.mu.Lock()+ defer f.mu.Unlock()+ tracked, cancel := context.WithCancelCause(ctx)+ f.tracked[activityID] = cancel+ return tracked+}++// requestCancel cancels an activity the way the activity service does for a user.+func (f *activitiesFake) requestCancel(activityID string) {+ f.mu.Lock()+ cancel := f.tracked[activityID]+ f.mu.Unlock()+ cancel(activitylib.ErrCanceled)+}++func (f *activitiesFake) UpdateActivity(_ context.Context, activityID string, _ activitylib.UpdateRequest) (*activitytypes.Activity, error) {+ return &activitytypes.Activity{ID: activityID}, nil+}++func (f *activitiesFake) AwaitActivitySlotBounded(context.Context, string, string) error { return nil }diff --git a/backend/pkg/scheduler/flow/flowtest/flowtest.go b/backend/pkg/scheduler/flow/flowtest/flowtest.gonew file mode 100644index 000000000..40f950dec--- /dev/null+++ b/backend/pkg/scheduler/flow/flowtest/flowtest.go@@ -0,0 +1,91 @@+// Package flowtest runs workflow engines on isolated Francis hosts for tests.+package flowtest++import (+ "context"+ "slices"+ "testing"+ "time"++ "github.com/italypaleale/francis/builtin/workflow"+ "github.com/libtnb/sqlite"+ "gorm.io/gorm"++ "github.com/getarcaneapp/arcane/backend/v2/internal/database"+ "github.com/getarcaneapp/arcane/backend/v2/internal/kv"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis"+ francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing"+)++// Harness is an engine with its host and coordinator, ready for workflows to be defined.+type Harness struct {+ Runtime *francis.Runtime+ Coordinator *runs.Coordinator+ Engine *flow.Engine+}++// New builds a harness on an in-memory host, or on the given database URL.+func New(t testing.TB, activities flow.Activities, databaseURLs ...string) *Harness {+ t.Helper()+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})+ if err != nil {+ t.Fatal(err)+ }+ sqlDB, err := db.DB()+ if err != nil {+ t.Fatal(err)+ }+ sqlDB.SetMaxOpenConns(1)+ t.Cleanup(func() { _ = sqlDB.Close() })+ if migrateErr := db.AutoMigrate(&kv.KVEntry{}); migrateErr != nil {+ t.Fatal(migrateErr)+ }+ runtime := francistest.New(t, databaseURLs...)+ coordinator := runs.New(kv.NewKVService(&database.DB{DB: db}), runtime.Service(), time.UTC)+ if registerErr := coordinator.Register(runtime); registerErr != nil {+ t.Fatal(registerErr)+ }+ engine, err := flow.New(t.Context(), runtime, coordinator, activities)+ if err != nil {+ t.Fatal(err)+ }+ return &Harness{Runtime: runtime, Coordinator: coordinator, Engine: engine}+}++// Start starts the host and the engine once every workflow is defined.+func (h *Harness) Start(t testing.TB) {+ t.Helper()+ francistest.Start(t, h.Runtime)+ if err := h.Engine.Start(t.Context()); err != nil {+ t.Fatal(err)+ }+ h.Engine.Ready()+ t.Cleanup(func() { _ = h.Engine.Stop(context.WithoutCancel(t.Context())) })+}++// AssertDefinitions fails when a workflow's graph no longer matches the+// Fingerprint its definition declares, which means it changed without a version bump.+func AssertDefinitions(t testing.TB, h *Harness) {+ t.Helper()+ for _, wf := range h.Engine.Workflows() {+ definition := wf.Francis()+ service := definition.Service(h.Runtime.Service())+ // A fresh host has no instances, so resetting installs this binary's fingerprint.+ if err := service.ForgetVersion(t.Context(), definition.Version()); err != nil {+ t.Fatal(err)+ }+ infos, err := service.Definitions(t.Context())+ if err != nil {+ t.Fatal(err)+ }+ index := slices.IndexFunc(infos, func(info workflow.DefinitionInfo) bool { return info.Version == definition.Version() })+ if index < 0 {+ t.Fatalf("workflow %q version %d is not registered", definition.Name(), definition.Version())+ }+ if got := infos[index].Fingerprint; got != wf.Fingerprint() {+ t.Errorf("workflow %q no longer matches its Fingerprint: bump Version if this change is not yet released, then set Fingerprint to %q", definition.Name(), got)+ }+ }+}diff --git a/backend/pkg/scheduler/flow/job.go b/backend/pkg/scheduler/flow/job.gonew file mode 100644index 000000000..2b4d5743e--- /dev/null+++ b/backend/pkg/scheduler/flow/job.go@@ -0,0 +1,76 @@+package flow++import (+ "context"+ "errors"++ "github.com/getarcaneapp/arcane/types/v2/scheduler"++ activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"+)++// Job runs a workflow as a scheduler job. The coordinator's single flight per+// job and environment already prevents overlapping runs.+type Job struct {+ Engine *Engine+ Workflow *Workflow+ JobName string+ // Payload is every instance's input, such as the entity a per-entity job runs for.+ Payload any+ // Activity overrides the definition's activity template, such as with the entity's name.+ Activity activitylib.StartRequest+ ScheduleFn func(ctx context.Context) string+ ShouldRunFn func(ctx context.Context) bool+ // FallbackFn recovers a run whose instance cannot resume; nil fails the run so+ // a retry starts a fresh instance under a new attempt.+ FallbackFn func(ctx context.Context, previous scheduler.Run) (scheduler.Outcome, error)+ ValidateRetryFn func(ctx context.Context, run scheduler.Run) error+}++// Name is the scheduler job name.+func (j *Job) Name() string { return j.JobName }++// Schedule is the job's current cron expression.+func (j *Job) Schedule(ctx context.Context) string { return j.ScheduleFn(ctx) }++// ShouldSchedule reports whether the job runs; a nil ShouldRunFn always does.+func (j *Job) ShouldSchedule(ctx context.Context) bool {+ return j.ShouldRunFn == nil || j.ShouldRunFn(ctx)+}++// Run skips when the job's condition no longer holds, then runs the workflow.+func (j *Job) Run(ctx context.Context) (scheduler.Outcome, error) {+ if !j.ShouldSchedule(ctx) {+ return scheduler.Outcome{Status: scheduler.Skipped}, nil+ }+ return j.Engine.Run(ctx, j.Workflow, j.Payload, j.Activity, nil)+}++// Reconcile resumes an interrupted run's instance, falling back to FallbackFn when it cannot be resumed.+// A job disabled since the run started makes no further changes.+func (j *Job) Reconcile(ctx context.Context, previous scheduler.Run) (scheduler.Outcome, error) {+ if !j.ShouldSchedule(ctx) {+ const message = "Job is disabled; the interrupted run was not resumed"+ if outcome, finished, err := j.Engine.Retire(ctx, j.Workflow, previous, message); finished {+ return outcome, err+ }+ return scheduler.Outcome{Status: scheduler.Failed, Message: message}, nil+ }+ outcome, err := j.Engine.Resume(ctx, j.Workflow, previous)+ if !errors.Is(err, ErrNotResumable) {+ return outcome, err+ }+ if j.FallbackFn != nil {+ return j.FallbackFn(ctx, previous)+ }+ // Running again here could reuse the superseded instance's ID within this attempt.+ return scheduler.Outcome{Status: scheduler.Failed, Message: "Workflow instance could not be resumed; retry the run"}, nil+}++// ValidateRetry rejects a retry that ValidateRetryFn reports unsafe.+func (j *Job) ValidateRetry(ctx context.Context, run scheduler.Run) error {+ if j.ValidateRetryFn == nil {+ return nil+ }+ return j.ValidateRetryFn(ctx, run)+}diff --git a/backend/pkg/scheduler/flow/monitor.go b/backend/pkg/scheduler/flow/monitor.gonew file mode 100644index 000000000..8010d759a--- /dev/null+++ b/backend/pkg/scheduler/flow/monitor.go@@ -0,0 +1,337 @@+package flow++import (+ "context"+ "errors"+ "fmt"+ "log/slog"+ "slices"+ "sync"+ "time"++ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"+ "github.com/getarcaneapp/arcane/types/v2/scheduler"+ "github.com/italypaleale/francis/actor"+ "github.com/italypaleale/francis/builtin/workflow"++ activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"+)++const (+ minPollInterval = 100 * time.Millisecond+ maxPollInterval = time.Second+)++type (+ // instanceMonitor is the single poller for one root instance. It owns the+ // instance's activity and broadcasts step changes and the final outcome.+ instanceMonitor struct {+ mu sync.Mutex+ steps []scheduler.StepOutcome+ outcome *scheduler.Outcome+ changed chan struct{}+ cancel context.CancelFunc+ // children caches the workflow serving each child instance.+ children map[string]*Workflow+ }++ // activityProjection remembers what was last written to the activity.+ activityProjection struct {+ progress int+ step string+ statuses map[string]scheduler.RunStatus+ }+)++func (e *Engine) monitor(wf *Workflow, instanceID string) *instanceMonitor {+ e.mu.Lock()+ defer e.mu.Unlock()+ if monitor := e.monitors[instanceID]; monitor != nil {+ return monitor+ }+ ctx, cancel := context.WithCancel(e.ctx)+ monitor := &instanceMonitor{changed: make(chan struct{}), cancel: cancel, children: map[string]*Workflow{}}+ e.monitors[instanceID] = monitor+ go e.poll(ctx, wf, instanceID, monitor)+ return monitor+}++func (e *Engine) poll(ctx context.Context, wf *Workflow, instanceID string, monitor *instanceMonitor) {+ activityID := wf.activityFor(instanceID)+ var canceled <-chan struct{}+ trackedCtx := ctx+ if activityID != "" {+ // A starting instance's activity is already tracked; one found on Start is tracked here.+ e.mu.Lock()+ trackedCtx = e.tracked[activityID]+ delete(e.tracked, activityID)+ e.mu.Unlock()+ if trackedCtx == nil {+ trackedCtx = e.activities.Track(ctx, activityID)+ }+ canceled = trackedCtx.Done()+ }+ userCanceled := false+ projection := activityProjection{progress: -1, statuses: map[string]scheduler.RunStatus{}}+ delay := minPollInterval+ for {+ status, err := wf.svc.GetStatus(ctx, instanceID)+ switch {+ case err == nil:+ steps := e.projectSteps(ctx, monitor, status)+ if status.Status.IsTerminal() {+ outcome := decodeOutcome(status, steps, activityID)+ // The interrupted task may fail the instance before the cancel turn lands.+ if userCanceled && outcome.Status == scheduler.Failed {+ outcome.Status, outcome.Message = scheduler.Canceled, activitylib.ErrCanceled.Error()+ }+ e.finish(ctx, wf, instanceID, monitor, outcome)+ return+ }+ if monitor.publish(steps, nil) {+ e.projectActivity(ctx, wf, activityID, &projection, steps)+ delay = minPollInterval+ }+ case errors.Is(err, workflow.ErrInstanceNotFound):+ e.finish(ctx, wf, instanceID, monitor, scheduler.Outcome{Status: scheduler.Failed, Message: "Workflow instance no longer exists", ActivityID: activityID})+ return+ case ctx.Err() == nil:+ slog.DebugContext(ctx, "Failed to read workflow status", "instanceId", instanceID, "error", err)+ }+ timer := time.NewTimer(delay)+ select {+ case <-ctx.Done():+ timer.Stop()+ return+ case <-canceled:+ canceled = nil+ if errors.Is(context.Cause(trackedCtx), activitylib.ErrCanceled) {+ userCanceled = true+ if cancelErr := wf.svc.Cancel(ctx, instanceID, activitylib.ErrCanceled.Error()); cancelErr != nil {+ slog.WarnContext(ctx, "Failed to cancel workflow", "instanceId", instanceID, "error", cancelErr)+ }+ e.cancelRoot(instanceID, activitylib.ErrCanceled)+ }+ case <-timer.C:+ }+ timer.Stop()+ delay = min(delay*2, maxPollInterval)+ }+}++func (e *Engine) finish(ctx context.Context, wf *Workflow, instanceID string, monitor *instanceMonitor, outcome scheduler.Outcome) {+ e.mu.Lock()+ if e.monitors[instanceID] == monitor {+ delete(e.monitors, instanceID)+ }+ e.mu.Unlock()+ e.releaseRoot(instanceID)+ if !isInstanceOf(instanceID, wf.def.Name) {+ if err := e.runtime.Service().DeleteState(context.WithoutCancel(ctx), evidenceActorType, instanceID); err != nil && !errors.Is(err, actor.ErrStateNotFound) {+ slog.WarnContext(ctx, "Failed to delete workflow progress", "instanceId", instanceID, "error", err)+ }+ }+ activityID := wf.activityFor(instanceID)+ if activityID != "" {+ status := activitytypes.StatusSuccess+ var errMessage *string+ switch outcome.Status {+ case scheduler.Canceled:+ status = activitytypes.StatusCancelled+ case scheduler.Failed, scheduler.Partial, scheduler.NeedsAttention:+ status = activitytypes.StatusFailed+ errMessage = new(outcome.Message)+ case scheduler.Queued, scheduler.Waiting, scheduler.Running, scheduler.Retrying, scheduler.Succeeded, scheduler.Skipped:+ }+ if _, err := e.activities.CompleteActivity(context.WithoutCancel(ctx), activityID, status, outcome.Message, errMessage); err != nil {+ slog.WarnContext(ctx, "Failed to complete workflow activity", "instanceId", instanceID, "error", err)+ }+ }+ monitor.publish(outcome.Steps, &outcome)+ monitor.cancel()+}++// stop ends a monitor whose instance was superseded.+func (m *instanceMonitor) stop() {+ m.cancel()+ m.publish(nil, &scheduler.Outcome{Status: scheduler.Failed, Message: "Workflow instance was superseded"})+}++// publish records a change and wakes waiters; it reports whether anything changed.+func (m *instanceMonitor) publish(steps []scheduler.StepOutcome, outcome *scheduler.Outcome) bool {+ m.mu.Lock()+ defer m.mu.Unlock()+ if m.outcome != nil || (outcome == nil && stepsEqual(m.steps, steps)) {+ return false+ }+ m.steps = steps+ m.outcome = outcome+ close(m.changed)+ m.changed = make(chan struct{})+ return true+}++func (m *instanceMonitor) snapshot() ([]scheduler.StepOutcome, *scheduler.Outcome, <-chan struct{}) {+ m.mu.Lock()+ defer m.mu.Unlock()+ return m.steps, m.outcome, m.changed+}++// projectSteps maps Francis steps onto run steps; a child step reports its child's fan-out totals.+func (e *Engine) projectSteps(ctx context.Context, monitor *instanceMonitor, status workflow.InstanceStatus) []scheduler.StepOutcome {+ steps := make([]scheduler.StepOutcome, 0, len(status.Steps))+ for _, view := range status.Steps {+ step := scheduler.StepOutcome{+ Name: view.Name,+ Total: view.Tasks,+ Completed: view.Completed,+ Failed: view.Failed,+ Attempts: view.Attempts,+ Message: view.Error,+ }+ if view.Kind == workflow.KindChild && len(view.ChildIDs) == 1 {+ if child, ok := e.childStatus(ctx, monitor, view.ChildIDs[0]); ok {+ for _, childStep := range child.Steps {+ if childStep.Kind == workflow.KindForEach {+ step.Total, step.Completed, step.Failed = childStep.Tasks, childStep.Completed, childStep.Failed+ }+ }+ }+ }+ step.Status = stepStatus(view.Status, step.Failed)+ if !view.StartedAt.IsZero() {+ step.StartedAt = new(view.StartedAt)+ }+ if !view.CompletedAt.IsZero() {+ step.FinishedAt = new(view.CompletedAt)+ }+ steps = append(steps, step)+ }+ return steps+}++// childStatus reads a child instance, resolving its workflow once per monitor.+func (e *Engine) childStatus(ctx context.Context, monitor *instanceMonitor, childID string) (workflow.InstanceStatus, bool) {+ if wf := monitor.children[childID]; wf != nil {+ status, err := wf.svc.GetStatus(ctx, childID)+ return status, err == nil+ }+ for _, wf := range e.Workflows() {+ if status, err := wf.svc.GetStatus(ctx, childID); err == nil {+ monitor.children[childID] = wf+ return status, true+ }+ }+ return workflow.InstanceStatus{}, false+}++func stepStatus(status workflow.StepStatus, failed int) scheduler.RunStatus {+ switch status {+ case workflow.StepRunning, workflow.StepCompensating:+ return scheduler.Running+ case workflow.StepCompleted:+ if failed > 0 {+ return scheduler.Partial+ }+ return scheduler.Succeeded+ case workflow.StepFailed, workflow.StepCompensationFailed:+ return scheduler.Failed+ case workflow.StepSkipped:+ return scheduler.Skipped+ case workflow.StepCompensated:+ return scheduler.Canceled+ case workflow.StepPending:+ return scheduler.Queued+ }+ return scheduler.Queued+}++// decodeOutcome decodes a terminal instance into the run outcome contract.+func decodeOutcome(status workflow.InstanceStatus, steps []scheduler.StepOutcome, activityID string) scheduler.Outcome {+ var outcome scheduler.Outcome+ switch status.Status {+ case workflow.StatusCompleted:+ if err := status.DecodeOutput(&outcome); err != nil {+ outcome = scheduler.Outcome{Status: scheduler.Failed, Message: "Workflow returned an invalid outcome: " + err.Error()}+ }+ if outcome.Status == "" {+ outcome.Status = scheduler.Succeeded+ }+ case workflow.StatusCancelled:+ outcome = scheduler.Outcome{Status: scheduler.Canceled, Message: status.Cause}+ case workflow.StatusFailed, workflow.StatusPending, workflow.StatusRunning, workflow.StatusSuspended, workflow.StatusCompensating:+ outcome = scheduler.Outcome{Status: scheduler.Failed, Message: status.Cause}+ }+ outcome.Steps = steps+ if outcome.ActivityID == "" {+ outcome.ActivityID = activityID+ }+ return outcome+}++// projectActivity writes overall progress and step transitions to the activity.+func (e *Engine) projectActivity(ctx context.Context, wf *Workflow, activityID string, projection *activityProjection, steps []scheduler.StepOutcome) {+ if activityID == "" || len(steps) == 0 {+ return+ }+ done := 0.0+ current := ""+ for _, step := range steps {+ previous, seen := projection.statuses[step.Name]+ if step.Status != scheduler.Queued && (!seen || previous != step.Status) {+ projection.statuses[step.Name] = step.Status+ if step.Status != scheduler.Running {+ e.appendStepMessage(ctx, activityID, wf.stepLabel(step.Name), step)+ }+ }+ switch {+ case step.Status == scheduler.Running:+ current = wf.stepLabel(step.Name)+ if step.Total > 0 {+ done += float64(step.Completed+step.Failed) / float64(step.Total)+ }+ case step.Status != scheduler.Queued:+ done+++ }+ }+ progress := int(done * 100 / float64(len(steps)))+ if progress == projection.progress && current == projection.step {+ return+ }+ projection.progress, projection.step = progress, current+ if _, err := e.activities.UpdateActivity(ctx, activityID, activitylib.UpdateRequest{Progress: &progress, Step: ¤t}); err != nil {+ slog.DebugContext(ctx, "Failed to update workflow activity progress", "activityId", activityID, "error", err)+ }+}++func (e *Engine) appendStepMessage(ctx context.Context, activityID, label string, step scheduler.StepOutcome) {+ level := activitytypes.MessageLevelInfo+ message := fmt.Sprintf("%s: %s", label, step.Status)+ if step.Total > 1 {+ message += fmt.Sprintf(" (%d/%d", step.Completed, step.Total)+ if step.Failed > 0 {+ message += fmt.Sprintf(", %d failed", step.Failed)+ }+ message += ")"+ }+ switch step.Status {+ case scheduler.Failed:+ level = activitytypes.MessageLevelError+ if step.Message != "" {+ message += ": " + step.Message+ }+ case scheduler.Partial:+ level = activitytypes.MessageLevelWarning+ case scheduler.Queued, scheduler.Waiting, scheduler.Running, scheduler.Retrying, scheduler.Succeeded, scheduler.Skipped, scheduler.NeedsAttention, scheduler.Canceled:+ }+ if _, err := e.activities.AppendMessage(ctx, activityID, activitylib.AppendMessageRequest{Level: level, Message: message}); err != nil {+ slog.DebugContext(ctx, "Failed to append workflow step message", "activityId", activityID, "error", err)+ }+}++func stepsEqual(left, right []scheduler.StepOutcome) bool {+ return slices.EqualFunc(left, right, func(a, b scheduler.StepOutcome) bool {+ return a.Name == b.Name && a.Status == b.Status && a.Total == b.Total && a.Completed == b.Completed &&+ a.Failed == b.Failed && a.Attempts == b.Attempts && a.Message == b.Message+ })+}diff --git a/backend/pkg/scheduler/flow/task.go b/backend/pkg/scheduler/flow/task.gonew file mode 100644index 000000000..5e6b8e9bc--- /dev/null+++ b/backend/pkg/scheduler/flow/task.go@@ -0,0 +1,262 @@+package flow++import (+ "cmp"+ "context"+ "encoding/json"+ "errors"+ "fmt"+ "log/slog"+ "net"+ "strings"+ "time"+ "unicode/utf8"++ "github.com/getarcaneapp/arcane/types/v2/scheduler"+ "github.com/italypaleale/francis/actor"+ "github.com/italypaleale/francis/builtin/workflow"++ "github.com/getarcaneapp/arcane/backend/v2/internal/common"+ activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils"+)++const (+ // maxErrorLength bounds task errors recorded in the workflow journal.+ maxErrorLength = 1024+ // slotWait bounds how long a resumed task waits for its activity slot before it is handed back.+ slotWait = 20 * time.Second+)++type (+ // Binding ties an instance, and its children, to the run that started it.+ Binding struct {+ EnvironmentID string `json:"environmentId,omitempty"`+ JobID string `json:"jobId,omitempty"`+ RunID string `json:"runId,omitempty"`+ RootID string `json:"rootId"`+ }++ // envelope is every instance's input: its run binding plus the caller's payload.+ envelope struct {+ Binding Binding `json:"binding"`+ Payload json.RawMessage `json:"payload,omitempty"`+ }++ // Task is what an engine handler receives: the Francis task plus its binding.+ Task struct {+ workflow.Task+ binding Binding+ payload json.RawMessage+ // recovered marks a standalone task whose instance an earlier process started.+ recovered bool+ }++ // Result is one fan-out slot: the task's output, or the error that failed it.+ Result[T any] struct {+ Value T+ Err string+ }+)++// Handler wraps a step handler. Its ctx carries the bound run exactly as a job's+// Run does, is canceled when the user cancels the root activity, and its errors+// are classified for Francis retries.+func (e *Engine) Handler(fn func(ctx context.Context, t Task) (any, error)) workflow.StepOption {+ return workflow.WithRun(func(ctx context.Context, wt workflow.Task) (output any, err error) {+ // Francis does not recover handler panics, which would take the process down.+ defer func() {+ if panicErr := utils.PanicToError(recover()); panicErr != nil {+ slog.ErrorContext(ctx, "Workflow task panicked", "workflow", wt.Workflow(), "step", wt.Step(), "error", panicErr)+ output, err = nil, fmt.Errorf("%w: task panicked: %w", actor.ErrJobPermanentFailure, panicErr)+ }+ }()+ var input envelope+ if decodeErr := wt.DecodeInput(&input); decodeErr != nil {+ return nil, fmt.Errorf("%w: decode workflow input: %w", actor.ErrJobPermanentFailure, decodeErr)+ }+ // Francis neither interrupts a running job on Cancel nor at host shutdown, so the engine does both.+ ctx, cancel := context.WithCancelCause(ctx)+ defer cancel(nil)+ root := e.rootContext(input.Binding.RootID)+ defer context.AfterFunc(root, func() { cancel(context.Cause(root)) })()+ taskCtx, err := e.taskContext(ctx, input.Binding)+ if err != nil {+ return nil, classify(ctx, err)+ }+ // A resumed instance's activity lost its slot with the old process, so its first task takes it back;+ // a held slot returns at once. While none frees up, the task is handed back without spending an attempt.+ if root := e.rootWorkflow(input.Binding.RootID); root != nil && root.def.Activity.Queue && !root.def.Activity.DeferSlot {+ environmentID := cmp.Or(root.def.Activity.EnvironmentID, input.Binding.EnvironmentID, "0")+ // The wait stays well inside any attempt timeout, so waiting for a slot never spends an attempt.+ activityID := root.activityFor(input.Binding.RootID)+ slotCtx, cancelSlot := context.WithTimeout(taskCtx, slotWait)+ unlockSlot := e.slots.Lock(activityID)+ slotErr := e.activities.AwaitActivitySlotBounded(slotCtx, activityID, environmentID)+ unlockSlot()+ cancelSlot()+ if slotErr != nil {+ if ctx.Err() != nil {+ return nil, classify(ctx, ctx.Err())+ }+ return nil, actor.ErrJobRejected+ }+ }+ e.mu.Lock()+ _, submitted := e.submitted[input.Binding.RootID]+ e.mu.Unlock()+ recovered := input.Binding.RunID == "" && !submitted+ output, err = fn(taskCtx, Task{Task: wt, binding: input.Binding, payload: input.Payload, recovered: recovered})+ // An interrupted handler may report a result its interruption caused, so its output is never kept.+ if err = cmp.Or(err, ctx.Err()); err != nil {+ return nil, classify(ctx, err)+ }+ return output, nil+ })+}++// taskContext rebuilds a task's job execution ctx, fencing run-bound tasks.+func (e *Engine) taskContext(ctx context.Context, binding Binding) (context.Context, error) {+ if binding.RunID == "" {+ return e.standaloneContext(ctx, binding)+ }+ run, err := e.coordinator.Get(ctx, binding.EnvironmentID, binding.JobID, binding.RunID)+ if errors.Is(err, runs.ErrRunNotFound) {+ return nil, fmt.Errorf("%w: run %s no longer exists", actor.ErrJobPermanentFailure, binding.RunID)+ }+ if err != nil {+ return nil, common.Classify(common.ErrUnavailable, fmt.Errorf("load run %s: %w", binding.RunID, err))+ }+ if !runs.ActiveWorkflow(run, binding.RootID) {+ // A run waiting for its next attempt reattaches to this instance, so the instance's tasks wait with it.+ attempt := run+ attempt.Status = scheduler.Running+ if (run.Status == scheduler.Queued || run.Status == scheduler.Waiting || run.Status == scheduler.Retrying) && runs.ActiveWorkflow(attempt, binding.RootID) {+ return nil, actor.ErrJobRejected+ }+ return nil, fmt.Errorf("%w: run %s no longer follows instance %s", actor.ErrJobPermanentFailure, binding.RunID, binding.RootID)+ }+ // A task recovered at startup waits until reconcile re-authorizes its run and attaches to it.+ e.mu.Lock()+ _, admitted := e.admitted[binding.RunID]+ e.mu.Unlock()+ if !admitted {+ return nil, actor.ErrJobRejected+ }+ return e.coordinator.ExecutionContext(ctx, run, binding.RootID), nil+}++// standaloneContext gives a standalone task the execution a run would give it, with+// target progress kept in the engine's state for the instance.+func (e *Engine) standaloneContext(ctx context.Context, binding Binding) (context.Context, error) {+ select {+ case <-e.ready:+ default:+ // Work started before startup recovery collects protected records could be failed as interrupted.+ return nil, actor.ErrJobRejected+ }+ service := e.runtime.Service()+ run := scheduler.Run{ID: binding.RootID, EnvironmentID: cmp.Or(binding.EnvironmentID, "0"), Status: scheduler.Running}+ if err := service.GetState(ctx, evidenceActorType, binding.RootID, &run.Outcome.Targets); err != nil && !errors.Is(err, actor.ErrStateNotFound) {+ return nil, common.Classify(common.ErrUnavailable, fmt.Errorf("load instance %s progress: %w", binding.RootID, err))+ }+ ctx = utils.WithActivityBatchID(ctx, binding.RootID)+ return jobcontext.WithExecution(ctx, run, func(target scheduler.TargetOutcome) error {+ defer e.evidence.Lock(binding.RootID)()+ var targets []scheduler.TargetOutcome+ if err := service.GetState(ctx, evidenceActorType, binding.RootID, &targets); err != nil && !errors.Is(err, actor.ErrStateNotFound) {+ return err+ }+ return service.SetState(ctx, evidenceActorType, binding.RootID, jobcontext.Merge(targets, target), nil)+ }), nil+}++// classify maps handler errors onto Francis delivery semantics.+func classify(ctx context.Context, err error) error {+ switch {+ case errors.Is(err, actor.ErrJobRejected), errors.Is(err, actor.ErrJobPermanentFailure):+ return err+ case errors.Is(context.Cause(ctx), activitylib.ErrCanceled):+ // A user cancel must not be redelivered.+ return fmt.Errorf("%w: %w", actor.ErrJobPermanentFailure, activitylib.ErrCanceled)+ case errors.Is(ctx.Err(), context.Canceled):+ // Host shutdown: redeliver without spending an attempt.+ return actor.ErrJobRejected+ case Retryable(err):+ return errors.New(truncate(err.Error()))+ default:+ return fmt.Errorf("%w: %s", actor.ErrJobPermanentFailure, truncate(err.Error()))+ }+}++// Retryable reports whether another attempt could succeed. Sources classify+// transient failures; only network timeouts are assumed transient here.+func Retryable(err error) bool {+ var netErr net.Error+ return errors.Is(err, common.ErrUnavailable) || errors.Is(err, common.ErrTimeout) ||+ errors.Is(err, context.DeadlineExceeded) || (errors.As(err, &netErr) && netErr.Timeout())+}++// Payload decodes the instance payload.+func (t Task) Payload(into any) error {+ if len(t.payload) == 0 {+ return nil+ }+ return json.Unmarshal(t.payload, into)+}++// Recovered reports a standalone task whose instance an earlier process started, so no caller is waiting on it+// and the request's own checks did not run in this process.+func (t Task) Recovered() bool { return t.recovered }++// ActivityID returns the activity owned by the task's root instance.+func (t Task) ActivityID() string { return t.binding.RootID }++// ChildInput returns the output for a step that precedes a Child step, so the+// child inherits this instance's run binding.+func (t Task) ChildInput(payload any) (any, error) {+ encoded, err := json.Marshal(payload)+ if err != nil {+ return nil, err+ }+ return envelope{Binding: t.binding, Payload: encoded}, nil+}++// Results decodes a fan-out step's output slot by slot. Francis encodes a failed+// slot as an object whose only key is "error", so successful outputs must never+// have that shape.+func Results[T any](t Task, step string) ([]Result[T], error) {+ var slots []json.RawMessage+ if err := t.DecodeOutput(step, &slots); err != nil {+ return nil, err+ }+ results := make([]Result[T], len(slots))+ for index, slot := range slots {+ var failure map[string]json.RawMessage+ if json.Unmarshal(slot, &failure) == nil && len(failure) == 1 {+ var message string+ if raw, ok := failure["error"]; ok && json.Unmarshal(raw, &message) == nil {+ results[index].Err = strings.TrimPrefix(message, actor.ErrJobPermanentFailure.Error()+": ")+ continue+ }+ }+ if err := json.Unmarshal(slot, &results[index].Value); err != nil {+ return nil, fmt.Errorf("decode %s output %d: %w", step, index, err)+ }+ }+ return results, nil+}++// truncate shortens a message to maxErrorLength bytes on a rune boundary.+func truncate(message string) string {+ if len(message) <= maxErrorLength {+ return message+ }+ end := maxErrorLength+ for end > 0 && !utf8.RuneStart(message[end]) {+ end--+ }+ return message[:end] + "…"+}diff --git a/backend/pkg/scheduler/image_update_watcher.go b/backend/pkg/scheduler/image_update_watcher.goindex e72e81e42..a9062be2d 100644--- a/backend/pkg/scheduler/image_update_watcher.go+++ b/backend/pkg/scheduler/image_update_watcher.go@@ -9,7 +9,6 @@ import ("sync/atomic""time"- "github.com/getarcaneapp/arcane/types/v2/containerregistry"imageupdatetypes "github.com/getarcaneapp/arcane/types/v2/imageupdate"schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler""github.com/moby/moby/api/types/events"@@ -18,7 +17,6 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/docker"- "github.com/getarcaneapp/arcane/backend/v2/internal/environment""github.com/getarcaneapp/arcane/backend/v2/internal/imageupdate""github.com/getarcaneapp/arcane/backend/v2/internal/project""github.com/getarcaneapp/arcane/backend/v2/internal/settings"@@ -34,11 +32,7 @@ const ()type imageUpdateScannerInternal interface {- CheckAllImages(ctx context.Context, limit int, externalCreds []containerregistry.Credential) (map[string]*imageupdatetypes.Response, error)-}--type registryCredentialLoaderInternal interface {- GetEnabledRegistryCredentials(ctx context.Context) ([]containerregistry.Credential, error)+ RunImageCheck(ctx context.Context, request imageupdatetypes.CheckRequest) (schedulertypes.Outcome, error)}type pollingSettingReaderInternal interface {@@ -58,7 +52,6 @@ type projectImageRefsBackfillerInternal interface {type ImageUpdateWatcher struct {imageUpdateService imageUpdateScannerInternalsettingsService pollingSettingReaderInternal- environmentService registryCredentialLoaderInternaldockerService dockerEventBusProviderInternalprojectService projectImageRefsBackfillerInternaldispatchMu sync.RWMutex@@ -89,12 +82,11 @@ type ImageUpdateWatcher struct {func NewImageUpdateWatcher(cfg *config.Config,imageUpdateService *imageupdate.ImageUpdateService,settingsService *settings.SettingsService,- environmentService *environment.EnvironmentService,dockerService *docker.DockerClientService,projectService *project.ProjectService) (*ImageUpdateWatcher,error,) {- if imageUpdateService == nil || settingsService == nil || environmentService == nil || dockerService == nil || projectService == nil {+ if imageUpdateService == nil || settingsService == nil || dockerService == nil || projectService == nil {return nil, errors.New("image update watcher dependencies unavailable")}location := time.UTC@@ -104,7 +96,6 @@ func NewImageUpdateWatcher(cfg *config.Config,return &ImageUpdateWatcher{imageUpdateService: imageUpdateService,settingsService: settingsService,- environmentService: environmentService,dockerService: dockerService,projectService: projectService,location: location,@@ -396,36 +387,13 @@ func (w *ImageUpdateWatcher) executeScanInternal(ctx context.Context) error {}slog.InfoContext(ctx, "image scan run started")- creds, err := w.environmentService.GetEnabledRegistryCredentials(ctx)- if err != nil {- slog.WarnContext(ctx, "failed to load registry credentials for image scan", "error", err.Error())- creds = nil- }-- results, err := w.imageUpdateService.CheckAllImages(ctx, 0, creds)+ outcome, err := w.imageUpdateService.RunImageCheck(ctx, imageupdatetypes.CheckRequest{All: true})if err != nil {return fmt.Errorf("image scan failed: %w", err)}-- updates := 0- scanErrors := 0- targets := make([]schedulertypes.TargetOutcome, 0, len(results))- for imageID, result := range results {- if result == nil {- continue- }- if result.Error != "" {- scanErrors++- targets = append(targets, schedulertypes.TargetOutcome{ID: imageID, Status: schedulertypes.Failed, Message: result.Error})- continue- }- if result.HasUpdate {- updates++- }- }- slog.InfoContext(ctx, "image scan run completed", "checked", len(results), "updates", updates, "errors", scanErrors)- if scanErrors > 0 {- return &schedulertypes.OutcomeError{Outcome: schedulertypes.Outcome{Status: schedulertypes.Partial, Message: "Some image update checks failed", Targets: targets}}+ slog.InfoContext(ctx, "image scan run completed", "status", outcome.Status, "message", outcome.Message)+ if outcome.Status != schedulertypes.Succeeded {+ return &schedulertypes.OutcomeError{Outcome: outcome}}return nil}diff --git a/backend/pkg/scheduler/image_update_watcher_test.go b/backend/pkg/scheduler/image_update_watcher_test.goindex afd665389..2b27b58f4 100644--- a/backend/pkg/scheduler/image_update_watcher_test.go+++ b/backend/pkg/scheduler/image_update_watcher_test.go@@ -11,7 +11,6 @@ import ("testing/synctest""time"- "github.com/getarcaneapp/arcane/types/v2/containerregistry""github.com/getarcaneapp/arcane/types/v2/imageupdate"schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler""github.com/moby/moby/api/types/events"@@ -33,7 +32,7 @@ type imageUpdateScannerFakeInternal struct {releaseCh <-chan struct{}}-func (s *imageUpdateScannerFakeInternal) CheckAllImages(ctx context.Context, _ int, _ []containerregistry.Credential) (map[string]*imageupdate.Response, error) {+func (s *imageUpdateScannerFakeInternal) RunImageCheck(ctx context.Context, _ imageupdate.CheckRequest) (schedulertypes.Outcome, error) {s.mu.Lock()s.calls++call := s.calls@@ -70,7 +69,7 @@ func (s *imageUpdateScannerFakeInternal) CheckAllImages(ctx context.Context, _ ipanic("deliberate image scan panic")}- return map[string]*imageupdate.Response{}, err+ return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, err}func (s *imageUpdateScannerFakeInternal) countInternal() int {@@ -120,12 +119,6 @@ func (s *pollingSettingReaderFakeInternal) setEnabledInternal(enabled bool) {s.mu.Unlock()}-type registryCredentialLoaderFakeInternal struct{}--func (registryCredentialLoaderFakeInternal) GetEnabledRegistryCredentials(context.Context) ([]containerregistry.Credential, error) {- return nil, nil-}-type dockerEventBusProviderFakeInternal struct {eventBus *bus.DockerEventBus}@@ -197,7 +190,6 @@ func newImageUpdateWatcherForTestInternal(t *testing.T,watcher := &ImageUpdateWatcher{imageUpdateService: scanner,settingsService: settings,- environmentService: registryCredentialLoaderFakeInternal{},dockerService: dockerEventBusProviderFakeInternal{eventBus: eventBus},projectService: backfiller,trigger: make(chan struct{}, 1),diff --git a/backend/pkg/scheduler/jobcontext/context.go b/backend/pkg/scheduler/jobcontext/context.goindex 38fc7bb86..e28c50510 100644--- a/backend/pkg/scheduler/jobcontext/context.go+++ b/backend/pkg/scheduler/jobcontext/context.go@@ -3,6 +3,7 @@ package jobcontextimport ("context"+ "slices"schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler")@@ -12,6 +13,7 @@ type (runKeyInternal struct{})+// WithExecution binds the run a job executes and, when persist is set, how its target progress is saved.func WithExecution(ctx context.Context, run schedulertypes.Run, persist func(schedulertypes.TargetOutcome) error) context.Context {ctx = context.WithValue(ctx, runKeyInternal{}, run)if persist != nil {@@ -20,6 +22,7 @@ func WithExecution(ctx context.Context, run schedulertypes.Run, persist func(schreturn ctx}+// Progress saves target progress for the executing run; it is a no-op outside one.func Progress(ctx context.Context, target schedulertypes.TargetOutcome) error {if persist, ok := ctx.Value(progressKeyInternal{}).(func(schedulertypes.TargetOutcome) error); ok {return persist(target)@@ -27,6 +30,20 @@ func Progress(ctx context.Context, target schedulertypes.TargetOutcome) error {return nil}+// Merge records target progress in place of the entry with its ID, keeping recovery data the update omits.+func Merge(targets []schedulertypes.TargetOutcome, target schedulertypes.TargetOutcome) []schedulertypes.TargetOutcome {+ index := slices.IndexFunc(targets, func(existing schedulertypes.TargetOutcome) bool { return existing.ID == target.ID })+ if index < 0 {+ return append(targets, target)+ }+ if len(target.RecoveryData) == 0 {+ target.RecoveryData = targets[index].RecoveryData+ }+ targets[index] = target+ return targets+}++// Run returns the run being executed, as of when its execution started.func Run(ctx context.Context) (schedulertypes.Run, bool) {run, ok := ctx.Value(runKeyInternal{}).(schedulertypes.Run)return run, okdiff --git a/backend/pkg/scheduler/runs/coordinator.go b/backend/pkg/scheduler/runs/coordinator.goindex f4b63ac40..65ee97444 100644--- a/backend/pkg/scheduler/runs/coordinator.go+++ b/backend/pkg/scheduler/runs/coordinator.go@@ -9,6 +9,7 @@ import ("fmt""io""log/slog"+ "slices""strings""sync""sync/atomic"@@ -20,6 +21,8 @@ import (kit "go.getarcane.app/kit/pkg""github.com/getarcaneapp/arcane/backend/v2/internal/kv"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis")@@ -367,6 +370,7 @@ func (q *Coordinator) persistOutcomeInternal(ctx context.Context, run st.Run, oulast := ¤t.Attempts[len(current.Attempts)-1]last.FinishedAt = &nowlast.Outcome = outcome+ last.Outcome.Steps = nil}return nil})@@ -508,9 +512,53 @@ func mergeTargetProgressInternal(previous, outcome st.Outcome) st.Outcome {}}outcome.ActivityID = cmp.Or(outcome.ActivityID, previous.ActivityID)+ if len(outcome.Steps) == 0 {+ outcome.Steps = previous.Steps+ }return outcome}+// ExecutionContext binds durable progress for work executing a run. An empty+// instanceID fences writes by run owner; otherwise by the active workflow instance.+func (q *Coordinator) ExecutionContext(ctx context.Context, run st.Run, instanceID string) context.Context {+ ctx = utils.WithActivityBatchID(ctx, run.ID)+ return jobcontext.WithExecution(ctx, run, func(target st.TargetOutcome) error {+ return q.UpdateRun(ctx, run, func(current *st.Run) error {+ if !fencedInternal(current, run, instanceID) {+ return ErrRunConflict+ }+ current.Outcome.Targets = jobcontext.Merge(current.Outcome.Targets, target)+ return nil+ })+ })+}++// RecordSteps projects workflow step progress onto a running run.+func (q *Coordinator) RecordSteps(ctx context.Context, run st.Run, instanceID string, steps []st.StepOutcome) error {+ return q.UpdateRun(ctx, run, func(current *st.Run) error {+ if !fencedInternal(current, run, instanceID) {+ return ErrRunConflict+ }+ current.Outcome.Steps = steps+ return nil+ })+}++// fencedInternal reports whether work may still write to a run.+func fencedInternal(current *st.Run, run st.Run, instanceID string) bool {+ if instanceID == "" {+ return current.Status == st.Running && current.Owner == run.Owner+ }+ return ActiveWorkflow(*current, instanceID)+}++// ActiveWorkflow reports whether a running run still follows a workflow instance.+func ActiveWorkflow(run st.Run, instanceID string) bool {+ return run.Status == st.Running && slices.ContainsFunc(run.Outcome.Targets, func(target st.TargetOutcome) bool {+ return target.ResourceType == st.WorkflowTarget && target.ID == instanceID && !target.Status.Terminal()+ })+}+func (q *Coordinator) Active() bool { return q.enabled.Load() }// Resolve releases legacy reviewed work without discarding execution evidence.diff --git a/backend/pkg/scheduler/schedule/cron.go b/backend/pkg/scheduler/schedule/cron.goindex b074bef1d..d62afb519 100644--- a/backend/pkg/scheduler/schedule/cron.go+++ b/backend/pkg/scheduler/schedule/cron.go@@ -2,7 +2,10 @@package scheduleimport (+ "cmp"+ "context""fmt"+ "log/slog""strings""github.com/robfig/cron/v3"@@ -25,6 +28,16 @@ func NormalizeSixField(value, subject string) (string, error) {return normalized, nil}+// Or returns spec when it parses and fallback otherwise; job names the schedule in the warning.+func Or(ctx context.Context, spec, fallback, job string) string {+ spec = cmp.Or(spec, fallback)+ if _, err := Parser().Parse(spec); err != nil {+ slog.WarnContext(ctx, "Invalid cron expression, using default", "job", job, "invalidSchedule", spec, "error", err)+ return fallback+ }+ return spec+}+// Parser is shared by schedule validation, execution, and next-run display.func Parser() cron.Parser {return cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor)diff --git a/backend/pkg/scheduler/scheduled_prune_job.go b/backend/pkg/scheduler/scheduled_prune_job.goindex 6d8e65035..a97477542 100644--- a/backend/pkg/scheduler/scheduled_prune_job.go+++ b/backend/pkg/scheduler/scheduled_prune_job.go@@ -1,7 +1,6 @@package schedulerimport (- "cmp""context""log/slog"@@ -39,15 +38,7 @@ func (j *ScheduledPruneJob) ShouldSchedule(ctx context.Context) bool {}func (j *ScheduledPruneJob) Schedule(ctx context.Context) string {- schedule := cmp.Or(j.settingsService.GetStringSetting(ctx, "scheduledPruneInterval", "0 0 0 * * *"), "0 0 0 * * *")-- parser := scheduleutil.Parser()- if _, err := parser.Parse(schedule); err != nil {- slog.WarnContext(ctx, "Invalid cron expression for scheduled-prune, using default", "invalidSchedule", schedule, "error", err)- return "0 0 0 * * *"- }-- return schedule+ return scheduleutil.Or(ctx, j.settingsService.GetStringSetting(ctx, "scheduledPruneInterval", "0 0 0 * * *"), "0 0 0 * * *", "scheduled-prune")}func (j *ScheduledPruneJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {diff --git a/backend/pkg/scheduler/vulnerability_feature_test.go b/backend/pkg/scheduler/vulnerability_feature_test.godeleted file mode 100644index 674c7b962..000000000--- a/backend/pkg/scheduler/vulnerability_feature_test.go+++ /dev/null@@ -1,50 +0,0 @@-package scheduler--import (- "testing"-- "github.com/getarcaneapp/arcane/types/v2/features"- schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"- "github.com/stretchr/testify/require"-)--func TestVulnerabilityFeatureGatesScheduledScansAndPatches(t *testing.T) {- tests := []struct {- name string- scheduleEnabled bool- featureEnabled bool- reenable bool- wantScheduled bool- }{- {name: "schedules disabled", featureEnabled: true},- {name: "schedules enabled", scheduleEnabled: true, featureEnabled: true, wantScheduled: true},- {name: "feature disabled", scheduleEnabled: true},- {name: "both disabled"},- {name: "feature re-enabled", scheduleEnabled: true, reenable: true, wantScheduled: true},- }- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- settingsSvc := setupScheduledPruneSettingsServiceInternal(t)- require.NoError(t, settingsSvc.SetBoolSetting(t.Context(), "vulnerabilityScanEnabled", tt.scheduleEnabled))- require.NoError(t, settingsSvc.SetBoolSetting(t.Context(), "imageAutoPatchEnabled", tt.scheduleEnabled))- require.NoError(t, settingsSvc.SetBoolSetting(t.Context(), features.VulnerabilityManagementSettingKey, tt.featureEnabled))- if tt.reenable {- require.NoError(t, settingsSvc.SetBoolSetting(t.Context(), features.VulnerabilityManagementSettingKey, true))- }- jobs := []interface {- schedulertypes.Job- schedulertypes.ConditionalJob- }{NewVulnerabilityScanJob(nil, settingsSvc), NewAutoPatchJob(nil, settingsSvc)}- for _, job := range jobs {- t.Run(job.Name(), func(t *testing.T) {- require.Equal(t, tt.wantScheduled, job.ShouldSchedule(t.Context()))- if !tt.wantScheduled {- outcome, err := job.Run(t.Context())- require.NoError(t, err)- require.Equal(t, schedulertypes.Skipped, outcome.Status)- }- })- }- })- }-}diff --git a/backend/pkg/scheduler/vulnerability_scan_job.go b/backend/pkg/scheduler/vulnerability_scan_job.goindex aa54f0f3f..b4a5c31ab 100644--- a/backend/pkg/scheduler/vulnerability_scan_job.go+++ b/backend/pkg/scheduler/vulnerability_scan_job.go@@ -1,99 +1,33 @@package schedulerimport (- "cmp""context"- "errors"- "log/slog"- "github.com/getarcaneapp/arcane/types/v2""github.com/getarcaneapp/arcane/types/v2/features"- schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"- "github.com/getarcaneapp/arcane/types/v2/user"- "go.getarcane.app/kit/pkg"- "github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow"scheduleutil "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/schedule")-const VulnerabilityScanJobName = "vulnerability-scan"--// systemUser is used as the actor for scheduled vulnerability scans.-var vulnerabilityScanSystemUser = user.Actor{- Username: "System",-}--// VulnerabilityScanJob periodically scans all Docker images for vulnerabilities-// using Trivy. It is opt-in via the "vulnerabilityScanEnabled" setting.-type VulnerabilityScanJob struct {- vulnerabilityService *vulnerability.VulnerabilityService- settingsService *settings.SettingsService-}--// NewVulnerabilityScanJob creates a new VulnerabilityScanJob.-func NewVulnerabilityScanJob(vulnerabilityService *vulnerability.VulnerabilityService, settingsService *settings.SettingsService) *VulnerabilityScanJob {- return &VulnerabilityScanJob{- vulnerabilityService: vulnerabilityService,- settingsService: settingsService,- }-}--func (j *VulnerabilityScanJob) Name() string {- return VulnerabilityScanJobName-}--func (j *VulnerabilityScanJob) ShouldSchedule(ctx context.Context) bool {- return j.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) && j.settingsService.GetBoolSetting(ctx, "vulnerabilityScanEnabled", false)-}--// Schedule returns the cron expression for the job. Defaults to daily at midnight.-func (j *VulnerabilityScanJob) Schedule(ctx context.Context) string {- schedule := cmp.Or(j.settingsService.GetStringSetting(ctx, "vulnerabilityScanInterval", "0 0 0 * * *"), "0 0 0 * * *")-- parser := scheduleutil.Parser()- if _, err := parser.Parse(schedule); err != nil {- slog.WarnContext(ctx, "Invalid cron expression for vulnerability-scan, using default", "invalidSchedule", schedule, "error", err)- return "0 0 0 * * *"- }-- return schedule-}--func (j *VulnerabilityScanJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {- if !j.ShouldSchedule(ctx) {- slog.DebugContext(ctx, "scheduled vulnerability scan disabled; skipping run")- return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil- }-- slog.InfoContext(ctx, "scheduled vulnerability scan started")-- // Remove scan records for images that no longer exist (fixes e.g. "5/3 images scanned").- deleted, cleanupErr := j.vulnerabilityService.CleanupOrphanedScanRecords(ctx)- if cleanupErr != nil {- slog.WarnContext(ctx, "orphaned vulnerability scan cleanup failed", "error", cleanupErr)- }- if deleted > 0 {- slog.InfoContext(ctx, "cleaned up orphaned vulnerability scan records", "deleted", deleted)- }-- scanned, failed, err := j.vulnerabilityService.ScanAllImages(ctx, types.LocalDockerEnvironmentID, vulnerabilityScanSystemUser, false)- if errors.Is(err, common.ErrFeatureDisabled) {- status := kit.Ternary(scanned > 0, schedulertypes.Partial, schedulertypes.Skipped)- return schedulertypes.Outcome{Status: status, Message: err.Error()}, nil- }- if err != nil {- slog.ErrorContext(ctx, "scheduled vulnerability scan failed", "error", err)- return schedulertypes.Outcome{}, err- }+const (+ VulnerabilityScanJobName = "vulnerability-scan"+ vulnerabilityScanDefaultCron = "0 0 0 * * *"+)- slog.InfoContext(ctx, "scheduled vulnerability scan completed",- "scanned", scanned,- "failed", failed,- )- if failed > 0 || cleanupErr != nil {- return schedulertypes.Outcome{Status: schedulertypes.Partial, Message: "Some scans or orphaned record cleanup failed"}, cleanupErr+// NewVulnerabilityScanJob runs the durable vulnerability scan workflow. It is+// opt-in via the "vulnerabilityScanEnabled" setting and defaults to daily at midnight.+func NewVulnerabilityScanJob(engine *flow.Engine, vulnerabilityService *vulnerability.VulnerabilityService, settingsService *settings.SettingsService) *flow.Job {+ return &flow.Job{+ Engine: engine,+ Workflow: vulnerabilityService.ScanWorkflow(),+ JobName: VulnerabilityScanJobName,+ ScheduleFn: func(ctx context.Context) string {+ return scheduleutil.Or(ctx, settingsService.GetStringSetting(ctx, "vulnerabilityScanInterval", vulnerabilityScanDefaultCron), vulnerabilityScanDefaultCron, VulnerabilityScanJobName)+ },+ ShouldRunFn: func(ctx context.Context) bool {+ return settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) && settingsService.GetBoolSetting(ctx, "vulnerabilityScanEnabled", false)+ },}- return schedulertypes.Outcome{Status: schedulertypes.Succeeded}, nil}diff --git a/backend/pkg/utils/francis/runtime.go b/backend/pkg/utils/francis/runtime.goindex 01b4c7762..e1e15bdfe 100644--- a/backend/pkg/utils/francis/runtime.go+++ b/backend/pkg/utils/francis/runtime.go@@ -10,6 +10,7 @@ import ("fmt""log/slog""net"+ "slices""strconv""strings""sync"@@ -17,6 +18,7 @@ import ("time""github.com/italypaleale/francis/actor"+ "github.com/italypaleale/francis/builtin/workflow""github.com/italypaleale/francis/components""github.com/italypaleale/francis/host/local""github.com/libtnb/sqlite"@@ -34,6 +36,7 @@ type Runtime struct {mu sync.Mutexoptions []local.HostOptionregistrations []registration+ workflows []*workflow.Workflowservice *actor.Serviceready chan struct{}done chan struct{}@@ -147,6 +150,20 @@ func (r *Runtime) RegisterActor(actorType string, factory actor.Factory, optionsreturn nil}+// RegisterWorkflow adds a Francis workflow, registered on every host attempt.+func (r *Runtime) RegisterWorkflow(wf *workflow.Workflow) error {+ r.mu.Lock()+ defer r.mu.Unlock()+ if r.started {+ return errors.New("workflows must be registered before startup")+ }+ if slices.ContainsFunc(r.workflows, func(existing *workflow.Workflow) bool { return existing.Name() == wf.Name() }) {+ return fmt.Errorf("workflow %q is already registered", wf.Name())+ }+ r.workflows = append(r.workflows, wf)+ return nil+}+// Start waits for host registration. appCtx owns the running host; ctx only// limits startup. onFailure must cancel the application without blocking.func (r *Runtime) Start(ctx, appCtx context.Context, onFailure func(error)) error {@@ -233,16 +250,26 @@ func (r *Runtime) runHost(ctx context.Context) (*local.Host, chan error, error)if err != nil {return nil, nil, err}+ var registerErr errorfor _, registration := range r.registrations {- if registerActorErr := host.RegisterActor(registration.actorType, registration.factory, registration.options...); registerActorErr != nil {- // Running a canceled host closes its provider-owned connections.- cleanupCtx, cancel := context.WithCancel(ctx)- cancel()- if cleanupErr := host.Run(cleanupCtx); cleanupErr != nil {- return nil, nil, errors.Join(registerActorErr, fmt.Errorf("clean up unregistered actor host: %w", cleanupErr))- }- return nil, nil, registerActorErr+ if registerErr = host.RegisterActor(registration.actorType, registration.factory, registration.options...); registerErr != nil {+ break+ }+ }+ for _, wf := range r.workflows {+ if registerErr != nil {+ break+ }+ registerErr = host.RegisterBuiltInActor(wf)+ }+ if registerErr != nil {+ // Running a canceled host closes its provider-owned connections.+ cleanupCtx, cancel := context.WithCancel(ctx)+ cancel()+ if cleanupErr := host.Run(cleanupCtx); cleanupErr != nil {+ return nil, nil, errors.Join(registerErr, fmt.Errorf("clean up unregistered actor host: %w", cleanupErr))}+ return nil, nil, registerErr}localErrors := make(chan error, 1)go func() { localErrors <- host.Run(ctx) }()diff --git a/backend/pkg/utils/handlerutil/handlerutil.go b/backend/pkg/utils/handlerutil/handlerutil.goindex bc681ac38..3d86b33bc 100644--- a/backend/pkg/utils/handlerutil/handlerutil.go+++ b/backend/pkg/utils/handlerutil/handlerutil.go@@ -30,6 +30,15 @@ type Out[T any] struct {Body base.ApiResponse[T]}+// MessageOutput is a success response carrying a message and, when an activity tracks the work, its ID.+func MessageOutput(message, activityID string) *Out[base.MessageResponse] {+ data := base.MessageResponse{Message: message}+ if activityID = strings.TrimSpace(activityID); activityID != "" {+ data.ActivityID = &activityID+ }+ return &Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: data}}+}+// Page is the huma response envelope for endpoints returning base.Paginated[T].type Page[T any] struct {Body base.Paginated[T]diff --git a/cli/pkg/images/updates/cmd.go b/cli/pkg/images/updates/cmd.goindex 0c5c03aa9..062157566 100644--- a/cli/pkg/images/updates/cmd.go+++ b/cli/pkg/images/updates/cmd.go@@ -26,6 +26,9 @@ var UpdatesCmd = &cobra.Command{var checkAll bool+// checkAllBodyInternal is an empty JSON object; older servers reject check-all without a body.+var checkAllBodyInternal = struct{}{}+var checkCmd = &cobra.Command{Use: "check [image-ref...]",Short: "Check image references for updates",@@ -44,7 +47,7 @@ var checkCmd = &cobra.Command{return errors.New("--all cannot be combined with image references")}- result, checkAllUpdatesErr := c.PostJSON[imageupdate.BatchResponse](cmd.Context(), types.ImageUpdatesCheckAll(c.EnvID()), imageupdate.CheckAllImagesRequest{})+ result, checkAllUpdatesErr := c.PostJSON[imageupdate.BatchResponse](cmd.Context(), types.ImageUpdatesCheckAll(c.EnvID()), checkAllBodyInternal)if checkAllUpdatesErr != nil {return fmt.Errorf("failed to check all updates: %w", checkAllUpdatesErr)}@@ -97,8 +100,7 @@ var checkAllCmd = &cobra.Command{return err}- // The handler declares a non-pointer Body, so an empty body is a 400.- result, err := c.PostJSON[imageupdate.BatchResponse](cmd.Context(), types.ImageUpdatesCheckAll(c.EnvID()), imageupdate.CheckAllImagesRequest{})+ result, err := c.PostJSON[imageupdate.BatchResponse](cmd.Context(), types.ImageUpdatesCheckAll(c.EnvID()), checkAllBodyInternal)if err != nil {return fmt.Errorf("failed to check all updates: %w", err)}diff --git a/frontend/messages/en.json b/frontend/messages/en.jsonindex f170995ea..1f7463d76 100644--- a/frontend/messages/en.json+++ b/frontend/messages/en.json@@ -3706,6 +3706,22 @@"jobs_upgrade_required": "Upgrade this agent to use durable jobs.","jobs_offline_status": "This environment is offline. Showing its last confirmed job status.","jobs_run_targets": "Target outcomes",+ "jobs_run_steps": "Steps",+ "jobs_step_progress": "{completed}/{total}",+ "jobs_step_failed": "{count} failed",+ "jobs_step_prepare": "Prepare check",+ "jobs_step_discover": "Discover images",+ "jobs_step_check": "Check images",+ "jobs_step_scan": "Scan images",+ "jobs_step_patch": "Patch images",+ "jobs_step_sync": "Sync from Git",+ "jobs_step_backup": "Back up",+ "jobs_step_tags": "Check tag policies",+ "jobs_step_finalize": "Finalize",+ "jobs_step_resume": "Check for saved plan",+ "jobs_step_candidates": "Select candidates",+ "jobs_step_plan": "Plan updates",+ "jobs_step_apply": "Apply updates","jobs_worker_health": "Worker health","jobs_run_previous": "Previous page","jobs_run_next": "Next page",diff --git a/frontend/src/lib/components/activity/activity-detail-panel.svelte b/frontend/src/lib/components/activity/activity-detail-panel.svelteindex ab15065a2..57169ed39 100644--- a/frontend/src/lib/components/activity/activity-detail-panel.svelte+++ b/frontend/src/lib/components/activity/activity-detail-panel.svelte@@ -42,7 +42,7 @@<Dialog.Description>{jobId}</Dialog.Description></Dialog.Header>{#if historyOpen}- <JobRunHistory {jobId} {environmentId} initialRunId={runId} />+ <JobRunHistory {jobId} {environmentId} initialRunId={runId} onOpenActivity={() => (historyOpen = false)} />{/if}</Dialog.Content></Dialog.Root>diff --git a/frontend/src/lib/components/job-card/job-detail-panel.svelte b/frontend/src/lib/components/job-card/job-detail-panel.svelteindex bf73c2d41..a2ab86735 100644--- a/frontend/src/lib/components/job-card/job-detail-panel.svelte+++ b/frontend/src/lib/components/job-card/job-detail-panel.svelte@@ -157,7 +157,12 @@</div>{/if}{#key activeHistoryJobId}- <JobRunHistory jobId={activeHistoryJobId} {environmentId} onUpdate={onScheduleUpdate} />+ <JobRunHistory+ jobId={activeHistoryJobId}+ {environmentId}+ onUpdate={onScheduleUpdate}+ onOpenActivity={() => (open = false)}+ />{/key}{:else}{#each errors as text (text)}diff --git a/frontend/src/lib/components/job-card/job-run-history.svelte b/frontend/src/lib/components/job-card/job-run-history.svelteindex 79ec8098e..f6792ffa0 100644--- a/frontend/src/lib/components/job-card/job-run-history.svelte+++ b/frontend/src/lib/components/job-card/job-run-history.svelte@@ -13,15 +13,28 @@import { extractApiErrorMessage } from '#lib/utils/api.js';import { formatDateTimeShort } from '#lib/utils/formatting.js';- import { jobStatusLabel, jobStatusTone } from './job-status';+ import { jobStatusLabel, jobStatusTone, jobStepLabel } from './job-status';// Renders the run list and selected-run detail; the host decides where it lives (panel or dialog).let {jobId,environmentId,initialRunId,- onUpdate- }: { jobId: string; environmentId: string; initialRunId?: string; onUpdate?: () => void } = $props();+ onUpdate,+ onOpenActivity+ }: {+ jobId: string;+ environmentId: string;+ initialRunId?: string;+ onUpdate?: () => void;+ /** Lets the hosting dialog close so the Activity Center is not opened behind it. */+ onOpenActivity?: () => void;+ } = $props();++ function openActivity(activityId: string, activityEnvironmentId: string) {+ onOpenActivity?.();+ activityStore.openCenter(activityId, undefined, activityEnvironmentId);+ }let page = $state(1);// Hosts re-key this component per job, so the initial run only seeds state once.// svelte-ignore state_referenced_locally@@ -48,6 +61,8 @@}}));const selectedRun = $derived(selected ? detail.data : undefined);+ // The workflow target only links a run to its workflow instance.+ const targets = $derived(selectedRun?.outcome.targets?.filter((target) => target.resourceType !== 'workflow') ?? []);const hasPrev = $derived(page > 1);const hasNext = $derived(!!runs.data && page * runs.data.limit < runs.data.total);const metadata = $derived(@@ -111,10 +126,8 @@label = m.jobs_activity_output(),activityEnvironmentId = environmentId)}- {#if activityId}<Button- variant="link"- size="inline"- onclick={() => activityStore.openCenter(activityId, undefined, activityEnvironmentId)}>{label}</Button+ {#if activityId}<Button variant="link" size="inline" onclick={() => openActivity(activityId, activityEnvironmentId)}+ >{label}</Button>{/if}{/snippet}@@ -181,9 +194,27 @@{@render outcomeMessage(attempt.outcome.message)}</div>{/each}- {#if run.outcome.targets?.length}+ {#if run.outcome.steps?.length}+ <h4 class="font-medium">{m.jobs_run_steps()}</h4>+ <ol class="divide-y divide-border/50 rounded-lg border border-border/60">+ {#each run.outcome.steps as step (step.name)}+ <li class="space-y-1 px-2 py-1.5">+ <div class="flex flex-wrap items-center justify-between gap-2">+ <span>{jobStepLabel(step.name)}</span>+ <span class="flex items-center gap-2 text-xs text-muted-foreground tabular-nums">+ {#if step.total}{m.jobs_step_progress({ completed: step.completed ?? 0, total: step.total })}{/if}+ {#if step.failed}<span class="text-destructive">{m.jobs_step_failed({ count: step.failed })}</span>{/if}+ <Badge variant={jobStatusTone(step.status)} size="sm">{jobStatusLabel(step.status)}</Badge>+ </span>+ </div>+ {@render outcomeMessage(step.message)}+ </li>+ {/each}+ </ol>+ {/if}+ {#if targets.length}<h4 class="font-medium">{m.jobs_run_targets()}</h4>- {#each run.outcome.targets as target (target.id)}+ {#each targets as target (target.id)}<div class="space-y-1 rounded-lg border border-border/60 p-2"><p class="break-all">{target.id} · {jobStatusLabel(target.status)}</p>{@render outcomeMessage(target.message)}diff --git a/frontend/src/lib/components/job-card/job-status.ts b/frontend/src/lib/components/job-card/job-status.tsindex 1b7c3527f..1f37847f6 100644--- a/frontend/src/lib/components/job-card/job-status.ts+++ b/frontend/src/lib/components/job-card/job-status.ts@@ -20,6 +20,26 @@ export function jobStatusLabel(status: string, isWorker = false): string {return labels[status]?.() ?? m.jobs_status_unknown();}+/** Label for a workflow step, falling back to its name. */+export function jobStepLabel(name: string): string {+ const labels: Record<string, () => string> = {+ prepare: m.jobs_step_prepare,+ discover: m.jobs_step_discover,+ check: m.jobs_step_check,+ scan: m.jobs_step_scan,+ patch: m.jobs_step_patch,+ sync: m.jobs_step_sync,+ backup: m.jobs_step_backup,+ tags: m.jobs_step_tags,+ finalize: m.jobs_step_finalize,+ resume: m.jobs_step_resume,+ candidates: m.jobs_step_candidates,+ plan: m.jobs_step_plan,+ apply: m.jobs_step_apply+ };+ return labels[name]?.() ?? name;+}+export function jobNameLabel(job: JobStatus): string {const [group = '', ...target] = job.id.split(':');const labels: Record<string, () => string> = {diff --git a/frontend/src/lib/types/job.ts b/frontend/src/lib/types/job.tsindex 8016367f0..ca0925684 100644--- a/frontend/src/lib/types/job.ts+++ b/frontend/src/lib/types/job.ts@@ -9,8 +9,25 @@ export type JobRunStatus =| 'failed'| 'needs_attention'| 'canceled';-export type JobTargetOutcome = { id: string; status: JobRunStatus; message?: string; activityId?: string };-export type JobOutcome = { status: JobRunStatus; message?: string; activityId?: string; targets?: JobTargetOutcome[] };+export type JobTargetOutcome = { id: string; resourceType?: string; status: JobRunStatus; message?: string; activityId?: string };+export type JobStepOutcome = {+ name: string;+ status: JobRunStatus;+ total?: number;+ completed?: number;+ failed?: number;+ attempts?: number;+ message?: string;+ startedAt?: string;+ finishedAt?: string;+};+export type JobOutcome = {+ status: JobRunStatus;+ message?: string;+ activityId?: string;+ targets?: JobTargetOutcome[];+ steps?: JobStepOutcome[];+};export type JobAttempt = { number: number; startedAt: string; finishedAt?: string; outcome: JobOutcome };export type JobRun = {id: string;diff --git a/types/backup/durable_run.go b/types/backup/durable_run.godeleted file mode 100644index 755a04a87..000000000--- a/types/backup/durable_run.go+++ /dev/null@@ -1,24 +0,0 @@-package backup--import "github.com/getarcaneapp/arcane/types/v2/scheduler"--// DurableRunCommand freezes the input accepted by a backup domain.-type DurableRunCommand struct {- UserID string `msgpack:"userID"`- EnvironmentID string `msgpack:"environmentID"`- Permission string `msgpack:"permission"`- RequestedWithKey string `msgpack:"requestedWithKey"`- Kind string `msgpack:"kind"`- RunID string `msgpack:"runID"`- ActivityID string `msgpack:"activityID"`- Payload []byte `msgpack:"payload"`-}--// DurableRunState records dispatch and effect evidence across host restarts.-type DurableRunState struct {- Command DurableRunCommand `msgpack:"command"`- Started bool `msgpack:"started"`- Status scheduler.RunStatus `msgpack:"status"`- Targets []scheduler.TargetOutcome `msgpack:"targets"`- Error string `msgpack:"error"`-}diff --git a/types/backup/requester.go b/types/backup/requester.gonew file mode 100644index 000000000..fa525fe5d--- /dev/null+++ b/types/backup/requester.go@@ -0,0 +1,9 @@+package backup++// Requester is who started a manual backup; its workflow re-checks their permission on every delivery.+type Requester struct {+ UserID string `json:"userId"`+ APIKeyID string `json:"apiKeyId,omitempty"`+ EnvironmentID string `json:"environmentId"`+ Permission string `json:"permission"`+}diff --git a/types/backup/system_volume_backup.go b/types/backup/system_volume_backup.goindex 20cb2a54c..660874474 100644--- a/types/backup/system_volume_backup.go+++ b/types/backup/system_volume_backup.go@@ -104,6 +104,7 @@ type HistoryEntry struct {Format string `json:"format,omitempty"`LocalSnapshotID string `json:"localSnapshotId,omitempty"`RemoteSnapshotID string `json:"remoteSnapshotId,omitempty"`+ RemoteInstanceID string `json:"remoteInstanceId,omitempty"`S3DestinationID string `json:"s3DestinationId,omitempty"`S3DestinationName string `json:"s3DestinationName,omitempty"`PolicyID string `json:"policyId,omitempty"`diff --git a/types/imageupdate/image_update.go b/types/imageupdate/image_update.goindex 348d3f1ff..0c68d68ea 100644--- a/types/imageupdate/image_update.go+++ b/types/imageupdate/image_update.go@@ -126,4 +126,11 @@ type CheckAllImagesRequest struct {Credentials []containerregistry.Credential `json:"credentials,omitempty"`}+// CheckRequest selects the images an update check covers. All discovers every+// local image; otherwise only ImageRefs are checked, and an empty list checks nothing.+type CheckRequest struct {+ ImageRefs []string `json:"imageRefs,omitempty"`+ All bool `json:"all,omitempty"`+}+type BatchResponse map[string]*Responsediff --git a/types/meta/job_metadata.go b/types/meta/job_metadata.goindex 6e0569af8..93070107c 100644--- a/types/meta/job_metadata.go+++ b/types/meta/job_metadata.go@@ -89,13 +89,7 @@ var jobMetadataRegistry = map[string]JobMetadata{ManagerOnly: false,IsContinuous: false,CanRunManually: true,- Prerequisites: []JobPrerequisiteMetadata{- {- SettingKey: "analyticsEnabled",- Label: "Analytics enabled",- SettingsURL: "/settings/general",- },- },+ Prerequisites: []JobPrerequisiteMetadata{},},"auto-update": {ID: "auto-update",diff --git a/types/scheduler/run.go b/types/scheduler/run.goindex 7ec541ed5..d969cec17 100644--- a/types/scheduler/run.go+++ b/types/scheduler/run.go@@ -23,6 +23,9 @@ const (Canceled RunStatus = "canceled")+// WorkflowTarget marks the target that records a run's workflow instance.+const WorkflowTarget = "workflow"+func (s RunStatus) Terminal() bool {return s == Succeeded || s == Partial || s == Skipped || s == Failed || s == Canceled}@@ -36,11 +39,25 @@ type TargetOutcome struct {ActivityID string `json:"activityId,omitempty"`}+// StepOutcome is one workflow step as projected onto its run.+type StepOutcome struct {+ Name string `json:"name"`+ Status RunStatus `json:"status"`+ Total int `json:"total,omitempty"`+ Completed int `json:"completed,omitempty"`+ Failed int `json:"failed,omitempty"`+ Attempts int `json:"attempts,omitempty"`+ Message string `json:"message,omitempty"`+ StartedAt *time.Time `json:"startedAt,omitempty"`+ FinishedAt *time.Time `json:"finishedAt,omitempty"`+}+type Outcome struct {Status RunStatus `json:"status"`Message string `json:"message,omitempty"`ActivityID string `json:"activityId,omitempty"`Targets []TargetOutcome `json:"targets,omitempty"`+ Steps []StepOutcome `json:"steps,omitempty"`}type Request struct {diff --git a/types/updater/durable.go b/types/updater/durable.goindex 952db96b4..ce2fbca7b 100644--- a/types/updater/durable.go+++ b/types/updater/durable.go@@ -1,21 +1,5 @@package updater-// SingleUpdateCommand contains the persisted inputs for an asynchronous update.-type SingleUpdateCommand struct {- ContainerID string- ActivityID string- UserID string- KeyID string-}--type SingleUpdateState struct {- Command SingleUpdateCommand- Status string- Target *FrozenUpdateTarget- Result *Result- Failure string-}-// FrozenUpdateTarget identifies the selected container and immutable image intent.type FrozenUpdateTarget struct {ContainerID stringdiff --git a/types/volume/backup.go b/types/volume/backup.goindex 016cc04c1..b23b3412a 100644--- a/types/volume/backup.go+++ b/types/volume/backup.go@@ -81,6 +81,29 @@ type BackupEntry struct {Type backup.ManagementType `json:"type" doc:"Whether the backup was system-managed or volume-managed"`}+// Entry is the backup as the API lists it.+func (b *Backup) Entry() BackupEntry {+ return BackupEntry{+ RemoteAvailable: b.RemoteAvailable,+ ActivityID: b.ActivityID,+ ID: b.ID,+ VolumeName: b.VolumeName,+ Size: b.Size,+ CreatedAt: b.CreatedAt.Format(time.RFC3339),+ Status: b.Status,+ Trigger: b.Trigger,+ Destination: b.Destination,+ Format: b.Format,+ LocalSnapshotID: b.LocalSnapshotID,+ RemoteSnapshotID: b.RemoteSnapshotID,+ S3DestinationID: b.S3DestinationID,+ S3DestinationName: b.S3DestinationName,+ PolicyID: b.PolicyID,+ Error: b.Error,+ Type: b.Type,+ }+}+type BackupPolicy struct {ID string `json:"id"`VolumeName string `json:"volumeName"`