Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
refactor(system): use composite literals, drop DI nil checks and mappers
Kyle Mendellcdfe49e8cbe0
diff --git a/backend/internal/backup/service.go b/backend/internal/backup/service.goindex cabd3c168..2c8b8dfa6 100644--- a/backend/internal/backup/service.go+++ b/backend/internal/backup/service.go@@ -161,9 +161,6 @@ func NewEngine(admission *runs.Admission, imageService *image.ImageService) *Eng// TryAcquireRun takes the admission lease for one backup resource, reporting false while another run holds it.func (e *Engine) TryAcquireRun(ctx context.Context, scope, id string) (*runs.Lease, bool, error) {- if e == nil || e.admission == nil {- return nil, false, errors.New("backup engine is unavailable")- }return e.admission.TryAcquire(ctx, scheduler.AdmissionKey{Scope: scope, ID: id})}@@ -307,12 +304,7 @@ func qualifySnapshotListing(files []string, prefix string) []string {return qualified}for index, file := range qualified {- directory := strings.HasSuffix(file, "/")- relative := strings.TrimPrefix(strings.TrimPrefix(file, "./"), "/")- qualified[index] = path.Join(prefix, relative)- if directory {- qualified[index] += "/"- }+ qualified[index] = path.Join(prefix, strings.TrimPrefix(strings.TrimPrefix(file, "./"), "/")) + kit.Ternary(strings.HasSuffix(file, "/"), "/", "")}return qualified}@@ -393,11 +385,7 @@ func (e *Engine) FindRunSnapshot(ctx context.Context, dockerClient *client.Clien}var found Snapshotfor _, snapshot := range snapshots {- match := snapshotID != "" && snapshot.ID == snapshotID- if snapshotID == "" {- match = slices.Contains(snapshot.Tags, RunSnapshotTag(runID))- }- if !match {+ if !kit.Ternary(snapshotID == "", slices.Contains(snapshot.Tags, RunSnapshotTag(runID)), snapshot.ID == snapshotID) {continue}if found.ID != "" {@@ -558,12 +546,19 @@ func (e *Engine) Replicate(ctx context.Context, dockerClient *client.Client, frodefer func() {_, _ = dockerClient.VolumeRemove(context.WithoutCancel(ctx), temporaryVolume, client.VolumeRemoveOptions{Force: true})}()- copyMount := mount.Mount{Type: mount.TypeVolume, Source: temporaryVolume, Target: "/volume"}- if err := e.RestoreSnapshot(ctx, dockerClient, from, password, fromSnapshotID, copyMount, RestoreOptions{DeleteExtra: true}); err != nil {+ if err := e.RestoreSnapshot(ctx, dockerClient, from, password, fromSnapshotID, mount.Mount{+ Type: mount.TypeVolume,+ Source: temporaryVolume,+ Target: "/volume",+ }, RestoreOptions{DeleteExtra: true}); err != nil {return Snapshot{}, fmt.Errorf("failed to load Rustic snapshot for replication: %w", err)}- copyMount.ReadOnly = true- snapshot, err := e.CreateSnapshot(ctx, dockerClient, to, password, label, RootSnapshotInput(copyMount, tags...))+ snapshot, err := e.CreateSnapshot(ctx, dockerClient, to, password, label, RootSnapshotInput(mount.Mount{+ Type: mount.TypeVolume,+ Source: temporaryVolume,+ Target: "/volume",+ ReadOnly: true,+ }, tags...))if err != nil {return Snapshot{}, fmt.Errorf("failed to replicate Rustic snapshot: %w", err)}@@ -582,22 +577,26 @@ func (e *Engine) run(ctx context.Context, dockerClient *client.Client, repositor// runContainer runs one Rustic command; callers hold the repository lock.func (e *Engine) runContainer(ctx context.Context, dockerClient *client.Client, repository Repository, password string, command []string, extraMounts ...mount.Mount) (string, error) {if _, err := dockerClient.ImageInspect(ctx, rustic.DefaultImage); err != nil {- if e.imageService == nil {- return "", errors.New("image service is unavailable")- }if pullErr := e.imageService.PullImage(ctx, rustic.DefaultImage, io.Discard, user.SystemUser, nil); pullErr != nil {return "", fmt.Errorf("failed to pull official Rustic image: %w", pullErr)}}- var networkMode container.NetworkMode- arcane, err := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient)- if err != nil || arcane == nil || arcane.ID == "" {- slog.DebugContext(ctx, "backup engine: running Rustic on the default network", "error", err)- } else {- networkMode = container.NetworkMode("container:" + arcane.ID)- }- mounts := append(slices.Clone(repository.Mounts), extraMounts...)- return rustic.Run(ctx, dockerClient, password, command, repository.Environment, mounts, networkMode)+ return rustic.Run(+ ctx,+ dockerClient,+ password,+ command,+ repository.Environment,+ append(slices.Clone(repository.Mounts), extraMounts...),+ func() container.NetworkMode {+ arcane, err := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient)+ if err != nil || kit.FromPtr(arcane).ID == "" {+ slog.DebugContext(ctx, "backup engine: running Rustic on the default network", "error", err)+ return ""+ }+ return container.NetworkMode("container:" + arcane.ID)+ }(),+ )}var (@@ -719,8 +718,13 @@ func (s *RecoveryKeyStore) save(ctx context.Context, id, recoveryKey string) errfunc AcceptedTarget(backupID string) scheduler.TargetOutcome {data, _ := json.Marshal(struct {BackupID string `json:"backupId"`- }{backupID})- return scheduler.TargetOutcome{ResourceType: "backup", ID: "accepted:" + backupID, Status: scheduler.Queued, RecoveryData: data}+ }{BackupID: backupID})+ return scheduler.TargetOutcome{+ ResourceType: "backup",+ ID: "accepted:" + backupID,+ Status: scheduler.Queued,+ RecoveryData: data,+ }}// AcceptedBackupID returns the backup a manual backup workflow's run owns, from its AcceptedTarget.@@ -736,9 +740,6 @@ func AcceptedBackupID(run scheduler.Run) string {// lockRepository serializes commands against one repository and returns its unlock func.func (e *Engine) lockRepository(ctx context.Context, id string) (func(), error) {- if e == nil {- return nil, errors.New("backup engine is unavailable")- }if strings.TrimSpace(id) == "" {return nil, errors.New("backup repository ID is required")}@@ -815,8 +816,6 @@ func (r PolicyReconciliation[P, U]) Run(ctx context.Context, updates []U) errorswitch {case r.Persist != nil:persistErr = r.Persist(ctx, policies)- case r.DB == nil:- persistErr = errors.New("backup policy database is unavailable")default:persistErr = r.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {for i := range policies {@@ -850,27 +849,30 @@ func ValidatePolicyUpdate(ctx context.Context, domain string, update backup.Updaif err != nil {return update, err}- update.Schedule = normalizedif update.RetentionCount < 0 || update.RetentionCount > 3650 {return update, errors.New("retentionCount must be between 0 and 3650")}if !update.LocalEnabled && !update.S3Enabled {return update, fmt.Errorf("select at least one %s backup destination", domain)}+ if update.S3Enabled && strings.TrimSpace(update.S3DestinationID) == "" {+ return update, fmt.Errorf("select an S3 destination for %s backups", domain)+ }if update.S3Enabled {- if strings.TrimSpace(update.S3DestinationID) == "" {- return update, fmt.Errorf("select an S3 destination for %s backups", domain)- }- if s3Service == nil {- return update, errors.New("S3 backup destinations are unavailable")- }if _, configurationErr := s3Service.Configuration(ctx, update.S3DestinationID); configurationErr != nil {return update, fmt.Errorf("select a valid S3 destination for %s backups", domain)}- } else {- update.S3DestinationID = ""}- return update, nil+ return backup.UpdateBackupPolicy{+ ID: update.ID,+ Enabled: update.Enabled,+ Schedule: normalized,+ RetentionCount: update.RetentionCount,+ StopContainers: update.StopContainers,+ LocalEnabled: update.LocalEnabled,+ S3Enabled: update.S3Enabled,+ S3DestinationID: kit.Ternary(update.S3Enabled, update.S3DestinationID, ""),+ }, nil}var ErrRemoteRepositoryMissing = errors.New("S3 backup storage is missing")@@ -883,7 +885,7 @@ func RemoteSnapshotChecker(ctx context.Context, destinations *s3.S3DestinationSechecked := make(map[string]*backup.RepositoryObservation)deadline := time.Now().Add(10 * time.Second)return func(destinationID, root, snapshotID string) *bool {- if destinations == nil || root == "" || destinationID == "" || snapshotID == "" {+ if root == "" || destinationID == "" || snapshotID == "" {return nil}key := destinationID + ":" + root + ":" + snapshotID@@ -913,9 +915,6 @@ func RemoteSnapshotChecker(ctx context.Context, destinations *s3.S3DestinationSe// CheckRemoteRepository probes the destination's repository config within a bounded budget.// Transport and credential failures are returned as errors so callers never forget snapshots blindly.func CheckRemoteRepository(ctx context.Context, destinations *s3.S3DestinationService, destinationID, root string) (backup.RepositoryObservation, error) {- if destinations == nil {- return backup.RepositoryObservation{}, errors.New("S3 destinations are unavailable")- }configuration, err := destinations.Configuration(ctx, destinationID)if err != nil {return backup.RepositoryObservation{}, err@@ -932,15 +931,11 @@ func DisableMissingRemote(remoteErr error, localEnabled bool, s3Enabled, enabledif !errors.Is(remoteErr, ErrRemoteRepositoryMissing) {return false, nil}- field, flag := "enabled", enabled- if localEnabled {- field, flag = "s3_enabled", s3Enabled- }- saved, err := save(field)+ saved, err := save(kit.Ternary(localEnabled, "s3_enabled", "enabled"))if err != nil || !saved {return false, err}- *flag = false+ *kit.Ternary(localEnabled, s3Enabled, enabled) = falsereturn true, nil}diff --git a/backend/internal/s3/handler.go b/backend/internal/s3/handler.goindex 5519b880e..1bde751ba 100644--- a/backend/internal/s3/handler.go+++ b/backend/internal/s3/handler.go@@ -168,15 +168,11 @@ func (h *s3DestinationHandler) test(ctx context.Context, input *testS3Destinatio}func (h *s3DestinationHandler) triggerRemoteSync(ctx context.Context, reason string) {- if h.syncRemoteDestinations == nil {- return- }- detachedCtx := context.WithoutCancel(ctx)go func(syncCtx context.Context, syncReason string) {if err := h.syncRemoteDestinations(syncCtx); err != nil {slog.WarnContext(syncCtx, s3RemoteSyncFailureMessage, "reason", syncReason, "error", err.Error())}- }(detachedCtx, reason)+ }(context.WithoutCancel(ctx), reason)}func (h *s3DestinationHandler) testConfiguration(ctx context.Context, input *testS3DestinationConfigurationInput) (*handlerutil.Out[base.MessageResponse], error) {diff --git a/backend/internal/s3/service.go b/backend/internal/s3/service.goindex 9812cbf27..6a9fa4c30 100644--- a/backend/internal/s3/service.go+++ b/backend/internal/s3/service.go@@ -1,6 +1,7 @@package s3import (+ "cmp""context""errors""fmt"@@ -8,8 +9,8 @@ import ("strings""github.com/getarcaneapp/arcane/types/v2/backup"- "github.com/samber/mo""go.getarcane.app/kit/normalization"+ kit "go.getarcane.app/kit/pkg""go.getarcane.app/sys/crypto""gorm.io/gorm"@@ -22,9 +23,9 @@ import (var (ErrS3DestinationNotFound = errors.New("S3 destination not found")ErrS3DestinationInUse = errors.New("S3 destination is in use")- // errS3SecretRequiredInternal enforces the fresh-secret rule: a stored+ // errS3SecretRequired enforces the fresh-secret rule: a stored// secret is never combined with changed connection or trust settings.- errS3SecretRequiredInternal = errors.New("re-enter the secret access key when changing the endpoint, bucket, region, access key ID, SSL mode, or path style")+ errS3SecretRequired = errors.New("re-enter the secret access key when changing the endpoint, bucket, region, access key ID, SSL mode, or path style"))type S3DestinationService struct {@@ -39,7 +40,7 @@ func NewS3DestinationService(db *database.DB, checkRemoteReferences func(contextreturn &S3DestinationService{db: db, checkRemoteReferences: checkRemoteReferences}}-func destinationConfigurationInternal(id string, input backup.CreateS3Destination) s3.Configuration {+func destinationConfiguration(id string, input backup.CreateS3Destination) s3.Configuration {return s3.Configuration{ID: id,Name: input.Name,@@ -54,14 +55,6 @@ func destinationConfigurationInternal(id string, input backup.CreateS3Destinatio}.Normalized()}-func s3DestinationsToDTOsInternal(destinations []S3Destination) []backup.S3Destination {- result := make([]backup.S3Destination, len(destinations))- for i := range destinations {- result[i] = destinations[i].ToDTO()- }- return result-}-func (s *S3DestinationService) ListS3Destinations(ctx context.Context, params pagination.QueryParams) ([]backup.S3Destination, pagination.Response, error) {var destinations []S3Destinationquery := s.db.WithContext(ctx).Model(&S3Destination{})@@ -73,7 +66,11 @@ func (s *S3DestinationService) ListS3Destinations(ctx context.Context, params paif err != nil {return nil, pagination.Response{}, fmt.Errorf("failed to list S3 destinations: %w", err)}- return s3DestinationsToDTOsInternal(destinations), response, nil+ result := make([]backup.S3Destination, len(destinations))+ for i := range destinations {+ result[i] = destinations[i].ToDTO()+ }+ return result, response, nil}func (s *S3DestinationService) ListAllS3Destinations(ctx context.Context) ([]backup.S3Destination, error) {@@ -81,15 +78,16 @@ func (s *S3DestinationService) ListAllS3Destinations(ctx context.Context) ([]bacif err := s.db.WithContext(ctx).Order("name ASC").Find(&destinations).Error; err != nil {return nil, fmt.Errorf("failed to list S3 destinations: %w", err)}- return s3DestinationsToDTOsInternal(destinations), nil+ result := make([]backup.S3Destination, len(destinations))+ for i := range destinations {+ result[i] = destinations[i].ToDTO()+ }+ return result, nil}// DestinationsByID indexes destinations to decorate backup records, such as with their names. Decoration never fails-// its caller, so a missing service or failed listing yields an empty index.+// its caller, so a failed listing yields an empty index.func (s *S3DestinationService) DestinationsByID(ctx context.Context) map[string]backup.S3Destination {- if s == nil {- return nil- }destinations, err := s.ListAllS3Destinations(ctx)if err != nil {slog.WarnContext(ctx, "could not list S3 destinations", "error", err)@@ -102,7 +100,7 @@ func (s *S3DestinationService) DestinationsByID(ctx context.Context) map[string]return indexed}-func (s *S3DestinationService) getS3DestinationModelInternal(ctx context.Context, id string) (*S3Destination, error) {+func (s *S3DestinationService) getS3DestinationModel(ctx context.Context, id string) (*S3Destination, error) {var destination S3Destinationif err := s.db.WithContext(ctx).Where("id = ?", strings.TrimSpace(id)).First(&destination).Error; err != nil {if errors.Is(err, gorm.ErrRecordNotFound) {@@ -114,7 +112,7 @@ func (s *S3DestinationService) getS3DestinationModelInternal(ctx context.Context}func (s *S3DestinationService) GetS3Destination(ctx context.Context, id string) (*backup.S3Destination, error) {- destination, err := s.getS3DestinationModelInternal(ctx, id)+ destination, err := s.getS3DestinationModel(ctx, id)if err != nil {return nil, err}@@ -122,26 +120,11 @@ func (s *S3DestinationService) GetS3Destination(ctx context.Context, id string)return &dto, nil}-// applyS3ConfigurationInternal copies configuration onto destination and-// reports whether any field actually changed.-func applyS3ConfigurationInternal(destination *S3Destination, configuration s3.Configuration, encryptedSecret string) bool {- changed := utils.ApplyChanged(&destination.Name, mo.Some(configuration.Name))- changed = utils.ApplyChanged(&destination.Endpoint, mo.Some(configuration.Endpoint)) || changed- changed = utils.ApplyChanged(&destination.Bucket, mo.Some(configuration.Bucket)) || changed- changed = utils.ApplyChanged(&destination.Region, mo.Some(configuration.Region)) || changed- changed = utils.ApplyChanged(&destination.AccessKeyID, mo.Some(configuration.AccessKeyID)) || changed- changed = utils.ApplyChanged(&destination.SecretAccessKey, mo.Some(encryptedSecret)) || changed- changed = utils.ApplyChanged(&destination.Prefix, mo.Some(configuration.Prefix)) || changed- changed = utils.ApplyChanged(&destination.UseSSL, mo.Some(configuration.UseSSL)) || changed- changed = utils.ApplyChanged(&destination.ForcePathStyle, mo.Some(configuration.ForcePathStyle)) || changed- return changed-}-func (s *S3DestinationService) CreateS3Destination(ctx context.Context, input backup.CreateS3Destination) (*backup.S3Destination, error) {if err := normalization.Normalize(&input); err != nil {return nil, err}- configuration := destinationConfigurationInternal("", input)+ configuration := destinationConfiguration("", input)if err := configuration.Validate(true); err != nil {return nil, err}@@ -149,8 +132,17 @@ func (s *S3DestinationService) CreateS3Destination(ctx context.Context, input baif err != nil {return nil, fmt.Errorf("failed to encrypt S3 secret access key: %w", err)}- destination := &S3Destination{}- applyS3ConfigurationInternal(destination, configuration, encryptedSecret)+ destination := &S3Destination{+ Name: configuration.Name,+ Endpoint: configuration.Endpoint,+ Bucket: configuration.Bucket,+ Region: configuration.Region,+ AccessKeyID: configuration.AccessKeyID,+ SecretAccessKey: encryptedSecret,+ Prefix: configuration.Prefix,+ UseSSL: configuration.UseSSL,+ ForcePathStyle: configuration.ForcePathStyle,+ }if createDestinationErr := s.db.WithContext(ctx).Create(destination).Error; createDestinationErr != nil {return nil, fmt.Errorf("failed to create S3 destination: %w", createDestinationErr)}@@ -158,45 +150,54 @@ func (s *S3DestinationService) CreateS3Destination(ctx context.Context, input bareturn &dto, nil}-// storedConfigurationInternal returns the persisted configuration without the-// decrypted secret, for comparing connection fields against an update.-func storedConfigurationInternal(destination *S3Destination) s3.Configuration {- return s3.Configuration{- ID: destination.ID,- Name: destination.Name,- Endpoint: destination.Endpoint,- Bucket: destination.Bucket,- Region: destination.Region,- AccessKeyID: destination.AccessKeyID,- Prefix: destination.Prefix,- UseSSL: destination.UseSSL,- ForcePathStyle: destination.ForcePathStyle,- }.Normalized()-}-func (s *S3DestinationService) UpdateS3Destination(ctx context.Context, id string, input backup.UpdateS3Destination) (*backup.S3Destination, error) {if err := normalization.Normalize(&input); err != nil {return nil, err}- configuration := destinationConfigurationInternal("", input)+ configuration := destinationConfiguration("", input)if err := configuration.Validate(false); err != nil {return nil, err}- destination, err := s.getS3DestinationModelInternal(ctx, id)+ destination, err := s.getS3DestinationModel(ctx, id)if err != nil {return nil, err}- if configuration.SecretAccessKey == "" && !s3.ConnectionFieldsEqual(storedConfigurationInternal(destination), configuration) {- return nil, errS3SecretRequiredInternal+ // The persisted configuration without the+ // decrypted secret, for comparing connection fields against an update.+ if configuration.SecretAccessKey == "" && !s3.ConnectionFieldsEqual(s3.Configuration{+ ID: destination.ID,+ Name: destination.Name,+ Endpoint: destination.Endpoint,+ Bucket: destination.Bucket,+ Region: destination.Region,+ AccessKeyID: destination.AccessKeyID,+ Prefix: destination.Prefix,+ UseSSL: destination.UseSSL,+ ForcePathStyle: destination.ForcePathStyle,+ }.Normalized(), configuration) {+ return nil, errS3SecretRequired}- encryptedSecret := destination.SecretAccessKey- if configuration.SecretAccessKey != "" {- encryptedSecret, err = crypto.Encrypt(configuration.SecretAccessKey)- if err != nil {- return nil, fmt.Errorf("failed to encrypt S3 secret access key: %w", err)+ encryptedSecret, err := func() (string, error) {+ if configuration.SecretAccessKey == "" {+ return destination.SecretAccessKey, nil}+ return crypto.Encrypt(configuration.SecretAccessKey)+ }()+ if err != nil {+ return nil, fmt.Errorf("failed to encrypt S3 secret access key: %w", err)+ }+ *destination = S3Destination{ //exhaustruct:enforce+ BaseModel: destination.BaseModel,+ Name: configuration.Name,+ Endpoint: configuration.Endpoint,+ Bucket: configuration.Bucket,+ Region: configuration.Region,+ AccessKeyID: configuration.AccessKeyID,+ SecretAccessKey: encryptedSecret,+ Prefix: configuration.Prefix,+ UseSSL: configuration.UseSSL,+ ForcePathStyle: configuration.ForcePathStyle,}- applyS3ConfigurationInternal(destination, configuration, encryptedSecret)if updateDestinationErr := s.db.WithContext(ctx).Save(destination).Error; updateDestinationErr != nil {return nil, fmt.Errorf("failed to update S3 destination: %w", updateDestinationErr)}@@ -205,7 +206,7 @@ func (s *S3DestinationService) UpdateS3Destination(ctx context.Context, id strin}func (s *S3DestinationService) DeleteS3Destination(ctx context.Context, id string) error {- destination, err := s.getS3DestinationModelInternal(ctx, id)+ destination, err := s.getS3DestinationModel(ctx, id)if err != nil {return err}@@ -216,10 +217,8 @@ func (s *S3DestinationService) DeleteS3Destination(ctx context.Context, id strinif inUse {return fmt.Errorf("%w: remove it from backup configurations and retained backups first", ErrS3DestinationInUse)}- if s.checkRemoteReferences != nil {- if checkRemoteReferencesErr := s.checkRemoteReferences(ctx, id); checkRemoteReferencesErr != nil {- return fmt.Errorf("%w: %s", ErrS3DestinationInUse, checkRemoteReferencesErr.Error())- }+ if checkRemoteReferencesErr := s.checkRemoteReferences(ctx, id); checkRemoteReferencesErr != nil {+ return fmt.Errorf("%w: %s", ErrS3DestinationInUse, checkRemoteReferencesErr.Error())}if deleteDestinationErr := s.db.WithContext(ctx).Delete(destination).Error; deleteDestinationErr != nil {return fmt.Errorf("failed to delete S3 destination: %w", deleteDestinationErr)@@ -266,41 +265,56 @@ func (s *S3DestinationService) SyncS3Destinations(ctx context.Context, destinatiif err := s.db.WithContext(ctx).Find(&existing).Error; err != nil {return fmt.Errorf("failed to load existing S3 destinations: %w", err)}- existingByID := make(map[string]*S3Destination, len(existing))- for i := range existing {- existingByID[existing[i].ID] = &existing[i]+ existingByID := make(map[string]S3Destination, len(existing))+ for _, destination := range existing {+ existingByID[destination.ID] = destination}syncedIDs := make(map[string]struct{}, len(destinations))for _, item := range destinations {- configuration := destinationConfigurationInternal(item.ID, item.CreateS3Destination)+ configuration := destinationConfiguration(item.ID, item.CreateS3Destination)if configuration.ID == "" {return errors.New("S3 destination ID is required")}if err := configuration.Validate(true); err != nil {return fmt.Errorf("invalid S3 destination %s: %w", configuration.ID, err)}- destination, exists := existingByID[configuration.ID]- if !exists {- destination = &S3Destination{ID: configuration.ID}- }+ current, exists := existingByID[configuration.ID]// The manager resends the full set every health tick; unchanged rows// keep their ciphertext and skip the write.- secretChanged, err := utils.ApplyEncrypted(&destination.SecretAccessKey, configuration.SecretAccessKey)+ encryptedSecret := current.SecretAccessKey+ secretChanged, err := utils.ApplyEncrypted(&encryptedSecret, configuration.SecretAccessKey)if err != nil {return fmt.Errorf("failed to apply S3 secret access key for %s: %w", configuration.ID, err)}- changed := applyS3ConfigurationInternal(destination, configuration, destination.SecretAccessKey) || secretChanged || !exists- if !item.CreatedAt.IsZero() && !destination.CreatedAt.Equal(item.CreatedAt) {- destination.CreatedAt = item.CreatedAt- changed = true- }- if item.UpdatedAt != nil && (destination.UpdatedAt == nil || !destination.UpdatedAt.Equal(*item.UpdatedAt)) {- destination.UpdatedAt = item.UpdatedAt- changed = true- }+ changed := !exists || secretChanged ||+ current.Name != configuration.Name ||+ current.Endpoint != configuration.Endpoint ||+ current.Bucket != configuration.Bucket ||+ current.Region != configuration.Region ||+ current.AccessKeyID != configuration.AccessKeyID ||+ current.Prefix != configuration.Prefix ||+ current.UseSSL != configuration.UseSSL ||+ current.ForcePathStyle != configuration.ForcePathStyle ||+ (!item.CreatedAt.IsZero() && !current.CreatedAt.Equal(item.CreatedAt)) ||+ (item.UpdatedAt != nil && !kit.FromPtr(current.UpdatedAt).Equal(*item.UpdatedAt))if changed {- if syncDestinationErr := s.db.WithContext(ctx).Save(destination).Error; syncDestinationErr != nil {+ if syncDestinationErr := s.db.WithContext(ctx).Save(&S3Destination{ //exhaustruct:enforce+ BaseModel: database.BaseModel{+ ID: configuration.ID,+ CreatedAt: kit.Ternary(item.CreatedAt.IsZero(), current.CreatedAt, item.CreatedAt),+ UpdatedAt: kit.Ternary(item.UpdatedAt != nil, item.UpdatedAt, current.UpdatedAt),+ },+ Name: configuration.Name,+ Endpoint: configuration.Endpoint,+ Bucket: configuration.Bucket,+ Region: configuration.Region,+ AccessKeyID: configuration.AccessKeyID,+ SecretAccessKey: encryptedSecret,+ Prefix: configuration.Prefix,+ UseSSL: configuration.UseSSL,+ ForcePathStyle: configuration.ForcePathStyle,+ }).Error; syncDestinationErr != nil {return fmt.Errorf("failed to sync S3 destination %s: %w", configuration.ID, syncDestinationErr)}}@@ -323,7 +337,7 @@ func (s *S3DestinationService) SyncS3Destinations(ctx context.Context, destinati// Configuration returns a destination's decrypted runtime configuration for backup domains.func (s *S3DestinationService) Configuration(ctx context.Context, id string) (s3.Configuration, error) {- destination, err := s.getS3DestinationModelInternal(ctx, id)+ destination, err := s.getS3DestinationModel(ctx, id)if err != nil {return s3.Configuration{}, err}@@ -350,28 +364,35 @@ func (s *S3DestinationService) TestS3Destination(ctx context.Context, id string,if err != nil {return err}- if input != nil {- if normalizeErr := normalization.Normalize(input); normalizeErr != nil {- return normalizeErr- }- updated := destinationConfigurationInternal("", *input)- updated.ID = configuration.ID- if updated.SecretAccessKey == "" {- // The stored secret may only sign requests against the stored- // connection settings; any change requires a fresh secret.- if !s3.ConnectionFieldsEqual(configuration, updated) {- return errS3SecretRequiredInternal- }- updated.SecretAccessKey = configuration.SecretAccessKey- }- configuration = updated- }- return s3.TestConnection(ctx, configuration)+ if input == nil {+ return s3.TestConnection(ctx, configuration)+ }+ if normalizeErr := normalization.Normalize(input); normalizeErr != nil {+ return normalizeErr+ }+ updated := destinationConfiguration(configuration.ID, *input)+ // The stored secret may only sign requests against the stored+ // connection settings; any change requires a fresh secret.+ if updated.SecretAccessKey == "" && !s3.ConnectionFieldsEqual(configuration, updated) {+ return errS3SecretRequired+ }+ return s3.TestConnection(ctx, s3.Configuration{+ ID: updated.ID,+ Name: updated.Name,+ Endpoint: updated.Endpoint,+ Bucket: updated.Bucket,+ Region: updated.Region,+ AccessKeyID: updated.AccessKeyID,+ SecretAccessKey: cmp.Or(updated.SecretAccessKey, configuration.SecretAccessKey),+ Prefix: updated.Prefix,+ UseSSL: updated.UseSSL,+ ForcePathStyle: updated.ForcePathStyle,+ })}func (s *S3DestinationService) TestS3DestinationConfiguration(ctx context.Context, input backup.CreateS3Destination) error {if err := normalization.Normalize(&input); err != nil {return err}- return s3.TestConnection(ctx, destinationConfigurationInternal("", input))+ return s3.TestConnection(ctx, destinationConfiguration("", input))}diff --git a/backend/internal/s3/service_test.go b/backend/internal/s3/service_test.goindex e4b6dc405..acb492c31 100644--- a/backend/internal/s3/service_test.go+++ b/backend/internal/s3/service_test.go@@ -1,6 +1,7 @@package s3import (+ "context""io""net/http""net/http/httptest"@@ -23,7 +24,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/utils/s3")-func setupS3DestinationServiceTestInternal(t *testing.T) (*S3DestinationService, *gorm.DB) {+func setupS3DestinationServiceTest(t *testing.T) (*S3DestinationService, *gorm.DB) {t.Helper()gormDB, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{})@@ -40,11 +41,11 @@ func setupS3DestinationServiceTestInternal(t *testing.T) (*S3DestinationService,EncryptionKey: "test-encryption-key-for-s3-destination-32bytes",Environment: "test",})- return NewS3DestinationService(&database.DB{DB: gormDB}, nil), gormDB+ return NewS3DestinationService(&database.DB{DB: gormDB}, func(context.Context, string) error { return nil }), gormDB}func TestS3DestinationService_CRUD(t *testing.T) {- service, gormDB := setupS3DestinationServiceTestInternal(t)+ service, gormDB := setupS3DestinationServiceTest(t)ctx := t.Context()created, err := service.CreateS3Destination(ctx, backup.CreateS3Destination{@@ -99,7 +100,7 @@ func TestS3DestinationService_CRUD(t *testing.T) {}func TestS3DestinationService_DeleteRejectsConfiguredDestination(t *testing.T) {- service, gormDB := setupS3DestinationServiceTestInternal(t)+ service, gormDB := setupS3DestinationServiceTest(t)ctx := t.Context()destination, err := service.CreateS3Destination(ctx, backup.CreateS3Destination{Name: "In use",@@ -117,7 +118,7 @@ func TestS3DestinationService_DeleteRejectsConfiguredDestination(t *testing.T) {}func TestS3DestinationService_SyncS3Destinations(t *testing.T) {- service, gormDB := setupS3DestinationServiceTestInternal(t)+ service, gormDB := setupS3DestinationServiceTest(t)ctx := t.Context()legacy, err := service.CreateS3Destination(ctx, backup.CreateS3Destination{@@ -213,7 +214,7 @@ func TestS3DestinationService_TestS3DestinationRoundTrip(t *testing.T) {}))defer server.Close()- service, _ := setupS3DestinationServiceTestInternal(t)+ service, _ := setupS3DestinationServiceTest(t)require.NoError(t, service.TestS3DestinationConfiguration(t.Context(), backup.CreateS3Destination{Name: "Unsaved destination",Endpoint: server.URL,@@ -344,7 +345,7 @@ func TestS3DestinationService_TestS3DestinationRoundTrip(t *testing.T) {}func TestDestinationsByID(t *testing.T) {- service, db := setupS3DestinationServiceTestInternal(t)+ service, db := setupS3DestinationServiceTest(t)require.Empty(t, service.DestinationsByID(t.Context()))destination := S3Destination{Name: "Offsite", Bucket: "backups", SecretAccessKey: "encrypted"}require.NoError(t, db.Create(&destination).Error)@@ -354,8 +355,7 @@ func TestDestinationsByID(t *testing.T) {require.Equal(t, "backups", indexed[destination.ID].Bucket)require.True(t, indexed[destination.ID].SecretConfigured)require.Empty(t, indexed["missing"].Name)- // Listing failures and a missing service leave records undecorated rather than failing.+ // Listing failures leave records undecorated rather than failing.require.NoError(t, db.Migrator().DropTable(&S3Destination{}))require.Nil(t, service.DestinationsByID(t.Context()))- require.Nil(t, (*S3DestinationService)(nil).DestinationsByID(t.Context()))}diff --git a/backend/internal/system/children/backup/children/snapshots/service.go b/backend/internal/system/children/backup/children/snapshots/service.goindex e93ffe36a..697caac22 100644--- a/backend/internal/system/children/backup/children/snapshots/service.go+++ b/backend/internal/system/children/backup/children/snapshots/service.go@@ -192,8 +192,11 @@ func (session systemBackupSnapshotSession) inspectProjectSnapshot(ctx context.Coif err != nil {return systemBackupSnapshot{}, fmt.Errorf("inspect %s project files: %w", location.name, err)}- snapshot.entries = projectEntriesFromSnapshot(listed, snapshot.layout, "", true)- return snapshot, nil+ return systemBackupSnapshot{+ systemBackupSnapshotLocation: snapshot.systemBackupSnapshotLocation,+ layout: snapshot.layout,+ entries: projectEntriesFromSnapshot(listed, snapshot.layout, "", true),+ }, nil}// inspectProjectManifest opens a snapshot for project browsing;@@ -319,12 +322,12 @@ func (session systemBackupSnapshotSession) openSafetySnapshot(systemBackupSafetySnapshot,error,) {- location := source.systemBackupSnapshotLocation- location.name = "safety"- if source.destination == backuptypes.SystemBackupDestinationS3 {- location.snapshotID = safetyRun.RemoteSnapshotID- } else {- location.snapshotID = safetyRun.LocalSnapshotID+ location := systemBackupSnapshotLocation{ //exhaustruct:enforce+ name: "safety",+ destination: source.destination,+ s3DestinationID: source.s3DestinationID,+ repository: source.repository,+ snapshotID: kit.Ternary(source.destination == backuptypes.SystemBackupDestinationS3, safetyRun.RemoteSnapshotID, safetyRun.LocalSnapshotID),}if location.snapshotID == "" {return systemBackupSafetySnapshot{}, errors.New("pre-restore system backup has no snapshot in the selected repository")@@ -337,7 +340,11 @@ func (session systemBackupSnapshotSession) openSafetySnapshot(for _, entry := range snapshot.entries {paths[entry.Path] = struct{}{}}- return systemBackupSafetySnapshot{systemBackupSnapshotLocation: location, layout: snapshot.layout, paths: paths}, nil+ return systemBackupSafetySnapshot{+ systemBackupSnapshotLocation: location,+ layout: snapshot.layout,+ paths: paths,+ }, nil}func (session systemBackupSnapshotSession) restoreEntry(@@ -437,9 +444,7 @@ func (session systemBackupSnapshotSession) restoreSelected(restored := make([]backuptypes.BackupFileEntry, 0, len(selected))for _, selectedEntry := range selected {if err := session.restoreEntry(ctx, snapshots, selectedEntry, destination); err != nil {- affected := slices.Clone(restored)- affected = append(affected, selectedEntry)- rollbackErr := session.rollback(context.WithoutCancel(ctx), safety, affected, destination)+ rollbackErr := session.rollback(context.WithoutCancel(ctx), safety, append(slices.Clone(restored), selectedEntry), destination)restoreErr := fmt.Errorf("failed to restore project path %s from system backup: %w", selectedEntry.Path, err)if rollbackErr != nil {return errors.Join(restoreErr, fmt.Errorf("failed to roll back project files from pre-restore system backup: %w", rollbackErr))@@ -492,12 +497,11 @@ func (s *Service) RestoreFiles(// The safety backup mirrors the restore source so the rollback snapshot// lives in the repository the restore is already reading from.source := snapshots[0]- safetyRequest := backuptypes.CreateSystemBackupRequest{Destination: backuptypes.SystemBackupDestinationLocal, RecoveryKey: key}- if source.destination == backuptypes.SystemBackupDestinationS3 {- safetyRequest.Destination = backuptypes.SystemBackupDestinationS3- safetyRequest.S3DestinationID = source.s3DestinationID- }- safetyRun, err := createSafety(ctx, safetyRequest)+ safetyRun, err := createSafety(ctx, backuptypes.CreateSystemBackupRequest{+ Destination: kit.Ternary(source.destination == backuptypes.SystemBackupDestinationS3, backuptypes.SystemBackupDestinationS3, backuptypes.SystemBackupDestinationLocal),+ S3DestinationID: kit.Ternary(source.destination == backuptypes.SystemBackupDestinationS3, source.s3DestinationID, ""),+ RecoveryKey: key,+ })if err != nil {return fmt.Errorf("failed to create pre-restore system backup: %w", err)}@@ -552,7 +556,12 @@ func (s *Service) PlanRollback(return nil, err}session := s.snapshotSession(dockerClient, key)- safetyLocation := systemBackupSnapshotLocation{name: "safety", destination: backuptypes.SystemBackupDestinationLocal, repository: localRepository, snapshotID: localSnapshotID}+ safetyLocation := systemBackupSnapshotLocation{+ name: "safety",+ destination: backuptypes.SystemBackupDestinationLocal,+ repository: localRepository,+ snapshotID: localSnapshotID,+ }safetyLayout, err := session.readSnapshotLayout(ctx, safetyLocation)if err != nil {return nil, fmt.Errorf("open pre-restore system backup: %w", err)@@ -631,38 +640,24 @@ func sourceMounts(mounts []container.MountPoint, inContainer bool, containerPathif primary == nil {return nil, fmt.Errorf("%s must be mounted into the Arcane container from a bind or named volume for system backups", containerPath)}- result := append([]mount.Mount{*primary}, docker.NestedMounts(mounts, containerPath, target)...)- for index := range result {- result[index].ReadOnly = true+ result := make([]mount.Mount, 0, 1+len(docker.NestedMounts(mounts, containerPath, target)))+ for _, resolved := range append([]mount.Mount{*primary}, docker.NestedMounts(mounts, containerPath, target)...) {+ result = append(result, mount.Mount{ //exhaustruct:enforce+ Type: resolved.Type,+ Source: resolved.Source,+ Target: resolved.Target,+ ReadOnly: true,+ Consistency: resolved.Consistency,+ BindOptions: resolved.BindOptions,+ VolumeOptions: resolved.VolumeOptions,+ ImageOptions: resolved.ImageOptions,+ TmpfsOptions: resolved.TmpfsOptions,+ ClusterOptions: resolved.ClusterOptions,+ })}return result, nil}-// bindVolumeSubpaths rewrites volume subpath mounts as binds of the-// subpath on daemons before API 1.45, which ignore subpaths and mount the-// volume root instead.-func bindVolumeSubpaths(ctx context.Context, dockerClient *client.Client, mounts []mount.Mount) error {- if compat.IsDockerAPIVersionAtLeast(compat.DetectDockerAPIVersion(ctx, dockerClient), "1.45") {- return nil- }- for index, resolved := range mounts {- if resolved.Type != mount.TypeVolume || resolved.VolumeOptions == nil || resolved.VolumeOptions.Subpath == "" {- continue- }- inspect, err := dockerClient.VolumeInspect(ctx, resolved.Source, client.VolumeInspectOptions{})- if err != nil {- return fmt.Errorf("inspect volume %s for system backup: %w", resolved.Source, err)- }- mounts[index] = mount.Mount{- Type: mount.TypeBind,- Source: filepath.Join(inspect.Volume.Mountpoint, resolved.VolumeOptions.Subpath),- Target: resolved.Target,- ReadOnly: resolved.ReadOnly,- }- }- return nil-}-// nestedDataPath returns the data directory relative to the projects// directory when the projects mount contains it on the host: the same bind// path prefix or the same named volume and subpath. Mounting one host@@ -678,14 +673,11 @@ func nestedDataPath(mounts []container.MountPoint, dataDirectory, projectsDirectif resolved == nil {return "", false, nil}- source := path.Clean(filepath.ToSlash(resolved.Source))- if resolved.Type == mount.TypeVolume {- source = "volume:" + resolved.Source- if resolved.VolumeOptions != nil && resolved.VolumeOptions.Subpath != "" {- source = path.Join(source, resolved.VolumeOptions.Subpath)- }- }- sources = append(sources, source)+ sources = append(sources, kit.Ternary(+ resolved.Type == mount.TypeVolume,+ path.Join("volume:"+resolved.Source, kit.FromPtr(resolved.VolumeOptions).Subpath),+ path.Clean(filepath.ToSlash(resolved.Source)),+ ))}projectsSource, dataSource := sources[0], sources[1]if !kit.FilePathMatches(dataSource, projectsSource) {@@ -707,42 +699,54 @@ func (s *Service) backupSourceLayout(ctx context.Context, dockerClient *client.Cif err != nil {return backupSourceLayout{}, err}- layout := backupSourceLayout{- dataDirectory: filepath.Dir(databaseFile),- projectsDirectory: s.projectsDirectory(ctx),- databaseName: filepath.Base(databaseFile),- sources: []string{snapshotDataPath},+ dataDirectory, projectsDirectory, databaseName := filepath.Dir(databaseFile), s.projectsDirectory(ctx), filepath.Base(databaseFile)+ excludes := []string{".arcane-snapshot-*", RecoveryRequestName}+ for _, name := range []string{databaseName, francis.StorePath(databaseName)} {+ for _, sidecar := range sqliteSidecars {+ excludes = append(excludes, name+sidecar)+ }}- projectsPath, external, err := projectsSnapshotPath(layout.dataDirectory, layout.projectsDirectory)+ projectsPath, external, err := projectsSnapshotPath(dataDirectory, projectsDirectory)if err != nil {return backupSourceLayout{}, err}- layout.projectsPath = projectsPathmounts, inContainer, err := currentMounts(ctx, dockerClient)if err != nil {return backupSourceLayout{}, err}- layout.mounts, err = sourceMounts(mounts, inContainer, layout.dataDirectory, snapshotDataPath)+ dataMounts, err := sourceMounts(mounts, inContainer, dataDirectory, snapshotDataPath)if err != nil {return backupSourceLayout{}, fmt.Errorf("resolve Arcane data source: %w", err)}if !external {- return layout, nil- }- projectMounts, err := sourceMounts(mounts, inContainer, layout.projectsDirectory, snapshotProjectsPath)+ return backupSourceLayout{+ dataDirectory: dataDirectory,+ projectsDirectory: projectsDirectory,+ databaseName: databaseName,+ projectsPath: projectsPath,+ mounts: dataMounts,+ sources: []string{snapshotDataPath},+ excludes: excludes,+ }, nil+ }+ projectMounts, err := sourceMounts(mounts, inContainer, projectsDirectory, snapshotProjectsPath)if err != nil {return backupSourceLayout{}, fmt.Errorf("resolve projects source: %w", err)}- layout.mounts = append(layout.mounts, projectMounts...)- layout.sources = append(layout.sources, snapshotProjectsPath)- relative, nested, err := nestedDataPath(mounts, layout.dataDirectory, layout.projectsDirectory)+ relative, nested, err := nestedDataPath(mounts, dataDirectory, projectsDirectory)if err != nil {return backupSourceLayout{}, err}- if nested {- layout.nestedDataPath = path.Join(snapshotProjectsPath, relative)- }- return layout, nil+ return backupSourceLayout{+ dataDirectory: dataDirectory,+ projectsDirectory: projectsDirectory,+ databaseName: databaseName,+ projectsPath: projectsPath,+ nestedDataPath: kit.Ternary(nested, path.Join(snapshotProjectsPath, relative), ""),+ mounts: append(dataMounts, projectMounts...),+ sources: []string{snapshotDataPath, snapshotProjectsPath},+ excludes: excludes,+ }, nil}var (@@ -840,16 +844,18 @@ func snapshotLayoutFromManifest(manifest recovery.Manifest, manifestRoot string)if err != nil {return snapshotLayout{}, err}- layout := snapshotLayout{dataPath: manifestRoot, databaseName: path.Base(databasePath)}relative, err := projectsRelativePathFromManifest(manifest)if errors.Is(err, errProjectsOutsideData) {- return layout, nil+ return snapshotLayout{dataPath: manifestRoot, databaseName: path.Base(databasePath)}, nil}if err != nil {return snapshotLayout{}, fmt.Errorf("resolve projects directory: %w", err)}- layout.projectsPath = path.Join(manifestRoot, relative)- return layout, nil+ return snapshotLayout{+ dataPath: manifestRoot,+ projectsPath: path.Join(manifestRoot, relative),+ databaseName: path.Base(databasePath),+ }, nilcase recovery.ManifestFormatVersion:dataPath, err := kit.NormalizeRelativePath(strings.TrimPrefix(manifest.DataPath, "/"))if err != nil {@@ -955,9 +961,10 @@ func restoreTarget(mounts []container.MountPoint, containerPath, target, excludecandidates := slices.DeleteFunc(slices.Clone(mounts), func(m container.MountPoint) bool {return exclude != "" && kit.FilePathMatches(m.Destination, exclude)})- result := recovery.RestoreTarget{Mounts: []mount.Mount{*enclosing}, Path: destination}- result.Mounts = append(result.Mounts, docker.NestedMounts(candidates, containerPath, destination)...)- return result, nil+ return recovery.RestoreTarget{+ Mounts: append([]mount.Mount{*enclosing}, docker.NestedMounts(candidates, containerPath, destination)...),+ Path: destination,+ }, nil}// projectsRestoreDestination is the current projects directory as a@@ -975,11 +982,17 @@ func (s *Service) projectsRestoreDestination(ctx context.Context, dockerClient *return projectsRestoreDestination{}, err}if !inContainer {- hostTarget := recovery.RestoreTarget{- Mounts: []mount.Mount{{Type: mount.TypeBind, Source: directory, Target: selectiveProjectsRestoreTarget}},- Path: selectiveProjectsRestoreTarget,- }- return projectsRestoreDestination{directory: directory, target: hostTarget}, nil+ return projectsRestoreDestination{+ directory: directory,+ target: recovery.RestoreTarget{+ Mounts: []mount.Mount{{+ Type: mount.TypeBind,+ Source: directory,+ Target: selectiveProjectsRestoreTarget,+ }},+ Path: selectiveProjectsRestoreTarget,+ },+ }, nil}target, err := restoreTarget(mounts, directory, selectiveProjectsRestoreTarget, "")if err != nil {@@ -989,12 +1002,15 @@ func (s *Service) projectsRestoreDestination(ctx context.Context, dockerClient *if err != nil {return projectsRestoreDestination{}, err}- dataDirectory := filepath.Dir(databaseFile)- dataPath, _, err := nestedDataPath(mounts, dataDirectory, directory)+ dataPath, _, err := nestedDataPath(mounts, filepath.Dir(databaseFile), directory)if err != nil {return projectsRestoreDestination{}, err}- return projectsRestoreDestination{directory: directory, dataPath: dataPath, target: target}, nil+ return projectsRestoreDestination{+ directory: directory,+ dataPath: dataPath,+ target: target,+ }, nil}// restoreStages plans the Rustic restores that put a snapshot's data@@ -1032,8 +1048,12 @@ func restoreStages(projectsDirectory, dataDirectory,)}- stages := make([]recovery.RestoreStage, 0, 2)- stages = append(stages, recovery.RestoreStage{Repository: repository, SnapshotID: snapshotID, SourcePath: layout.dataPath, Target: dataTarget})+ stages := []recovery.RestoreStage{{+ Repository: repository,+ SnapshotID: snapshotID,+ SourcePath: layout.dataPath,+ Target: dataTarget,+ }}if !separate {return stages, nil}@@ -1041,7 +1061,12 @@ func restoreStages(if err != nil {return nil, err}- return append(stages, recovery.RestoreStage{Repository: repository, SnapshotID: snapshotID, SourcePath: layout.projectsPath, Target: projectsTarget}), nil+ return append(stages, recovery.RestoreStage{+ Repository: repository,+ SnapshotID: snapshotID,+ SourcePath: layout.projectsPath,+ Target: projectsTarget,+ }), nil}// Create archives a consistent system snapshot without blocking actor leases during the upload.@@ -1091,12 +1116,6 @@ func (s *Service) Create(return backup.Snapshot{}, fmt.Errorf("failed to write recovery manifest: %w", err)}defer func() { _ = os.Remove(manifestPath) }()- layout.excludes = []string{".arcane-snapshot-*", RecoveryRequestName}- for name := range databases {- for _, sidecar := range sqliteSidecars {- layout.excludes = append(layout.excludes, name+sidecar)- }- }files, err := snapshotSourceFiles(ctx, layout)if err != nil {return backup.Snapshot{}, err@@ -1106,6 +1125,7 @@ func (s *Service) Create(return backup.Snapshot{}, err}// Mount VACUUM INTO copies over the live databases so the snapshot is consistent.+ snapshotMounts := slices.Clone(layout.mounts)for name, db := range databases {stagedDatabase := filepath.Join(stage, name)if stageErr := stageDatabase(ctx, db, filepath.Join(layout.dataDirectory, name), stagedDatabase); stageErr != nil {@@ -1115,19 +1135,40 @@ func (s *Service) Create(if mountErr != nil {return backup.Snapshot{}, mountErr}- layout.mounts = append(layout.mounts, stagedMounts...)- }- if bindErr := bindVolumeSubpaths(ctx, dockerClient, layout.mounts); bindErr != nil {- return backup.Snapshot{}, bindErr+ snapshotMounts = append(snapshotMounts, stagedMounts...)+ }+ // Rewrite volume subpath mounts as binds of the+ // subpath on daemons before API 1.45, which ignore subpaths and mount the+ // volume root instead.+ if !compat.IsDockerAPIVersionAtLeast(compat.DetectDockerAPIVersion(ctx, dockerClient), "1.45") {+ for index, resolved := range snapshotMounts {+ if resolved.Type != mount.TypeVolume || kit.FromPtr(resolved.VolumeOptions).Subpath == "" {+ continue+ }+ inspect, inspectErr := dockerClient.VolumeInspect(ctx, resolved.Source, client.VolumeInspectOptions{})+ if inspectErr != nil {+ return backup.Snapshot{}, fmt.Errorf("inspect volume %s for system backup: %w", resolved.Source, inspectErr)+ }+ snapshotMounts[index] = mount.Mount{+ Type: mount.TypeBind,+ Source: filepath.Join(inspect.Volume.Mountpoint, resolved.VolumeOptions.Subpath),+ Target: resolved.Target,+ ReadOnly: resolved.ReadOnly,+ }+ }}- input := backup.CreateSnapshotInput{Mounts: layout.mounts, Sources: layout.sources}+ globs := make([]string, 0, len(layout.excludes)+1)for _, excluded := range layout.excludes {- input.Globs = append(input.Globs, "!"+snapshotDataPath+"/"+excluded)+ globs = append(globs, "!"+snapshotDataPath+"/"+excluded)}if layout.nestedDataPath != "" {- input.Globs = append(input.Globs, "!"+layout.nestedDataPath)+ globs = append(globs, "!"+layout.nestedDataPath)}- snapshot, err := s.engine.CreateSnapshot(ctx, dockerClient, repository, recoveryKey, "arcane-system-recovery", input)+ snapshot, err := s.engine.CreateSnapshot(ctx, dockerClient, repository, recoveryKey, "arcane-system-recovery", backup.CreateSnapshotInput{+ Mounts: snapshotMounts,+ Sources: layout.sources,+ Globs: globs,+ })if err != nil {return backup.Snapshot{}, err}@@ -1150,11 +1191,11 @@ func (s *Service) Create(}}if err == nil {- var confirmed backup.Snapshot- confirmed, err = s.engine.ConfirmRunSnapshot(ctx, dockerClient, repository, recoveryKey, backupID, snapshot.ID)- if err == nil {+ confirmed, confirmErr := s.engine.ConfirmRunSnapshot(ctx, dockerClient, repository, recoveryKey, backupID, snapshot.ID)+ if confirmErr == nil {return confirmed, nil}+ err = confirmErr}// Tagging changes the ID; only remove the original, unconfirmed snapshot.cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Minute)@@ -1171,9 +1212,6 @@ func (s *Service) Create(// project folders hold live container data.func snapshotSourceFiles(ctx context.Context, layout backupSourceLayout) (map[string]os.FileInfo, error) {projectsInfo, statErr := os.Stat(layout.projectsDirectory)- if statErr != nil {- projectsInfo = nil- }actorDatabase := francis.StorePath(layout.databaseName)files := make(map[string]os.FileInfo)visit := func(filePath string, entry fs.DirEntry, walkErr error) error {@@ -1207,7 +1245,7 @@ func snapshotSourceFiles(ctx context.Context, layout backupSourceLayout) (map[stif err != nil {return err}- if entry.IsDir() && relative != "." && projectsInfo != nil && os.SameFile(info, projectsInfo) {+ if entry.IsDir() && relative != "." && statErr == nil && os.SameFile(info, projectsInfo) {return filepath.SkipDir}files[filePath] = infodiff --git a/backend/internal/system/children/backup/children/volumes/handler.go b/backend/internal/system/children/backup/children/volumes/handler.goindex 243f69b7c..05d48376e 100644--- a/backend/internal/system/children/backup/children/volumes/handler.go+++ b/backend/internal/system/children/backup/children/volumes/handler.go@@ -2,11 +2,12 @@ package volumesimport ("context"- "errors""github.com/danielgtaylor/huma/v2""github.com/getarcaneapp/arcane/types/v2/backup"+ kit "go.getarcane.app/kit/pkg"+ "github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/pkg/utils""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil")@@ -66,20 +67,14 @@ func (h *Handler) ListSystemVolumeOptions(ctx context.Context, _ *struct{}) (*Sy}func (h *Handler) RunSystemVolumeBackups(ctx context.Context, input *RunSystemVolumeBackupsInput) (*SystemVolumeBackupRunOutput, error) {- request := backup.RunSystemVolumeBackupsRequest{}- if input.Body != nil {- request = *input.Body- }user, err := handlerutil.RequireUser(ctx)if err != nil {return nil, err}- result, err := h.service.Start(utils.ActivityRuntimeContext(ctx, h.appCtx), *user, request)- if errors.Is(err, h.service.alreadyRunning) {- return nil, huma.Error409Conflict(err.Error())- }+ result, err := h.service.Start(utils.ActivityRuntimeContext(ctx, h.appCtx), *user, kit.FromPtr(input.Body))if err != nil {- return nil, huma.Error500InternalServerError(err.Error())+ apiErr := common.ToAPIError(err)+ return nil, huma.NewError(apiErr.HTTPStatus(), apiErr.Message)}return &SystemVolumeBackupRunOutput{Body: *result}, nil}diff --git a/backend/internal/system/children/backup/children/volumes/service.go b/backend/internal/system/children/backup/children/volumes/service.goindex ba9047407..35e54f236 100644--- a/backend/internal/system/children/backup/children/volumes/service.go+++ b/backend/internal/system/children/backup/children/volumes/service.go@@ -1,6 +1,7 @@package volumesimport (+ "cmp""context""encoding/json/v2""errors"@@ -98,15 +99,20 @@ func NewService(deps Dependencies) *Service {func (s *Service) RegisterWorkflows(engine *flow.Engine) error {var err errors.flow = engine- template := activitylib.StartRequest{Type: activitytypes.TypeResourceAction, EnvironmentID: "0", ResourceType: new("system_backup"), ResourceID: new("volumes"), ResourceName: new("Volumes")}s.backupWorkflow, err = engine.Define(flow.Definition{Name: "system-volume-backup",Version: 1,Fingerprint: "9b4a5e46a984446bc08df39a71e563e031548db3af1825a2413e8c96515db6ab",Concurrency: 1,Timeout: 24 * time.Hour,- Activity: template,- Labels: map[string]string{"backup": "Backing up volumes"},+ Activity: activitylib.StartRequest{+ Type: activitytypes.TypeResourceAction,+ EnvironmentID: "0",+ ResourceType: new("system_backup"),+ ResourceID: new("volumes"),+ ResourceName: new("Volumes"),+ },+ Labels: map[string]string{"backup": "Backing up volumes"},Steps: []workflow.StepSpec{workflow.Step("backup", engine.Handler(func(ctx context.Context, t flow.Task) (any, error) {var input selectionInputif decodeErr := t.Payload(&input); decodeErr != nil {@@ -122,15 +128,23 @@ func (s *Service) RegisterWorkflows(engine *flow.Engine) error {if err != nil {return err}- template.StartedBy, template.Step, template.LatestMessage = &usertypes.SystemUser, "Backing up volumes", "Creating scheduled system-managed volume backups"s.policyWorkflow, err = engine.Define(flow.Definition{Name: "system-volume-backup-policy",Version: 1,Fingerprint: "94a13d1bac530e6eddcb73c27990a07d69915dc947ce5b559166eb3fb85e39ef",Concurrency: 1,Timeout: 24 * time.Hour,- Activity: template,- Labels: map[string]string{"backup": "Backing up volumes"},+ Activity: activitylib.StartRequest{+ Type: activitytypes.TypeResourceAction,+ EnvironmentID: "0",+ ResourceType: new("system_backup"),+ ResourceID: new("volumes"),+ ResourceName: new("Volumes"),+ StartedBy: &usertypes.SystemUser,+ Step: "Backing up volumes",+ LatestMessage: "Creating scheduled system-managed volume backups",+ },+ Labels: map[string]string{"backup": "Backing up volumes"},Steps: []workflow.StepSpec{workflow.Step("backup", engine.Handler(func(ctx context.Context, t flow.Task) (any, error) {previous, _ := jobcontext.Run(ctx)if slices.ContainsFunc(previous.Outcome.Targets, func(target scheduler.TargetOutcome) bool { return target.ID == systemVolumePlanTarget }) {@@ -147,21 +161,15 @@ func (s *Service) RegisterWorkflows(engine *flow.Engine) error {}func (s *Service) loadSystemVolumeBackupPolicies() (*backuptypes.SystemVolumeBackupPolicyCollection, error) {- collection := &backuptypes.SystemVolumeBackupPolicyCollection{Policies: []backuptypes.SystemVolumeBackupPolicy{}}- if s.settings == nil {- return collection, nil- }- raw := strings.TrimSpace(s.settings.GetSettingsConfig().SystemVolumeBackupConfig.Value)- if raw == "" {- return collection, nil- }- if err := json.Unmarshal([]byte(raw), collection); err != nil {- return nil, fmt.Errorf("decode system-managed volume backup policies: %w", err)- }- if collection.Policies == nil {- collection.Policies = []backuptypes.SystemVolumeBackupPolicy{}+ var collection backuptypes.SystemVolumeBackupPolicyCollection+ if raw := strings.TrimSpace(s.settings.GetSettingsConfig().SystemVolumeBackupConfig.Value); raw != "" {+ if err := json.Unmarshal([]byte(raw), &collection); err != nil {+ return nil, fmt.Errorf("decode system-managed volume backup policies: %w", err)+ }}- return collection, nil+ return &backuptypes.SystemVolumeBackupPolicyCollection{+ Policies: kit.Ternary(collection.Policies == nil, []backuptypes.SystemVolumeBackupPolicy{}, collection.Policies),+ }, nil}func (s *Service) systemVolumeBackupPolicy(policyID string) (*backuptypes.SystemVolumeBackupPolicy, error) {@@ -182,34 +190,57 @@ func (s *Service) GetConfig(ctx context.Context) (*backuptypes.SystemVolumeBackureturn nil, err}destinations := s.s3Destinations.DestinationsByID(ctx)- for i := range collection.Policies {- policy := &collection.Policies[i]- policy.S3DestinationName = destinations[policy.S3DestinationID].Name- if s.db == nil {- continue- }+ policies := make([]backuptypes.SystemVolumeBackupPolicy, 0, len(collection.Policies))+ for _, policy := range collection.Policies {var lastRun volume.VolumeBackuprunErr := s.db.WithContext(ctx).Where("policy_id LIKE ?", backuptypes.SystemVolumePolicyPrefix+policy.ID+":%").Order("created_at DESC").First(&lastRun).Error- if runErr == nil {- policy.LastRun = &backuptypes.SystemBackupRun{- ID: lastRun.ID, Size: lastRun.Size, CreatedAt: lastRun.CreatedAt, Status: string(lastRun.Status),- Trigger: string(lastRun.Trigger), Destination: backuptypes.SystemBackupDestination(lastRun.Destination),- LocalSnapshotID: lastRun.LocalSnapshotID, RemoteSnapshotID: lastRun.RemoteSnapshotID,- S3DestinationID: lastRun.S3DestinationID, S3DestinationName: destinations[lastRun.S3DestinationID].Name,- PolicyID: lastRun.PolicyID, Error: lastRun.Error,- }- } else if !errors.Is(runErr, gorm.ErrRecordNotFound) {+ if runErr != nil && !errors.Is(runErr, gorm.ErrRecordNotFound) {return nil, fmt.Errorf("load latest system-managed volume backup: %w", runErr)}+ policies = append(policies, backuptypes.SystemVolumeBackupPolicy{ //exhaustruct:enforce+ ID: policy.ID,+ Enabled: policy.Enabled,+ Schedule: policy.Schedule,+ RetentionCount: policy.RetentionCount,+ StopContainers: policy.StopContainers,+ LocalEnabled: policy.LocalEnabled,+ S3Enabled: policy.S3Enabled,+ S3DestinationID: policy.S3DestinationID,+ S3DestinationName: destinations[policy.S3DestinationID].Name,+ SelectionMode: policy.SelectionMode,+ VolumeNames: policy.VolumeNames,+ IgnoreAnonymous: policy.IgnoreAnonymous,+ LastRun: func() *backuptypes.SystemBackupRun {+ if runErr != nil {+ return policy.LastRun+ }+ return &backuptypes.SystemBackupRun{+ ID: lastRun.ID,+ Size: lastRun.Size,+ CreatedAt: lastRun.CreatedAt,+ Status: string(lastRun.Status),+ Trigger: string(lastRun.Trigger),+ Destination: backuptypes.SystemBackupDestination(lastRun.Destination),+ LocalSnapshotID: lastRun.LocalSnapshotID,+ RemoteSnapshotID: lastRun.RemoteSnapshotID,+ S3DestinationID: lastRun.S3DestinationID,+ S3DestinationName: destinations[lastRun.S3DestinationID].Name,+ PolicyID: lastRun.PolicyID,+ Error: lastRun.Error,+ }+ }(),+ })}- return collection, nil+ return &backuptypes.SystemVolumeBackupPolicyCollection{Policies: policies}, nil}func (s *Service) normalizeSystemVolumePolicyUpdate(ctx context.Context, input backuptypes.UpdateSystemVolumeBackupPolicy) (backuptypes.SystemVolumeBackupPolicy, error) {- mode := input.SelectionMode- if mode != backuptypes.SystemVolumeSelectionAll && mode != backuptypes.SystemVolumeSelectionAllowlist && mode != backuptypes.SystemVolumeSelectionBlocklist {+ if !slices.Contains(+ []backuptypes.SystemVolumeSelectionMode{backuptypes.SystemVolumeSelectionAll, backuptypes.SystemVolumeSelectionAllowlist, backuptypes.SystemVolumeSelectionBlocklist},+ input.SelectionMode,+ ) {return backuptypes.SystemVolumeBackupPolicy{}, errors.New("selectionMode must be all, allowlist, or blocklist")}update, err := backup.ValidatePolicyUpdate(ctx, "system-managed volume", input.UpdateBackupPolicy, s.s3Destinations)@@ -218,26 +249,27 @@ func (s *Service) normalizeSystemVolumePolicyUpdate(ctx context.Context, input b}names := kit.Unique(kit.TrimNonEmpty(input.VolumeNames))slices.Sort(names)- if mode == backuptypes.SystemVolumeSelectionAll || names == nil {- names = []string{}- }return backuptypes.SystemVolumeBackupPolicy{- ID: input.ID, Enabled: update.Enabled, Schedule: update.Schedule, RetentionCount: update.RetentionCount,- StopContainers: update.StopContainers, LocalEnabled: update.LocalEnabled, S3Enabled: update.S3Enabled,- S3DestinationID: update.S3DestinationID, SelectionMode: mode,- VolumeNames: names, IgnoreAnonymous: input.IgnoreAnonymous,+ ID: input.ID,+ Enabled: update.Enabled,+ Schedule: update.Schedule,+ RetentionCount: update.RetentionCount,+ StopContainers: update.StopContainers,+ LocalEnabled: update.LocalEnabled,+ S3Enabled: update.S3Enabled,+ S3DestinationID: update.S3DestinationID,+ SelectionMode: input.SelectionMode,+ VolumeNames: kit.Ternary(input.SelectionMode == backuptypes.SystemVolumeSelectionAll || names == nil, []string{}, names),+ IgnoreAnonymous: input.IgnoreAnonymous,}, nil}func (s *Service) UpdateConfig(ctx context.Context, updates []backuptypes.UpdateSystemVolumeBackupPolicy) (*backuptypes.SystemVolumeBackupPolicyCollection, error) {- if s.settings == nil {- return nil, errors.New("settings service is unavailable")- }existing, err := s.loadSystemVolumeBackupPolicies()if err != nil {return nil, err}- reconcile := backup.PolicyReconciliation[backuptypes.SystemVolumeBackupPolicy, backuptypes.UpdateSystemVolumeBackupPolicy]{+ if runErr := (backup.PolicyReconciliation[backuptypes.SystemVolumeBackupPolicy, backuptypes.UpdateSystemVolumeBackupPolicy]{Domain: "system-managed volume",Existing: existing.Policies,ID: func(policy *backuptypes.SystemVolumeBackupPolicy) string { return policy.ID },@@ -250,8 +282,21 @@ func (s *Service) UpdateConfig(ctx context.Context, updates []backuptypes.Updateif normalizeErr != nil {return normalizeErr}- normalized.ID = policy.ID- *policy = normalized+ *policy = backuptypes.SystemVolumeBackupPolicy{ //exhaustruct:enforce+ ID: policy.ID,+ Enabled: normalized.Enabled,+ Schedule: normalized.Schedule,+ RetentionCount: normalized.RetentionCount,+ StopContainers: normalized.StopContainers,+ LocalEnabled: normalized.LocalEnabled,+ S3Enabled: normalized.S3Enabled,+ S3DestinationID: normalized.S3DestinationID,+ S3DestinationName: normalized.S3DestinationName,+ SelectionMode: normalized.SelectionMode,+ VolumeNames: normalized.VolumeNames,+ IgnoreAnonymous: normalized.IgnoreAnonymous,+ LastRun: normalized.LastRun,+ }return nil},Persist: func(ctx context.Context, policies []backuptypes.SystemVolumeBackupPolicy) error {@@ -265,8 +310,7 @@ func (s *Service) UpdateConfig(ctx context.Context, updates []backuptypes.Updates.jobs.Unregister(ctx, systemVolumeBackupJobPrefix+policyID)},Reschedule: s.rescheduleSystemVolumeBackup,- }- if runErr := reconcile.Run(ctx, updates); runErr != nil {+ }).Run(ctx, updates); runErr != nil {return nil, runErr}return s.GetConfig(ctx)@@ -274,9 +318,6 @@ func (s *Service) UpdateConfig(ctx context.Context, updates []backuptypes.Update// ListOptions returns live choices plus configured names that are currently unavailable.func (s *Service) ListOptions(ctx context.Context) ([]backuptypes.SystemVolumeBackupOption, error) {- if s.volumes == nil {- return nil, errors.New("volume service is unavailable")- }options, err := s.volumes.ListBackupVolumeOptions(ctx)if err != nil {return nil, err@@ -305,44 +346,53 @@ func (s *Service) ListOptions(ctx context.Context) ([]backuptypes.SystemVolumeBa// prepareSystemVolumeBackups resolves the requested policy, takes the system admission, and selects the live volumes.func (s *Service) prepareSystemVolumeBackups(ctx context.Context, request backuptypes.RunSystemVolumeBackupsRequest) (selectionInput, *runs.Lease, error) {- if s.volumes == nil {- return selectionInput{}, nil, errors.New("volume service is unavailable")- }- var input selectionInput- switch {- case request.PolicyID != "" && request.Custom != nil:+ if request.PolicyID != "" && request.Custom != nil {return selectionInput{}, nil, errors.New("select a saved policy or custom configuration, not both")- case request.PolicyID != "":- policy, err := s.systemVolumeBackupPolicy(request.PolicyID)- if err != nil {- return selectionInput{}, nil, err- }- if policy == nil {- return selectionInput{}, nil, errors.New("system-managed volume backup policy not found")- }- input.Policy = *policy- default:- custom := backuptypes.UpdateSystemVolumeBackupPolicy{- Enabled: true, Schedule: defaultSystemVolumeSchedule, LocalEnabled: true,- SelectionMode: backuptypes.SystemVolumeSelectionAll, VolumeNames: []string{}, IgnoreAnonymous: true,- }- if run := request.Custom; run != nil {- localEnabled := run.Destination == backuptypes.SystemBackupDestinationLocal || run.Destination == backuptypes.SystemBackupDestinationLocalS3- s3Enabled := run.Destination == backuptypes.SystemBackupDestinationS3 || run.Destination == backuptypes.SystemBackupDestinationLocalS3- if !localEnabled && !s3Enabled {- return selectionInput{}, nil, errors.New("destination must be local, s3, or local_s3")- }- custom = backuptypes.UpdateSystemVolumeBackupPolicy{- Enabled: true, Schedule: defaultSystemVolumeSchedule, StopContainers: run.StopContainers,- LocalEnabled: localEnabled, S3Enabled: s3Enabled, S3DestinationID: run.S3DestinationID,- SelectionMode: run.SelectionMode, VolumeNames: run.VolumeNames, IgnoreAnonymous: run.IgnoreAnonymous,+ }+ if request.Custom != nil && !slices.Contains(+ []backuptypes.SystemBackupDestination{backuptypes.SystemBackupDestinationLocal, backuptypes.SystemBackupDestinationS3, backuptypes.SystemBackupDestinationLocalS3},+ request.Custom.Destination,+ ) {+ return selectionInput{}, nil, errors.New("destination must be local, s3, or local_s3")+ }+ policy, err := func() (*backuptypes.SystemVolumeBackupPolicy, error) {+ if request.PolicyID != "" {+ stored, loadErr := s.systemVolumeBackupPolicy(request.PolicyID)+ if loadErr == nil && stored == nil {+ return nil, errors.New("system-managed volume backup policy not found")}- }- policy, err := s.normalizeSystemVolumePolicyUpdate(ctx, custom)- if err != nil {- return selectionInput{}, nil, err- }- input.Policy, input.ManualPolicy = policy, true+ return stored, loadErr+ }+ if request.Custom == nil {+ normalized, normalizeErr := s.normalizeSystemVolumePolicyUpdate(ctx, backuptypes.UpdateSystemVolumeBackupPolicy{+ UpdateBackupPolicy: backuptypes.UpdateBackupPolicy{+ Enabled: true,+ Schedule: defaultSystemVolumeSchedule,+ LocalEnabled: true,+ },+ SelectionMode: backuptypes.SystemVolumeSelectionAll,+ VolumeNames: []string{},+ IgnoreAnonymous: true,+ })+ return &normalized, normalizeErr+ }+ normalized, normalizeErr := s.normalizeSystemVolumePolicyUpdate(ctx, backuptypes.UpdateSystemVolumeBackupPolicy{+ UpdateBackupPolicy: backuptypes.UpdateBackupPolicy{+ Enabled: true,+ Schedule: defaultSystemVolumeSchedule,+ StopContainers: request.Custom.StopContainers,+ LocalEnabled: request.Custom.Destination == backuptypes.SystemBackupDestinationLocal || request.Custom.Destination == backuptypes.SystemBackupDestinationLocalS3,+ S3Enabled: request.Custom.Destination == backuptypes.SystemBackupDestinationS3 || request.Custom.Destination == backuptypes.SystemBackupDestinationLocalS3,+ S3DestinationID: request.Custom.S3DestinationID,+ },+ SelectionMode: request.Custom.SelectionMode,+ VolumeNames: request.Custom.VolumeNames,+ IgnoreAnonymous: request.Custom.IgnoreAnonymous,+ })+ return &normalized, normalizeErr+ }()+ if err != nil {+ return selectionInput{}, nil, err}lease, admitted, err := s.engine.TryAcquireRun(ctx, backup.SystemAdmissionScope, backup.SystemAdmissionID)if err != nil {@@ -356,14 +406,17 @@ func (s *Service) prepareSystemVolumeBackups(ctx context.Context, request backuplease.Release(ctx)return selectionInput{}, nil, err}- input.Candidates = selectSystemVolumeBackupCandidates(input.Policy, options)- return input, lease, nil+ return selectionInput{+ Policy: *policy,+ ManualPolicy: request.PolicyID == "",+ Candidates: selectSystemVolumeBackupCandidates(*policy, options),+ }, lease, nil}func (s *Service) executeSystemVolumeBackups(ctx context.Context, input selectionInput, trigger volume.VolumeBackupTrigger, activityID string,) (result *backuptypes.SystemVolumeBackupRunResult, err error) {- result = &backuptypes.SystemVolumeBackupRunResult{Matched: len(input.Candidates), Failures: make([]backuptypes.SystemVolumeBackupFailure, 0)}+ succeeded, failed, skipped, failures := 0, 0, 0, make([]backuptypes.SystemVolumeBackupFailure, 0)reportProgress := func(ctx context.Context) {if activityID == "" {return@@ -373,18 +426,36 @@ func (s *Service) executeSystemVolumeBackups(names[i] = candidate.Name}progress := 100- if result.Matched > 0 {- progress = 100 * (result.Succeeded + result.Failed + result.Skipped) / result.Matched- }- _, updateErr := s.activity.UpdateActivity(ctx, activityID, activitylib.UpdateRequest{Progress: &progress, Metadata: database.JSON{- "action": "run_system_volume_backups", "policyId": input.Policy.ID, "volumeNames": names,- "matched": result.Matched, "succeeded": result.Succeeded, "failed": result.Failed, "skipped": result.Skipped, "failures": result.Failures,- }})+ if len(input.Candidates) > 0 {+ progress = 100 * (succeeded + failed + skipped) / len(input.Candidates)+ }+ _, updateErr := s.activity.UpdateActivity(ctx, activityID, activitylib.UpdateRequest{+ Progress: &progress,+ Metadata: database.JSON{+ "action": "run_system_volume_backups",+ "policyId": input.Policy.ID,+ "volumeNames": names,+ "matched": len(input.Candidates),+ "succeeded": succeeded,+ "failed": failed,+ "skipped": skipped,+ "failures": failures,+ },+ })if updateErr != nil {slog.WarnContext(ctx, "Failed to report system-managed volume backup progress", "activityId", activityID, "policyId", input.Policy.ID, "error", updateErr)}}defer reportProgress(context.WithoutCancel(ctx))+ defer func() {+ result = &backuptypes.SystemVolumeBackupRunResult{+ Matched: len(input.Candidates),+ Succeeded: succeeded,+ Failed: failed,+ Skipped: skipped,+ Failures: failures,+ }+ }()defer utils.RecoverToError(&err, "system-managed volume backup")policy := backuptypes.UpdateBackupPolicy{@@ -398,23 +469,23 @@ func (s *Service) executeSystemVolumeBackups(return target.ID == candidate.Name && (target.Status == scheduler.Succeeded || target.Status == scheduler.Skipped)})if done >= 0 && previous.Outcome.Targets[done].Status == scheduler.Succeeded {- result.Succeeded+++ succeeded++continue}if done >= 0 {- result.Skipped+++ skipped++continue}if cancellationErr := ctx.Err(); cancellationErr != nil {- return result, cancellationErr+ return nil, cancellationErr}reportProgress(ctx)overridden, backupErr := s.volumes.HasEnabledBackupPolicy(ctx, candidate.Name)if backupErr == nil && overridden {if skipProgressErr := jobcontext.Progress(ctx, scheduler.TargetOutcome{ID: candidate.Name, Status: scheduler.Skipped}); skipProgressErr != nil {- return result, skipProgressErr+ return nil, skipProgressErr}- result.Skipped+++ skipped++continue}if backupErr == nil {@@ -422,24 +493,27 @@ func (s *Service) executeSystemVolumeBackups(_, backupErr = s.volumes.CreateSystemManagedBackup(ctx, candidate.Name, usertypes.SystemUser, trigger, seriesID, policy)}if errors.Is(backupErr, volume.ErrVolumeBackupAlreadyRunning) {- result.Skipped+++ skipped++continue}if backupErr != nil && ctx.Err() != nil {// Shutdown interrupted this volume; hand the task back instead of counting it failed.- return result, ctx.Err()+ return nil, ctx.Err()}if backupErr != nil {- result.Failed++- result.Failures = append(result.Failures, backuptypes.SystemVolumeBackupFailure{VolumeName: candidate.Name, Error: backupErr.Error()})+ failed+++ failures = append(failures, backuptypes.SystemVolumeBackupFailure{+ VolumeName: candidate.Name,+ Error: backupErr.Error(),+ })continue}if progressErr := jobcontext.Progress(ctx, scheduler.TargetOutcome{ID: candidate.Name, Status: scheduler.Succeeded}); progressErr != nil {- return result, progressErr+ return nil, progressErr}- result.Succeeded+++ succeeded++}- return result, nil+ return nil, nil}func (s *Service) runScheduledSystemVolumeBackup(ctx context.Context, policyID, activityID string) (scheduler.Outcome, error) {@@ -450,11 +524,13 @@ func (s *Service) runScheduledSystemVolumeBackup(ctx context.Context, policyID,if policy == nil || !policy.Enabled {return scheduler.Outcome{Status: scheduler.Skipped}, nil}- var remoteErr error- if policy.S3Enabled {+ remoteErr := func() error {+ if !policy.S3Enabled {+ return nil+ }root := path.Join(backup.VolumeRoot, s.settings.GetSettingsConfig().InstanceID.Value)- remoteErr = backup.CheckScheduledRemote(ctx, s.db, s.s3Destinations, "volume_backups", policy.S3DestinationID, root)- }+ return backup.CheckScheduledRemote(ctx, s.db, s.s3Destinations, "volume_backups", policy.S3DestinationID, root)+ }()remoteDisabled, err := backup.DisableMissingRemote(remoteErr, policy.LocalEnabled, &policy.S3Enabled, &policy.Enabled, func(field string) (bool, error) {collection, loadErr := s.loadSystemVolumeBackupPolicies()if loadErr != nil {@@ -467,8 +543,22 @@ func (s *Service) runScheduledSystemVolumeBackup(ctx context.Context, policyID,if index < 0 {return false, nil}- current := &collection.Policies[index]- *kit.Ternary(field == "enabled", ¤t.Enabled, ¤t.S3Enabled) = false+ current := collection.Policies[index]+ collection.Policies[index] = backuptypes.SystemVolumeBackupPolicy{ //exhaustruct:enforce+ ID: current.ID,+ Enabled: current.Enabled && field != "enabled",+ Schedule: current.Schedule,+ RetentionCount: current.RetentionCount,+ StopContainers: current.StopContainers,+ LocalEnabled: current.LocalEnabled,+ S3Enabled: current.S3Enabled && field != "s3_enabled",+ S3DestinationID: current.S3DestinationID,+ S3DestinationName: current.S3DestinationName,+ SelectionMode: current.SelectionMode,+ VolumeNames: current.VolumeNames,+ IgnoreAnonymous: current.IgnoreAnonymous,+ LastRun: current.LastRun,+ }encoded, encodeErr := json.Marshal(collection)if encodeErr != nil {return false, fmt.Errorf("encode policies: %w", encodeErr)@@ -484,22 +574,30 @@ func (s *Service) runScheduledSystemVolumeBackup(ctx context.Context, policyID,if remoteDisabled && !policy.LocalEnabled {return scheduler.Outcome{Status: scheduler.NeedsAttention, Message: backup.RemoteDisabledMessage}, nil}- input, lease, err := s.prepareSystemVolumeBackups(ctx, backuptypes.RunSystemVolumeBackupsRequest{PolicyID: policyID})+ result, err := func() (*backuptypes.SystemVolumeBackupRunResult, error) {+ input, lease, prepareErr := s.prepareSystemVolumeBackups(ctx, backuptypes.RunSystemVolumeBackupsRequest{PolicyID: policyID})+ if prepareErr != nil {+ return nil, prepareErr+ }+ defer lease.Release(ctx)+ frozen, marshalErr := json.Marshal(input)+ if marshalErr != nil {+ return nil, marshalErr+ }+ if progressErr := jobcontext.Progress(ctx, scheduler.TargetOutcome{+ ResourceType: "backup_plan",+ ID: systemVolumePlanTarget,+ Status: scheduler.Succeeded,+ RecoveryData: frozen,+ }); progressErr != nil {+ return nil, progressErr+ }+ return s.executeSystemVolumeBackups(ctx, input, volume.VolumeBackupTriggerScheduled, activityID)+ }()if errors.Is(err, s.alreadyRunning) {slog.InfoContext(ctx, "Scheduled system-managed volume backups skipped; a system backup is running", "policyId", policyID)return scheduler.Outcome{Status: scheduler.Skipped, Message: "Skipped: a system backup is running"}, nil}- var result *backuptypes.SystemVolumeBackupRunResult- if err == nil {- defer lease.Release(ctx)- var frozen []byte- if frozen, err = json.Marshal(input); err == nil {- err = jobcontext.Progress(ctx, scheduler.TargetOutcome{ResourceType: "backup_plan", ID: systemVolumePlanTarget, Status: scheduler.Succeeded, RecoveryData: frozen})- }- if err == nil {- result, err = s.executeSystemVolumeBackups(ctx, input, volume.VolumeBackupTriggerScheduled, activityID)- }- }if err != nil {slog.ErrorContext(ctx, "Scheduled system-managed volume backups failed", "policyId", policyID, "error", err)return scheduler.Outcome{}, err@@ -561,48 +659,66 @@ func (s *Service) backUpSelection(ctx context.Context, input selectionInput, triif outcome.Status != scheduler.Succeeded {return nil, errors.New(outcome.Message)}- for i := range previous.Outcome.Targets {- if previous.Outcome.Targets[i].ID == candidate.Name {- previous.Outcome.Targets[i].Status = scheduler.Succeeded+ for i, target := range previous.Outcome.Targets {+ if target.ID == candidate.Name {+ previous.Outcome.Targets[i] = scheduler.TargetOutcome{ //exhaustruct:enforce+ RecoveryData: target.RecoveryData,+ ResourceType: target.ResourceType,+ ID: target.ID,+ Status: scheduler.Succeeded,+ Message: target.Message,+ ActivityID: target.ActivityID,+ }}}}- progressCtx := ctx- ctx = jobcontext.WithExecution(ctx, previous, func(target scheduler.TargetOutcome) error { return jobcontext.Progress(progressCtx, target) })- return s.executeSystemVolumeBackups(ctx, input, trigger, activityID)+ return s.executeSystemVolumeBackups(+ jobcontext.WithExecution(ctx, previous, func(target scheduler.TargetOutcome) error { return jobcontext.Progress(ctx, target) }),+ input,+ trigger,+ activityID,+ )}// selectionOutcome reports a run's result; failed volumes or disabled remote storage make it partial.func selectionOutcome(result *backuptypes.SystemVolumeBackupRunResult, remoteDisabled bool) scheduler.Outcome {- outcome := scheduler.Outcome{Status: scheduler.Succeeded, Message: "System-managed volume backups completed"}- if result.Failed > 0 {- outcome.Status, outcome.Message = scheduler.Partial, fmt.Sprintf("%d volume backups failed", result.Failed)- }- if remoteDisabled {- outcome.Status, outcome.Message = scheduler.Partial, backup.RemoteDisabledMessage- }+ var targets []scheduler.TargetOutcomefor _, failure := range result.Failures {- outcome.Targets = append(outcome.Targets, scheduler.TargetOutcome{ID: failure.VolumeName, Status: scheduler.Failed, Message: failure.Error})+ targets = append(targets, scheduler.TargetOutcome{+ ID: failure.VolumeName,+ Status: scheduler.Failed,+ Message: failure.Error,+ })+ }+ return scheduler.Outcome{+ Status: kit.Ternary(result.Failed > 0 || remoteDisabled, scheduler.Partial, scheduler.Succeeded),+ Message: cmp.Or(+ kit.Ternary(remoteDisabled, backup.RemoteDisabledMessage, ""),+ kit.Ternary(result.Failed > 0, fmt.Sprintf("%d volume backups failed", result.Failed), ""),+ "System-managed volume backups completed",+ ),+ Targets: targets,}- return outcome}func (s *Service) rescheduleSystemVolumeBackup(ctx context.Context, policy *backuptypes.SystemVolumeBackupPolicy) {- if policy == nil {- return- }- jobID := systemVolumeBackupJobPrefix + policy.IDif !policy.Enabled {- s.jobs.Unregister(ctx, jobID)+ s.jobs.Unregister(ctx, systemVolumeBackupJobPrefix+policy.ID)return}policyID := policy.IDs.jobs.Add(ctx, &flow.Job{Engine: s.flow,Workflow: s.policyWorkflow,- JobName: s.jobs.JobName(jobID),+ JobName: s.jobs.JobName(systemVolumeBackupJobPrefix + policyID),Payload: policyID,- Activity: activitylib.StartRequest{Metadata: database.JSON{"action": "scheduled_system_volume_backups", "policyId": policyID, "schedule": policy.Schedule}},+ Activity: activitylib.StartRequest{+ Metadata: database.JSON{+ "action": "scheduled_system_volume_backups",+ "policyId": policyID,+ "schedule": policy.Schedule,+ },+ },ScheduleFn: func(context.Context) string {current, err := s.systemVolumeBackupPolicy(policyID)if err != nil || current == nil {@@ -622,16 +738,21 @@ func (s *Service) Start(ctx context.Context, user usertypes.Actor, request backuif err := s.engine.Authorize(ctx, requester); err != nil {return nil, err}- input, lease, err := s.prepareSystemVolumeBackups(ctx, request)+ prepared, lease, err := s.prepareSystemVolumeBackups(ctx, request)if err != nil {return nil, err}- input.Requester = &requester- names := make([]string, len(input.Candidates))- for i, candidate := range input.Candidates {+ names := make([]string, len(prepared.Candidates))+ for i, candidate := range prepared.Candidates {names[i] = candidate.Name}- input.Claim = uuid.NewV7().String()+ input := selectionInput{+ Policy: prepared.Policy,+ ManualPolicy: prepared.ManualPolicy,+ Candidates: prepared.Candidates,+ Requester: &requester,+ Claim: uuid.NewV7().String(),+ }release := s.engine.Hold(backup.SystemAdmissionScope, backup.SystemAdmissionID, input.Claim, lease)activityID, err := s.flow.Submit(ctx, s.backupWorkflow, input, activitylib.StartRequest{StartedBy: &user, Step: "Backing up volumes", LatestMessage: "Creating system-managed volume backups",diff --git a/backend/internal/system/children/backup/handler.go b/backend/internal/system/children/backup/handler.goindex c98d526f1..e20567497 100644--- a/backend/internal/system/children/backup/handler.go+++ b/backend/internal/system/children/backup/handler.go@@ -2,7 +2,6 @@ package backupimport ("context"- "errors""github.com/danielgtaylor/huma/v2"activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"@@ -108,7 +107,7 @@ func NewHandler(service *Service, activityService *activity.ActivityService, appfunc (h *Handler) ListHistory(ctx context.Context, input *ListBackupHistoryInput) (*handlerutil.Page[backup.HistoryEntry], error) {params := handlerutil.PaginationParams(input.Start, input.Limit, input.Sort, input.Order, input.Search)- params.Filters = map[string]string{"type": input.Type}+ params.Filters["type"] = input.Typehistory, page, err := h.service.ListBackupHistory(ctx, params)if err != nil {return nil, huma.Error500InternalServerError(err.Error())@@ -222,11 +221,9 @@ func (h *Handler) Create(ctx context.Context, input *CreateSystemBackupInput) (*return nil, err}run, err := h.service.StartBackup(utils.ActivityRuntimeContext(ctx, h.appCtx), *user, input.Body)- if errors.Is(err, ErrSystemBackupAlreadyRunning) {- return nil, huma.Error409Conflict(err.Error())- }if err != nil {- return nil, huma.Error500InternalServerError(err.Error())+ apiErr := common.ToAPIError(err)+ return nil, huma.NewError(apiErr.HTTPStatus(), apiErr.Message)}return &SystemBackupOutput{Body: *run}, nil}@@ -274,14 +271,9 @@ func (h *Handler) RestoreFiles(ctx context.Context, input *RestoreSystemBackupFi}, func(activityCtx context.Context) error {return h.service.RestoreBackupFiles(activityCtx, input.ID, input.Body, *user)})- if errors.Is(err, ErrSystemBackupAlreadyRunning) {- return nil, huma.Error409Conflict(err.Error())- }- if errors.Is(err, common.ErrBadRequest) {- return nil, huma.Error400BadRequest(err.Error())- }if err != nil {- return nil, huma.Error500InternalServerError(err.Error())+ apiErr := common.ToAPIError(err)+ return nil, huma.NewError(apiErr.HTTPStatus(), apiErr.Message)}return handlerutil.MessageOutput("Arcane project files restored successfully", activityID), nil}diff --git a/backend/internal/system/children/backup/module.go b/backend/internal/system/children/backup/module.goindex b926b2882..a2baedbd7 100644--- a/backend/internal/system/children/backup/module.go+++ b/backend/internal/system/children/backup/module.go@@ -4,6 +4,7 @@ import ("net/http""github.com/danielgtaylor/huma/v2"+ kit "go.getarcane.app/kit/pkg""github.com/getarcaneapp/arcane/backend/v2/internal/middleware""github.com/getarcaneapp/arcane/backend/v2/internal/system/children/backup/children/volumes"@@ -211,9 +212,5 @@ func RegisterRoutes(api huma.API, h *Handler) {h.Delete,)- var volumeService *volumes.Service- if h.service != nil {- volumeService = h.service.volumes- }- volumes.RegisterRoutes(api, volumes.NewHandler(volumeService, h.appCtx))+ volumes.RegisterRoutes(api, volumes.NewHandler(kit.FromPtr(h.service).volumes, h.appCtx))}diff --git a/backend/internal/system/children/backup/service.go b/backend/internal/system/children/backup/service.goindex 15ace75d9..cb3522171 100644--- a/backend/internal/system/children/backup/service.go+++ b/backend/internal/system/children/backup/service.go@@ -66,7 +66,7 @@ const (systemRecoveryHelperPath = "/app/arcane-recovery-helper")-var ErrSystemBackupAlreadyRunning = errors.New("an Arcane system backup is already running")+var ErrSystemBackupAlreadyRunning = common.Classify(common.ErrConflict, errors.New("an Arcane system backup is already running"))// Store persists system backup runs and policies through their public DTOs;// the system domain owns the persistence models.@@ -167,40 +167,53 @@ type preparedSystemBackup struct {}func NewService(deps Dependencies) *Service {- service := &Service{- store: deps.Store, dockerService: deps.Docker, volumeService: deps.Volumes, engine: deps.Engine,- s3Destinations: deps.S3Destinations, activityService: deps.Activity, settingsService: deps.Settings, config: deps.Config,- recoveryKeys: deps.RecoveryKeys, resolveRuntimeOptions: deps.ResolveRuntimeOptions,- jobs: entityjobs.New("system-backup:", backup.SystemAdmissionScope),- }- var counterErr error- service.operations, counterErr = otel.Meter(tracing.InstrumentationName).Int64Counter("arcane.backups",+ operations, counterErr := otel.Meter(tracing.InstrumentationName).Int64Counter("arcane.backups",metric.WithDescription("Backup operations by outcome"), metric.WithUnit("{backup}"))if counterErr != nil {otel.Handle(counterErr)}- service.snapshots = snapshots.NewService(- deps.Engine,- deps.SQLDB,- deps.ActorStore,- service.localRepository,- service.remoteRepository,- service.databaseFile,- func(ctx context.Context) string {- return projects.ResolveConfiguredContainerDirectory(service.projectsSetting(ctx), "/app/data/projects")- },- service.recoveryEnvironment,- )- service.volumes = volumes.NewService(volumes.Dependencies{- DB: deps.DB,- Engine: deps.Engine,- Volumes: deps.Volumes,- S3Destinations: deps.S3Destinations,- Activity: deps.Activity,- Settings: deps.Settings,- Jobs: service.jobs,- AlreadyRunning: ErrSystemBackupAlreadyRunning,- })+ jobs := entityjobs.New("system-backup:", backup.SystemAdmissionScope)+ var service *Service+ service = &Service{+ store: deps.Store,+ dockerService: deps.Docker,+ volumeService: deps.Volumes,+ engine: deps.Engine,+ s3Destinations: deps.S3Destinations,+ activityService: deps.Activity,+ settingsService: deps.Settings,+ config: deps.Config,+ recoveryKeys: deps.RecoveryKeys,+ resolveRuntimeOptions: deps.ResolveRuntimeOptions,+ jobs: jobs,+ operations: operations,+ snapshots: snapshots.NewService(+ deps.Engine,+ deps.SQLDB,+ deps.ActorStore,+ func(ctx context.Context, dockerClient *client.Client, readOnly bool) (backup.Repository, error) {+ return service.localRepository(ctx, dockerClient, readOnly)+ },+ func(ctx context.Context, destinationID string) (backup.Repository, error) {+ return service.remoteRepository(ctx, destinationID)+ },+ func() (string, error) { return service.databaseFile() },+ func(ctx context.Context) string {+ return projects.ResolveConfiguredContainerDirectory(service.projectsSetting(ctx), "/app/data/projects")+ },+ func(ctx context.Context) map[string]string { return service.recoveryEnvironment(ctx) },+ ),+ volumes: volumes.NewService(volumes.Dependencies{+ DB: deps.DB,+ Engine: deps.Engine,+ Volumes: deps.Volumes,+ S3Destinations: deps.S3Destinations,+ Activity: deps.Activity,+ Settings: deps.Settings,+ Jobs: jobs,+ AlreadyRunning: ErrSystemBackupAlreadyRunning,+ }),+ }return service}@@ -274,29 +287,40 @@ func (s *Service) acquireRun(ctx context.Context) (*runs.Lease, error) {func (s *Service) RegisterWorkflows(engine *flow.Engine) error {var err errors.flow = engine- template := activitylib.StartRequest{Type: activitytypes.TypeResourceAction, EnvironmentID: "0", ResourceType: new("system_backup"), ResourceName: new("Arcane")}s.backupWorkflow, err = engine.Define(flow.Definition{Name: "system-backup",Version: 1,Fingerprint: "9b44e4c38b7fcf8951468cc43a51b66c644d1104f1a56fa49bf8c065dc51c046",Concurrency: 1,Timeout: 24 * time.Hour,- Activity: template,- Labels: map[string]string{"backup": "Creating system backup"},- Steps: []workflow.StepSpec{workflow.Step("backup", engine.Handler(s.runManualBackup), workflow.WithMaxAttempts(1))},+ Activity: activitylib.StartRequest{+ Type: activitytypes.TypeResourceAction,+ EnvironmentID: "0",+ ResourceType: new("system_backup"),+ ResourceName: new("Arcane"),+ },+ Labels: map[string]string{"backup": "Creating system backup"},+ Steps: []workflow.StepSpec{workflow.Step("backup", engine.Handler(s.runManualBackup), workflow.WithMaxAttempts(1))},})if err != nil {return err}- template.StartedBy, template.Step, template.LatestMessage = &usertypes.SystemUser, "Creating scheduled system backup", "Creating scheduled Arcane system backup"s.policyWorkflow, err = engine.Define(flow.Definition{Name: "system-backup-policy",Version: 1,Fingerprint: "42fa9de20fefac192148a5f6d68132e34ae87948160b5bb0cc5876d506843bcf",Concurrency: 1,Timeout: 24 * time.Hour,- Activity: template,- Labels: map[string]string{"backup": "Creating scheduled system backup"},+ Activity: activitylib.StartRequest{+ Type: activitytypes.TypeResourceAction,+ EnvironmentID: "0",+ ResourceType: new("system_backup"),+ ResourceName: new("Arcane"),+ StartedBy: &usertypes.SystemUser,+ Step: "Creating scheduled system backup",+ LatestMessage: "Creating scheduled Arcane system backup",+ },+ Labels: map[string]string{"backup": "Creating scheduled system backup"},Steps: []workflow.StepSpec{workflow.Step("backup", engine.Handler(func(ctx context.Context, t flow.Task) (any, error) {var policyID stringif decodeErr := t.Payload(&policyID); decodeErr != nil {@@ -402,58 +426,87 @@ func (s *Service) prepareBackup(ctx context.Context, trigger string, request bacif err != nil {return nil, err}- var checkpoint systemBackupRecovery- destinationID := strings.TrimSpace(request.S3DestinationID)- if request.PolicyID != "" {- policy, policyErr := s.store.Policy(ctx, request.PolicyID)- if policyErr != nil {- return nil, policyErr- }- if policy == nil {- return nil, errors.New("system backup policy not found")+ selected, err := func() (*backuptypes.SystemBackupPolicy, error) {+ if request.PolicyID != "" {+ policy, policyErr := s.store.Policy(ctx, request.PolicyID)+ if policyErr == nil && policy == nil {+ return nil, errors.New("system backup policy not found")+ }+ return policy, policyErr}- checkpoint.LocalEnabled, checkpoint.S3Enabled, destinationID = policy.LocalEnabled, policy.S3Enabled, policy.S3DestinationID- } else {switch request.Destination {case backuptypes.SystemBackupDestinationLocal, "":- checkpoint.LocalEnabled = true+ return &backuptypes.SystemBackupPolicy{+ LocalEnabled: true,+ S3DestinationID: strings.TrimSpace(request.S3DestinationID),+ }, nilcase backuptypes.SystemBackupDestinationS3:- checkpoint.S3Enabled = true+ return &backuptypes.SystemBackupPolicy{+ S3Enabled: true,+ S3DestinationID: strings.TrimSpace(request.S3DestinationID),+ }, nilcase backuptypes.SystemBackupDestinationLocalS3:- checkpoint.LocalEnabled, checkpoint.S3Enabled = true, true+ return &backuptypes.SystemBackupPolicy{+ LocalEnabled: true,+ S3Enabled: true,+ S3DestinationID: strings.TrimSpace(request.S3DestinationID),+ }, nildefault:return nil, errors.New("unknown system backup destination")}+ }()+ if err != nil {+ return nil, err}- if !checkpoint.LocalEnabled && !checkpoint.S3Enabled {+ if !selected.LocalEnabled && !selected.S3Enabled {return nil, errors.New("select at least one system backup destination")}- if checkpoint.S3Enabled && destinationID == "" {+ if selected.S3Enabled && selected.S3DestinationID == "" {return nil, errors.New("select an S3 destination for the system backup")}- destination := backuptypes.SystemBackupDestinationS3- switch {- case checkpoint.LocalEnabled && checkpoint.S3Enabled:- destination = backuptypes.SystemBackupDestinationLocalS3- case checkpoint.LocalEnabled:- destination = backuptypes.SystemBackupDestinationLocal- }run := &backuptypes.SystemBackupRun{- ID: "system-" + uuid.New().String(), CreatedAt: time.Now().UTC(), Status: backuptypes.SystemBackupStatusRunning,- Trigger: trigger, Destination: destination, S3DestinationID: destinationID, PolicyID: request.PolicyID,+ ID: "system-" + uuid.New().String(),+ CreatedAt: time.Now().UTC(),+ Status: backuptypes.SystemBackupStatusRunning,+ Trigger: trigger,+ Destination: func() backuptypes.SystemBackupDestination {+ switch {+ case selected.LocalEnabled && selected.S3Enabled:+ return backuptypes.SystemBackupDestinationLocalS3+ case selected.LocalEnabled:+ return backuptypes.SystemBackupDestinationLocal+ default:+ return backuptypes.SystemBackupDestinationS3+ }+ }(),+ S3DestinationID: selected.S3DestinationID,+ PolicyID: request.PolicyID,}if createErr := s.store.CreateRun(ctx, run); createErr != nil {return nil, createErr}- checkpoint.BackupID = run.ID+ checkpoint := systemBackupRecovery{+ BackupID: run.ID,+ LocalEnabled: selected.LocalEnabled,+ S3Enabled: selected.S3Enabled,+ }recoveryData, err := json.Marshal(checkpoint)if err == nil {- err = jobcontext.Progress(ctx, scheduler.TargetOutcome{ResourceType: "system_backup", ID: cmp.Or(run.PolicyID, run.ID), Status: scheduler.Running, RecoveryData: recoveryData})+ err = jobcontext.Progress(ctx, scheduler.TargetOutcome{+ ResourceType: "system_backup",+ ID: cmp.Or(run.PolicyID, run.ID),+ Status: scheduler.Running,+ RecoveryData: recoveryData,+ })}if err != nil {return nil, err}- return &preparedSystemBackup{run: run, recoveryKey: recoveryKey, checkpoint: checkpoint}, nil+ return &preparedSystemBackup{+ run: run,+ recoveryKey: recoveryKey,+ checkpoint: checkpoint,+ }, nil}// executeBackup snapshots straight to S3 for S3-only runs, otherwise keeps a local snapshot and optionally replicates it to S3, then records the run's result.@@ -465,21 +518,42 @@ func (s *Service) executeBackup(ctx context.Context, prepared *preparedSystemBacattribute.String("arcane.backup.s3_destination_id", run.S3DestinationID))}()defer func() {- if err = cmp.Or(err, ctx.Err()); err != nil {- run.Status, run.Error = backuptypes.SystemBackupStatusFailed, err.Error()- } else {- run.Status, run.Error = backuptypes.SystemBackupStatusSucceeded, ""+ err = cmp.Or(err, ctx.Err())+ *run = backuptypes.SystemBackupRun{ //exhaustruct:enforce+ RemoteAvailable: run.RemoteAvailable,+ ActivityID: run.ActivityID,+ ID: run.ID,+ Size: run.Size,+ CreatedAt: run.CreatedAt,+ Status: kit.Ternary(err != nil, backuptypes.SystemBackupStatusFailed, backuptypes.SystemBackupStatusSucceeded),+ Trigger: run.Trigger,+ Destination: run.Destination,+ LocalSnapshotID: run.LocalSnapshotID,+ RemoteSnapshotID: run.RemoteSnapshotID,+ S3DestinationID: run.S3DestinationID,+ S3DestinationName: run.S3DestinationName,+ PolicyID: run.PolicyID,+ Error: kit.Ternary(err != nil, fmt.Sprint(err), ""),}if saveErr := s.store.SaveRun(context.WithoutCancel(ctx), run); saveErr != nil {err = errors.Join(err, fmt.Errorf("failed to save system backup result: %w", saveErr))}if err == nil {- err = jobcontext.Progress(ctx, scheduler.TargetOutcome{ResourceType: "system_backup", ID: cmp.Or(run.PolicyID, run.ID), Status: scheduler.Succeeded})+ err = jobcontext.Progress(ctx, scheduler.TargetOutcome{+ ResourceType: "system_backup",+ ID: cmp.Or(run.PolicyID, run.ID),+ Status: scheduler.Succeeded,+ })}}()defer utils.RecoverToError(&err, "system backup")progress := func(stage string, status scheduler.RunStatus, message string) error {- return jobcontext.Progress(ctx, scheduler.TargetOutcome{ResourceType: "backup_destination", ID: run.ID + ":" + stage, Status: status, Message: message})+ return jobcontext.Progress(ctx, scheduler.TargetOutcome{+ ResourceType: "backup_destination",+ ID: run.ID + ":" + stage,+ Status: status,+ Message: message,+ })}dockerClient, err := s.dockerService.GetClient(ctx)if err != nil {@@ -489,28 +563,57 @@ func (s *Service) executeBackup(ctx context.Context, prepared *preparedSystemBacreturn run, progressErr}// S3-only runs write straight to S3 so no local repository is created.- s3Only := !checkpoint.LocalEnabled- var repository backup.Repository- if s3Only {- repository, err = s.remoteRepository(ctx, run.S3DestinationID)- } else {- repository, err = s.localRepository(ctx, dockerClient, false)- }- var staged backup.Snapshot- if err == nil {- staged, err = s.snapshots.Create(ctx, dockerClient, repository, key, run.ID)+ repository, err := func() (backup.Repository, error) {+ if !checkpoint.LocalEnabled {+ return s.remoteRepository(ctx, run.S3DestinationID)+ }+ return s.localRepository(ctx, dockerClient, false)+ }()+ if err != nil {+ return run, fmt.Errorf("failed to create %s system recovery snapshot: %w", kit.Ternary(checkpoint.LocalEnabled, "local", "S3"), err)}+ staged, err := s.snapshots.Create(ctx, dockerClient, repository, key, run.ID)if err != nil {- return run, fmt.Errorf("failed to create %s system recovery snapshot: %w", kit.Ternary(s3Only, "S3", "local"), err)+ return run, fmt.Errorf("failed to create %s system recovery snapshot: %w", kit.Ternary(checkpoint.LocalEnabled, "local", "S3"), err)}if progressErr := progress("stage", scheduler.Succeeded, staged.ID); progressErr != nil {return run, progressErr}- if s3Only {- run.RemoteSnapshotID, run.Size = staged.ID, staged.Size+ if !checkpoint.LocalEnabled {+ *run = backuptypes.SystemBackupRun{ //exhaustruct:enforce+ RemoteAvailable: run.RemoteAvailable,+ ActivityID: run.ActivityID,+ ID: run.ID,+ Size: staged.Size,+ CreatedAt: run.CreatedAt,+ Status: run.Status,+ Trigger: run.Trigger,+ Destination: run.Destination,+ LocalSnapshotID: run.LocalSnapshotID,+ RemoteSnapshotID: staged.ID,+ S3DestinationID: run.S3DestinationID,+ S3DestinationName: run.S3DestinationName,+ PolicyID: run.PolicyID,+ Error: run.Error,+ }return run, s.store.SaveRun(ctx, run)}- run.LocalSnapshotID, run.Size = staged.ID, staged.Size+ *run = backuptypes.SystemBackupRun{ //exhaustruct:enforce+ RemoteAvailable: run.RemoteAvailable,+ ActivityID: run.ActivityID,+ ID: run.ID,+ Size: staged.Size,+ CreatedAt: run.CreatedAt,+ Status: run.Status,+ Trigger: run.Trigger,+ Destination: run.Destination,+ LocalSnapshotID: staged.ID,+ RemoteSnapshotID: run.RemoteSnapshotID,+ S3DestinationID: run.S3DestinationID,+ S3DestinationName: run.S3DestinationName,+ PolicyID: run.PolicyID,+ Error: run.Error,+ }if saveErr := s.store.SaveRun(ctx, run); saveErr != nil {return run, saveErr}@@ -531,7 +634,22 @@ func (s *Service) executeBackup(ctx context.Context, prepared *preparedSystemBacif err != nil {return run, fmt.Errorf("failed to create S3 system recovery snapshot: %w", err)}- run.RemoteSnapshotID, run.Size = snapshot.ID, cmp.Or(run.Size, snapshot.Size)+ *run = backuptypes.SystemBackupRun{ //exhaustruct:enforce+ RemoteAvailable: run.RemoteAvailable,+ ActivityID: run.ActivityID,+ ID: run.ID,+ Size: cmp.Or(run.Size, snapshot.Size),+ CreatedAt: run.CreatedAt,+ Status: run.Status,+ Trigger: run.Trigger,+ Destination: run.Destination,+ LocalSnapshotID: run.LocalSnapshotID,+ RemoteSnapshotID: snapshot.ID,+ S3DestinationID: run.S3DestinationID,+ S3DestinationName: run.S3DestinationName,+ PolicyID: run.PolicyID,+ Error: run.Error,+ }if saveErr := s.store.SaveRun(ctx, run); saveErr != nil {return run, saveErr}@@ -545,11 +663,26 @@ func (s *Service) ListBackups(ctx context.Context, params pagination.QueryParams}names := s.s3Destinations.DestinationsByID(ctx)remoteAvailable := backup.RemoteSnapshotChecker(ctx, s.s3Destinations)- for i := range result {- result[i].S3DestinationName = names[result[i].S3DestinationID].Name- result[i].RemoteAvailable = remoteAvailable(result[i].S3DestinationID, "arcane-system-recovery", result[i].RemoteSnapshotID)- }- return result, response, nil+ decorated := make([]backuptypes.SystemBackupRun, 0, len(result))+ for _, run := range result {+ decorated = append(decorated, backuptypes.SystemBackupRun{ //exhaustruct:enforce+ RemoteAvailable: remoteAvailable(run.S3DestinationID, "arcane-system-recovery", run.RemoteSnapshotID),+ ActivityID: run.ActivityID,+ ID: run.ID,+ Size: run.Size,+ CreatedAt: run.CreatedAt,+ Status: run.Status,+ Trigger: run.Trigger,+ Destination: run.Destination,+ LocalSnapshotID: run.LocalSnapshotID,+ RemoteSnapshotID: run.RemoteSnapshotID,+ S3DestinationID: run.S3DestinationID,+ S3DestinationName: names[run.S3DestinationID].Name,+ PolicyID: run.PolicyID,+ Error: run.Error,+ })+ }+ return decorated, response, nil}// BrowseBackupFiles returns one page from the backup's logical project tree.@@ -643,12 +776,14 @@ func (s *Service) deleteRuns(ctx context.Context, backupRuns []*backuptypes.Syst}}// A run without snapshots (a failed attempt) is just a row; deleting it needs neither the key nor Rustic.- key := ""- if len(localRuns) > 0 || len(remoteGroups) > 0 {- var err error- if key, err = s.recoveryKey(ctx, recoveryKey); err != nil {- return err+ key, err := func() (string, error) {+ if len(localRuns) == 0 && len(remoteGroups) == 0 {+ return "", nil}+ return s.recoveryKey(ctx, recoveryKey)+ }()+ if err != nil {+ return err}dockerClient, err := s.dockerService.GetClient(ctx)if err != nil {@@ -664,12 +799,65 @@ func (s *Service) deleteRuns(ctx context.Context, backupRuns []*backuptypes.SystdeleteErr = fmt.Errorf("failed to delete local snapshots: %w", repoErr)} else {for _, run := range localRuns {- run.LocalSnapshotID = ""+ *run = backuptypes.SystemBackupRun{ //exhaustruct:enforce+ RemoteAvailable: run.RemoteAvailable,+ ActivityID: run.ActivityID,+ ID: run.ID,+ Size: run.Size,+ CreatedAt: run.CreatedAt,+ Status: run.Status,+ Trigger: run.Trigger,+ Destination: run.Destination,+ LocalSnapshotID: "",+ RemoteSnapshotID: run.RemoteSnapshotID,+ S3DestinationID: run.S3DestinationID,+ S3DestinationName: run.S3DestinationName,+ PolicyID: run.PolicyID,+ Error: run.Error,+ }}}}for destinationID, group := range remoteGroups {- deleteErr = errors.Join(deleteErr, s.forgetRemoteSnapshots(ctx, dockerClient, key, destinationID, group))+ snapshotIDs := make([]string, len(group))+ for index, run := range group {+ snapshotIDs[index] = run.RemoteSnapshotID+ }+ repository, repoErr := s.remoteRepository(ctx, destinationID)+ if repoErr == nil {+ observation, checkErr := backup.CheckRemoteRepository(ctx, s.s3Destinations, destinationID, "arcane-system-recovery")+ switch {+ case checkErr != nil:+ repoErr = checkErr+ case !observation.Available:+ // Nothing remains to forget; drop the references so the run can be removed.+ slog.WarnContext(ctx, "S3 repository is missing; dropping remote snapshot references", "destinationId", destinationID, "snapshots", len(snapshotIDs))+ default:+ repoErr = s.engine.ForgetSnapshots(ctx, dockerClient, repository, key, snapshotIDs)+ }+ }+ if repoErr != nil {+ deleteErr = errors.Join(deleteErr, fmt.Errorf("failed to delete S3 snapshots: %w", repoErr))+ continue+ }+ for _, run := range group {+ *run = backuptypes.SystemBackupRun{ //exhaustruct:enforce+ RemoteAvailable: run.RemoteAvailable,+ ActivityID: run.ActivityID,+ ID: run.ID,+ Size: run.Size,+ CreatedAt: run.CreatedAt,+ Status: run.Status,+ Trigger: run.Trigger,+ Destination: run.Destination,+ LocalSnapshotID: run.LocalSnapshotID,+ RemoteSnapshotID: "",+ S3DestinationID: "",+ S3DestinationName: run.S3DestinationName,+ PolicyID: run.PolicyID,+ Error: run.Error,+ }+ }}for _, run := range backupRuns {if run.LocalSnapshotID == "" && run.RemoteSnapshotID == "" {@@ -678,13 +866,30 @@ func (s *Service) deleteRuns(ctx context.Context, backupRuns []*backuptypes.Syst}continue}- switch {- case run.LocalSnapshotID != "" && run.RemoteSnapshotID != "":- run.Destination = backuptypes.SystemBackupDestinationLocalS3- case run.LocalSnapshotID != "":- run.Destination = backuptypes.SystemBackupDestinationLocal- default:- run.Destination = backuptypes.SystemBackupDestinationS3+ *run = backuptypes.SystemBackupRun{ //exhaustruct:enforce+ RemoteAvailable: run.RemoteAvailable,+ ActivityID: run.ActivityID,+ ID: run.ID,+ Size: run.Size,+ CreatedAt: run.CreatedAt,+ Status: run.Status,+ Trigger: run.Trigger,+ Destination: func() backuptypes.SystemBackupDestination {+ switch {+ case run.LocalSnapshotID != "" && run.RemoteSnapshotID != "":+ return backuptypes.SystemBackupDestinationLocalS3+ case run.LocalSnapshotID != "":+ return backuptypes.SystemBackupDestinationLocal+ default:+ return backuptypes.SystemBackupDestinationS3+ }+ }(),+ LocalSnapshotID: run.LocalSnapshotID,+ RemoteSnapshotID: run.RemoteSnapshotID,+ S3DestinationID: run.S3DestinationID,+ S3DestinationName: run.S3DestinationName,+ PolicyID: run.PolicyID,+ Error: run.Error,}if saveErr := s.store.SaveRun(ctx, run); saveErr != nil {deleteErr = errors.Join(deleteErr, saveErr)@@ -693,34 +898,6 @@ func (s *Service) deleteRuns(ctx context.Context, backupRuns []*backuptypes.Systreturn deleteErr}-// forgetRemoteSnapshots forgets one S3 destination's snapshots and drops the runs' references to them.-func (s *Service) forgetRemoteSnapshots(ctx context.Context, dockerClient *client.Client, key, destinationID string, group []*backuptypes.SystemBackupRun) error {- snapshotIDs := make([]string, len(group))- for index, run := range group {- snapshotIDs[index] = run.RemoteSnapshotID- }- repository, repoErr := s.remoteRepository(ctx, destinationID)- var observation backuptypes.RepositoryObservation- if repoErr == nil {- observation, repoErr = backup.CheckRemoteRepository(ctx, s.s3Destinations, destinationID, "arcane-system-recovery")- }- switch {- case repoErr != nil:- case !observation.Available:- // Nothing remains to forget; drop the references so the run can be removed.- slog.WarnContext(ctx, "S3 repository is missing; dropping remote snapshot references", "destinationId", destinationID, "snapshots", len(snapshotIDs))- default:- repoErr = s.engine.ForgetSnapshots(ctx, dockerClient, repository, key, snapshotIDs)- }- if repoErr != nil {- return fmt.Errorf("failed to delete S3 snapshots: %w", repoErr)- }- for _, run := range group {- run.RemoteSnapshotID, run.S3DestinationID = "", ""- }- return nil-}-// PruneLocalRepository frees the data of deleted local system backups once Rustic's keep-delete window passes.func (s *Service) PruneLocalRepository(ctx context.Context) error {key, err := s.recoveryKeys.Get(ctx)@@ -820,8 +997,22 @@ func (s *Service) UploadBackup(ctx context.Context, id string, request backuptypif err != nil {return nil, fmt.Errorf("failed to upload system recovery snapshot: %w", err)}- run.RemoteSnapshotID, run.S3DestinationID = snapshot.ID, request.S3DestinationID- run.Destination = backuptypes.SystemBackupDestinationLocalS3+ *run = backuptypes.SystemBackupRun{ //exhaustruct:enforce+ RemoteAvailable: run.RemoteAvailable,+ ActivityID: run.ActivityID,+ ID: run.ID,+ Size: run.Size,+ CreatedAt: run.CreatedAt,+ Status: run.Status,+ Trigger: run.Trigger,+ Destination: backuptypes.SystemBackupDestinationLocalS3,+ LocalSnapshotID: run.LocalSnapshotID,+ RemoteSnapshotID: snapshot.ID,+ S3DestinationID: request.S3DestinationID,+ S3DestinationName: run.S3DestinationName,+ PolicyID: run.PolicyID,+ Error: run.Error,+ }if saveUploadedBackupErr := s.store.SaveRun(ctx, run); saveUploadedBackupErr != nil {return nil, fmt.Errorf("failed to save uploaded system backup: %w", saveUploadedBackupErr)}@@ -882,26 +1073,35 @@ func (s *Service) RestoreBackup(ctx context.Context, id, recoveryKey string, useif err != nil {return err}- // Rustic needs a named network to reach network-local S3 once Arcane stops; bridge and none carry no DNS.- rusticNetwork := ""- if current.NetworkSettings != nil {- names := slices.Sorted(maps.Keys(current.NetworkSettings.Networks))- if index := slices.IndexFunc(names, func(name string) bool { return name != "bridge" && name != "none" }); index >= 0 {- rusticNetwork = names[index]- }- }- request := recovery.RestoreRequest{- BackupID: run.ID, ContainerID: current.ID, ContainerImage: current.Config.Image, SnapshotID: plan.SnapshotID, RecoveryKey: key,- LocalSnapshotID: run.LocalSnapshotID, RemoteSnapshotID: run.RemoteSnapshotID, S3DestinationID: run.S3DestinationID,- Size: run.Size, NetworkMode: rusticNetwork,+ requestData, err := json.Marshal(recovery.RestoreRequest{+ BackupID: run.ID,+ ContainerID: current.ID,+ ContainerImage: current.Config.Image,+ SnapshotID: plan.SnapshotID,+ RecoveryKey: key,+ LocalSnapshotID: run.LocalSnapshotID,+ RemoteSnapshotID: run.RemoteSnapshotID,+ S3DestinationID: run.S3DestinationID,+ Size: run.Size,+ // Rustic needs a named network to reach network-local S3 once Arcane stops; bridge and none carry no DNS.+ NetworkMode: func() string {+ names := slices.Sorted(maps.Keys(kit.FromPtr(current.NetworkSettings).Networks))+ if index := slices.IndexFunc(names, func(name string) bool { return name != "bridge" && name != "none" }); index >= 0 {+ return names[index]+ }+ return ""+ }(),SafetyBackup: &recovery.SafetyBackup{- ID: safetyBackup.ID, LocalSnapshotID: safetyBackup.LocalSnapshotID,- Size: safetyBackup.Size, CreatedAt: safetyBackup.CreatedAt,+ ID: safetyBackup.ID,+ LocalSnapshotID: safetyBackup.LocalSnapshotID,+ Size: safetyBackup.Size,+ CreatedAt: safetyBackup.CreatedAt,},- ProjectsSetting: s.projectsSetting(ctx), ProjectsIncluded: plan.ProjectsIncluded,- Stages: plan.Stages, RollbackStages: rollbackStages,- }- requestData, err := json.Marshal(request)+ ProjectsSetting: s.projectsSetting(ctx),+ ProjectsIncluded: plan.ProjectsIncluded,+ Stages: plan.Stages,+ RollbackStages: rollbackStages,+ })if err != nil {return fmt.Errorf("encode recovery request: %w", err)}@@ -934,27 +1134,41 @@ func (s *Service) RestoreBackup(ctx context.Context, id, recoveryKey string, useif err != nil {return fmt.Errorf("resolve Arcane executable: %w", err)}- helperExecutable, mounts := executablePath, append(slices.Clone(runtimeMounts), *appDataMount)- if executableMount := docker.MountForSubpath(current.Mounts, executablePath, systemRecoveryHelperPath); executableMount != nil {- if executableMount.Type != mount.TypeBind {- return errors.New("resolve recovery helper executable: the Arcane executable must come from the image or a bind mount")- }- executableMount.ReadOnly = true- helperExecutable, mounts = systemRecoveryHelperPath, append(mounts, *executableMount)- }- hostConfig := &container.HostConfig{AutoRemove: true, Mounts: mounts, NetworkMode: networkMode}- if current.HostConfig != nil {- hostConfig.SecurityOpt = append([]string{}, current.HostConfig.SecurityOpt...)- hostConfig.Privileged = current.HostConfig.Privileged+ executableMount := docker.MountForSubpath(current.Mounts, executablePath, systemRecoveryHelperPath)+ if executableMount != nil && executableMount.Type != mount.TypeBind {+ return errors.New("resolve recovery helper executable: the Arcane executable must come from the image or a bind mount")+ }+ mounts := append(slices.Clone(runtimeMounts), *appDataMount)+ if executableMount != nil {+ mounts = append(mounts, mount.Mount{+ Type: executableMount.Type,+ Source: executableMount.Source,+ Target: executableMount.Target,+ ReadOnly: true,+ Consistency: executableMount.Consistency,+ BindOptions: executableMount.BindOptions,+ VolumeOptions: executableMount.VolumeOptions,+ ImageOptions: executableMount.ImageOptions,+ TmpfsOptions: executableMount.TmpfsOptions,+ ClusterOptions: executableMount.ClusterOptions,+ })}created, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{Config: &container.Config{- Image: current.Config.Image, Cmd: []string{helperExecutable, "recovery-restore", "--request", requestFile}, User: "0:0",+ Image: current.Config.Image,+ Cmd: []string{kit.Ternary(executableMount != nil, systemRecoveryHelperPath, executablePath), "recovery-restore", "--request", requestFile},+ User: "0:0",Env: containerEnv,Labels: map[string]string{"com.getarcaneapp.arcane.recovery": "true", "com.getarcaneapp.arcane": "true"},},- HostConfig: hostConfig,- Name: fmt.Sprintf("%s-recovery-%d", strings.TrimPrefix(current.Name, "/"), time.Now().Unix()),+ HostConfig: &container.HostConfig{+ AutoRemove: true,+ Mounts: mounts,+ NetworkMode: networkMode,+ SecurityOpt: slices.Clone(kit.FromPtr(current.HostConfig).SecurityOpt),+ Privileged: kit.FromPtr(current.HostConfig).Privileged,+ },+ Name: fmt.Sprintf("%s-recovery-%d", strings.TrimPrefix(current.Name, "/"), time.Now().Unix()),})if err != nil {return fmt.Errorf("create recovery helper container: %w", err)@@ -1005,10 +1219,8 @@ func (s *Service) SetRecoveryKey(ctx context.Context, recoveryKey string) (*backif setErr := s.recoveryKeys.Set(ctx, recoveryKey); setErr != nil {return nil, setErr}- if s.volumeService != nil {- if migrateRepositoryPasswordsErr := s.volumeService.MigrateRepositoryPasswords(ctx); migrateRepositoryPasswordsErr != nil {- slog.WarnContext(ctx, "failed to re-key volume backup repositories to the recovery key", "error", migrateRepositoryPasswordsErr.Error())- }+ if migrateRepositoryPasswordsErr := s.volumeService.MigrateRepositoryPasswords(ctx); migrateRepositoryPasswordsErr != nil {+ slog.WarnContext(ctx, "failed to re-key volume backup repositories to the recovery key", "error", migrateRepositoryPasswordsErr.Error())}return &backuptypes.SystemBackupRecoveryKeyStatus{Configured: true}, nil}@@ -1022,24 +1234,49 @@ func (s *Service) GetPolicies(ctx context.Context) (*backuptypes.SystemBackupPolif err != nil {return nil, err}- result := &backuptypes.SystemBackupPolicyCollection{- Policies: make([]backuptypes.SystemBackupPolicy, 0, len(policies)), RecoveryKeyStored: configured,- }destinations := s.s3Destinations.DestinationsByID(ctx)- for i := range policies {- lastRun, runErr := s.store.LatestPolicyRun(ctx, policies[i].ID)+ decorated := make([]backuptypes.SystemBackupPolicy, 0, len(policies))+ for _, policy := range policies {+ lastRun, runErr := s.store.LatestPolicyRun(ctx, policy.ID)if runErr != nil {return nil, fmt.Errorf("failed to load latest system backup: %w", runErr)}- if lastRun != nil {- lastRun.S3DestinationName = destinations[lastRun.S3DestinationID].Name- }- dto := policies[i]- dto.LastRun = lastRun- dto.S3DestinationName = destinations[dto.S3DestinationID].Name- result.Policies = append(result.Policies, dto)- }- return result, nil+ decorated = append(decorated, backuptypes.SystemBackupPolicy{ //exhaustruct:enforce+ ID: policy.ID,+ Enabled: policy.Enabled,+ Schedule: policy.Schedule,+ RetentionCount: policy.RetentionCount,+ LocalEnabled: policy.LocalEnabled,+ S3Enabled: policy.S3Enabled,+ S3DestinationID: policy.S3DestinationID,+ S3DestinationName: destinations[policy.S3DestinationID].Name,+ LastRun: func() *backuptypes.SystemBackupRun {+ if lastRun == nil {+ return nil+ }+ return &backuptypes.SystemBackupRun{ //exhaustruct:enforce+ RemoteAvailable: lastRun.RemoteAvailable,+ ActivityID: lastRun.ActivityID,+ ID: lastRun.ID,+ Size: lastRun.Size,+ CreatedAt: lastRun.CreatedAt,+ Status: lastRun.Status,+ Trigger: lastRun.Trigger,+ Destination: lastRun.Destination,+ LocalSnapshotID: lastRun.LocalSnapshotID,+ RemoteSnapshotID: lastRun.RemoteSnapshotID,+ S3DestinationID: lastRun.S3DestinationID,+ S3DestinationName: destinations[lastRun.S3DestinationID].Name,+ PolicyID: lastRun.PolicyID,+ Error: lastRun.Error,+ }+ }(),+ })+ }+ return &backuptypes.SystemBackupPolicyCollection{+ Policies: decorated,+ RecoveryKeyStored: configured,+ }, nil}func (s *Service) UpdatePolicies(ctx context.Context, updates []backuptypes.UpdateSystemBackupPolicy) (*backuptypes.SystemBackupPolicyCollection, error) {@@ -1052,15 +1289,23 @@ func (s *Service) UpdatePolicies(ctx context.Context, updates []backuptypes.Updaif normalizeErr != nil {return normalizeErr}- update = normalized- if update.Enabled && !configured {+ if normalized.Enabled && !configured {return errors.New("configure a recovery key before enabling scheduled system backups")}- if update.Enabled && !s.SupportedDatabaseProvider() {+ if normalized.Enabled && !s.SupportedDatabaseProvider() {return errors.New("scheduled system backups require the SQLite database provider")}- policy.Enabled, policy.Schedule, policy.RetentionCount = update.Enabled, update.Schedule, update.RetentionCount- policy.LocalEnabled, policy.S3Enabled, policy.S3DestinationID = update.LocalEnabled, update.S3Enabled, update.S3DestinationID+ *policy = backuptypes.SystemBackupPolicy{ //exhaustruct:enforce+ ID: policy.ID,+ Enabled: normalized.Enabled,+ Schedule: normalized.Schedule,+ RetentionCount: normalized.RetentionCount,+ LocalEnabled: normalized.LocalEnabled,+ S3Enabled: normalized.S3Enabled,+ S3DestinationID: normalized.S3DestinationID,+ S3DestinationName: policy.S3DestinationName,+ LastRun: policy.LastRun,+ }return nil}if runErr := s.store.ReconcilePolicies(ctx, updates, build, s.jobs.Unregister, s.rescheduleSystemBackupPolicy); runErr != nil {@@ -1083,10 +1328,12 @@ func (s *Service) runScheduledBackup(ctx context.Context, policyID string) (scheif policy == nil || !policy.Enabled {return scheduler.Outcome{Status: scheduler.Skipped}, nil}- var remoteErr error- if policy.S3Enabled {- remoteErr = s.store.CheckScheduledRemote(ctx, policy.S3DestinationID)- }+ remoteErr := func() error {+ if !policy.S3Enabled {+ return nil+ }+ return s.store.CheckScheduledRemote(ctx, policy.S3DestinationID)+ }()remoteDisabled, disableErr := backup.DisableMissingRemote(remoteErr, policy.LocalEnabled, &policy.S3Enabled, &policy.Enabled, func(field string) (bool, error) {return s.store.DisablePolicyRemote(ctx, policy, field)})@@ -1133,17 +1380,19 @@ func (s *Service) resumeScheduledBackup(ctx context.Context, previous scheduler.}if current != nil && current.RetentionCount > 0 {if applyRetentionErr := s.applyRetention(ctx, current.ID, current.RetentionCount, current.S3Enabled); applyRetentionErr != nil {- outcome.Status = scheduler.Partial- return outcome, applyRetentionErr+ return scheduler.Outcome{+ Status: scheduler.Partial,+ Message: outcome.Message,+ ActivityID: outcome.ActivityID,+ Targets: outcome.Targets,+ Steps: outcome.Steps,+ }, applyRetentionErr}}return outcome, nil}func (s *Service) rescheduleSystemBackupPolicy(ctx context.Context, policy *backuptypes.SystemBackupPolicy) {- if policy == nil {- return- }if !policy.Enabled {s.jobs.Unregister(ctx, policy.ID)return@@ -1157,9 +1406,13 @@ func (s *Service) rescheduleSystemBackupPolicy(ctx context.Context, policy *backActivity: activitylib.StartRequest{ResourceID: new(policyID),Metadata: database.JSON{- "action": "scheduled_system_backup", "policyId": policyID, "schedule": policy.Schedule,- "retentionCount": policy.RetentionCount, "localEnabled": policy.LocalEnabled,- "s3Enabled": policy.S3Enabled, "s3DestinationId": policy.S3DestinationID,+ "action": "scheduled_system_backup",+ "policyId": policyID,+ "schedule": policy.Schedule,+ "retentionCount": policy.RetentionCount,+ "localEnabled": policy.LocalEnabled,+ "s3Enabled": policy.S3Enabled,+ "s3DestinationId": policy.S3DestinationID,},},ScheduleFn: func(ctx context.Context) string {@@ -1207,14 +1460,7 @@ func (s *Service) applyRetention(ctx context.Context, policyID string, keep int,// projectsSetting returns the effective projectsDirectory setting.func (s *Service) projectsSetting(ctx context.Context) string {- fallback := ""- if s.config != nil {- fallback = s.config.ProjectsDirectory- }- if s.settingsService == nil {- return fallback- }- return s.settingsService.GetStringSetting(ctx, "projectsDirectory", fallback)+ return s.settingsService.GetStringSetting(ctx, "projectsDirectory", s.config.ProjectsDirectory)}// StartBackup prepares a manual backup, then submits its workflow.@@ -1241,10 +1487,21 @@ func (s *Service) StartBackup(ctx context.Context, user usertypes.Actor, requestreturn nil, errors.Join(holdErr, s.store.FailRun(context.WithoutCancel(ctx), run.ID, holdErr.Error()))}}- input := manualBackupInput{Checkpoint: prepared.checkpoint, Requester: requester}- activityID, err := s.flow.Submit(ctx, s.backupWorkflow, input, activitylib.StartRequest{- ResourceID: new("arcane"), StartedBy: &user, Step: "Creating system backup", LatestMessage: "Creating Arcane system backup",- Metadata: database.JSON{"action": "create_system_backup", "backupId": run.ID, "destination": run.Destination, "s3DestinationId": run.S3DestinationID, "policyId": run.PolicyID},+ activityID, err := s.flow.Submit(ctx, s.backupWorkflow, manualBackupInput{+ Checkpoint: prepared.checkpoint,+ Requester: requester,+ }, activitylib.StartRequest{+ ResourceID: new("arcane"),+ StartedBy: &user,+ Step: "Creating system backup",+ LatestMessage: "Creating Arcane system backup",+ Metadata: database.JSON{+ "action": "create_system_backup",+ "backupId": run.ID,+ "destination": run.Destination,+ "s3DestinationId": run.S3DestinationID,+ "policyId": run.PolicyID,+ },}, backup.AcceptedTarget(run.ID))if err != nil {release(ctx)@@ -1256,9 +1513,22 @@ func (s *Service) StartBackup(ctx context.Context, user usertypes.Actor, requests.watchBackup(background, run.ID, activityID)release(background)}()- dto := *run- dto.ActivityID = activityID- return &dto, nil+ return &backuptypes.SystemBackupRun{ //exhaustruct:enforce+ RemoteAvailable: run.RemoteAvailable,+ ActivityID: activityID,+ ID: run.ID,+ Size: run.Size,+ CreatedAt: run.CreatedAt,+ Status: run.Status,+ Trigger: run.Trigger,+ Destination: run.Destination,+ LocalSnapshotID: run.LocalSnapshotID,+ RemoteSnapshotID: run.RemoteSnapshotID,+ S3DestinationID: run.S3DestinationID,+ S3DestinationName: run.S3DestinationName,+ PolicyID: run.PolicyID,+ Error: run.Error,+ }, nil}// WatchAcceptedBackups restores, at startup, the watch over manual backups whose saved workflows resume.@@ -1310,11 +1580,21 @@ func (s *Service) reconcileBackup(ctx context.Context, previous scheduler.Run, treturn outcome, err}confirm := func() (scheduler.Outcome, error) {- target.Status = scheduler.Succeeded- if progressErr := jobcontext.Progress(ctx, target); progressErr != nil {+ confirmed := scheduler.TargetOutcome{ //exhaustruct:enforce+ RecoveryData: target.RecoveryData,+ ResourceType: target.ResourceType,+ ID: target.ID,+ Status: scheduler.Succeeded,+ Message: target.Message,+ ActivityID: target.ActivityID,+ }+ if progressErr := jobcontext.Progress(ctx, confirmed); progressErr != nil {return outcome, progressErr}- return scheduler.Outcome{Status: scheduler.Succeeded, Targets: []scheduler.TargetOutcome{target}}, nil+ return scheduler.Outcome{+ Status: scheduler.Succeeded,+ Targets: []scheduler.TargetOutcome{confirmed},+ }, nil}run, err := s.store.Run(ctx, checkpoint.BackupID)if err != nil {@@ -1347,28 +1627,45 @@ func (s *Service) reconcileBackup(ctx context.Context, previous scheduler.Run, t}if staged.ID == "" && run.RemoteSnapshotID == "" {// Nothing survived the interruption, so the backup runs again.- if _, err = s.executeBackup(ctx, &preparedSystemBackup{run: run, recoveryKey: key, checkpoint: checkpoint}); err != nil {+ if _, err = s.executeBackup(ctx, &preparedSystemBackup{+ run: run,+ recoveryKey: key,+ checkpoint: checkpoint,+ }); err != nil {return outcome, err}return confirm()}- if checkpoint.LocalEnabled {- run.LocalSnapshotID, run.Size = staged.ID, staged.Size- }// A missing remote snapshot implies a staged one; the rerun above handled neither surviving.+ remoteSnapshotID := run.RemoteSnapshotIDif checkpoint.S3Enabled && run.RemoteSnapshotID == "" {snapshot, replicateErr := s.engine.Replicate(ctx, dockerClient, local, staged.ID, remote, key, "arcane-system-recovery", backup.RunSnapshotTag(run.ID))if replicateErr != nil {return outcome, replicateErr}- run.RemoteSnapshotID = snapshot.ID+ remoteSnapshotID = snapshot.ID}if !checkpoint.LocalEnabled && staged.ID != "" {if forgetErr := s.engine.ForgetSnapshots(ctx, dockerClient, local, key, []string{staged.ID}); forgetErr != nil {return outcome, forgetErr}}- run.Status, run.Error = backuptypes.SystemBackupStatusSucceeded, ""+ *run = backuptypes.SystemBackupRun{ //exhaustruct:enforce+ RemoteAvailable: run.RemoteAvailable,+ ActivityID: run.ActivityID,+ ID: run.ID,+ Size: kit.Ternary(checkpoint.LocalEnabled, staged.Size, run.Size),+ CreatedAt: run.CreatedAt,+ Status: backuptypes.SystemBackupStatusSucceeded,+ Trigger: run.Trigger,+ Destination: run.Destination,+ LocalSnapshotID: kit.Ternary(checkpoint.LocalEnabled, staged.ID, run.LocalSnapshotID),+ RemoteSnapshotID: remoteSnapshotID,+ S3DestinationID: run.S3DestinationID,+ S3DestinationName: run.S3DestinationName,+ PolicyID: run.PolicyID,+ Error: "",+ }if saveErr := s.store.SaveRun(ctx, run); saveErr != nil {return outcome, saveErr}@@ -1421,7 +1718,22 @@ func (s *Service) observeBackupSnapshots(return staged, local, remote, findErr}if found {- run.RemoteSnapshotID, run.Size = snapshot.ID, cmp.Or(run.Size, snapshot.Size)+ *run = backuptypes.SystemBackupRun{ //exhaustruct:enforce+ RemoteAvailable: run.RemoteAvailable,+ ActivityID: run.ActivityID,+ ID: run.ID,+ Size: cmp.Or(run.Size, snapshot.Size),+ CreatedAt: run.CreatedAt,+ Status: run.Status,+ Trigger: run.Trigger,+ Destination: run.Destination,+ LocalSnapshotID: run.LocalSnapshotID,+ RemoteSnapshotID: snapshot.ID,+ S3DestinationID: run.S3DestinationID,+ S3DestinationName: run.S3DestinationName,+ PolicyID: run.PolicyID,+ Error: run.Error,+ }}}// S3-only runs snapshot straight to S3, so only runs from older versions left a local staged snapshot to recover,@@ -1499,11 +1811,20 @@ func (s *Service) runManualBackup(ctx context.Context, t flow.Task) (_ any, errif err != nil {return nil, err}- target := scheduler.TargetOutcome{ResourceType: "system_backup", ID: runID, Status: scheduler.Running, RecoveryData: checkpoint, ActivityID: t.ActivityID()}- if progressErr := jobcontext.Progress(ctx, target); progressErr != nil {+ if progressErr := jobcontext.Progress(ctx, scheduler.TargetOutcome{+ ResourceType: "system_backup",+ ID: runID,+ Status: scheduler.Running,+ RecoveryData: checkpoint,+ ActivityID: t.ActivityID(),+ }); progressErr != nil {return nil, progressErr}- if _, executeErr := s.executeBackup(ctx, &preparedSystemBackup{run: run, recoveryKey: key, checkpoint: input.Checkpoint}); executeErr != nil {+ if _, executeErr := s.executeBackup(ctx, &preparedSystemBackup{+ run: run,+ recoveryKey: key,+ checkpoint: input.Checkpoint,+ }); executeErr != nil {return nil, executeErr}return succeeded, nil@@ -1517,23 +1838,47 @@ func (s *Service) ListBackupHistory(ctx context.Context, params pagination.Query}destinations := s.s3Destinations.DestinationsByID(ctx)remoteAvailable := backup.RemoteSnapshotChecker(ctx, s.s3Destinations)- instanceID := ""- if s.settingsService != nil {- instanceID = s.settingsService.GetSettingsConfig().InstanceID.Value- }- for i := range history {- history[i].S3DestinationName = destinations[history[i].S3DestinationID].Name+ instanceID := s.settingsService.GetSettingsConfig().InstanceID.Value+ decorated := make([]backuptypes.HistoryEntry, 0, len(history))+ for _, entry := range history {// Volume backups discovered from another instance live under that instance's root.- root := "arcane-system-recovery"- if history[i].ResourceType == "volume" {- root = ""- if owner := cmp.Or(history[i].RemoteInstanceID, instanceID); owner != "" {- root = path.Join(backup.VolumeRoot, owner)+ root := func() string {+ owner := cmp.Or(entry.RemoteInstanceID, instanceID)+ switch {+ case entry.ResourceType != "volume":+ return "arcane-system-recovery"+ case owner == "":+ return ""+ default:+ return path.Join(backup.VolumeRoot, owner)}- }- history[i].RemoteAvailable = remoteAvailable(history[i].S3DestinationID, root, history[i].RemoteSnapshotID)- }-- page.GrandTotalItems = page.TotalItems- return history, page, nil+ }()+ decorated = append(decorated, backuptypes.HistoryEntry{ //exhaustruct:enforce+ RemoteAvailable: remoteAvailable(entry.S3DestinationID, root, entry.RemoteSnapshotID),+ ID: entry.ID,+ Size: entry.Size,+ CreatedAt: entry.CreatedAt,+ Status: entry.Status,+ Trigger: entry.Trigger,+ Destination: entry.Destination,+ Format: entry.Format,+ LocalSnapshotID: entry.LocalSnapshotID,+ RemoteSnapshotID: entry.RemoteSnapshotID,+ RemoteInstanceID: entry.RemoteInstanceID,+ S3DestinationID: entry.S3DestinationID,+ S3DestinationName: destinations[entry.S3DestinationID].Name,+ PolicyID: entry.PolicyID,+ Error: entry.Error,+ Type: entry.Type,+ ResourceType: entry.ResourceType,+ ResourceName: entry.ResourceName,+ })+ }+ return decorated, pagination.Response{+ TotalPages: page.TotalPages,+ TotalItems: page.TotalItems,+ CurrentPage: page.CurrentPage,+ ItemsPerPage: page.ItemsPerPage,+ GrandTotalItems: page.TotalItems,+ }, nil}diff --git a/backend/internal/system/children/backup/service_test.go b/backend/internal/system/children/backup/service_test.goindex 3657612a3..c03bebae2 100644--- a/backend/internal/system/children/backup/service_test.go+++ b/backend/internal/system/children/backup/service_test.go@@ -12,27 +12,33 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/s3"+ "github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/backend/v2/pkg/pagination""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow/flowtest")-func TestProjectsSettingFallsBackToConfig(t *testing.T) {- service := &Service{config: &config.Config{ProjectsDirectory: "/app/data/projects:/host/projects"}}- require.Equal(t, "/app/data/projects:/host/projects", service.projectsSetting(t.Context()))-}-func TestHistoryDestinationDecoration(t *testing.T) {db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})require.NoError(t, err)- require.NoError(t, db.AutoMigrate(&s3.S3Destination{}))+ require.NoError(t, db.AutoMigrate(&s3.S3Destination{}, &settings.SettingVariable{}))destination := s3.S3Destination{Name: "Offsite", Bucket: "backups"}require.NoError(t, db.Create(&destination).Error)- service := &Service{- s3Destinations: s3.NewS3DestinationService(&database.DB{DB: db}, nil),- store: Store{ListHistory: func(context.Context, pagination.QueryParams) ([]backup.HistoryEntry, pagination.Response, error) {- return []backup.HistoryEntry{{S3DestinationID: destination.ID}, {S3DestinationID: "missing"}}, pagination.Response{}, nil- }},- }+ store := &database.DB{DB: db}+ settingsService, err := settings.NewSettingsService(t.Context(), store)+ require.NoError(t, err)+ t.Cleanup(func() { require.NoError(t, settingsService.Stop(context.WithoutCancel(t.Context()))) })+ service := NewService(Dependencies{+ S3Destinations: s3.NewS3DestinationService(store, func(context.Context, string) error { return nil }),+ Settings: settingsService,+ Store: Store{+ ListHistory: func(context.Context, pagination.QueryParams) ([]backup.HistoryEntry, pagination.Response, error) {+ return []backup.HistoryEntry{+ {S3DestinationID: destination.ID},+ {S3DestinationID: "missing"},+ }, pagination.Response{}, nil+ },+ },+ })history, _, err := service.ListBackupHistory(t.Context(), pagination.QueryParams{})require.NoError(t, err)require.Equal(t, "Offsite", history[0].S3DestinationName)@@ -58,7 +64,16 @@ func TestRecoveryEnvironmentIncludesRuntimeSecrets(t *testing.T) {OidcClientSecret: "oidc-secret",FilePerm: 0o640,}- environment := (&Service{config: cfg}).recoveryEnvironment(t.Context())+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})+ require.NoError(t, err)+ require.NoError(t, db.AutoMigrate(&settings.SettingVariable{}))+ settingsService, err := settings.NewSettingsService(t.Context(), &database.DB{DB: db})+ require.NoError(t, err)+ t.Cleanup(func() { require.NoError(t, settingsService.Stop(context.WithoutCancel(t.Context()))) })+ environment := NewService(Dependencies{+ Config: cfg,+ Settings: settingsService,+ }).recoveryEnvironment(t.Context())require.Equal(t, "jwt-secret", environment["JWT_SECRET"])require.Equal(t, "encryption-secret", environment["ENCRYPTION_KEY"])require.Equal(t, "admin-key", environment["ADMIN_STATIC_API_KEY"])diff --git a/backend/internal/system/children/prune/service.go b/backend/internal/system/children/prune/service.goindex 41900a9bc..2b0ea382b 100644--- a/backend/internal/system/children/prune/service.go+++ b/backend/internal/system/children/prune/service.go@@ -21,6 +21,7 @@ import ("github.com/moby/moby/client""github.com/samber/mo""go.getarcane.app/docker/compat"+ kit "go.getarcane.app/kit/pkg""golang.org/x/sync/errgroup""github.com/getarcaneapp/arcane/backend/v2/internal/activity"@@ -74,47 +75,54 @@ func (s *Service) PruneAll(ctx context.Context, environmentID string, req system"buildCache", req.BuildCache,)- prune := s.beginSystemPruneInternal(ctx, environmentID, req)- activityID := prune.activityID- result := &system.PruneAllResult{Success: true, ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()}+ prune := s.beginSystemPrune(ctx, environmentID, req)if prune.started.IsAbsent() {- slog.InfoContext(ctx, "System prune already running", "environmentId", environmentID, "activityId", activityID)- return result, false, nil+ slog.InfoContext(ctx, "System prune already running", "environmentId", environmentID, "activityId", prune.activityID)+ return &system.PruneAllResult{+ Success: true,+ ActivityID: mo.EmptyableToOption(strings.TrimSpace(prune.activityID)).ToPointer(),+ }, false, nil}- defer s.finishSystemPruneInternal(environmentID)+ defer s.finishSystemPrune(environmentID)- ctx = s.activityService.Track(ctx, activityID)- if err := jobcontext.Progress(ctx, scheduler.TargetOutcome{ID: "scheduled-prune", Status: scheduler.Running, ActivityID: activityID}); err != nil {- result.Success = false- result.Errors = append(result.Errors, err.Error())- s.completeSystemPruneActivityInternal(ctx, activityID, result)+ ctx = s.activityService.Track(ctx, prune.activityID)+ if err := jobcontext.Progress(ctx, scheduler.TargetOutcome{+ ID: "scheduled-prune",+ Status: scheduler.Running,+ ActivityID: prune.activityID,+ }); err != nil {+ result := &system.PruneAllResult{+ Success: false,+ Errors: []string{err.Error()},+ ActivityID: mo.EmptyableToOption(strings.TrimSpace(prune.activityID)).ToPointer(),+ }+ s.completeSystemPruneActivity(ctx, prune.activityID, result)return result, true, err}- s.runSystemPruneInternal(ctx, req, activityID, result)-- return result, true, nil+ return s.runSystemPrune(ctx, req, prune.activityID), true, nil}func (s *Service) StartPruneAll(ctx context.Context, environmentID string, req system.PruneAllRequest) *system.PruneAllResult {- prune := s.beginSystemPruneInternal(ctx, environmentID, req)- activityID := prune.activityID+ prune := s.beginSystemPrune(ctx, environmentID, req)if prune.started.IsAbsent() {- slog.InfoContext(ctx, "System prune already running", "environmentId", environmentID, "activityId", activityID)- return &system.PruneAllResult{Success: true, ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()}+ slog.InfoContext(ctx, "System prune already running", "environmentId", environmentID, "activityId", prune.activityID)+ return &system.PruneAllResult{+ Success: true,+ ActivityID: mo.EmptyableToOption(strings.TrimSpace(prune.activityID)).ToPointer(),+ }}- backgroundCtx := utils.ActivityRuntimeContext(ctx, nil)- backgroundCtx = s.activityService.Track(backgroundCtx, activityID)-+ backgroundCtx := s.activityService.Track(utils.ActivityRuntimeContext(ctx, nil), prune.activityID)go func() {- defer s.finishSystemPruneInternal(environmentID)-- result := &system.PruneAllResult{Success: true, ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()}- s.runSystemPruneInternal(backgroundCtx, req, activityID, result)+ defer s.finishSystemPrune(environmentID)+ s.runSystemPrune(backgroundCtx, req, prune.activityID)}()- return &system.PruneAllResult{Success: true, ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()}+ return &system.PruneAllResult{+ Success: true,+ ActivityID: mo.EmptyableToOption(strings.TrimSpace(prune.activityID)).ToPointer(),+ }}type systemPruneBeginResult struct {@@ -122,7 +130,7 @@ type systemPruneBeginResult struct {started mo.Option[struct{}]}-func (s *Service) beginSystemPruneInternal(ctx context.Context, environmentID string, req system.PruneAllRequest) systemPruneBeginResult {+func (s *Service) beginSystemPrune(ctx context.Context, environmentID string, req system.PruneAllRequest) systemPruneBeginResult {s.pruneMu.Lock()defer s.pruneMu.Unlock()@@ -130,29 +138,55 @@ func (s *Service) beginSystemPruneInternal(ctx context.Context, environmentID streturn systemPruneBeginResult{activityID: activityID, started: mo.None[struct{}]()}}- activityID := s.startSystemPruneActivityInternal(ctx, environmentID, req)+ localActivity, err := s.activityService.StartActivity(ctx, activity.StartActivityRequest{+ EnvironmentID: environmentID,+ Type: activitytypes.TypeSystemPrune,+ ResourceType: new("system"),+ ResourceName: new("Docker resources"),+ Step: "Preparing prune",+ LatestMessage: "System prune started",+ Metadata: database.JSON{+ "containers": req.Containers,+ "images": req.Images,+ "volumes": req.Volumes,+ "networks": req.Networks,+ "buildCache": req.BuildCache,+ },+ })+ if err != nil {+ slog.DebugContext(ctx, "failed to start system prune activity", "error", err)+ }+ activityID := kit.Ternary(err != nil, "", kit.FromPtr(localActivity).ID)s.runningPrunes[environmentID] = activityIDreturn systemPruneBeginResult{activityID: activityID, started: mo.Some(struct{}{})}}-func (s *Service) finishSystemPruneInternal(environmentID string) {+func (s *Service) finishSystemPrune(environmentID string) {s.pruneMu.Lock()defer s.pruneMu.Unlock()delete(s.runningPrunes, environmentID)}-func (s *Service) runSystemPruneInternal(ctx context.Context, req system.PruneAllRequest, activityID string, result *system.PruneAllResult) {+func (s *Service) runSystemPrune(ctx context.Context, req system.PruneAllRequest, activityID string) *system.PruneAllResult {var mu sync.Mutex+ var errs []string+ addError := func(format string, err error) {+ mu.Lock()+ defer mu.Unlock()+ errs = append(errs, fmt.Sprintf(format, err))+ }+ var containers, images, buildCache, volumes, networks system.PruneAllResult// 1. Prune Containers first (sequential) as it may free up other resourcesif req.Containers != nil && req.Containers.Mode != system.PruneContainerModeNone {- s.appendSystemPruneActivityMessageInternal(ctx, activityID, "Pruning containers", 15)+ s.appendSystemPruneActivityMessage(ctx, activityID, "Pruning containers", 15)slog.InfoContext(ctx, "Pruning containers...", "mode", req.Containers.Mode, "until", req.Containers.Until)- if err := s.pruneContainersInternal(ctx, *req.Containers, result); err != nil {- result.Errors = append(result.Errors, fmt.Sprintf("Container pruning failed: %v", err))- result.Success = false+ part, err := s.pruneContainers(ctx, *req.Containers)+ if err != nil {+ addError("Container pruning failed: %v", err)}+ containers = part}// 2. Prune other resources in parallel@@ -162,21 +196,13 @@ func (s *Service) runSystemPruneInternal(ctx context.Context, req system.PruneAlg.Go(func() (workerErr error) {defer utils.RecoverToError(&workerErr, "system info worker")- s.appendSystemPruneActivityMessageInternal(groupCtx, activityID, "Pruning images", 40)+ s.appendSystemPruneActivityMessage(groupCtx, activityID, "Pruning images", 40)slog.InfoContext(groupCtx, "Pruning images...", "mode", req.Images.Mode, "until", req.Images.Until)- localResult := &system.PruneAllResult{}- if err := s.pruneImagesInternal(groupCtx, *req.Images, localResult); err != nil {- mu.Lock()- result.Errors = append(result.Errors, fmt.Sprintf("Image pruning failed: %v", err))- result.Success = false- mu.Unlock()- } else {- mu.Lock()- result.ImagesDeleted = append(result.ImagesDeleted, localResult.ImagesDeleted...)- result.SpaceReclaimed += localResult.SpaceReclaimed- result.ImageSpaceReclaimed += localResult.ImageSpaceReclaimed- mu.Unlock()+ part, err := s.pruneImages(groupCtx, *req.Images)+ if err != nil {+ addError("Image pruning failed: %v", err)}+ images = partreturn nil})}@@ -185,23 +211,16 @@ func (s *Service) runSystemPruneInternal(ctx context.Context, req system.PruneAlg.Go(func() (workerErr error) {defer utils.RecoverToError(&workerErr, "system info worker")- s.appendSystemPruneActivityMessageInternal(groupCtx, activityID, "Pruning build cache", 45)+ s.appendSystemPruneActivityMessage(groupCtx, activityID, "Pruning build cache", 45)slog.InfoContext(groupCtx, "Pruning build cache...", "mode", req.BuildCache.Mode, "until", req.BuildCache.Until)- localResult := &system.PruneAllResult{}- if err := s.pruneBuildCacheInternal(groupCtx, *req.BuildCache, localResult); err != nil {+ part, err := s.pruneBuildCache(groupCtx, *req.BuildCache)+ if err != nil {slog.WarnContext(groupCtx, "Build cache pruning encountered an error", "error", err.Error())// Surface the failure like every other prune type so a build cache that// could not be reclaimed is reported instead of silently left behind.- mu.Lock()- result.Errors = append(result.Errors, fmt.Sprintf("Build cache pruning failed: %v", err))- result.Success = false- mu.Unlock()- } else {- mu.Lock()- result.SpaceReclaimed += localResult.SpaceReclaimed- result.BuildCacheSpaceReclaimed += localResult.BuildCacheSpaceReclaimed- mu.Unlock()+ addError("Build cache pruning failed: %v", err)}+ buildCache = partreturn nil})}@@ -210,21 +229,13 @@ func (s *Service) runSystemPruneInternal(ctx context.Context, req system.PruneAlg.Go(func() (workerErr error) {defer utils.RecoverToError(&workerErr, "system info worker")- s.appendSystemPruneActivityMessageInternal(groupCtx, activityID, "Pruning volumes", 55)+ s.appendSystemPruneActivityMessage(groupCtx, activityID, "Pruning volumes", 55)slog.InfoContext(groupCtx, "Pruning volumes...", "mode", req.Volumes.Mode)- localResult := &system.PruneAllResult{}- if err := s.pruneVolumesInternal(groupCtx, *req.Volumes, localResult); err != nil {- mu.Lock()- result.Errors = append(result.Errors, fmt.Sprintf("Volume pruning failed: %v", err))- result.Success = false- mu.Unlock()- } else {- mu.Lock()- result.VolumesDeleted = append(result.VolumesDeleted, localResult.VolumesDeleted...)- result.SpaceReclaimed += localResult.SpaceReclaimed- result.VolumeSpaceReclaimed += localResult.VolumeSpaceReclaimed- mu.Unlock()+ part, err := s.pruneVolumes(groupCtx, *req.Volumes)+ if err != nil {+ addError("Volume pruning failed: %v", err)}+ volumes = partreturn nil})}@@ -233,19 +244,13 @@ func (s *Service) runSystemPruneInternal(ctx context.Context, req system.PruneAlg.Go(func() (workerErr error) {defer utils.RecoverToError(&workerErr, "system info worker")- s.appendSystemPruneActivityMessageInternal(groupCtx, activityID, "Pruning networks", 65)+ s.appendSystemPruneActivityMessage(groupCtx, activityID, "Pruning networks", 65)slog.InfoContext(groupCtx, "Pruning networks...", "mode", req.Networks.Mode, "until", req.Networks.Until)- localResult := &system.PruneAllResult{}- if err := s.pruneNetworksInternal(groupCtx, *req.Networks, localResult); err != nil {- mu.Lock()- result.Errors = append(result.Errors, fmt.Sprintf("Network pruning failed: %v", err))- result.Success = false- mu.Unlock()- } else {- mu.Lock()- result.NetworksDeleted = append(result.NetworksDeleted, localResult.NetworksDeleted...)- mu.Unlock()+ part, err := s.pruneNetworks(groupCtx, *req.Networks)+ if err != nil {+ addError("Network pruning failed: %v", err)}+ networks = partreturn nil})}@@ -254,6 +259,20 @@ func (s *Service) runSystemPruneInternal(ctx context.Context, req system.PruneAlslog.ErrorContext(ctx, "Prune operations failed", "error", err)}+ result := &system.PruneAllResult{+ ContainersPruned: containers.ContainersPruned,+ ImagesDeleted: images.ImagesDeleted,+ VolumesDeleted: volumes.VolumesDeleted,+ NetworksDeleted: networks.NetworksDeleted,+ SpaceReclaimed: containers.SpaceReclaimed + images.SpaceReclaimed + buildCache.SpaceReclaimed + volumes.SpaceReclaimed,+ ContainerSpaceReclaimed: containers.ContainerSpaceReclaimed,+ ImageSpaceReclaimed: images.ImageSpaceReclaimed,+ VolumeSpaceReclaimed: volumes.VolumeSpaceReclaimed,+ BuildCacheSpaceReclaimed: buildCache.BuildCacheSpaceReclaimed,+ Success: len(errs) == 0,+ Errors: errs,+ ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer(),+ }slog.InfoContext(ctx,"Selective prune operation completed",@@ -282,37 +301,12 @@ func (s *Service) runSystemPruneInternal(ctx context.Context, req system.PruneAlresult.Errors,),)- s.completeSystemPruneActivityInternal(ctx, activityID, result)-}--func (s *Service) startSystemPruneActivityInternal(ctx context.Context, environmentID string, req system.PruneAllRequest) string {- if s.activityService == nil {- return ""- }- localActivity, err := s.activityService.StartActivity(ctx, activity.StartActivityRequest{- EnvironmentID: environmentID,- Type: activitytypes.TypeSystemPrune,- ResourceType: new("system"),- ResourceName: new("Docker resources"),- Step: "Preparing prune",- LatestMessage: "System prune started",- Metadata: database.JSON{- "containers": req.Containers,- "images": req.Images,- "volumes": req.Volumes,- "networks": req.Networks,- "buildCache": req.BuildCache,- },- })- if err != nil {- slog.DebugContext(ctx, "failed to start system prune activity", "error", err)- return ""- }- return localActivity.ID+ s.completeSystemPruneActivity(ctx, activityID, result)+ return result}-func (s *Service) appendSystemPruneActivityMessageInternal(ctx context.Context, activityID, message string, progress int) {- if s.activityService == nil || activityID == "" {+func (s *Service) appendSystemPruneActivityMessage(ctx context.Context, activityID, message string, progress int) {+ if activityID == "" {return}if _, err := s.activityService.AppendMessage(ctx, activityID, activity.AppendActivityMessageRequest{@@ -325,81 +319,79 @@ func (s *Service) appendSystemPruneActivityMessageInternal(ctx context.Context,}}-func (s *Service) completeSystemPruneActivityInternal(ctx context.Context, activityID string, result *system.PruneAllResult) {- if s.activityService == nil || activityID == "" || result == nil {+func (s *Service) completeSystemPruneActivity(ctx context.Context, activityID string, result *system.PruneAllResult) {+ if activityID == "" {return}- status := activitytypes.StatusSuccess- message := "System prune completed"- var errMessage *string- if !result.Success || len(result.Errors) > 0 {- if activitylib.CancelledByContext(ctx) {- status = activitytypes.StatusCancelled- message = "System prune cancelled"- } else {- status = activitytypes.StatusFailed- message = "System prune completed with errors"- errMessage = new(strings.Join(result.Errors, "; "))+ status, message, errMessage := func() (activitytypes.Status, string, *string) {+ switch {+ case result.Success && len(result.Errors) == 0:+ return activitytypes.StatusSuccess, "System prune completed", nil+ case activitylib.CancelledByContext(ctx):+ return activitytypes.StatusCancelled, "System prune cancelled", nil+ default:+ return activitytypes.StatusFailed, "System prune completed with errors", new(strings.Join(result.Errors, "; "))}- }+ }()if _, err := s.activityService.CompleteActivity(utils.ActivityRuntimeContext(ctx, nil), activityID, status, message, errMessage); err != nil {slog.DebugContext(ctx, "failed to complete system prune activity", "activityId", activityID, "error", err)}}-func (s *Service) pruneContainersInternal(ctx context.Context, options system.PruneContainersOptions, result *system.PruneAllResult) error {+func (s *Service) pruneContainers(ctx context.Context, options system.PruneContainersOptions) (system.PruneAllResult, error) {dockerClient, err := s.dockerClient(ctx)if err != nil {- return fmt.Errorf("failed to connect to Docker: %w", err)+ return system.PruneAllResult{}, fmt.Errorf("failed to connect to Docker: %w", err)}filterArgs := make(client.Filters)if options.Mode == system.PruneContainerModeOlderThan {if strings.TrimSpace(options.Until) == "" {- return errors.New("container prune mode olderThan requires until")+ return system.PruneAllResult{}, errors.New("container prune mode olderThan requires until")}filterArgs = filterArgs.Add("until", options.Until)}report, err := dockerClient.ContainerPrune(ctx, client.ContainerPruneOptions{Filters: filterArgs})if err != nil {- return fmt.Errorf("failed to prune containers: %w", err)+ return system.PruneAllResult{}, fmt.Errorf("failed to prune containers: %w", err)}- result.ContainersPruned = report.Report.ContainersDeleted- result.SpaceReclaimed += report.Report.SpaceReclaimed- result.ContainerSpaceReclaimed += report.Report.SpaceReclaimed- return nil+ return system.PruneAllResult{+ ContainersPruned: report.Report.ContainersDeleted,+ SpaceReclaimed: report.Report.SpaceReclaimed,+ ContainerSpaceReclaimed: report.Report.SpaceReclaimed,+ }, nil}-func (s *Service) pruneImagesInternal(ctx context.Context, options system.PruneImagesOptions, result *system.PruneAllResult) error {+func (s *Service) pruneImages(ctx context.Context, options system.PruneImagesOptions) (system.PruneAllResult, error) {dockerClient, err := s.dockerClient(ctx)if err != nil {- return fmt.Errorf("failed to connect to Docker: %w", err)+ return system.PruneAllResult{}, fmt.Errorf("failed to connect to Docker: %w", err)}filterArgs := make(client.Filters)switch options.Mode {case system.PruneImageModeNone:- return errors.New("image prune mode none is not allowed")+ return system.PruneAllResult{}, errors.New("image prune mode none is not allowed")case system.PruneImageModeDangling:filterArgs = filterArgs.Add("dangling", "true")case system.PruneImageModeAll:filterArgs = filterArgs.Add("dangling", "false")case system.PruneImageModeOlderThan:if strings.TrimSpace(options.Until) == "" {- return errors.New("image prune mode olderThan requires until")+ return system.PruneAllResult{}, errors.New("image prune mode olderThan requires until")}filterArgs = filterArgs.Add("dangling", "false")filterArgs = filterArgs.Add("until", options.Until)default:- return fmt.Errorf("unsupported image prune mode: %s", options.Mode)+ return system.PruneAllResult{}, fmt.Errorf("unsupported image prune mode: %s", options.Mode)}report, err := dockerClient.ImagePrune(ctx, client.ImagePruneOptions{Filters: filterArgs})if err != nil {- return fmt.Errorf("failed to prune images: %w", err)+ return system.PruneAllResult{}, fmt.Errorf("failed to prune images: %w", err)}slog.InfoContext(ctx, "Image pruning completed", "imagesDeleted", len(report.Report.ImagesDeleted), "bytesReclaimed", report.Report.SpaceReclaimed)@@ -418,117 +410,113 @@ func (s *Service) pruneImagesInternal(ctx context.Context, options system.PruneIslog.WarnContext(ctx, "Failed to delete image update records", "count", len(idsToDelete), "error", deleteRecordsForImagesErr.Error())}- result.ImagesDeleted = idsToDelete- result.SpaceReclaimed += report.Report.SpaceReclaimed- result.ImageSpaceReclaimed += report.Report.SpaceReclaimed- return nil+ return system.PruneAllResult{+ ImagesDeleted: idsToDelete,+ SpaceReclaimed: report.Report.SpaceReclaimed,+ ImageSpaceReclaimed: report.Report.SpaceReclaimed,+ }, nil}-func (s *Service) pruneBuildCacheInternal(ctx context.Context, options system.PruneBuildCacheOptions, result *system.PruneAllResult) error {+func (s *Service) pruneBuildCache(ctx context.Context, options system.PruneBuildCacheOptions) (system.PruneAllResult, error) {dockerClient, err := s.dockerClient(ctx)if err != nil {- result.Errors = append(result.Errors, fmt.Errorf("build cache pruning failed (connection): %w", err).Error())slog.ErrorContext(ctx, "Error connecting to Docker for build cache prune", "error", err.Error())- return fmt.Errorf("failed to connect to Docker for build cache prune: %w", err)- }-- pruneOptions := client.BuildCachePruneOptions{- All: options.Mode == system.PruneBuildCacheModeAll,+ return system.PruneAllResult{}, fmt.Errorf("failed to connect to Docker for build cache prune: %w", err)}- if options.Mode == system.PruneBuildCacheModeOlderThan {- if strings.TrimSpace(options.Until) == "" {- return errors.New("build cache prune mode olderThan requires until")- }- pruneOptions.Filters = make(client.Filters)- pruneOptions.Filters = pruneOptions.Filters.Add("until", options.Until)+ if options.Mode == system.PruneBuildCacheModeOlderThan && strings.TrimSpace(options.Until) == "" {+ return system.PruneAllResult{}, errors.New("build cache prune mode olderThan requires until")}slog.DebugContext(ctx, "starting build cache pruning", "mode", options.Mode, "until", options.Until)- report, err := dockerClient.BuildCachePrune(ctx, pruneOptions)+ report, err := dockerClient.BuildCachePrune(ctx, client.BuildCachePruneOptions{+ All: options.Mode == system.PruneBuildCacheModeAll,+ Filters: kit.Ternary(options.Mode == system.PruneBuildCacheModeOlderThan, make(client.Filters).Add("until", options.Until), nil),+ })if err != nil {- result.Errors = append(result.Errors, fmt.Errorf("build cache pruning failed: %w", err).Error())slog.ErrorContext(ctx, "Error pruning build cache", "error", err.Error())- return fmt.Errorf("failed to prune build cache: %w", err)+ return system.PruneAllResult{}, fmt.Errorf("failed to prune build cache: %w", err)}slog.InfoContext(ctx, "build cache pruning completed", "cacheEntriesDeleted", len(report.Report.CachesDeleted), "bytesReclaimed", report.Report.SpaceReclaimed)- result.SpaceReclaimed += report.Report.SpaceReclaimed- result.BuildCacheSpaceReclaimed += report.Report.SpaceReclaimed- return nil+ return system.PruneAllResult{+ SpaceReclaimed: report.Report.SpaceReclaimed,+ BuildCacheSpaceReclaimed: report.Report.SpaceReclaimed,+ }, nil}-func (s *Service) pruneVolumesInternal(ctx context.Context, options system.PruneVolumesOptions, result *system.PruneAllResult) error {- allVolumes := options.Mode == system.PruneVolumeModeAll- report, err := s.volumeService.PruneVolumesWithOptions(ctx, allVolumes)+func (s *Service) pruneVolumes(ctx context.Context, options system.PruneVolumesOptions) (system.PruneAllResult, error) {+ report, err := s.volumeService.PruneVolumesWithOptions(ctx, options.Mode == system.PruneVolumeModeAll)if err != nil {- return err+ return system.PruneAllResult{}, err}slog.InfoContext(ctx, "Volume prune completed", "volumesDeleted", len(report.VolumesDeleted), "spaceReclaimed", report.SpaceReclaimed)- result.VolumesDeleted = report.VolumesDeleted- result.SpaceReclaimed += report.SpaceReclaimed- result.VolumeSpaceReclaimed += report.SpaceReclaimed- return nil+ return system.PruneAllResult{+ VolumesDeleted: report.VolumesDeleted,+ SpaceReclaimed: report.SpaceReclaimed,+ VolumeSpaceReclaimed: report.SpaceReclaimed,+ }, nil}-func (s *Service) pruneNetworksInternal(ctx context.Context, options system.PruneNetworksOptions, result *system.PruneAllResult) error {+func (s *Service) pruneNetworks(ctx context.Context, options system.PruneNetworksOptions) (system.PruneAllResult, error) {dockerClient, err := s.dockerClient(ctx)if err != nil {- return fmt.Errorf("failed to connect to Docker: %w", err)+ return system.PruneAllResult{}, fmt.Errorf("failed to connect to Docker: %w", err)}filterArgs := make(client.Filters)if options.Mode == system.PruneNetworkModeOlderThan {if strings.TrimSpace(options.Until) == "" {- return errors.New("network prune mode olderThan requires until")+ return system.PruneAllResult{}, errors.New("network prune mode olderThan requires until")}filterArgs = filterArgs.Add("until", options.Until)}report, err := dockerClient.NetworkPrune(ctx, client.NetworkPruneOptions{Filters: filterArgs})if err != nil {- return fmt.Errorf("failed to prune networks: %w", err)+ return system.PruneAllResult{}, fmt.Errorf("failed to prune networks: %w", err)}slog.InfoContext(ctx, "Network prune completed", "networksDeleted", len(report.Report.NetworksDeleted))- result.NetworksDeleted = report.Report.NetworksDeleted- return nil+ return system.PruneAllResult{NetworksDeleted: report.Report.NetworksDeleted}, nil}-type pruneResourceInternal struct {+type pruneResource struct {Kind string `json:"kind"`ID string `json:"id"`Size int64 `json:"size"`Names []string `json:"names,omitempty"`}-type prunePlanInternal struct {- Request system.PruneAllRequest `json:"request"`- Resources []pruneResourceInternal `json:"resources"`+type prunePlan struct {+ Request system.PruneAllRequest `json:"request"`+ Resources []pruneResource `json:"resources"`}// PruneScheduled freezes exact targets before applying destructive work.func (s *Service) PruneScheduled(ctx context.Context, environmentID string, req system.PruneAllRequest) (*system.PruneAllResult, bool, error) {- prune := s.beginSystemPruneInternal(ctx, environmentID, req)- result := &system.PruneAllResult{Success: true}- if prune.activityID != "" {- result.ActivityID = &prune.activityID- }+ prune := s.beginSystemPrune(ctx, environmentID, req)if prune.started.IsAbsent() {- return result, false, nil+ return &system.PruneAllResult{+ Success: true,+ ActivityID: mo.EmptyableToOption(prune.activityID).ToPointer(),+ }, false, nil}- defer s.finishSystemPruneInternal(environmentID)+ defer s.finishSystemPrune(environmentID)fail := func(err error) (*system.PruneAllResult, bool, error) {- result.Success = false- result.Errors = append(result.Errors, err.Error())- s.completeSystemPruneActivityInternal(ctx, prune.activityID, result)+ result := &system.PruneAllResult{+ Success: false,+ Errors: []string{err.Error()},+ ActivityID: mo.EmptyableToOption(prune.activityID).ToPointer(),+ }+ s.completeSystemPruneActivity(ctx, prune.activityID, result)return result, true, err}- plan, err := s.preparePrunePlanInternal(ctx, req)+ plan, err := s.preparePrunePlan(ctx, req)if err != nil {return fail(err)}@@ -549,186 +537,188 @@ func (s *Service) PruneScheduled(ctx context.Context, environmentID string, reqreturn fail(progressErr)}previous, _ := jobcontext.Run(ctx)- err = s.executePrunePlanInternal(ctx, plan, previous, result, false)- s.completeSystemPruneActivityInternal(ctx, prune.activityID, result)+ result, err := s.executePrunePlan(ctx, plan, previous, prune.activityID, false)+ s.completeSystemPruneActivity(ctx, prune.activityID, result)if err == nil && result.Success {err = jobcontext.Progress(ctx, scheduler.TargetOutcome{ID: "scheduled-prune", Status: scheduler.Succeeded, ActivityID: prune.activityID})}return result, true, err}-func (s *Service) preparePrunePlanInternal(ctx context.Context, req system.PruneAllRequest) (prunePlanInternal, error) {- plan := prunePlanInternal{Request: req}+func (s *Service) preparePrunePlan(ctx context.Context, req system.PruneAllRequest) (prunePlan, error) {dockerClient, err := s.dockerClient(ctx)if err != nil {- return plan, err+ return prunePlan{}, err}if req.Volumes != nil {s.volumeService.CleanupHelperContainers(ctx)}- usage, err := dockerClient.DiskUsage(ctx, client.DiskUsageOptions{Containers: true, Images: true, Volumes: true, BuildCache: req.BuildCache != nil, Verbose: true})+ usage, err := dockerClient.DiskUsage(ctx, client.DiskUsageOptions{+ Containers: true,+ Images: true,+ Volumes: true,+ BuildCache: req.BuildCache != nil,+ Verbose: true,+ })if err != nil {- return plan, err+ return prunePlan{}, err}- if selectPruneContainersErr := s.selectPruneContainersInternal(&plan, usage); selectPruneContainersErr != nil {- return plan, selectPruneContainersErr+ containers, err := s.selectPruneContainers(req, usage)+ if err != nil {+ return prunePlan{}, err}- selectedContainers, imageRefs, volumeRefs := pruneSelectedReferencesInternal(plan.Resources, usage)- if selectPruneImagesErr := s.selectPruneImagesInternal(&plan, usage, imageRefs); selectPruneImagesErr != nil {- return plan, selectPruneImagesErr+ selectedContainers, imageRefs, volumeRefs := pruneSelectedReferences(containers, usage)+ images, err := s.selectPruneImages(req, usage, imageRefs)+ if err != nil {+ return prunePlan{}, err}- s.selectPruneVolumesInternal(&plan, usage, volumeRefs)- if selectPruneNetworksErr := s.selectPruneNetworksInternal(ctx, dockerClient, &plan, selectedContainers); selectPruneNetworksErr != nil {- return plan, selectPruneNetworksErr+ networks, err := s.selectPruneNetworks(ctx, dockerClient, req, selectedContainers)+ if err != nil {+ return prunePlan{}, err}- if selectPruneBuildCacheErr := s.selectPruneBuildCacheInternal(&plan, usage); selectPruneBuildCacheErr != nil {- return plan, selectPruneBuildCacheErr+ buildCache, err := s.selectPruneBuildCache(req, usage)+ if err != nil {+ return prunePlan{}, err}- return plan, nil+ return prunePlan{+ Request: req,+ Resources: slices.Concat(containers, images, s.selectPruneVolumes(req, usage, volumeRefs), networks, buildCache),+ }, nil}-func (s *Service) selectPruneContainersInternal(plan *prunePlanInternal, usage client.DiskUsageResult) error {- req := plan.Request- var err error+func (s *Service) selectPruneContainers(req system.PruneAllRequest, usage client.DiskUsageResult) ([]pruneResource, error) {if req.Containers == nil || req.Containers.Mode == system.PruneContainerModeNone {- return nil+ return nil, nil}- until := time.Time{}- if req.Containers.Mode == system.PruneContainerModeOlderThan {- until, err = s.pruneCutoffInternal(req.Containers.Until)- if err != nil {- return err- }+ until, err := s.pruneCutoff(req.Containers.Mode == system.PruneContainerModeOlderThan, req.Containers.Until)+ if err != nil {+ return nil, err}+ var resources []pruneResourcefor _, item := range usage.Containers.Items {if item.State == "running" || item.State == "paused" || item.State == "restarting" || (!until.IsZero() && !time.Unix(item.Created, 0).Before(until)) {continue}- plan.Resources = append(plan.Resources, pruneResourceInternal{Kind: "container", ID: item.ID, Size: item.SizeRw})+ resources = append(resources, pruneResource{+ Kind: "container",+ ID: item.ID,+ Size: item.SizeRw,+ })}- return nil+ return resources, nil}-func (s *Service) selectPruneImagesInternal(plan *prunePlanInternal, usage client.DiskUsageResult, imageRefs map[string]int) error {- req := plan.Request- var err error+func (s *Service) selectPruneImages(req system.PruneAllRequest, usage client.DiskUsageResult, imageRefs map[string]int) ([]pruneResource, error) {if req.Images == nil || req.Images.Mode == system.PruneImageModeNone {- return nil+ return nil, nil}- until := time.Time{}- if req.Images.Mode == system.PruneImageModeOlderThan {- until, err = s.pruneCutoffInternal(req.Images.Until)- if err != nil {- return err- }+ until, err := s.pruneCutoff(req.Images.Mode == system.PruneImageModeOlderThan, req.Images.Until)+ if err != nil {+ return nil, err}+ var resources []pruneResourcefor _, item := range usage.Images.Items {- if item.Containers > int64(- imageRefs[item.ID],- ) || (req.Images.Mode == system.PruneImageModeDangling && len(- item.RepoTags,- ) > 0) || (!until.IsZero() && !time.Unix(- item.Created,- 0,- ).Before(- until,- )) {+ if item.Containers > int64(imageRefs[item.ID]) ||+ (req.Images.Mode == system.PruneImageModeDangling && len(item.RepoTags) > 0) ||+ (!until.IsZero() && !time.Unix(item.Created, 0).Before(until)) {continue}- plan.Resources = append(plan.Resources, pruneResourceInternal{Kind: "image", ID: item.ID, Size: item.Size, Names: slices.Clone(item.RepoTags)})+ resources = append(resources, pruneResource{+ Kind: "image",+ ID: item.ID,+ Size: item.Size,+ Names: slices.Clone(item.RepoTags),+ })}- return nil+ return resources, nil}-func (s *Service) selectPruneVolumesInternal(plan *prunePlanInternal, usage client.DiskUsageResult, volumeRefs map[string]int) {- req := plan.Request- if req.Volumes != nil && req.Volumes.Mode != system.PruneVolumeModeNone {- for _, item := range usage.Volumes.Items {- // The backup volume is unused between backups but holds them, so it is never pruned.- if item.UsageData == nil || item.UsageData.RefCount > int64(volumeRefs[item.Name]) ||- strings.EqualFold(item.Labels[libarcane.InternalResourceLabel], "true") || s.volumeService.IsBackupVolume(item.Name) {- continue- }- if req.Volumes.Mode == system.PruneVolumeModeAnonymous {- if _, anonymous := item.Labels["com.docker.volume.anonymous"]; !anonymous {- continue- }- }- plan.Resources = append(plan.Resources, pruneResourceInternal{Kind: "volume", ID: item.Name, Size: item.UsageData.Size})+func (s *Service) selectPruneVolumes(req system.PruneAllRequest, usage client.DiskUsageResult, volumeRefs map[string]int) []pruneResource {+ if req.Volumes == nil || req.Volumes.Mode == system.PruneVolumeModeNone {+ return nil+ }+ var resources []pruneResource+ for _, item := range usage.Volumes.Items {+ // The backup volume is unused between backups but holds them, so it is never pruned.+ if item.UsageData == nil || item.UsageData.RefCount > int64(volumeRefs[item.Name]) ||+ strings.EqualFold(item.Labels[libarcane.InternalResourceLabel], "true") || s.volumeService.IsBackupVolume(item.Name) {+ continue+ }+ if _, anonymous := item.Labels["com.docker.volume.anonymous"]; req.Volumes.Mode == system.PruneVolumeModeAnonymous && !anonymous {+ continue}+ resources = append(resources, pruneResource{+ Kind: "volume",+ ID: item.Name,+ Size: item.UsageData.Size,+ })}+ return resources}-func (s *Service) selectPruneNetworksInternal(ctx context.Context, dockerClient *client.Client, plan *prunePlanInternal, selectedContainers map[string]bool) error {- req := plan.Request- var err error+func (s *Service) selectPruneNetworks(ctx context.Context, dockerClient *client.Client, req system.PruneAllRequest, selectedContainers map[string]bool) ([]pruneResource, error) {if req.Networks == nil || req.Networks.Mode == system.PruneNetworkModeNone {- return nil+ return nil, nil}- until := time.Time{}- if req.Networks.Mode == system.PruneNetworkModeOlderThan {- until, err = s.pruneCutoffInternal(req.Networks.Until)- if err != nil {- return err- }+ until, err := s.pruneCutoff(req.Networks.Mode == system.PruneNetworkModeOlderThan, req.Networks.Until)+ if err != nil {+ return nil, err}networks, err := dockerClient.NetworkList(ctx, client.NetworkListOptions{})if err != nil {- return err+ return nil, err}+ var resources []pruneResource+nextNetwork:for _, item := range networks.Items {if item.Name == "bridge" || item.Name == "host" || item.Name == "none" || item.Ingress || item.Scope != "local" || (!until.IsZero() && !item.Created.Before(until)) {continue}inspect, networkInspectWithCompatibilityErr := compat.NetworkInspectWithCompatibility(ctx, dockerClient, item.ID, client.NetworkInspectOptions{})if networkInspectWithCompatibilityErr != nil {- return networkInspectWithCompatibilityErr+ return nil, networkInspectWithCompatibilityErr}- eligible := truefor id := range inspect.Network.Containers {if !selectedContainers[id] {- eligible = false- break+ continue nextNetwork}}- if !eligible {- continue- }- plan.Resources = append(plan.Resources, pruneResourceInternal{Kind: "network", ID: item.ID})+ resources = append(resources, pruneResource{+ Kind: "network",+ ID: item.ID,+ })}- return nil+ return resources, nil}-func (s *Service) selectPruneBuildCacheInternal(plan *prunePlanInternal, usage client.DiskUsageResult) error {- req := plan.Request- var err error+func (s *Service) selectPruneBuildCache(req system.PruneAllRequest, usage client.DiskUsageResult) ([]pruneResource, error) {if req.BuildCache == nil || req.BuildCache.Mode == system.PruneBuildCacheModeNone {- return nil+ return nil, nil}- until := time.Time{}- if req.BuildCache.Mode == system.PruneBuildCacheModeOlderThan {- until, err = s.pruneCutoffInternal(req.BuildCache.Until)- if err != nil {- return err- }+ until, err := s.pruneCutoff(req.BuildCache.Mode == system.PruneBuildCacheModeOlderThan, req.BuildCache.Until)+ if err != nil {+ return nil, err}+ var resources []pruneResourcefor _, item := range usage.BuildCache.Items {if item.InUse || (!until.IsZero() && (item.LastUsedAt == nil || !item.LastUsedAt.Before(until))) {continue}- plan.Resources = append(plan.Resources, pruneResourceInternal{Kind: "build_cache", ID: item.ID, Size: item.Size})+ resources = append(resources, pruneResource{+ Kind: "build_cache",+ ID: item.ID,+ Size: item.Size,+ })}- return nil+ return resources, nil}-func pruneSelectedReferencesInternal(resources []pruneResourceInternal, usage client.DiskUsageResult) (map[string]bool, map[string]int, map[string]int) {+func pruneSelectedReferences(containers []pruneResource, usage client.DiskUsageResult) (map[string]bool, map[string]int, map[string]int) {selectedContainers := make(map[string]bool)imageRefs := make(map[string]int)volumeRefs := make(map[string]int)- for _, resource := range resources {- if resource.Kind == "container" {- selectedContainers[resource.ID] = true- }+ for _, resource := range containers {+ selectedContainers[resource.ID] = true}for _, item := range usage.Containers.Items {if !selectedContainers[item.ID] {@@ -746,17 +736,15 @@ func pruneSelectedReferencesInternal(resources []pruneResourceInternal, usage clreturn selectedContainers, imageRefs, volumeRefs}-func (s *Service) pruneCutoffInternal(raw string) (time.Time, error) {- now := time.Now()- location := s.location- if location == nil {- location = time.UTC+func (s *Service) pruneCutoff(olderThan bool, raw string) (time.Time, error) {+ if !olderThan {+ return time.Time{}, nil}if duration, err := time.ParseDuration(raw); err == nil && duration >= 0 {- return now.Add(-duration), nil+ return time.Now().Add(-duration), nil}for _, format := range []string{time.RFC3339Nano, "2006-01-02T15:04:05", "2006-01-02"} {- if value, err := time.ParseInLocation(format, raw, location); err == nil {+ if value, err := time.ParseInLocation(format, raw, s.location); err == nil {return value, nil}}@@ -766,133 +754,155 @@ func (s *Service) pruneCutoffInternal(raw string) (time.Time, error) {return time.Time{}, fmt.Errorf("unsupported prune cutoff %q", raw)}-func (s *Service) executePrunePlanInternal(ctx context.Context, plan prunePlanInternal, previous scheduler.Run, result *system.PruneAllResult, recovery bool) error {+func (s *Service) executePrunePlan(+ ctx context.Context, plan prunePlan, previous scheduler.Run, activityID string, recovery bool,+) (*system.PruneAllResult, error) {+ var containersPruned, imagesDeleted, volumesDeleted, networksDeleted, errs []string+ var spaceReclaimed, containerSpace, imageSpace, volumeSpace, buildCacheSpace uint64+ finish := func(err error) (*system.PruneAllResult, error) {+ if err != nil {+ errs = append(errs, err.Error())+ }+ return &system.PruneAllResult{+ ContainersPruned: containersPruned,+ ImagesDeleted: imagesDeleted,+ VolumesDeleted: volumesDeleted,+ NetworksDeleted: networksDeleted,+ SpaceReclaimed: spaceReclaimed,+ ContainerSpaceReclaimed: containerSpace,+ ImageSpaceReclaimed: imageSpace,+ VolumeSpaceReclaimed: volumeSpace,+ BuildCacheSpaceReclaimed: buildCacheSpace,+ Success: len(errs) == 0,+ Errors: errs,+ ActivityID: mo.EmptyableToOption(activityID).ToPointer(),+ }, err+ }dockerClient, err := s.dockerClient(ctx)if err != nil {- return err+ return finish(err)}for _, resource := range plan.Resources {if resource.Kind == "build_cache" {continue}- target := scheduler.TargetOutcome{ResourceType: resource.Kind, ID: "prune:" + resource.Kind + ":" + resource.ID, Status: scheduler.Running}- if pruneTargetCompletedInternal(previous, target.ID) {+ targetID := "prune:" + resource.Kind + ":" + resource.ID+ if slices.ContainsFunc(previous.Outcome.Targets, func(target scheduler.TargetOutcome) bool {+ return target.ID == targetID && (target.Status == scheduler.Succeeded || target.Status == scheduler.Skipped)+ }) {continue}- if progressErr := jobcontext.Progress(ctx, target); progressErr != nil {- return progressErr+ if progressErr := jobcontext.Progress(ctx, scheduler.TargetOutcome{+ ResourceType: resource.Kind,+ ID: targetID,+ Status: scheduler.Running,+ }); progressErr != nil {+ return finish(progressErr)}- eligible, exists, pruneResourceEligibleErr := s.pruneResourceEligibleInternal(ctx, dockerClient, plan, &resource)- if pruneResourceEligibleErr != nil {- return pruneResourceEligibleErr+ size, eligible, exists, eligibleErr := s.pruneResourceEligible(ctx, dockerClient, plan, resource)+ if eligibleErr != nil {+ return finish(eligibleErr)}- switch {- case !exists:- target.Status = scheduler.Succeeded- case !eligible:- target.Status = scheduler.Skipped- default:- pruneResourceEligibleErr = s.removePruneResourceInternal(ctx, dockerClient, resource)- if pruneResourceEligibleErr != nil && !errdefs.IsNotFound(pruneResourceEligibleErr) {- target.Status = scheduler.Failed- target.Message = pruneResourceEligibleErr.Error()- result.Success = false- result.Errors = append(result.Errors, pruneResourceEligibleErr.Error())- } else {- target.Status = scheduler.Succeeded- if pruneResourceEligibleErr != nil {- // Something else removed it first; do not claim its bytes.- resource.Size = 0- }- recordPrunedResourceInternal(resource, result)+ removeErr := func() error {+ if !exists || !eligible {+ return nil+ }+ return s.removePruneResource(ctx, dockerClient, resource)+ }()+ removeFailed := removeErr != nil && !errdefs.IsNotFound(removeErr)+ if removeFailed {+ errs = append(errs, removeErr.Error())+ }+ if exists && eligible && !removeFailed {+ // Something else removed it first; do not claim its bytes.+ claimed := kit.Ternary(removeErr == nil && size > 0, uint64(size), 0)+ spaceReclaimed += claimed+ switch resource.Kind {+ case "container":+ containersPruned = append(containersPruned, resource.ID)+ containerSpace += claimed+ case "image":+ imagesDeleted = append(imagesDeleted, resource.ID)+ imageSpace += claimed+ case "volume":+ volumesDeleted = append(volumesDeleted, resource.ID)+ volumeSpace += claimed+ case "network":+ networksDeleted = append(networksDeleted, resource.ID)}}- if targetProgressErr := jobcontext.Progress(ctx, target); targetProgressErr != nil {- return targetProgressErr+ if targetProgressErr := jobcontext.Progress(ctx, scheduler.TargetOutcome{+ ResourceType: resource.Kind,+ ID: targetID,+ Status: func() scheduler.RunStatus {+ switch {+ case !exists:+ return scheduler.Succeeded+ case !eligible:+ return scheduler.Skipped+ case removeFailed:+ return scheduler.Failed+ default:+ return scheduler.Succeeded+ }+ }(),+ Message: kit.Ternary(removeFailed, fmt.Sprint(removeErr), ""),+ }); targetProgressErr != nil {+ return finish(targetProgressErr)}}- if len(result.ImagesDeleted) > 0 {- if deleteRecordsForImagesErr := s.imageUpdateService.DeleteRecordsForImages(ctx, result.ImagesDeleted); deleteRecordsForImagesErr != nil {- return deleteRecordsForImagesErr+ if len(imagesDeleted) > 0 {+ if deleteRecordsForImagesErr := s.imageUpdateService.DeleteRecordsForImages(ctx, imagesDeleted); deleteRecordsForImagesErr != nil {+ return finish(deleteRecordsForImagesErr)}}- return s.executePruneCacheInternal(ctx, dockerClient, plan, result, recovery)+ reclaimed, err := s.executePruneCache(ctx, dockerClient, plan, recovery)+ spaceReclaimed += reclaimed+ buildCacheSpace += reclaimed+ return finish(err)}-func (s *Service) removePruneResourceInternal(ctx context.Context, dockerClient *client.Client, resource pruneResourceInternal) error {- var err error+func (s *Service) removePruneResource(ctx context.Context, dockerClient *client.Client, resource pruneResource) error {switch resource.Kind {case "container":- _, err = dockerClient.ContainerRemove(ctx, resource.ID, client.ContainerRemoveOptions{})+ _, err := dockerClient.ContainerRemove(ctx, resource.ID, client.ContainerRemoveOptions{})+ return errcase "image":- err = removePruneImageInternal(ctx, dockerClient, resource)+ for _, name := range resource.Names {+ inspected, inspectErr := dockerClient.ImageInspect(ctx, name)+ if errdefs.IsNotFound(inspectErr) {+ continue+ }+ if inspectErr != nil {+ return inspectErr+ }+ if inspected.ID != resource.ID {+ continue+ }+ if _, err := dockerClient.ImageRemove(ctx, name, client.ImageRemoveOptions{PruneChildren: false}); err != nil {+ return err+ }+ }+ _, err := dockerClient.ImageRemove(ctx, resource.ID, client.ImageRemoveOptions{PruneChildren: false})+ return errcase "volume":- err = s.volumeService.DeleteVolume(ctx, resource.ID, false, user.SystemUser)+ return s.volumeService.DeleteVolume(ctx, resource.ID, false, user.SystemUser)case "network":- err = s.networkService.RemoveNetwork(ctx, resource.ID, user.SystemUser)+ return s.networkService.RemoveNetwork(ctx, resource.ID, user.SystemUser)default:return fmt.Errorf("unknown prune target %q", resource.Kind)}- return err-}--func removePruneImageInternal(ctx context.Context, dockerClient *client.Client, resource pruneResourceInternal) error {- var err error- for _, name := range resource.Names {- inspected, inspectErr := dockerClient.ImageInspect(ctx, name)- if errdefs.IsNotFound(inspectErr) {- continue- }- if inspectErr != nil {- return inspectErr- }- if inspected.ID != resource.ID {- continue- }- _, err = dockerClient.ImageRemove(ctx, name, client.ImageRemoveOptions{PruneChildren: false})- if err != nil {- return err- }- }- _, err = dockerClient.ImageRemove(ctx, resource.ID, client.ImageRemoveOptions{PruneChildren: false})- return err-}--func recordPrunedResourceInternal(resource pruneResourceInternal, result *system.PruneAllResult) {- switch resource.Kind {- case "container":- result.ContainersPruned = append(result.ContainersPruned, resource.ID)- case "image":- result.ImagesDeleted = append(result.ImagesDeleted, resource.ID)- case "volume":- result.VolumesDeleted = append(result.VolumesDeleted, resource.ID)- case "network":- result.NetworksDeleted = append(result.NetworksDeleted, resource.ID)- }- if resource.Size <= 0 {- return- }- size := uint64(resource.Size)- result.SpaceReclaimed += size- switch resource.Kind {- case "container":- result.ContainerSpaceReclaimed += size- case "image":- result.ImageSpaceReclaimed += size- case "volume":- result.VolumeSpaceReclaimed += size- }}-func (s *Service) executePruneCacheInternal(ctx context.Context, dockerClient *client.Client, plan prunePlanInternal, result *system.PruneAllResult, recovery bool) error {+func (s *Service) executePruneCache(ctx context.Context, dockerClient *client.Client, plan prunePlan, recovery bool) (uint64, error) {request := plan.Request.BuildCacheif request == nil || request.Mode == system.PruneBuildCacheModeNone {- return nil+ return 0, nil}if recovery {usage, err := dockerClient.DiskUsage(ctx, client.DiskUsageOptions{BuildCache: true, Verbose: true})if err != nil {- return err+ return 0, err}selected := make(map[string]bool)for _, resource := range plan.Resources {@@ -902,11 +912,12 @@ func (s *Service) executePruneCacheInternal(ctx context.Context, dockerClient *c}for _, current := range usage.BuildCache.Items {if selected[current.ID] {- return errors.New("build cache prune outcome is unconfirmed")+ return 0, errors.New("build cache prune outcome is unconfirmed")}}- return nil+ return 0, nil}+ var reclaimed uint64for _, resource := range plan.Resources {if resource.Kind != "build_cache" {continue@@ -921,72 +932,66 @@ func (s *Service) executePruneCacheInternal(ctx context.Context, dockerClient *cFilters: filters,})if err != nil {- return err+ return reclaimed, err}- result.SpaceReclaimed += report.Report.SpaceReclaimed- result.BuildCacheSpaceReclaimed += report.Report.SpaceReclaimed+ reclaimed += report.Report.SpaceReclaimed}- return nil+ return reclaimed, nil}-// pruneResourceEligibleInternal re-checks a planned resource against live state and refreshes its size from that snapshot.-func (s *Service) pruneResourceEligibleInternal(ctx context.Context, dockerClient *client.Client, plan prunePlanInternal, resource *pruneResourceInternal) (eligible, exists bool, err error) {+// pruneResourceEligible re-checks a planned resource against live state and returns its size from that snapshot.+func (s *Service) pruneResourceEligible(ctx context.Context, dockerClient *client.Client, plan prunePlan, resource pruneResource) (size int64, eligible, exists bool, err error) {if resource.Kind == "network" {inspected, networkInspectWithCompatibilityErr := compat.NetworkInspectWithCompatibility(ctx, dockerClient, resource.ID, client.NetworkInspectOptions{})if errdefs.IsNotFound(networkInspectWithCompatibilityErr) {- return false, false, nil+ return resource.Size, false, false, nil}if networkInspectWithCompatibilityErr != nil {- return false, false, networkInspectWithCompatibilityErr+ return resource.Size, false, false, networkInspectWithCompatibilityErr}- return len(inspected.Network.Containers) == 0 && !inspected.Network.Ingress, true, nil+ return resource.Size, len(inspected.Network.Containers) == 0 && !inspected.Network.Ingress, true, nil}- usage, err := dockerClient.DiskUsage(ctx, client.DiskUsageOptions{Containers: resource.Kind == "container", Images: resource.Kind == "image", Volumes: resource.Kind == "volume", Verbose: true})+ usage, err := dockerClient.DiskUsage(ctx, client.DiskUsageOptions{+ Containers: resource.Kind == "container",+ Images: resource.Kind == "image",+ Volumes: resource.Kind == "volume",+ Verbose: true,+ })if err != nil {- return false, false, err+ return resource.Size, false, false, err}switch resource.Kind {case "container":for _, item := range usage.Containers.Items {if item.ID == resource.ID {- resource.Size = item.SizeRw- return item.State != "running" && item.State != "paused" && item.State != "restarting", true, nil+ return item.SizeRw, item.State != "running" && item.State != "paused" && item.State != "restarting", true, nil}}case "image":- return pruneImageEligibleInternal(usage, plan.Request.Images, resource)- case "volume":- for _, item := range usage.Volumes.Items {- if item.Name == resource.ID {- if item.UsageData == nil {- return false, true, nil- }- resource.Size = item.UsageData.Size- return item.UsageData.RefCount == 0 && !strings.EqualFold(item.Labels[libarcane.InternalResourceLabel], "true") &&- !s.volumeService.IsBackupVolume(item.Name), true, nil+ for _, item := range usage.Images.Items {+ if item.ID != resource.ID {+ continue}- }- }- return false, false, nil-}--func pruneImageEligibleInternal(usage client.DiskUsageResult, request *system.PruneImagesOptions, resource *pruneResourceInternal) (eligible, exists bool, err error) {- for _, item := range usage.Images.Items {- if item.ID == resource.ID {// Size - SharedSize is the unique layer size Docker counts as reclaimable for an unused image.- resource.Size = 0- if item.SharedSize >= 0 {- resource.Size = item.Size - item.SharedSize+ reclaimable := kit.Ternary(item.SharedSize >= 0, item.Size-item.SharedSize, 0)+ if slices.ContainsFunc(item.RepoTags, func(tag string) bool { return !slices.Contains(resource.Names, tag) }) {+ return reclaimable, false, true, nil}- for _, tag := range item.RepoTags {- if !slices.Contains(resource.Names, tag) {- return false, true, nil- }+ return reclaimable, item.Containers == 0 && (plan.Request.Images.Mode != system.PruneImageModeDangling || len(item.RepoTags) == 0), true, nil+ }+ case "volume":+ for _, item := range usage.Volumes.Items {+ if item.Name != resource.ID {+ continue+ }+ if item.UsageData == nil {+ return resource.Size, false, true, nil}- return item.Containers == 0 && (request.Mode != system.PruneImageModeDangling || len(item.RepoTags) == 0), true, nil+ return item.UsageData.Size, item.UsageData.RefCount == 0 && !strings.EqualFold(item.Labels[libarcane.InternalResourceLabel], "true") &&+ !s.volumeService.IsBackupVolume(item.Name), true, nil}}- return false, false, nil+ return resource.Size, false, false, nil}func (s *Service) ReconcileScheduledPrune(ctx context.Context, previous scheduler.Run) (scheduler.Outcome, error) {@@ -998,25 +1003,25 @@ func (s *Service) ReconcileScheduledPrune(ctx context.Context, previous scheduleif target.ID != "prune-plan" || len(target.RecoveryData) == 0 {continue}- var plan prunePlanInternal+ var plan prunePlanif err := json.Unmarshal(target.RecoveryData, &plan); err != nil {return outcome, err}- prune := s.beginSystemPruneInternal(ctx, previous.EnvironmentID, plan.Request)+ prune := s.beginSystemPrune(ctx, previous.EnvironmentID, plan.Request)if prune.started.IsAbsent() {return outcome, nil}- defer s.finishSystemPruneInternal(previous.EnvironmentID) //nolint:gocritic // The matching recovery target always returns before the loop advances.- result := &system.PruneAllResult{Success: true}- err := s.executePrunePlanInternal(ctx, plan, previous, result, true)+ defer s.finishSystemPrune(previous.EnvironmentID) //nolint:gocritic // The matching recovery target always returns before the loop advances.+ result, err := s.executePrunePlan(ctx, plan, previous, "", true)+ s.completeSystemPruneActivity(ctx, prune.activityID, result)if err != nil {- result.Success = false- result.Errors = append(result.Errors, err.Error())- }- s.completeSystemPruneActivityInternal(ctx, prune.activityID, result)- if err != nil {- outcome.Message = err.Error()- return outcome, nil+ return scheduler.Outcome{+ Status: outcome.Status,+ Message: err.Error(),+ ActivityID: outcome.ActivityID,+ Targets: outcome.Targets,+ Steps: outcome.Steps,+ }, nil}if !result.Success {return scheduler.Outcome{Status: scheduler.Partial, Targets: previous.Outcome.Targets}, nil@@ -1028,12 +1033,3 @@ func (s *Service) ReconcileScheduledPrune(ctx context.Context, previous schedule}return outcome, nil}--func pruneTargetCompletedInternal(previous scheduler.Run, targetID string) bool {- for _, target := range previous.Outcome.Targets {- if target.ID == targetID && (target.Status == scheduler.Succeeded || target.Status == scheduler.Skipped) {- return true- }- }- return false-}diff --git a/backend/internal/system/children/upgrade/handler.go b/backend/internal/system/children/upgrade/handler.goindex b1eaeb365..300d2abb1 100644--- a/backend/internal/system/children/upgrade/handler.go+++ b/backend/internal/system/children/upgrade/handler.go@@ -2,8 +2,8 @@ package upgradeimport ("context"- "errors""log/slog"+ "net/http""github.com/danielgtaylor/huma/v2""github.com/getarcaneapp/arcane/types/v2/base"@@ -97,12 +97,8 @@ func (h *Handler) TriggerUpgrade(ctx context.Context, input *TriggerUpgradeInputerr = h.service.TriggerUpgradeAsync(utils.ActivityRuntimeContext(ctx, h.appCtx), *user, kit.FromPtr(input.Body).TargetVersion)if err != nil {slog.ErrorContext(ctx, "System upgrade failed", "error", err, "user", user.Username)-- if errors.Is(err, common.ErrUpgradeInProgress) {- return nil, huma.Error409Conflict("Failed to initiate upgrade: " + err.Error())- }-- return nil, huma.Error500InternalServerError("Failed to initiate upgrade: " + err.Error())+ apiErr := common.ToAPIError(err)+ return nil, huma.NewError(apiErr.HTTPStatus(), "Failed to initiate upgrade: "+err.Error())}message := kit.Ternary(@@ -124,7 +120,7 @@ func (h *Handler) TriggerUpgrade(ctx context.Context, input *TriggerUpgradeInput// TriggerUpdateAll starts a fleet-wide update: every remote agent in turn, then the manager last.func (h *Handler) TriggerUpdateAll(ctx context.Context, input *TriggerUpdateAllInput) (*handlerutil.Out[system.EnvironmentUpdateJob], error) {- if h.cfg != nil && h.cfg.AgentMode {+ if h.cfg.AgentMode {return nil, huma.Error400BadRequest("update-all is managed on the Arcane manager")}@@ -137,8 +133,8 @@ func (h *Handler) TriggerUpdateAll(ctx context.Context, input *TriggerUpdateAllIjob, err := h.service.StartUpdateAll(ctx, *user)if err != nil {- if errors.Is(err, common.ErrUpdateAllInProgress) {- return nil, huma.Error409Conflict(err.Error())+ if apiErr := common.ToAPIError(err); apiErr.HTTPStatus() != http.StatusInternalServerError {+ return nil, huma.NewError(apiErr.HTTPStatus(), apiErr.Message)}return nil, huma.Error500InternalServerError("Failed to initiate upgrade: " + err.Error())}@@ -154,7 +150,7 @@ func (h *Handler) TriggerUpdateAll(ctx context.Context, input *TriggerUpdateAllI// GetUpdateAllStatus returns the latest update-all job for live progress polling.func (h *Handler) GetUpdateAllStatus(ctx context.Context, input *UpdateAllStatusInput) (*handlerutil.Out[system.EnvironmentUpdateJob], error) {- if h.cfg != nil && h.cfg.AgentMode {+ if h.cfg.AgentMode {return nil, huma.Error400BadRequest("update-all is managed on the Arcane manager")}diff --git a/backend/internal/system/children/upgrade/model.go b/backend/internal/system/children/upgrade/model.goindex c713f8be3..da000e3ed 100644--- a/backend/internal/system/children/upgrade/model.go+++ b/backend/internal/system/children/upgrade/model.go@@ -71,9 +71,18 @@ type EnvironmentUpdateResult struct {}// clearStage drops live-progress fields once the row leaves updating.-func (r *EnvironmentUpdateResult) clearStage() {- r.Stage = ""- r.StageStartedAt = nil+func (r EnvironmentUpdateResult) clearStage() EnvironmentUpdateResult {+ return EnvironmentUpdateResult{ //exhaustruct:enforce+ EnvironmentID: r.EnvironmentID,+ EnvironmentName: r.EnvironmentName,+ Status: r.Status,+ FromVersion: r.FromVersion,+ FromDigest: r.FromDigest,+ ToVersion: r.ToVersion,+ Stage: "",+ StageStartedAt: nil,+ Error: r.Error,+ }}// EnvironmentUpdateResults is a JSON-serialized slice of per-environment results,diff --git a/backend/internal/system/children/upgrade/service.go b/backend/internal/system/children/upgrade/service.goindex e243a302e..1c2a0644b 100644--- a/backend/internal/system/children/upgrade/service.go+++ b/backend/internal/system/children/upgrade/service.go@@ -253,30 +253,35 @@ func (s *Service) prepareUpgrade(ctx context.Context, user usertypes.Actor, targif err != nil {return nil, fmt.Errorf("failed to connect to Docker: %w", err)}- var current container.InspectResponse- if containerID := strings.TrimSpace(target.ContainerID); containerID != "" {- inspect, inspectErr := compat.ContainerInspectWithCompatibility(ctx, dockerClient, containerID, client.ContainerInspectOptions{})- if inspectErr != nil {- // Fall back to a prefix match across every container.- containers, listErr := dockerClient.ContainerList(ctx, client.ContainerListOptions{All: true})- if listErr != nil {- return nil, fmt.Errorf("inspect container: %w", listErr)- }- idx := slices.IndexFunc(containers.Items, func(c container.Summary) bool { return strings.HasPrefix(c.ID, containerID) })- if idx < 0 {- return nil, fmt.Errorf("inspect container: %w", common.Classify(common.ErrNotFound, errors.New("could not find Arcane container")))- }- if inspect, inspectErr = compat.ContainerInspectWithCompatibility(ctx, dockerClient, containers.Items[idx].ID, client.ContainerInspectOptions{}); inspectErr != nil {- return nil, fmt.Errorf("inspect container: %w", inspectErr)+ current, err := func() (container.InspectResponse, error) {+ containerID := strings.TrimSpace(target.ContainerID)+ if containerID == "" {+ self, inspectErr := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient)+ if inspectErr != nil {+ return container.InspectResponse{}, errors.New("get current container: arcane is not running in a Docker container")}+ return *self, nil}- current = inspect.Container- } else {- self, inspectErr := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient)- if inspectErr != nil {- return nil, errors.New("get current container: arcane is not running in a Docker container")+ inspect, inspectErr := compat.ContainerInspectWithCompatibility(ctx, dockerClient, containerID, client.ContainerInspectOptions{})+ if inspectErr == nil {+ return inspect.Container, nil+ }+ // Fall back to a prefix match across every container.+ containers, listErr := dockerClient.ContainerList(ctx, client.ContainerListOptions{All: true})+ if listErr != nil {+ return container.InspectResponse{}, fmt.Errorf("inspect container: %w", listErr)+ }+ idx := slices.IndexFunc(containers.Items, func(c container.Summary) bool { return strings.HasPrefix(c.ID, containerID) })+ if idx < 0 {+ return container.InspectResponse{}, fmt.Errorf("inspect container: %w", common.Classify(common.ErrNotFound, errors.New("could not find Arcane container")))}- current = *self+ if inspect, inspectErr = compat.ContainerInspectWithCompatibility(ctx, dockerClient, containers.Items[idx].ID, client.ContainerInspectOptions{}); inspectErr != nil {+ return container.InspectResponse{}, fmt.Errorf("inspect container: %w", inspectErr)+ }+ return inspect.Container, nil+ }()+ if err != nil {+ return nil, err}containerName := strings.TrimPrefix(current.Name, "/")@@ -286,9 +291,11 @@ func (s *Service) prepareUpgrade(ctx context.Context, user usertypes.Actor, targtargetImage := strings.TrimSpace(target.NewImageRef)if targetImage == "" {// A blank target resolves against the version check; a manager-supplied version outranks this instance's own.- info := kit.FromPtr(s.versionService.GetAppVersionInfo(ctx))- info.NewestVersion = cmp.Or(strings.TrimSpace(targetVersion), info.NewestVersion)- if targetImage, err = resolveSelfUpgradeTargetImage(config.Image, &info); err != nil {+ info := s.versionService.GetAppVersionInfo(ctx)+ if targetImage, err = resolveSelfUpgradeTargetImage(config.Image, &versiontypes.Info{+ NewestVersion: cmp.Or(strings.TrimSpace(targetVersion), kit.FromPtr(info).NewestVersion),+ NewestDigest: kit.FromPtr(info).NewestDigest,+ }); err != nil {return nil, fmt.Errorf("resolve upgrade target image: %w", err)}}@@ -387,18 +394,6 @@ func (s *Service) runPreparedUpgrade(ctx context.Context, prepared *preparedUpgrupgradeCmd = append(upgradeCmd, "--pull-image", prepared.pullImage)}- config := &container.Config{- Image: prepared.pullImage,- Cmd: upgradeCmd,- // Root for the Docker socket: the short-lived upgrader never goes through the runtime-identity drop.- User: "0:0",- Env: containerEnv,- Labels: map[string]string{- "com.getarcaneapp.arcane.upgrader": "true",- "com.getarcaneapp.arcane": "true",- },- }-mounts := append([]mount.Mount{}, runtimeMounts...)if appDataMount != nil {mounts = append(mounts, *appDataMount)@@ -409,42 +404,50 @@ func (s *Service) runPreparedUpgrade(ctx context.Context, prepared *preparedUpgrslog.InfoContext(ctx, "Keeping upgrader container after exit (ARCANE_UPGRADE_KEEP_CONTAINER=true)")}- hostConfig := &container.HostConfig{- AutoRemove: !keepUpgraderContainer,- Mounts: mounts,- NetworkMode: networkMode,- }// Inherit the security context (SELinux label, privileged) that lets Arcane reach the socket on hardened hosts.- if prepared.current.HostConfig != nil {- hostConfig.SecurityOpt = slices.Clone(prepared.current.HostConfig.SecurityOpt)- hostConfig.Privileged = prepared.current.HostConfig.Privileged- }+ securityOpt := slices.Clone(kit.FromPtr(prepared.current.HostConfig).SecurityOpt)// On SELinux-enforcing hosts the socket is container_var_run_t, so without a label opt the upgrader exits with EACCES.- hasLabelOpt := slices.ContainsFunc(hostConfig.SecurityOpt, func(opt string) bool { return strings.HasPrefix(strings.TrimSpace(opt), "label") })- if !hostConfig.Privileged && !hasLabelOpt {+ if !kit.FromPtr(prepared.current.HostConfig).Privileged &&+ !slices.ContainsFunc(securityOpt, func(opt string) bool { return strings.HasPrefix(strings.TrimSpace(opt), "label") }) {infoResult, infoErr := dockerClient.Info(ctx, client.InfoOptions{})if infoErr != nil {slog.DebugContext(ctx, "Failed to query daemon info for SELinux detection", "error", infoErr)} else if slices.Contains(infoResult.Info.SecurityOptions, "name=selinux") {- hostConfig.SecurityOpt = append(hostConfig.SecurityOpt, "label=disable")- }- }-- var networkingConfig *network.NetworkingConfig- if networkMode != "" && networkMode.IsUserDefined() {- networkingConfig = &network.NetworkingConfig{- EndpointsConfig: map[string]*network.EndpointSettings{- string(networkMode): {Aliases: []string{"arcane-upgrader"}},- },+ securityOpt = append(securityOpt, "label=disable")}}upgraderName := fmt.Sprintf("%s-upgrader-%d", prepared.containerName, time.Now().Unix())resp, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{- Config: config,- HostConfig: hostConfig,- NetworkingConfig: networkingConfig,- Name: upgraderName,+ Config: &container.Config{+ Image: prepared.pullImage,+ Cmd: upgradeCmd,+ // Root for the Docker socket: the short-lived upgrader never goes through the runtime-identity drop.+ User: "0:0",+ Env: containerEnv,+ Labels: map[string]string{+ "com.getarcaneapp.arcane.upgrader": "true",+ "com.getarcaneapp.arcane": "true",+ },+ },+ HostConfig: &container.HostConfig{+ AutoRemove: !keepUpgraderContainer,+ Mounts: mounts,+ NetworkMode: networkMode,+ SecurityOpt: securityOpt,+ Privileged: kit.FromPtr(prepared.current.HostConfig).Privileged,+ },+ NetworkingConfig: func() *network.NetworkingConfig {+ if networkMode == "" || !networkMode.IsUserDefined() {+ return nil+ }+ return &network.NetworkingConfig{+ EndpointsConfig: map[string]*network.EndpointSettings{+ string(networkMode): {Aliases: []string{"arcane-upgrader"}},+ },+ }+ }(),+ Name: upgraderName,})if err != nil {return "", fmt.Errorf("create upgrader container: %w", err)@@ -578,15 +581,15 @@ func (s *Service) StartUpdateAll(ctx context.Context, user usertypes.Actor) (*EnManagerVersionAtStart: info.CurrentVersion,ManagerDigestAtStart: info.CurrentDigest,ManagerTargetVersion: cmp.Or(info.NewestVersion, info.NewestDigest, info.CurrentVersion, info.CurrentDigest),+ // The manager row stays pending until the final manager step.+ Results: EnvironmentUpdateResults{{+ EnvironmentID: environment.LocalEnvironmentID,+ EnvironmentName: s.environments.ResolveEnvironmentName(ctx, environment.LocalEnvironmentID),+ Status: EnvironmentUpdateResultStatusPending,+ FromVersion: info.CurrentVersion,+ ToVersion: cmp.Or(info.NewestVersion, info.NewestDigest, info.CurrentVersion, info.CurrentDigest),+ }},}- // The manager row stays pending until the final manager step.- job.Results = EnvironmentUpdateResults{{- EnvironmentID: environment.LocalEnvironmentID,- EnvironmentName: s.environments.ResolveEnvironmentName(ctx, environment.LocalEnvironmentID),- Status: EnvironmentUpdateResultStatusPending,- FromVersion: info.CurrentVersion,- ToVersion: job.ManagerTargetVersion,- }}// Seed every remote row so the dialog shows the whole fleet at once; the agents step fills any gaps.if remotes, listErr := s.environments.ListRemoteEnvironments(ctx); listErr != nil {slog.WarnContext(ctx, "update-all: failed to pre-list remote environments for seeding", "error", listErr)@@ -600,9 +603,11 @@ func (s *Service) StartUpdateAll(ctx context.Context, user usertypes.Actor) (*Enreturn nil, fmt.Errorf("create update-all job: %w", createUpgradeJobErr)}- input := updateAllInput{JobID: job.ID}- input.KeyID, _ = ctx.Value(middleware.ContextKeyApiKeyID).(string)- instanceID, submitErr := s.flow.Submit(ctx, s.updateAllWorkflow, input, activitylib.StartRequest{})+ keyID, _ := ctx.Value(middleware.ContextKeyApiKeyID).(string)+ instanceID, submitErr := s.flow.Submit(ctx, s.updateAllWorkflow, updateAllInput{+ JobID: job.ID,+ KeyID: keyID,+ }, activitylib.StartRequest{})if submitErr != nil {s.markUpdateAllFailed(context.WithoutCancel(ctx), job, fmt.Sprintf("failed to start update-all: %v", submitErr))return nil, fmt.Errorf("start update-all: %w", submitErr)@@ -667,8 +672,7 @@ func resolveResumeAction(job *EnvironmentUpdateJob, currentVersion, currentDigesif now.Sub(job.CreatedAt) > updateAllStaleThreshold {return resumeAction{markStale: true}}- landed := upgradeLanded(job.ManagerVersionAtStart, job.ManagerDigestAtStart, job.ManagerTargetVersion, currentVersion, currentDigest)- return resumeAction{managerSucceeded: landed}+ return resumeAction{managerSucceeded: upgradeLanded(job.ManagerVersionAtStart, job.ManagerDigestAtStart, job.ManagerTargetVersion, currentVersion, currentDigest)}}// upgradeLanded reports whether the version or digest moved off its baseline, or already matches target (a version or@@ -727,12 +731,25 @@ func (s *Service) upgradeAgentTask(ctx context.Context, t flow.Task) (_ any, err// Only a later stage proves the agent accepted it.triggered := result.Status == EnvironmentUpdateResultStatusUpdating && result.Stage != "" && result.Stage != EnvironmentUpdateStageCheckingunconfirmed := triggered && result.Stage == EnvironmentUpdateStageStarting- result.Status = EnvironmentUpdateResultStatusUpdating+ settle := func(status EnvironmentUpdateResultStatus, message string) {+ *result = EnvironmentUpdateResult{ //exhaustruct:enforce+ EnvironmentID: result.EnvironmentID,+ EnvironmentName: result.EnvironmentName,+ Status: status,+ FromVersion: result.FromVersion,+ FromDigest: result.FromDigest,+ ToVersion: result.ToVersion,+ Stage: result.Stage,+ StageStartedAt: result.StageStartedAt,+ Error: message[:min(len(message), updateAllErrorMaxLen)],+ }+ }+ settle(EnvironmentUpdateResultStatusUpdating, result.Error)if persistErr := s.db.WithContext(ctx).Save(job).Error; persistErr != nil {slog.WarnContext(ctx, "update-all: failed to persist updating status", "jobId", job.ID, "environmentId", remote.ID, "error", persistErr)}defer func() {- result.clearStage()+ *result = result.clearStage()// Shutdown: leave the row updating at its stage so the redelivered task resumes it.if err = ctx.Err(); err != nil {return@@ -744,15 +761,17 @@ func (s *Service) upgradeAgentTask(ctx context.Context, t flow.Task) (_ any, errvar info versiontypes.Infoif triggered {- info.CurrentVersion, info.CurrentDigest = result.FromVersion, result.FromDigest+ info = versiontypes.Info{+ CurrentVersion: result.FromVersion,+ CurrentDigest: result.FromDigest,+ }} else {s.setUpdateStage(ctx, job, result, EnvironmentUpdateStageChecking)versionCtx, cancel := context.WithTimeout(ctx, updateAllAgentRequestTimeout)checkErr := s.environments.ProxyJSONRequest(versionCtx, remote.ID, http.MethodGet, "/api/app-version", nil, &info)cancel()if checkErr != nil {- msg := checkErr.Error()- result.Status, result.Error = updateAllAgentFailureStatus(checkErr), msg[:min(len(msg), updateAllErrorMaxLen)]+ settle(updateAllAgentFailureStatus(checkErr), checkErr.Error())return nil, nil}// The manager's newest release outranks the agent's own check, so the recorded target tracks what was sent.@@ -762,16 +781,23 @@ func (s *Service) upgradeAgentTask(ctx context.Context, t flow.Task) (_ any, errif marshalErr != nil {slog.WarnContext(ctx, "update-all: failed to marshal trigger body", "environmentId", remote.ID, "error", marshalErr)} else {- info.NewestVersion, triggerBody = newest, body+ info.NewestVersion, triggerBody = newest, body // keep: versiontypes.Info is decoded by ProxyJSONRequest and has ~20 fields}}- result.FromVersion, result.FromDigest = info.CurrentVersion, info.CurrentDigest- result.ToVersion = cmp.Or(info.NewestVersion, info.NewestDigest, info.CurrentVersion, info.CurrentDigest)-// The saved starting stage keeps a replay from sending the trigger again, so the trigger waits on it.- result.Stage, result.StageStartedAt = EnvironmentUpdateStageStarting, new(time.Now())+ *result = EnvironmentUpdateResult{ //exhaustruct:enforce+ EnvironmentID: result.EnvironmentID,+ EnvironmentName: result.EnvironmentName,+ Status: result.Status,+ FromVersion: info.CurrentVersion,+ FromDigest: info.CurrentDigest,+ ToVersion: cmp.Or(info.NewestVersion, info.NewestDigest, info.CurrentVersion, info.CurrentDigest),+ Stage: EnvironmentUpdateStageStarting,+ StageStartedAt: new(time.Now()),+ Error: result.Error,+ }if saveErr := s.db.WithContext(ctx).Save(job).Error; saveErr != nil {- result.Status, result.Error = EnvironmentUpdateResultStatusFailed, "Could not save upgrade progress: "+saveErr.Error()+ settle(EnvironmentUpdateResultStatusFailed, "Could not save upgrade progress: "+saveErr.Error())return nil, nil}triggerCtx, cancel := context.WithTimeout(ctx, updateAllAgentRequestTimeout)@@ -781,14 +807,23 @@ func (s *Service) upgradeAgentTask(ctx context.Context, t flow.Task) (_ any, errtriggerErr = resp.RequireSuccess()}if triggerErr != nil {- msg := triggerErr.Error()- result.Status, result.Error = EnvironmentUpdateResultStatusFailed, msg[:min(len(msg), updateAllErrorMaxLen)]+ settle(EnvironmentUpdateResultStatusFailed, triggerErr.Error())return nil, nil}// An agent already on the target finds nothing to swap in; confirming would only burn the poll window.if info.AlreadyOnNewest() {- result.Status, result.ToVersion = EnvironmentUpdateResultStatusUpToDate, info.CurrentVersion+ *result = EnvironmentUpdateResult{ //exhaustruct:enforce+ EnvironmentID: result.EnvironmentID,+ EnvironmentName: result.EnvironmentName,+ Status: EnvironmentUpdateResultStatusUpToDate,+ FromVersion: result.FromVersion,+ FromDigest: result.FromDigest,+ ToVersion: info.CurrentVersion,+ Stage: result.Stage,+ StageStartedAt: result.StageStartedAt,+ Error: result.Error,+ }return nil, nil}}@@ -820,15 +855,15 @@ func (s *Service) upgradeAgentTask(ctx context.Context, t flow.Task) (_ any, errstage(EnvironmentUpdateStageVerifying)// An unconfirmed request proves nothing by the agent merely matching the target, so it needs a real change.if upgradeLanded(info.CurrentVersion, info.CurrentDigest, kit.Ternary(unconfirmed, "", result.ToVersion), current.CurrentVersion, current.CurrentDigest) {- result.Status = EnvironmentUpdateResultStatusUpdated+ settle(EnvironmentUpdateResultStatusUpdated, result.Error)return nil, nil}}if unconfirmed {- result.Status, result.Error = EnvironmentUpdateResultStatusFailed, "Upgrade request was interrupted before the agent confirmed it"+ settle(EnvironmentUpdateResultStatusFailed, "Upgrade request was interrupted before the agent confirmed it")return nil, nil}- result.Status = EnvironmentUpdateResultStatusTriggered+ settle(EnvironmentUpdateResultStatusTriggered, result.Error)return nil, nil}@@ -843,28 +878,61 @@ func (s *Service) upgradeManagerTask(ctx context.Context, t flow.Task) (any, errif err != nil {return nil, err}- allowed := false- if job.Status == EnvironmentUpdateJobStatusRunning {- if allowed, err = s.authorizeUpdateAll(ctx, job, input.KeyID); err != nil {- return nil, err+ allowed, err := func() (bool, error) {+ if job.Status != EnvironmentUpdateJobStatusRunning {+ return false, nil}+ return s.authorizeUpdateAll(ctx, job, input.KeyID)+ }()+ if err != nil {+ return nil, err}if allowed {// Every agent task has finished, so an agent row still in progress belongs to a task that failed first.- for i := range job.Results {- pending := job.Results[i].Status == EnvironmentUpdateResultStatusPending || job.Results[i].Status == EnvironmentUpdateResultStatusUpdating- if job.Results[i].EnvironmentID != environment.LocalEnvironmentID && pending {- job.Results[i].clearStage()- job.Results[i].Status, job.Results[i].Error = EnvironmentUpdateResultStatusFailed, "Agent upgrade did not finish"+ results := make(EnvironmentUpdateResults, 0, len(job.Results))+ for _, row := range job.Results {+ switch {+ case row.EnvironmentID == environment.LocalEnvironmentID:+ results = append(results, EnvironmentUpdateResult{ //exhaustruct:enforce+ EnvironmentID: row.EnvironmentID,+ EnvironmentName: row.EnvironmentName,+ Status: EnvironmentUpdateResultStatusUpdating,+ FromVersion: row.FromVersion,+ FromDigest: row.FromDigest,+ ToVersion: row.ToVersion,+ Stage: EnvironmentUpdateStageStarting,+ StageStartedAt: new(time.Now()),+ Error: row.Error,+ })+ case row.Status == EnvironmentUpdateResultStatusPending || row.Status == EnvironmentUpdateResultStatusUpdating:+ results = append(results, EnvironmentUpdateResult{ //exhaustruct:enforce+ EnvironmentID: row.EnvironmentID,+ EnvironmentName: row.EnvironmentName,+ Status: EnvironmentUpdateResultStatusFailed,+ FromVersion: row.FromVersion,+ FromDigest: row.FromDigest,+ ToVersion: row.ToVersion,+ Stage: "",+ StageStartedAt: nil,+ Error: "Agent upgrade did not finish",+ })+ default:+ results = append(results, row)}}- manager := slices.IndexFunc(job.Results, func(r EnvironmentUpdateResult) bool { return r.EnvironmentID == environment.LocalEnvironmentID })- if manager >= 0 {- job.Results[manager].Status = EnvironmentUpdateResultStatusUpdating- job.Results[manager].Stage = EnvironmentUpdateStageStarting- job.Results[manager].StageStartedAt = new(time.Now())+ manager := slices.IndexFunc(results, func(r EnvironmentUpdateResult) bool { return r.EnvironmentID == environment.LocalEnvironmentID })+ *job = EnvironmentUpdateJob{ //exhaustruct:enforce+ BaseModel: job.BaseModel,+ Status: EnvironmentUpdateJobStatusPendingRestart,+ UserID: job.UserID,+ Username: job.Username,+ ManagerVersionAtStart: job.ManagerVersionAtStart,+ ManagerDigestAtStart: job.ManagerDigestAtStart,+ ManagerTargetVersion: job.ManagerTargetVersion,+ Results: results,+ Error: job.Error,+ CompletedAt: job.CompletedAt,}- job.Status = EnvironmentUpdateJobStatusPendingRestart// A replay that misses pending_restart would trigger the manager upgrade again, so the trigger waits on it.if savePendingRestartErr := s.db.WithContext(ctx).Save(job).Error; savePendingRestartErr != nil {return nil, fmt.Errorf("persist pending_restart before manager upgrade: %w", savePendingRestartErr)@@ -933,14 +1001,10 @@ func (s *Service) watchManagerUpgrader(ctx context.Context, job *EnvironmentUpda}// Let a recreate already underway stop this process; an unobserved run waits out the stale window instead.- hold := updateAllManagerNoRestartGrace- if exit == upgraderExitUnobserved {- hold = time.Until(job.CreatedAt.Add(updateAllStaleThreshold))- }select {case <-ctx.Done():return- case <-time.After(hold):+ case <-time.After(kit.Ternary(exit == upgraderExitUnobserved, time.Until(job.CreatedAt.Add(updateAllStaleThreshold)), updateAllManagerNoRestartGrace)):}job, err = s.loadUpdateAllJob(ctx, job.ID)@@ -973,11 +1037,23 @@ func upsertPendingResult(job *EnvironmentUpdateJob, envID, envName string) int {if idx := slices.IndexFunc(job.Results, func(r EnvironmentUpdateResult) bool { return r.EnvironmentID == envID }); idx >= 0 {return idx}- job.Results = append(job.Results, EnvironmentUpdateResult{+ pending := EnvironmentUpdateResult{EnvironmentID: envID,EnvironmentName: envName,Status: EnvironmentUpdateResultStatusPending,- })+ }+ *job = EnvironmentUpdateJob{ //exhaustruct:enforce+ BaseModel: job.BaseModel,+ Status: job.Status,+ UserID: job.UserID,+ Username: job.Username,+ ManagerVersionAtStart: job.ManagerVersionAtStart,+ ManagerDigestAtStart: job.ManagerDigestAtStart,+ ManagerTargetVersion: job.ManagerTargetVersion,+ Results: append(job.Results, pending),+ Error: job.Error,+ CompletedAt: job.CompletedAt,+ }return len(job.Results) - 1}@@ -998,8 +1074,17 @@ func (s *Service) setUpdateStage(ctx context.Context, job *EnvironmentUpdateJob,if result.Stage == stage {return}- result.Stage = stage- result.StageStartedAt = new(time.Now())+ *result = EnvironmentUpdateResult{ //exhaustruct:enforce+ EnvironmentID: result.EnvironmentID,+ EnvironmentName: result.EnvironmentName,+ Status: result.Status,+ FromVersion: result.FromVersion,+ FromDigest: result.FromDigest,+ ToVersion: result.ToVersion,+ Stage: stage,+ StageStartedAt: new(time.Now()),+ Error: result.Error,+ }if err := s.db.WithContext(ctx).Save(job).Error; err != nil {slog.WarnContext(ctx, "update-all: failed to persist stage", "jobId", job.ID, "environmentId", result.EnvironmentID, "stage", stage, "error", err)}@@ -1047,15 +1132,35 @@ func (s *Service) authorizeUpdateAll(ctx context.Context, job *EnvironmentUpdate}func (s *Service) markUpdateAllFailed(ctx context.Context, job *EnvironmentUpdateJob, reason string) {- job.Status = EnvironmentUpdateJobStatusFailed- job.Error = &reason- job.CompletedAt = new(time.Now())- for i := range job.Results {- job.Results[i].clearStage()- if job.Results[i].Status == EnvironmentUpdateResultStatusUpdating {- job.Results[i].Status = EnvironmentUpdateResultStatusFailed- job.Results[i].Error = reason+ results := make(EnvironmentUpdateResults, 0, len(job.Results))+ for _, row := range job.Results {+ if row.Status != EnvironmentUpdateResultStatusUpdating {+ results = append(results, row.clearStage())+ continue}+ results = append(results, EnvironmentUpdateResult{ //exhaustruct:enforce+ EnvironmentID: row.EnvironmentID,+ EnvironmentName: row.EnvironmentName,+ Status: EnvironmentUpdateResultStatusFailed,+ FromVersion: row.FromVersion,+ FromDigest: row.FromDigest,+ ToVersion: row.ToVersion,+ Stage: "",+ StageStartedAt: nil,+ Error: reason,+ })+ }+ *job = EnvironmentUpdateJob{ //exhaustruct:enforce+ BaseModel: job.BaseModel,+ Status: EnvironmentUpdateJobStatusFailed,+ UserID: job.UserID,+ Username: job.Username,+ ManagerVersionAtStart: job.ManagerVersionAtStart,+ ManagerDigestAtStart: job.ManagerDigestAtStart,+ ManagerTargetVersion: job.ManagerTargetVersion,+ Results: results,+ Error: &reason,+ CompletedAt: new(time.Now()),}if err := s.db.WithContext(ctx).Save(job).Error; err != nil {slog.WarnContext(ctx, "update-all: failed to mark job failed", "jobId", job.ID, "error", err)@@ -1084,22 +1189,38 @@ func (s *Service) markUpdateAllFailed(ctx context.Context, job *EnvironmentUpdat// finalizeUpdateAllJob settles the manager row, completes the job and records the audit event.func (s *Service) finalizeUpdateAllJob(ctx context.Context, job *EnvironmentUpdateJob, managerStatus EnvironmentUpdateResultStatus, managerVersion string) {- if idx := slices.IndexFunc(job.Results, func(r EnvironmentUpdateResult) bool { return r.EnvironmentID == environment.LocalEnvironmentID }); idx >= 0 {- job.Results[idx].Status = managerStatus- if managerStatus == EnvironmentUpdateResultStatusFailed {- job.Results[idx].Error = "manager version did not change after upgrade"- } else {- job.Results[idx].ToVersion = managerVersion- }- }- job.Status = EnvironmentUpdateJobStatusCompleted- job.CompletedAt = new(time.Now())+ results := make(EnvironmentUpdateResults, 0, len(job.Results))failed := 0- for i := range job.Results {- job.Results[i].clearStage()- if job.Results[i].Status == EnvironmentUpdateResultStatusFailed {+ for _, row := range job.Results {+ if row.EnvironmentID == environment.LocalEnvironmentID {+ row = EnvironmentUpdateResult{ //exhaustruct:enforce+ EnvironmentID: row.EnvironmentID,+ EnvironmentName: row.EnvironmentName,+ Status: managerStatus,+ FromVersion: row.FromVersion,+ FromDigest: row.FromDigest,+ ToVersion: kit.Ternary(managerStatus == EnvironmentUpdateResultStatusFailed, row.ToVersion, managerVersion),+ Stage: row.Stage,+ StageStartedAt: row.StageStartedAt,+ Error: kit.Ternary(managerStatus == EnvironmentUpdateResultStatusFailed, "manager version did not change after upgrade", row.Error),+ }+ }+ if row.Status == EnvironmentUpdateResultStatusFailed {failed++}+ results = append(results, row.clearStage())+ }+ *job = EnvironmentUpdateJob{ //exhaustruct:enforce+ BaseModel: job.BaseModel,+ Status: EnvironmentUpdateJobStatusCompleted,+ UserID: job.UserID,+ Username: job.Username,+ ManagerVersionAtStart: job.ManagerVersionAtStart,+ ManagerDigestAtStart: job.ManagerDigestAtStart,+ ManagerTargetVersion: job.ManagerTargetVersion,+ Results: results,+ Error: job.Error,+ CompletedAt: new(time.Now()),}if err := s.db.WithContext(ctx).Save(job).Error; err != nil {slog.WarnContext(ctx, "update-all: failed to finalize job", "jobId", job.ID, "error", err)diff --git a/backend/internal/system/handler.go b/backend/internal/system/handler.goindex 427de5f24..b479d584f 100644--- a/backend/internal/system/handler.go+++ b/backend/internal/system/handler.go@@ -99,9 +99,6 @@ func (h *SystemHandler) GetDockerInfo(ctx context.Context, input *GetDockerInfoI}info := infoResult.Info- cpuCount := info.NCPU- memTotal := info.MemTotal-// Apply cgroup limits only when running outside Docker (e.g. in LXC).// In Docker, --cpus/--memory are artificial operator constraints that// should not cap the host totals shown in the dashboard. The Docker@@ -111,21 +108,15 @@ func (h *SystemHandler) GetDockerInfo(ctx context.Context, input *GetDockerInfoI// dashboard shows what Arcane's host actually has available.if !cgroup.IsDockerContainer() {if cgroupLimits, detectLimitsErr := cgroup.DetectLimits(); detectLimitsErr == nil {- if limit := cgroupLimits.MemoryLimit; limit > 0 {- limitInt := limit- if memTotal == 0 || limitInt < memTotal {- memTotal = limitInt- }+ if limit := cgroupLimits.MemoryLimit; limit > 0 && (info.MemTotal == 0 || limit < info.MemTotal) {+ info.MemTotal = limit // keep: copying Docker's ~70-field system.Info as a literal is unreadable}- if cgroupLimits.CPUCount > 0 && (cpuCount == 0 || cgroupLimits.CPUCount < cpuCount) {- cpuCount = cgroupLimits.CPUCount+ if cgroupLimits.CPUCount > 0 && (info.NCPU == 0 || cgroupLimits.CPUCount < info.NCPU) {+ info.NCPU = cgroupLimits.CPUCount // keep: copying Docker's ~70-field system.Info as a literal is unreadable}}}- info.NCPU = cpuCount- info.MemTotal = memTotal-gitCommit, goVersion, buildTime := extractVersionDetailsFromComponents(version.Components)return &GetDockerInfoOutput{@@ -221,17 +212,17 @@ func (h *SystemHandler) ConvertDockerRun(ctx context.Context, input *ConvertDockreturn nil, huma.Error500InternalServerError("Failed to convert to Docker Compose format.")}- serviceName := ""- if len(result.Services) > 0 {- serviceName = result.Services[0].Name- }-return &ConvertDockerRunOutput{Body: system.ConvertDockerRunResponse{Success: true,DockerCompose: string(result.YAML),EnvVars: strings.TrimSuffix(string(result.EnvFile), "\n"),- ServiceName: serviceName,+ ServiceName: func() string {+ if len(result.Services) == 0 {+ return ""+ }+ return result.Services[0].Name+ }(),},}, nil}diff --git a/backend/internal/system/model.go b/backend/internal/system/model.goindex f8635beac..bf913b6c8 100644--- a/backend/internal/system/model.go+++ b/backend/internal/system/model.go@@ -48,16 +48,6 @@ func (b SystemBackupRun) ToDTO() backup.SystemBackupRun {}}-func systemBackupRunFromDTOInternal(dto backup.SystemBackupRun) SystemBackupRun {- run := SystemBackupRun{- Size: dto.Size, CreatedAt: dto.CreatedAt, Status: SystemBackupStatus(dto.Status), Trigger: SystemBackupTrigger(dto.Trigger),- Destination: dto.Destination, LocalSnapshotID: dto.LocalSnapshotID, RemoteSnapshotID: dto.RemoteSnapshotID,- S3DestinationID: dto.S3DestinationID, S3DestinationName: dto.S3DestinationName, PolicyID: dto.PolicyID, Error: dto.Error,- }- run.ID = dto.ID- return run-}-type SystemBackupPolicy struct {database.BaseModel@@ -77,8 +67,3 @@ func (p SystemBackupPolicy) ToDTO() backup.SystemBackupPolicy {LocalEnabled: p.LocalEnabled, S3Enabled: p.S3Enabled, S3DestinationID: p.S3DestinationID,}}--func (p *SystemBackupPolicy) applyDTOInternal(dto backup.SystemBackupPolicy) {- p.Enabled, p.Schedule, p.RetentionCount = dto.Enabled, dto.Schedule, dto.RetentionCount- p.LocalEnabled, p.S3Enabled, p.S3DestinationID = dto.LocalEnabled, dto.S3Enabled, dto.S3DestinationID-}diff --git a/backend/internal/system/module.go b/backend/internal/system/module.goindex 27df6978c..f3eb46dd9 100644--- a/backend/internal/system/module.go+++ b/backend/internal/system/module.go@@ -3,6 +3,7 @@package systemimport (+ "cmp""net/http""github.com/danielgtaylor/huma/v2"@@ -50,10 +51,7 @@ func (m *Module) RegisterRoutes(api huma.API, appCtx handlerutil.ActivityAppContif m == nil {m = &Module{}}- service := m.service- if service == nil {- service = &SystemService{}- }+ service := cmp.Or(m.service, &SystemService{})RegisterSystem(api, NewHandler(m.dockerService, m.service, appCtx.Context()))prune.RegisterRoutes(api, prune.NewHandler(service.prune, appCtx.Context()))diff --git a/backend/internal/system/service.go b/backend/internal/system/service.goindex d3519925d..287cc739e 100644--- a/backend/internal/system/service.go+++ b/backend/internal/system/service.go@@ -3,7 +3,6 @@ package systemimport ("cmp""context"- "database/sql""errors""fmt""log/slog"@@ -25,6 +24,7 @@ import ("github.com/samber/mo""go.getarcane.app/docker""go.getarcane.app/docker/compat"+ kit "go.getarcane.app/kit/pkg""go.getarcane.app/sys/cgroup""go.getarcane.app/updater""go.getarcane.app/updater/labels"@@ -91,10 +91,6 @@ func NewSystemService(actors *francis.Runtime,cfg *config.Config,) *SystemService {- location := time.UTC- if cfg != nil {- location = cfg.GetLocation()- }s := &SystemService{db: db,dockerService: dockerService,@@ -104,17 +100,13 @@ func NewSystemService(dockerHostMemoryCache: hot.NewHotCache[struct{}, dockerHostMemoryInfo](hot.LRU, 1).WithTTL(dockerHostMemoryCacheTTL).Build(),- prune: prune.NewService(location, dockerService.GetClient, activityService, imageUpdateService, volumeService, networkService),- upgrade: upgrade.NewService(db, dockerService, versionService, eventService, settingsService, projectService, resolveUpgraderRuntimeOptionsInternal),- }- var sqlDB func() (*sql.DB, error)- if db != nil {- sqlDB = db.SQLDB+ prune: prune.NewService(cfg.GetLocation(), dockerService.GetClient, activityService, imageUpdateService, volumeService, networkService),+ upgrade: upgrade.NewService(db, dockerService, versionService, eventService, settingsService, projectService, resolveUpgraderRuntimeOptions),}s.backup = systembackup.NewService(systembackup.Dependencies{- Store: s.backupStoreInternal(s3Destinations),+ Store: s.backupStore(s3Destinations),DB: db,- SQLDB: sqlDB,+ SQLDB: db.SQLDB,ActorStore: actors.Store,Docker: dockerService,Volumes: volumeService,@@ -124,7 +116,7 @@ func NewSystemService(Settings: settingsService,Config: cfg,RecoveryKeys: recoveryKeys,- ResolveRuntimeOptions: resolveUpgraderRuntimeOptionsInternal,+ ResolveRuntimeOptions: resolveUpgraderRuntimeOptions,})return s}@@ -188,184 +180,130 @@ func (s *SystemService) PruneUpgradeLogs(ctx context.Context, dataDir string, noreturn s.upgrade.PruneUpgradeLogs(ctx, dataDir, now)}-func (- s *SystemService,-) performBatchContainerAction(- ctx context.Context,- containers []mobycontainer.Summary,- actionName string,- shouldProcess func(- mobycontainer.Summary,- ) bool,- action func(- context.Context,- string,- ) error,-) *containertypes.ActionResult {- result := &containertypes.ActionResult{Success: true}- var mu sync.Mutex-- g, groupCtx := errgroup.WithContext(ctx)- // Limit concurrency to avoid overwhelming Docker daemon- g.SetLimit(5)-- for _, containerSummary := range containers {- c := containerSummary // capture loop var- if !shouldProcess(c) {- continue- }-- g.Go(func() (workerErr error) {- defer utils.RecoverToError(&workerErr, "system info worker")-- err := action(groupCtx, c.ID)-- mu.Lock()- defer mu.Unlock()-- if err != nil {- result.Failed = append(result.Failed, c.ID)- result.Errors = append(result.Errors, fmt.Sprintf("Failed to %s container %s: %v", actionName, c.ID, err))- result.Success = false- } else {- if actionName == "start" {- result.Started = append(result.Started, c.ID)- } else {- result.Stopped = append(result.Stopped, c.ID)- }- }- return nil- })- }-- _ = g.Wait()- return result-}-func (s *SystemService) StartAllContainers(ctx context.Context, environmentID string) (*containertypes.ActionResult, error) {- return s.startMatchingContainersInternal(ctx, environmentID, startMatchingContainersOptionsInternal{+ return s.performBatchContainerAction(ctx, environmentID, startMatchingContainersOptions{+ ActivityType: activitytypes.TypeContainerStart,+ ActionName: "start",ResourceName: "All containers",StartMessage: "Starting all containers",- FailureMessage: "Starting all containers failed",SuccessMessage: "Started all containers",ShouldStart: func(c mobycontainer.Summary) bool { return c.State != "running" },+ Action: func(ctx context.Context, id string) error {+ return s.containerService.StartContainer(ctx, id, usertypes.SystemUser)+ },})}func (s *SystemService) StartAllStoppedContainers(ctx context.Context, environmentID string) (*containertypes.ActionResult, error) {- return s.startMatchingContainersInternal(ctx, environmentID, startMatchingContainersOptionsInternal{+ return s.performBatchContainerAction(ctx, environmentID, startMatchingContainersOptions{+ ActivityType: activitytypes.TypeContainerStart,+ ActionName: "start",ResourceName: "Stopped containers",StartMessage: "Starting stopped containers",- FailureMessage: "Starting stopped containers failed",SuccessMessage: "Started stopped containers",ShouldStart: func(c mobycontainer.Summary) bool { return c.State == "exited" },+ Action: func(ctx context.Context, id string) error {+ return s.containerService.StartContainer(ctx, id, usertypes.SystemUser)+ },+ })+}++func (s *SystemService) StopAllContainers(ctx context.Context, environmentID string) (*containertypes.ActionResult, error) {+ return s.performBatchContainerAction(ctx, environmentID, startMatchingContainersOptions{+ ActivityType: activitytypes.TypeContainerStop,+ ActionName: "stop",+ ResourceName: "All containers",+ StartMessage: "Stopping all containers",+ SuccessMessage: "Stopped all containers",+ // Skip Arcane container+ ShouldStart: func(c mobycontainer.Summary) bool { return !labels.IsArcaneContainer(c.Labels) },+ Action: func(ctx context.Context, id string) error {+ return s.containerService.StopContainer(ctx, id, usertypes.SystemUser)+ },})}-type startMatchingContainersOptionsInternal struct {+type startMatchingContainersOptions struct {+ ActivityType activitytypes.Type+ ActionName stringResourceName stringStartMessage string- FailureMessage stringSuccessMessage stringShouldStart func(mobycontainer.Summary) bool+ Action func(context.Context, string) error}-func (s *SystemService) startMatchingContainersInternal(ctx context.Context, environmentID string, opts startMatchingContainersOptionsInternal) (*containertypes.ActionResult, error) {- activityID := s.startSystemContainerActivityInternal(ctx, environmentID, activitytypes.TypeContainerStart, opts.ResourceName, opts.StartMessage)- ctx = s.activityService.Track(ctx, activityID)- containers, _, _, _, err := s.dockerService.GetAllContainers(ctx)- if err != nil {- result := &containertypes.ActionResult{- Success: false,- Errors: []string{fmt.Sprintf("Failed to list containers: %v", err)},- ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer(),- }- s.completeSystemContainerActivityInternal(ctx, activityID, opts.FailureMessage, result)- return result, err- }-- result := s.performBatchContainerAction(ctx, containers, "start", opts.ShouldStart, func(ctx context.Context, id string) error {- return s.containerService.StartContainer(ctx, id, usertypes.SystemUser)- })- result.ActivityID = mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()- s.completeSystemContainerActivityInternal(ctx, activityID, opts.SuccessMessage, result)- return result, nil-}--func (s *SystemService) StopAllContainers(ctx context.Context, environmentID string) (*containertypes.ActionResult, error) {- activityID := s.startSystemContainerActivityInternal(ctx, environmentID, activitytypes.TypeContainerStop, "All containers", "Stopping all containers")- ctx = s.activityService.Track(ctx, activityID)- containers, _, _, _, err := s.dockerService.GetAllContainers(ctx)- if err != nil {- result := &containertypes.ActionResult{- Success: false,- Errors: []string{fmt.Sprintf("Failed to list containers: %v", err)},- ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer(),- }- s.completeSystemContainerActivityInternal(ctx, activityID, "Stopping all containers failed", result)- return result, err- }-- result := s.performBatchContainerAction(ctx, containers, "stop",- func(c mobycontainer.Summary) bool {- // Skip Arcane container- return !labels.IsArcaneContainer(c.Labels)- },- func(ctx context.Context, id string) error {- return s.containerService.StopContainer(ctx, id, usertypes.SystemUser)- })- result.ActivityID = mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()- s.completeSystemContainerActivityInternal(ctx, activityID, "Stopped all containers", result)- return result, nil-}--func (s *SystemService) startSystemContainerActivityInternal(ctx context.Context, environmentID string, activityType activitytypes.Type, resourceName, message string) string {- if s.activityService == nil {- return ""- }+func (s *SystemService) performBatchContainerAction(ctx context.Context, environmentID string, opts startMatchingContainersOptions) (*containertypes.ActionResult, error) {localActivity, err := s.activityService.StartActivity(ctx, activity.StartActivityRequest{EnvironmentID: environmentID,- Type: activityType,+ Type: opts.ActivityType,ResourceType: new("system"),- ResourceName: &resourceName,- Step: message,- LatestMessage: message,- Metadata: database.JSON{"scope": resourceName},+ ResourceName: &opts.ResourceName,+ Step: opts.StartMessage,+ LatestMessage: opts.StartMessage,+ Metadata: database.JSON{"scope": opts.ResourceName},})if err != nil {- slog.DebugContext(ctx, "failed to start system container activity", "type", activityType, "error", err)- return ""+ slog.DebugContext(ctx, "failed to start system container activity", "type", opts.ActivityType, "error", err)}- return localActivity.ID-}+ activityID := kit.Ternary(err != nil, "", kit.FromPtr(localActivity).ID)+ ctx = s.activityService.Track(ctx, activityID)-func (s *SystemService) completeSystemContainerActivityInternal(ctx context.Context, activityID, successMessage string, result *containertypes.ActionResult) {- if s.activityService == nil || activityID == "" || result == nil {- return+ var mu sync.Mutex+ var processed, failed, errs []string+ containers, _, _, _, listErr := s.dockerService.GetAllContainers(ctx)+ if listErr != nil {+ errs = append(errs, fmt.Sprintf("Failed to list containers: %v", listErr))+ } else {+ g, groupCtx := errgroup.WithContext(ctx)+ // Limit concurrency to avoid overwhelming Docker daemon+ g.SetLimit(5)+ for _, c := range containers {+ if !opts.ShouldStart(c) {+ continue+ }+ g.Go(func() (workerErr error) {+ defer utils.RecoverToError(&workerErr, "system info worker")+ actionErr := opts.Action(groupCtx, c.ID)+ mu.Lock()+ defer mu.Unlock()+ if actionErr != nil {+ failed = append(failed, c.ID)+ errs = append(errs, fmt.Sprintf("Failed to %s container %s: %v", opts.ActionName, c.ID, actionErr))+ return nil+ }+ processed = append(processed, c.ID)+ return nil+ })+ }+ _ = g.Wait()}- status := activitytypes.StatusSuccess- message := successMessage- var errMessage *string- if !result.Success || len(result.Errors) > 0 {- status = activitytypes.StatusFailed- message = strings.Join(result.Errors, "; ")- errMessage = &message+ result := &containertypes.ActionResult{+ Success: len(errs) == 0,+ Started: kit.Ternary(opts.ActionName == "start", processed, nil),+ Stopped: kit.Ternary(opts.ActionName == "start", nil, processed),+ Failed: failed,+ Errors: errs,+ ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer(),}-- if _, err := s.activityService.CompleteActivity(utils.ActivityRuntimeContext(ctx, nil), activityID, status, message, errMessage); err != nil {- slog.DebugContext(ctx, "failed to complete system container activity", "activityId", activityID, "error", err)+ if activityID != "" {+ _, completeErr := s.activityService.CompleteActivity(+ utils.ActivityRuntimeContext(ctx, nil),+ activityID,+ kit.Ternary(len(errs) > 0, activitytypes.StatusFailed, activitytypes.StatusSuccess),+ kit.Ternary(len(errs) > 0, strings.Join(errs, "; "), opts.SuccessMessage),+ kit.Ternary(len(errs) > 0, new(strings.Join(errs, "; ")), nil),+ )+ if completeErr != nil {+ slog.DebugContext(ctx, "failed to complete system container activity", "activityId", activityID, "error", completeErr)+ }}+ return result, listErr}func (s *SystemService) GetDiskUsagePath(ctx context.Context) string {- cfg := s.settingsService.GetSettingsConfig()- if cfg == nil {- return "/"- }-- path := cmp.Or(cfg.DiskUsagePath.Value, "/")- return path+ return cmp.Or(s.settingsService.GetSettingsConfig().DiskUsagePath.Value, "/")}const dockerHostMemoryCacheTTL = 30 * time.Second@@ -378,17 +316,16 @@ type dockerHostMemoryInfo struct {// GetDockerHostMemory reports enclosing guest memory when Docker shares its// host's cgroup namespace. A false result leaves the caller's baseline intact.func (s *SystemService) GetDockerHostMemory(ctx context.Context) (uint64, uint64, bool) {- if runtime.GOOS != "linux" || s.dockerService == nil || s.dockerHostMemoryCache == nil || !cgroup.IsDockerContainer() {+ if runtime.GOOS != "linux" || !cgroup.IsDockerContainer() {return 0, 0, false}info, found, err := s.dockerHostMemoryCache.GetWithLoaders(struct{}{}, func(_ []struct{}) (map[struct{}]dockerHostMemoryInfo, error) {- metadata, err := s.loadDockerHostMemoryInternal(ctx)+ metadata, err := s.loadDockerHostMemory(ctx)if err != nil {slog.DebugContext(ctx, "Docker host memory accounting unavailable", "error", err)- // Cache failures too, without retaining expired metadata.- metadata = dockerHostMemoryInfo{}}+ // Cache failures too, without retaining expired metadata.return map[struct{}]dockerHostMemoryInfo{{}: metadata}, nil})if err != nil || !found || info.total == 0 || ctx.Err() != nil {@@ -402,7 +339,7 @@ func (s *SystemService) GetDockerHostMemory(ctx context.Context) (uint64, uint64return min(used, info.total), info.total, true}-func (s *SystemService) loadDockerHostMemoryInternal(ctx context.Context) (dockerHostMemoryInfo, error) {+func (s *SystemService) loadDockerHostMemory(ctx context.Context) (dockerHostMemoryInfo, error) {ctx, cancel := context.WithTimeout(ctx, 2*time.Second)defer cancel()@@ -418,8 +355,7 @@ func (s *SystemService) loadDockerHostMemoryInternal(ctx context.Context) (dockeif err != nil {return dockerHostMemoryInfo{}, err}- if !strings.HasPrefix(inspect.Container.ID, containerID) || inspect.Container.HostConfig == nil ||- inspect.Container.HostConfig.CgroupnsMode != "host" {+ if !strings.HasPrefix(inspect.Container.ID, containerID) || kit.FromPtr(inspect.Container.HostConfig).CgroupnsMode != "host" {return dockerHostMemoryInfo{}, errors.New("current container does not share the Docker host cgroup namespace")}// Inspection expands hostname and mountinfo short IDs before cgroup validation.@@ -438,8 +374,8 @@ func (s *SystemService) loadDockerHostMemoryInternal(ctx context.Context) (dockereturn dockerHostMemoryInfo{root: root, total: uint64(info.Info.MemTotal)}, nil}-// resolveUpgraderRuntimeOptionsInternal determines how a helper container reaches the Docker daemon.-func resolveUpgraderRuntimeOptionsInternal(+// resolveUpgraderRuntimeOptions determines how a helper container reaches the Docker daemon.+func resolveUpgraderRuntimeOptions(ctx context.Context,dockerHost string,currentContainer *mobycontainer.InspectResponse,@@ -447,21 +383,18 @@ func resolveUpgraderRuntimeOptionsInternal(isRunningInDocker func() bool,selectReachableNetwork func(context.Context, *mobycontainer.InspectResponse, string) string,) ([]string, []mount.Mount, mobycontainer.NetworkMode, error) {- containerEnv := vuln.BuildDockerHostEnv(dockerHost)-scheme, socketPath, err := vuln.ParseDockerHost(dockerHost)if err != nil {return nil, nil, "", fmt.Errorf("resolve docker host %q: %w", dockerHost, err)}+ // The upgrader must reach the tcp DOCKER_HOST, so prefer a network the+ // daemon proxy is actually on over the plain auto heuristic (#3533).+ if scheme != "unix" && selectReachableNetwork != nil {+ return vuln.BuildDockerHostEnv(dockerHost), nil, mobycontainer.NetworkMode(selectReachableNetwork(ctx, currentContainer, dockerHost)), nil+ }if scheme != "unix" {- // The upgrader must reach the tcp DOCKER_HOST, so prefer a network the- // daemon proxy is actually on over the plain auto heuristic (#3533).- networkMode := docker.SelectAutoNetworkMode(currentContainer)- if selectReachableNetwork != nil {- networkMode = selectReachableNetwork(ctx, currentContainer, dockerHost)- }- return containerEnv, nil, mobycontainer.NetworkMode(networkMode), nil+ return vuln.BuildDockerHostEnv(dockerHost), nil, mobycontainer.NetworkMode(docker.SelectAutoNetworkMode(currentContainer)), nil}socketSource, err := vuln.ResolveUnixSocketSource(@@ -474,13 +407,11 @@ func resolveUpgraderRuntimeOptionsInternal(return nil, nil, "", fmt.Errorf("resolve unix socket source: %w", err)}- mounts := []mount.Mount{{+ return vuln.BuildDockerHostEnv(dockerHost), []mount.Mount{{Type: mount.TypeBind,Source: socketSource,Target: socketPath,- }}-- return containerEnv, mounts, "", nil+ }}, "", nil}const systemBackupHistoryUnion = "SELECT id, size, created_at, status, trigger, destination, '' AS format, local_snapshot_id, " +@@ -490,20 +421,68 @@ const systemBackupHistoryUnion = "SELECT id, size, created_at, status, trigger,"s3_destination_id, policy_id, error, CASE WHEN policy_id LIKE 'system-volume:%' THEN 'system' ELSE " +"'volume' END AS type, 'volume' AS resource_type, volume_name AS resource_name FROM volume_backups"-// backupStoreInternal persists system backup runs and policies for the backup child.-func (s *SystemService) backupStoreInternal(s3Destinations *s3.S3DestinationService) systembackup.Store {+// backupStore persists system backup runs and policies for the backup child.+func (s *SystemService) backupStore(s3Destinations *s3.S3DestinationService) systembackup.Store {return systembackup.Store{CreateRun: func(ctx context.Context, dto *backuptypes.SystemBackupRun) error {- run := systemBackupRunFromDTOInternal(*dto)+ run := SystemBackupRun{ //exhaustruct:enforce+ BaseModel: database.BaseModel{+ ID: dto.ID,+ CreatedAt: time.Time{},+ UpdatedAt: nil,+ },+ Size: dto.Size,+ CreatedAt: dto.CreatedAt,+ Status: SystemBackupStatus(dto.Status),+ Trigger: SystemBackupTrigger(dto.Trigger),+ Destination: dto.Destination,+ LocalSnapshotID: dto.LocalSnapshotID,+ RemoteSnapshotID: dto.RemoteSnapshotID,+ S3DestinationID: dto.S3DestinationID,+ S3DestinationName: dto.S3DestinationName,+ PolicyID: dto.PolicyID,+ Error: dto.Error,+ }if err := s.db.WithContext(ctx).Create(&run).Error; err != nil {return err}- dto.ID, dto.CreatedAt = run.ID, run.CreatedAt+ *dto = backuptypes.SystemBackupRun{ //exhaustruct:enforce+ RemoteAvailable: dto.RemoteAvailable,+ ActivityID: dto.ActivityID,+ ID: run.ID,+ Size: dto.Size,+ CreatedAt: run.CreatedAt,+ Status: dto.Status,+ Trigger: dto.Trigger,+ Destination: dto.Destination,+ LocalSnapshotID: dto.LocalSnapshotID,+ RemoteSnapshotID: dto.RemoteSnapshotID,+ S3DestinationID: dto.S3DestinationID,+ S3DestinationName: dto.S3DestinationName,+ PolicyID: dto.PolicyID,+ Error: dto.Error,+ }return nil},SaveRun: func(ctx context.Context, dto *backuptypes.SystemBackupRun) error {- run := systemBackupRunFromDTOInternal(*dto)- return s.db.WithContext(ctx).Save(&run).Error+ return s.db.WithContext(ctx).Save(&SystemBackupRun{ //exhaustruct:enforce+ BaseModel: database.BaseModel{+ ID: dto.ID,+ CreatedAt: time.Time{},+ UpdatedAt: nil,+ },+ Size: dto.Size,+ CreatedAt: dto.CreatedAt,+ Status: SystemBackupStatus(dto.Status),+ Trigger: SystemBackupTrigger(dto.Trigger),+ Destination: dto.Destination,+ LocalSnapshotID: dto.LocalSnapshotID,+ RemoteSnapshotID: dto.RemoteSnapshotID,+ S3DestinationID: dto.S3DestinationID,+ S3DestinationName: dto.S3DestinationName,+ PolicyID: dto.PolicyID,+ Error: dto.Error,+ }).Error},FailRun: func(ctx context.Context, id, message string) error {return s.db.WithContext(ctx).Model(&SystemBackupRun{}).Where("id = ?", id).Updates(map[string]any{"status": SystemBackupStatusFailed, "error": message}).Error@@ -578,9 +557,9 @@ func (s *SystemService) backupStoreInternal(s3Destinations *s3.S3DestinationServreturn count, err},Policies: func(ctx context.Context) ([]backuptypes.SystemBackupPolicy, error) {- policies, err := s.backupPoliciesInternal(ctx)- if err != nil {- return nil, err+ var policies []SystemBackupPolicy+ if err := s.db.WithContext(ctx).Order("created_at ASC").Find(&policies).Error; err != nil {+ return nil, fmt.Errorf("failed to load system backup policies: %w", err)}result := make([]backuptypes.SystemBackupPolicy, len(policies))for i := range policies {@@ -609,9 +588,9 @@ func (s *SystemService) backupStoreInternal(s3Destinations *s3.S3DestinationServunregister func(context.Context, string),reschedule func(context.Context, *backuptypes.SystemBackupPolicy),) error {- existing, err := s.backupPoliciesInternal(ctx)- if err != nil {- return err+ var existing []SystemBackupPolicy+ if err := s.db.WithContext(ctx).Order("created_at ASC").Find(&existing).Error; err != nil {+ return fmt.Errorf("failed to load system backup policies: %w", err)}return backup.PolicyReconciliation[SystemBackupPolicy, backuptypes.UpdateSystemBackupPolicy]{Domain: "system",@@ -625,7 +604,15 @@ func (s *SystemService) backupStoreInternal(s3Destinations *s3.S3DestinationServif buildErr := build(ctx, &dto, update); buildErr != nil {return buildErr}- policy.applyDTOInternal(dto)+ *policy = SystemBackupPolicy{ //exhaustruct:enforce+ BaseModel: policy.BaseModel,+ Enabled: dto.Enabled,+ Schedule: dto.Schedule,+ RetentionCount: dto.RetentionCount,+ LocalEnabled: dto.LocalEnabled,+ S3Enabled: dto.S3Enabled,+ S3DestinationID: dto.S3DestinationID,+ }return nil},Unregister: unregister,@@ -650,21 +637,17 @@ func (s *SystemService) backupStoreInternal(s3Destinations *s3.S3DestinationServ)}query = pagination.ApplyFilter(query, "type", params.Filters["type"])- if params.Sort == "" {- params.Sort = "createdAt"- params.Order = pagination.SortDesc- }var history []backuptypes.HistoryEntry- page, err := pagination.PaginateAndSortDB(params, query, &history)+ page, err := pagination.PaginateAndSortDB(pagination.QueryParams{+ SearchQuery: params.SearchQuery,+ SortParams: pagination.SortParams{+ Sort: cmp.Or(params.Sort, "createdAt"),+ Order: kit.Ternary(params.Sort == "", pagination.SortDesc, params.Order),+ },+ Params: params.Params,+ Filters: params.Filters,+ }, query, &history)return history, page, err},}}--func (s *SystemService) backupPoliciesInternal(ctx context.Context) ([]SystemBackupPolicy, error) {- var policies []SystemBackupPolicy- if err := s.db.WithContext(ctx).Order("created_at ASC").Find(&policies).Error; err != nil {- return nil, fmt.Errorf("failed to load system backup policies: %w", err)- }- return policies, nil-}diff --git a/backend/internal/system/service_test.go b/backend/internal/system/service_test.goindex bc2799023..da0069e85 100644--- a/backend/internal/system/service_test.go+++ b/backend/internal/system/service_test.go@@ -15,6 +15,7 @@ import ("github.com/moby/moby/api/types/container""github.com/moby/moby/api/types/mount""github.com/moby/moby/api/types/network"+ "github.com/moby/moby/client""github.com/stretchr/testify/require""go.getarcane.app/kit/pkg""go.getarcane.app/sys/crypto"@@ -24,8 +25,9 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/backup""github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/database"+ "github.com/getarcaneapp/arcane/backend/v2/internal/docker""github.com/getarcaneapp/arcane/backend/v2/internal/s3"- systembackup "github.com/getarcaneapp/arcane/backend/v2/internal/system/children/backup"+ "github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/backend/v2/internal/volume""github.com/getarcaneapp/arcane/backend/v2/pkg/pagination""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow/flowtest"@@ -33,7 +35,7 @@ import (francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing")-func TestResolveSystemUpgraderRuntimeOptionsInternal_TCPDockerHost(t *testing.T) {+func TestResolveSystemUpgraderRuntimeOptions_TCPDockerHost(t *testing.T) {currentContainer := &container.InspectResponse{HostConfig: &container.HostConfig{NetworkMode: "bridge"},NetworkSettings: &container.NetworkSettings{@@ -44,7 +46,7 @@ func TestResolveSystemUpgraderRuntimeOptionsInternal_TCPDockerHost(t *testing.T)},}- containerEnv, mounts, networkMode, err := resolveUpgraderRuntimeOptionsInternal(+ containerEnv, mounts, networkMode, err := resolveUpgraderRuntimeOptions(t.Context(),"tcp://docker-socket-proxy:2375",currentContainer,@@ -58,8 +60,8 @@ func TestResolveSystemUpgraderRuntimeOptionsInternal_TCPDockerHost(t *testing.T)require.Equal(t, container.NetworkMode("arcane-test"), networkMode)}-func TestResolveSystemUpgraderRuntimeOptionsInternal_UnixDockerHost(t *testing.T) {- containerEnv, mounts, networkMode, err := resolveUpgraderRuntimeOptionsInternal(+func TestResolveSystemUpgraderRuntimeOptions_UnixDockerHost(t *testing.T) {+ containerEnv, mounts, networkMode, err := resolveUpgraderRuntimeOptions(t.Context(),"unix:///var/run/docker.sock",nil,@@ -81,8 +83,8 @@ func TestResolveSystemUpgraderRuntimeOptionsInternal_UnixDockerHost(t *testing.T}, mounts)}-func TestResolveSystemUpgraderRuntimeOptionsInternal_DefaultDockerHost(t *testing.T) {- containerEnv, mounts, _, err := resolveUpgraderRuntimeOptionsInternal(+func TestResolveSystemUpgraderRuntimeOptions_DefaultDockerHost(t *testing.T) {+ containerEnv, mounts, _, err := resolveUpgraderRuntimeOptions(t.Context(),"",nil,@@ -103,8 +105,8 @@ func TestResolveSystemUpgraderRuntimeOptionsInternal_DefaultDockerHost(t *testin}, mounts)}-func TestResolveSystemUpgraderRuntimeOptionsInternal_UnixDockerHostResolutionError(t *testing.T) {- _, _, _, err := resolveUpgraderRuntimeOptionsInternal(+func TestResolveSystemUpgraderRuntimeOptions_UnixDockerHostResolutionError(t *testing.T) {+ _, _, _, err := resolveUpgraderRuntimeOptions(t.Context(),"unix:///var/run/docker.sock",nil,@@ -121,7 +123,7 @@ func TestResolveSystemUpgraderRuntimeOptionsInternal_UnixDockerHostResolutionErrfunc TestListBackupHistoryClassifiesAndFiltersOrigins(t *testing.T) {gormDB, err := gorm.Open(sqlite.Open("file:system-volume-history?mode=memory&cache=shared"), &gorm.Config{})require.NoError(t, err)- require.NoError(t, gormDB.AutoMigrate(&SystemBackupRun{}, &volume.VolumeBackup{}))+ require.NoError(t, gormDB.AutoMigrate(&SystemBackupRun{}, &volume.VolumeBackup{}, &s3.S3Destination{}, &settings.SettingVariable{}))now := time.Now().UTC()require.NoError(t, gormDB.Create(&SystemBackupRun{CreatedAt: now.Add(-2 * time.Minute), Status: SystemBackupStatusSucceeded, Trigger: SystemBackupTriggerManual,@@ -137,8 +139,14 @@ func TestListBackupHistoryClassifiesAndFiltersOrigins(t *testing.T) {Trigger: volume.VolumeBackupTriggerManual, Destination: volumetypes.BackupDestinationLocal,Format: volume.VolumeBackupFormatRustic,}).Error)- service := &SystemService{db: &database.DB{DB: gormDB}}- service.backup = systembackup.NewService(systembackup.Dependencies{Store: service.backupStoreInternal(nil)})+ db := &database.DB{DB: gormDB}+ settingsService, err := settings.NewSettingsService(t.Context(), db)+ require.NoError(t, err)+ t.Cleanup(func() { require.NoError(t, settingsService.Stop(context.WithoutCancel(t.Context()))) })+ service := NewSystemService(+ db, nil, nil, nil, nil, nil, settingsService, nil, nil, nil, nil, nil,+ s3.NewS3DestinationService(db, func(context.Context, string) error { return nil }), nil, nil, &config.Config{},+ )systemRows, page, err := service.backup.ListBackupHistory(t.Context(), pagination.QueryParams{Sort: "createdAt", Order: pagination.SortDesc, Limit: 20,@@ -162,7 +170,7 @@ func TestListBackupHistoryClassifiesAndFiltersOrigins(t *testing.T) {require.Equal(t, "cache", volumeRows[0].ResourceName)}-func newSystemBackupAdmissionGateForTestInternal(t testing.TB) *runs.Admission {+func newSystemBackupAdmissionGateForTest(t testing.TB) *runs.Admission {t.Helper()runtime := francistest.New(t)gate := runs.NewAdmission(runtime.Service(), t.Name())@@ -171,26 +179,26 @@ func newSystemBackupAdmissionGateForTestInternal(t testing.TB) *runs.Admission {return gate}-type systemBackupPolicySchedulerInternal struct {+type systemBackupPolicyScheduler struct {submitted []scheduler.Requestjobs map[string]scheduler.Job}-func (s *systemBackupPolicySchedulerInternal) AddJob(_ context.Context, job scheduler.Job) error {+func (s *systemBackupPolicyScheduler) AddJob(_ context.Context, job scheduler.Job) error {s.jobs[job.Name()] = jobreturn nil}-func (s *systemBackupPolicySchedulerInternal) RemoveJob(_ context.Context, name string) {+func (s *systemBackupPolicyScheduler) RemoveJob(_ context.Context, name string) {delete(s.jobs, name)}-func (s *systemBackupPolicySchedulerInternal) HasJob(name string) bool {+func (s *systemBackupPolicyScheduler) HasJob(name string) bool {_, ok := s.jobs[name]return ok}-func (s *systemBackupPolicySchedulerInternal) Submit(_ context.Context, request scheduler.Request) (scheduler.Run, error) {+func (s *systemBackupPolicyScheduler) Submit(_ context.Context, request scheduler.Request) (scheduler.Run, error) {s.submitted = append(s.submitted, request)return scheduler.Run{ID: request.RunID, JobID: request.JobID, EnvironmentID: request.EnvironmentID, Status: scheduler.Queued}, nil}@@ -198,25 +206,51 @@ func (s *systemBackupPolicySchedulerInternal) Submit(_ context.Context, requestfunc TestSystemBackupPoliciesRegisterIndependentJobs(t *testing.T) {gormDB, err := gorm.Open(sqlite.Open("file:system-backup-schedules?mode=memory&cache=shared"), &gorm.Config{})require.NoError(t, err)- require.NoError(t, gormDB.AutoMigrate(&SystemBackupPolicy{}, &SystemBackupRun{}, &backup.SystemBackupRecoveryConfig{}, &s3.S3Destination{}, &activity.Activity{}, &activity.ActivityMessage{}))+ require.NoError(t, gormDB.AutoMigrate(+ &SystemBackupPolicy{},+ &SystemBackupRun{},+ &backup.SystemBackupRecoveryConfig{},+ &s3.S3Destination{},+ &activity.Activity{},+ &activity.ActivityMessage{},+ &settings.SettingVariable{},+ &volume.VolumeBackup{},+ ))crypto.InitEncryption(&crypto.Config{EncryptionKey: "system-backup-policy-test-key-32bytes", Environment: "test"})db := &database.DB{DB: gormDB}recoveryKeys := backup.NewRecoveryKeyStore(db)- s3Destinations := s3.NewS3DestinationService(db, nil)- service := &SystemService{db: db}- service.backup = systembackup.NewService(systembackup.Dependencies{- Store: service.backupStoreInternal(s3Destinations),- DB: db,- SQLDB: db.SQLDB,- S3Destinations: s3Destinations,- Config: &config.Config{DatabaseURL: "file:system-backup-schedules-test.db"},- RecoveryKeys: recoveryKeys,- })+ s3Destinations := s3.NewS3DestinationService(db, func(context.Context, string) error { return nil })+ settingsService, err := settings.NewSettingsService(t.Context(), db)+ require.NoError(t, err)+ t.Cleanup(func() { require.NoError(t, settingsService.Stop(context.WithoutCancel(t.Context()))) })+ gate := newSystemBackupAdmissionGateForTest(t)+ dockerService := &docker.DockerClientService{Client: func() *client.Client {+ cli, _ := client.New(client.WithHost("tcp://127.0.0.1:1"), client.WithAPIVersion("1.41"))+ return cli+ }()}+ service := NewSystemService(+ db,+ dockerService,+ nil,+ nil,+ volume.NewVolumeService(db, dockerService, nil, nil, nil, nil, nil, nil, nil, nil),+ nil,+ settingsService,+ nil,+ nil,+ nil,+ nil,+ backup.NewEngine(gate, nil),+ s3Destinations,+ recoveryKeys,+ nil,+ &config.Config{DatabaseURL: "file:system-backup-schedules-test.db"},+ )harness := flowtest.New(t, activity.NewActivityService(db, nil))require.NoError(t, service.backup.RegisterWorkflows(harness.Engine))harness.Start(t)- jobScheduler := &systemBackupPolicySchedulerInternal{jobs: make(map[string]scheduler.Job)}- require.NoError(t, service.SetBackupScheduler(t.Context(), jobScheduler, newSystemBackupAdmissionGateForTestInternal(t)))+ jobScheduler := &systemBackupPolicyScheduler{jobs: make(map[string]scheduler.Job)}+ require.NoError(t, service.SetBackupScheduler(t.Context(), jobScheduler, gate))status, err := service.backup.SetRecoveryKey(t.Context(), "QWERTY-ABCDEF-234567-GHIJKL-MNOPQR-STUVWX-YZ2345-ZXCVBN")require.NoError(t, err)@@ -263,12 +297,12 @@ func TestSystemBackupPoliciesRegisterIndependentJobs(t *testing.T) {job, registered := jobScheduler.jobs[jobName]require.True(t, registered)// The scheduled run checks the remote first; a local policy then- // attempts its backup, which has no engine in this test.+ // attempts its backup, which has no reachable Docker in this test.outcome, runErr := job.Run(t.Context())require.NoError(t, runErr)if local {require.Equal(t, scheduler.Failed, outcome.Status)- require.Contains(t, outcome.Message, "backup engine is unavailable")+ require.Contains(t, outcome.Message, "failed to create local system recovery snapshot")} else {require.Equal(t, scheduler.NeedsAttention, outcome.Status)require.Equal(t, backup.RemoteDisabledMessage, outcome.Message)@@ -283,15 +317,13 @@ func TestSystemBackupPoliciesRegisterIndependentJobs(t *testing.T) {func TestSystemBackupPolicyRequiresConfiguredRecoveryKeyWhenEnabled(t *testing.T) {gormDB, err := gorm.Open(sqlite.Open("file:system-backup-key-required?mode=memory&cache=shared"), &gorm.Config{})require.NoError(t, err)- require.NoError(t, gormDB.AutoMigrate(&SystemBackupPolicy{}, &backup.SystemBackupRecoveryConfig{}))+ require.NoError(t, gormDB.AutoMigrate(&SystemBackupPolicy{}, &backup.SystemBackupRecoveryConfig{}, &s3.S3Destination{}))db := &database.DB{DB: gormDB}- service := &SystemService{db: db}- service.backup = systembackup.NewService(systembackup.Dependencies{- Store: service.backupStoreInternal(nil),- DB: db,- Config: &config.Config{DatabaseURL: "file:system-backup-key-required-test.db"},- RecoveryKeys: backup.NewRecoveryKeyStore(db),- })+ service := NewSystemService(+ db, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil,+ s3.NewS3DestinationService(db, func(context.Context, string) error { return nil }),+ backup.NewRecoveryKeyStore(db), nil, &config.Config{DatabaseURL: "file:system-backup-key-required-test.db"},+ )_, err = service.backup.UpdatePolicies(t.Context(), []backuptypes.UpdateSystemBackupPolicy{{Enabled: true, Schedule: "0 0 2 * * *", RetentionCount: 7, LocalEnabled: true,diff --git a/backend/internal/volume/children/backup/children/policies/service_test.go b/backend/internal/volume/children/backup/children/policies/service_test.goindex 389c88412..dc1f5be65 100644--- a/backend/internal/volume/children/backup/children/policies/service_test.go+++ b/backend/internal/volume/children/backup/children/policies/service_test.go@@ -56,7 +56,11 @@ func TestVolumeBackupPolicy_UpdateRegistersIndependentJobsAndSettings(t *testingrequire.NoError(t, gormDB.AutoMigrate(&VolumeBackupPolicy{}))db := &database.DB{DB: gormDB}jobScheduler := &volumeBackupPolicyScheduler{jobs: make(map[string]scheduler.Job)}- service := NewService(Dependencies{DB: db, LatestRun: noLatestRun})+ service := NewService(Dependencies{+ DB: db,+ S3Destinations: s3.NewS3DestinationService(db, func(context.Context, string) error { return nil }),+ LatestRun: noLatestRun,+ })gate := newVolumeAdmissionForTest(t)require.NoError(t, service.SetScheduler(t.Context(), jobScheduler, gate))@@ -96,7 +100,8 @@ func TestVolumeBackupPolicy_GetReturnsLastRunForEachPolicy(t *testing.T) {}service := NewService(Dependencies{- DB: &database.DB{DB: gormDB},+ DB: &database.DB{DB: gormDB},+ S3Destinations: s3.NewS3DestinationService(&database.DB{DB: gormDB}, func(context.Context, string) error { return nil }),LatestRun: func(_ context.Context, policyID string) (*volume.BackupEntry, error) {return latest[policyID], nil},