Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix: make backup restores and recovery key changes data-safe (#4487)
Kyle Mendellc2d8e8d6cab0
diff --git a/backend/cli/recovery/recovery.go b/backend/cli/recovery/recovery.goindex 5c35a45fb..2fd755108 100644--- a/backend/cli/recovery/recovery.go+++ b/backend/cli/recovery/recovery.go@@ -21,11 +21,13 @@ import ("github.com/spf13/cobra""go.getarcane.app/docker""go.getarcane.app/docker/compat"+ "go.getarcane.app/sys/crypto""gorm.io/gorm""gorm.io/gorm/clause""github.com/getarcaneapp/arcane/backend/v2/cli/upgrade""github.com/getarcaneapp/arcane/backend/v2/internal/activity"+ "github.com/getarcaneapp/arcane/backend/v2/internal/backup""github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/backend/v2/internal/system"@@ -131,6 +133,11 @@ func runRestoreInternal(cmd *cobra.Command, _ []string) error {// Projects were restored into the current directory, so the recovered// configuration must keep pointing there rather than at the backup-time path.manifest.Environment["PROJECTS_DIRECTORY"] = cmp.Or(request.ProjectsSetting, manifest.Environment["PROJECTS_DIRECTORY"])+ crypto.InitEncryption(&crypto.Config{+ EncryptionKey: manifest.Environment["ENCRYPTION_KEY"],+ Environment: manifest.Environment["ENVIRONMENT"],+ AgentMode: manifest.Environment["AGENT_MODE"] == "true",+ })if finalizeRestoredBackupErr := finalizeRestoredBackupInternal(ctx, manifest.Environment["DATABASE_URL"], manifest.BackupID, manifest.ActivityID, request); finalizeRestoredBackupErr != nil {return fmt.Errorf("finalize restored system backup: %w", finalizeRestoredBackupErr)}@@ -161,6 +168,11 @@ func finalizeRestoredBackupInternal(ctx context.Context, databaseURL, manifestBaif finalizeRestoredRunErr := finalizeRestoredRunInternal(db, manifestBackupID, request); finalizeRestoredRunErr != nil {return finalizeRestoredRunErr}+ if strings.TrimSpace(request.RecoveryKey) != "" {+ if setRecoveryKeyErr := backup.NewRecoveryKeyStore(&database.DB{DB: db}).Set(ctx, request.RecoveryKey); setRecoveryKeyErr != nil {+ return setRecoveryKeyErr+ }+ }if preserveSafetyBackupErr := preserveSafetyBackupInternal(db, request.SafetyBackup); preserveSafetyBackupErr != nil {return preserveSafetyBackupErr}@@ -223,23 +235,23 @@ func finalizeRestoredRunInternal(db *gorm.DB, manifestBackupID string, request rreturn db.Model(&system.SystemBackupRun{}).Where("id = ?", run.ID).Updates(values).Error}-func preserveSafetyBackupInternal(db *gorm.DB, backup *recoverytypes.SafetyBackup) error {- if backup == nil || strings.TrimSpace(backup.ID) == "" || strings.TrimSpace(backup.LocalSnapshotID) == "" {+func preserveSafetyBackupInternal(db *gorm.DB, safetyBackup *recoverytypes.SafetyBackup) error {+ if safetyBackup == nil || strings.TrimSpace(safetyBackup.ID) == "" || strings.TrimSpace(safetyBackup.LocalSnapshotID) == "" {return nil}onConflict := map[string]any{- "size": backup.Size, "updated_at": time.Now().UTC(), "status": system.SystemBackupStatusSucceeded,+ "size": safetyBackup.Size, "updated_at": time.Now().UTC(), "status": system.SystemBackupStatusSucceeded,"trigger": system.SystemBackupTriggerSafety, "destination": backuptypes.SystemBackupDestinationLocal,- "local_snapshot_id": backup.LocalSnapshotID, "error": "",+ "local_snapshot_id": safetyBackup.LocalSnapshotID, "error": "",}return db.Model(&system.SystemBackupRun{}).Clauses(clause.OnConflict{Columns: []clause.Column{{Name: "id"}}, DoUpdates: clause.Assignments(onConflict)}).Create(map[string]any{- "id": backup.ID, "size": backup.Size, "created_at": backup.CreatedAt, "updated_at": time.Now().UTC(),+ "id": safetyBackup.ID, "size": safetyBackup.Size, "created_at": safetyBackup.CreatedAt, "updated_at": time.Now().UTC(),"status": system.SystemBackupStatusSucceeded,"trigger": system.SystemBackupTriggerSafety,"destination": backuptypes.SystemBackupDestinationLocal,- "local_snapshot_id": backup.LocalSnapshotID, "remote_snapshot_id": "", "s3_destination_id": "",+ "local_snapshot_id": safetyBackup.LocalSnapshotID, "remote_snapshot_id": "", "s3_destination_id": "","policy_id": "", "error": "",}).Error}diff --git a/backend/cli/recovery/recovery_test.go b/backend/cli/recovery/recovery_test.goindex 64f58f03a..e6e459f45 100644--- a/backend/cli/recovery/recovery_test.go+++ b/backend/cli/recovery/recovery_test.go@@ -9,9 +9,11 @@ import (recoverytypes "github.com/getarcaneapp/arcane/types/v2/recovery""github.com/libtnb/sqlite""github.com/stretchr/testify/require"+ "go.getarcane.app/sys/crypto""gorm.io/gorm""github.com/getarcaneapp/arcane/backend/v2/internal/activity"+ "github.com/getarcaneapp/arcane/backend/v2/internal/backup""github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/backend/v2/internal/system"@@ -24,7 +26,7 @@ func newRestoredDatabaseForTestInternal(t *testing.T, runIDs ...string) (string,require.NoError(t, err)db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{})require.NoError(t, err)- require.NoError(t, db.AutoMigrate(&system.SystemBackupRun{}, &activity.Activity{}, &settings.SettingVariable{}))+ require.NoError(t, db.AutoMigrate(&system.SystemBackupRun{}, &activity.Activity{}, &settings.SettingVariable{}, &backup.SystemBackupRecoveryConfig{}))require.NoError(t, db.Create(&settings.SettingVariable{Key: "projectsDirectory", Value: "/app/data/projects"}).Error)for i, id := range runIDs {require.NoError(t, db.Create(&system.SystemBackupRun{@@ -46,9 +48,13 @@ func newRestoredDatabaseForTestInternal(t *testing.T, runIDs ...string) (string,func TestFinalizeRestoredBackupInternal(t *testing.T) {databaseURL, dsn := newRestoredDatabaseForTestInternal(t, "older", "selected")+ crypto.InitEncryption(&crypto.Config{EncryptionKey: "recovery-finalize-test-key-32bytes", Environment: "test"})+ recoveryKey, err := backup.GenerateRecoveryKey()+ require.NoError(t, err)require.NoError(t, finalizeRestoredBackupInternal(t.Context(), databaseURL, "selected", "activity-1", recoverytypes.RestoreRequest{BackupID: "request-id", RemoteSnapshotID: "snapshot-1", S3DestinationID: "destination-1", Size: 1234,+ RecoveryKey: recoveryKey,ProjectsSetting: "/srv/projects",SafetyBackup: &recoverytypes.SafetyBackup{ID: "safety", LocalSnapshotID: "safety-snapshot", Size: 4321,@@ -77,6 +83,9 @@ func TestFinalizeRestoredBackupInternal(t *testing.T) {var projectsSetting settings.SettingVariablerequire.NoError(t, db.First(&projectsSetting, "key = ?", "projectsDirectory").Error)require.Equal(t, "/srv/projects", projectsSetting.Value)+ storedKey, err := backup.NewRecoveryKeyStore(&database.DB{DB: db}).Get(t.Context())+ require.NoError(t, err)+ require.Equal(t, recoveryKey, storedKey)}func TestFinalizeRestoredBackupInternalFallsBackForLegacyManifest(t *testing.T) {diff --git a/backend/internal/backup/service.go b/backend/internal/backup/service.goindex 830492928..386d2988a 100644--- a/backend/internal/backup/service.go+++ b/backend/internal/backup/service.go@@ -497,7 +497,7 @@ func (e *Engine) ForgetSnapshots(ctx context.Context, dockerClient *client.Clien}defer unlock()output, err := e.runContainer(ctx, dockerClient, repository, password, []string{"snapshots", "--json"})- if len(snapshotIDs) == 0 && err != nil && strings.Contains(err.Error(), rusticRepositoryMissingMessage) {+ if err != nil && strings.Contains(err.Error(), rusticRepositoryMissingMessage) {return nil}if err != nil {@@ -540,6 +540,62 @@ func (e *Engine) ChangeRepositoryPassword(ctx context.Context, dockerClient *clireturn err}+func (e *Engine) RekeyRepository(ctx context.Context, dockerClient *client.Client, repository Repository, newPassword string, currentPasswords ...string) (string, error) {+ _, err := e.ListSnapshots(ctx, dockerClient, repository, newPassword)+ if err == nil || strings.Contains(err.Error(), rusticRepositoryMissingMessage) {+ return "", nil+ }+ for _, candidate := range currentPasswords {+ if candidate == "" || candidate == newPassword {+ continue+ }+ if changeErr := e.ChangeRepositoryPassword(ctx, dockerClient, repository, candidate, newPassword); changeErr == nil {+ return candidate, nil+ }+ }+ return "", fmt.Errorf("backup repository %s does not open with the recovery key: %w", repository.ID, err)+}++func (e *Engine) RekeyRepositories(ctx context.Context, dockerClient *client.Client, repositories []Repository, newPassword string, currentPasswords ...string) error {+ type rekeyed struct {+ repository Repository+ previous string+ }+ changed := make([]rekeyed, 0, len(repositories))+ for _, repository := range repositories {+ previous, err := e.RekeyRepository(ctx, dockerClient, repository, newPassword, currentPasswords...)+ if err != nil {+ errs := []error{err}+ var failedRollbackErrs []error+ for _, candidate := range currentPasswords {+ if candidate == "" || candidate == newPassword {+ continue+ }+ _, rollbackErr := e.RekeyRepository(context.WithoutCancel(ctx), dockerClient, repository, candidate, newPassword)+ if rollbackErr == nil {+ failedRollbackErrs = nil+ break+ }+ failedRollbackErrs = append(failedRollbackErrs, fmt.Errorf("failed to restore the previous key on backup repository %s: %w", repository.ID, rollbackErr))+ }+ errs = append(errs, failedRollbackErrs...)+ for _, done := range changed {+ if _, rollbackErr := e.RekeyRepository(context.WithoutCancel(ctx), dockerClient, done.repository, done.previous, newPassword); rollbackErr != nil {+ errs = append(errs, fmt.Errorf("failed to restore the previous key on backup repository %s: %w", done.repository.ID, rollbackErr))+ }+ }+ return errors.Join(errs...)+ }+ if previous != "" {+ changed = append(changed, rekeyed{+ repository: repository,+ previous: previous,+ })+ }+ }+ return nil+}+// Replicate copies one snapshot into another repository through a temporary volume, never the live data.// Rustic's native `copy` needs a TOML profile for the target, which the env-only Repository cannot express.func (e *Engine) Replicate(ctx context.Context, dockerClient *client.Client, from Repository, fromSnapshotID string, to Repository, password, label string, tags ...string) (Snapshot, error) {@@ -970,20 +1026,27 @@ func CheckScheduledRemote(ctx context.Context, db *database.DB, destinations *s3return fmt.Errorf("%w: %s", ErrRemoteRepositoryMissing, result.Reason)}-// ExpiredRunIDs returns the IDs of succeeded runs with snapshots that fall-// outside the newest keep entries for one policy, oldest last. Callers delete-// each run through their own delete path so snapshots are forgotten too.+// ExpiredRunIDs returns the IDs of succeeded and failed runs with snapshots that+// fall outside the newest keep entries of their status for one policy, oldest+// last within each status. Callers delete each run through their own delete+// path so snapshots are forgotten too.func ExpiredRunIDs(ctx context.Context, db *database.DB, table, policyID string, keep int) ([]string, error) {- var ids []string- err := db.WithContext(ctx).- Table(table).- Where(- "policy_id = ? AND status = ? AND (COALESCE(local_snapshot_id, '') <> '' OR COALESCE(remote_snapshot_id, '') <> '')",- policyID,- "succeeded",- ).- Order("created_at DESC").- Offset(keep).- Pluck("id", &ids).Error- return ids, err+ var expired []string+ for _, status := range []string{"succeeded", "failed"} {+ var ids []string+ if err := db.WithContext(ctx).+ Table(table).+ Where(+ "policy_id = ? AND status = ? AND (COALESCE(local_snapshot_id, '') <> '' OR COALESCE(remote_snapshot_id, '') <> '')",+ policyID,+ status,+ ).+ Order("created_at DESC").+ Offset(keep).+ Pluck("id", &ids).Error; err != nil {+ return nil, err+ }+ expired = append(expired, ids...)+ }+ return expired, nil}diff --git a/backend/internal/s3/service.go b/backend/internal/s3/service.goindex 6a9fa4c30..44c49090f 100644--- a/backend/internal/s3/service.go+++ b/backend/internal/s3/service.go@@ -164,7 +164,7 @@ func (s *S3DestinationService) UpdateS3Destination(ctx context.Context, id strin}// The persisted configuration without the// decrypted secret, for comparing connection fields against an update.- if configuration.SecretAccessKey == "" && !s3.ConnectionFieldsEqual(s3.Configuration{+ stored := s3.Configuration{ID: destination.ID,Name: destination.Name,Endpoint: destination.Endpoint,@@ -174,9 +174,15 @@ func (s *S3DestinationService) UpdateS3Destination(ctx context.Context, id strinPrefix: destination.Prefix,UseSSL: destination.UseSSL,ForcePathStyle: destination.ForcePathStyle,- }.Normalized(), configuration) {+ }.Normalized()+ if configuration.SecretAccessKey == "" && !s3.ConnectionFieldsEqual(stored, configuration) {return nil, errS3SecretRequired}+ if stored.Endpoint != configuration.Endpoint || stored.Bucket != configuration.Bucket || stored.Region != configuration.Region || stored.Prefix != configuration.Prefix {+ if ensureUnusedErr := s.ensureUnused(ctx, id); ensureUnusedErr != nil {+ return nil, fmt.Errorf("the bucket, prefix, endpoint or region cannot change while backups use the destination: %w", ensureUnusedErr)+ }+ }encryptedSecret, err := func() (string, error) {if configuration.SecretAccessKey == "" {return destination.SecretAccessKey, nil@@ -210,6 +216,16 @@ func (s *S3DestinationService) DeleteS3Destination(ctx context.Context, id strinif err != nil {return err}+ if ensureUnusedErr := s.ensureUnused(ctx, id); ensureUnusedErr != nil {+ return ensureUnusedErr+ }+ if deleteDestinationErr := s.db.WithContext(ctx).Delete(destination).Error; deleteDestinationErr != nil {+ return fmt.Errorf("failed to delete S3 destination: %w", deleteDestinationErr)+ }+ return nil+}++func (s *S3DestinationService) ensureUnused(ctx context.Context, id string) error {inUse, err := s.DestinationInUse(ctx, id)if err != nil {return err@@ -220,9 +236,6 @@ func (s *S3DestinationService) DeleteS3Destination(ctx context.Context, id strinif checkRemoteReferencesErr := s.checkRemoteReferences(ctx, id); checkRemoteReferencesErr != nil {return fmt.Errorf("%w: %s", ErrS3DestinationInUse, checkRemoteReferencesErr.Error())}- if deleteDestinationErr := s.db.WithContext(ctx).Delete(destination).Error; deleteDestinationErr != nil {- return fmt.Errorf("failed to delete S3 destination: %w", deleteDestinationErr)- }return nil}diff --git a/backend/internal/s3/service_test.go b/backend/internal/s3/service_test.goindex acb492c31..7c9597f93 100644--- a/backend/internal/s3/service_test.go+++ b/backend/internal/s3/service_test.go@@ -117,6 +117,41 @@ func TestS3DestinationService_DeleteRejectsConfiguredDestination(t *testing.T) {require.ErrorIs(t, err, ErrS3DestinationInUse)}+func TestS3DestinationService_UpdateRejectsLocationChangeWhileInUse(t *testing.T) {+ service, gormDB := setupS3DestinationServiceTest(t)+ ctx := t.Context()+ destination, err := service.CreateS3Destination(ctx, backup.CreateS3Destination{+ Name: "In use",+ Bucket: "arcane-backups",+ Region: "us-east-1",+ AccessKeyID: "access-key",+ SecretAccessKey: "secret-key",+ UseSSL: true,+ })+ require.NoError(t, err)+ require.NoError(t, gormDB.Exec("INSERT INTO volume_backups (s3_destination_id, remote_snapshot_id) VALUES (?, ?)", destination.ID, "snapshot-1").Error)++ _, err = service.UpdateS3Destination(ctx, destination.ID, backup.UpdateS3Destination{+ Name: destination.Name,+ Bucket: "other-bucket",+ Region: destination.Region,+ AccessKeyID: destination.AccessKeyID,+ SecretAccessKey: "secret-key",+ UseSSL: destination.UseSSL,+ })+ require.ErrorIs(t, err, ErrS3DestinationInUse)++ updated, err := service.UpdateS3Destination(ctx, destination.ID, backup.UpdateS3Destination{+ Name: "Renamed",+ Bucket: destination.Bucket,+ Region: destination.Region,+ AccessKeyID: destination.AccessKeyID,+ UseSSL: destination.UseSSL,+ })+ require.NoError(t, err)+ require.Equal(t, "Renamed", updated.Name)+}+func TestS3DestinationService_SyncS3Destinations(t *testing.T) {service, gormDB := setupS3DestinationServiceTest(t)ctx := t.Context()diff --git a/backend/internal/system/children/backup/handler.go b/backend/internal/system/children/backup/handler.goindex e20567497..75e3894d0 100644--- a/backend/internal/system/children/backup/handler.go+++ b/backend/internal/system/children/backup/handler.go@@ -238,7 +238,7 @@ func (h *Handler) Restore(ctx context.Context, input *RestoreSystemBackupInput)Step: "Preparing system restore", Message: "Preparing Arcane system restore", SuccessMessage: "Arcane system restore started",Metadata: database.JSON{"action": "restore_system_backup", "backupId": input.ID},}, func(activityCtx context.Context) error {- return h.service.RestoreBackup(activityCtx, input.ID, input.Body.RecoveryKey, *user)+ return h.service.RestoreBackup(activityCtx, input.ID, input.Body.RecoveryKey)})if err != nil {return nil, huma.Error500InternalServerError(err.Error())diff --git a/backend/internal/system/children/backup/service.go b/backend/internal/system/children/backup/service.goindex 1403bdd09..6570013d5 100644--- a/backend/internal/system/children/backup/service.go+++ b/backend/internal/system/children/backup/service.go@@ -83,6 +83,7 @@ type Store struct {RemoteSnapshotIDs func(ctx context.Context, destinationID string) ([]string, error)CountRuns func(ctx context.Context) (int64, error)CountVolumeRepositoryRuns func(ctx context.Context) (int64, error)+ CountRunningVolumeBackups func(ctx context.Context) (int64, error)Policies func(ctx context.Context) ([]backuptypes.SystemBackupPolicy, error)Policy func(ctx context.Context, id string) (*backuptypes.SystemBackupPolicy, error)ReconcilePolicies func(@@ -919,6 +920,11 @@ func (s *Service) PruneLocalRepository(ctx context.Context) error {}func (s *Service) DiscoverRemoteBackups(ctx context.Context, request backuptypes.DiscoverSystemBackupsRequest) (int, error) {+ lease, err := s.acquireRun(ctx)+ if err != nil {+ return 0, err+ }+ defer lease.Release(ctx)recoveryKey, err := s.recoveryKey(ctx, request.RecoveryKey)if err != nil {return 0, err@@ -1019,9 +1025,14 @@ func (s *Service) UploadBackup(ctx context.Context, id string, request backuptypreturn run, nil}-func (s *Service) RestoreBackup(ctx context.Context, id, recoveryKey string, user usertypes.Actor) (opErr error) {+func (s *Service) RestoreBackup(ctx context.Context, id, recoveryKey string) (opErr error) {ctx, finish := s.startOperation(ctx, "restore", attribute.String("arcane.backup.id", id))defer func() { finish(opErr) }()+ lease, err := s.acquireRun(ctx)+ if err != nil {+ return err+ }+ defer lease.Release(ctx)run, err := s.store.Run(ctx, id)if err != nil {return err@@ -1064,10 +1075,14 @@ func (s *Service) RestoreBackup(ctx context.Context, id, recoveryKey string, usereturn err}// A local safety snapshot is created before the detached helper is allowed to stop Arcane and replace its data.- safetyBackup, err := s.CreateBackup(ctx, user, backuptypes.SystemBackupTriggerSafety, backuptypes.CreateSystemBackupRequest{+ prepared, prepareErr := s.prepareBackup(ctx, backuptypes.SystemBackupTriggerSafety, backuptypes.CreateSystemBackupRequest{Destination: backuptypes.SystemBackupDestinationLocal,RecoveryKey: key,})+ if prepareErr != nil {+ return fmt.Errorf("failed to create pre-restore system backup: %w", prepareErr)+ }+ safetyBackup, err := s.executeBackup(ctx, prepared)if err != nil {return fmt.Errorf("failed to create pre-restore system backup: %w", err)}@@ -1201,38 +1216,103 @@ func (s *Service) SetRecoveryKey(ctx context.Context, recoveryKey string) (*backif err := backup.ValidateRecoveryKey(recoveryKey); err != nil {return nil, err}- // Repositories only open with the key that initialized them, so refuse a rotation while backups exist.- configured, err := s.recoveryKeys.Configured(ctx)+ lease, err := s.acquireRun(ctx)if err != nil {return nil, err}- if configured {- if current, keyErr := s.recoveryKeys.Get(ctx); keyErr == nil && current != recoveryKey {- localRuns, countLocalBackupsErr := s.store.CountRuns(ctx)- if countLocalBackupsErr != nil {- return nil, countLocalBackupsErr- }- if localRuns > 0 {- return nil, errors.New("delete the existing system backups before replacing the recovery key; they can only be opened with the current key")- }- volumeBackups, countVolumeBackupsErr := s.store.CountVolumeRepositoryRuns(ctx)- if countVolumeBackupsErr != nil {- return nil, countVolumeBackupsErr- }- if volumeBackups > 0 {- return nil, errors.New("delete the existing volume backups before replacing the recovery key; they can only be opened with the current key")- }+ defer lease.Release(ctx)+ previous, err := s.recoveryKeys.Get(ctx)+ if err != nil {+ if !errors.Is(err, backup.ErrRecoveryKeyNotConfigured) {+ slog.WarnContext(ctx, "could not read the current recovery key; existing backups must already open with the new key", "error", err.Error())}+ previous = ""}- if setErr := s.recoveryKeys.Set(ctx, recoveryKey); setErr != nil {- return nil, setErr+ if previous == recoveryKey {+ if setErr := s.recoveryKeys.Set(ctx, recoveryKey); setErr != nil {+ return nil, setErr+ }+ return &backuptypes.SystemBackupRecoveryKeyStatus{Configured: true}, nil+ }+ runningVolumeBackupsErr := common.Classify(common.ErrConflict, errors.New("wait for the running volume backups to finish before replacing the recovery key"))+ runningVolumeBackups, countErr := s.store.CountRunningVolumeBackups(ctx)+ if countErr != nil {+ return nil, countErr}- if migrateRepositoryPasswordsErr := s.volumeService.MigrateRepositoryPasswords(ctx); migrateRepositoryPasswordsErr != nil {- slog.WarnContext(ctx, "failed to re-key volume backup repositories to the recovery key", "error", migrateRepositoryPasswordsErr.Error())+ if runningVolumeBackups > 0 {+ return nil, runningVolumeBackupsErr+ }+ if rekeyErr := s.rekeyRepositories(ctx, previous, recoveryKey); rekeyErr != nil {+ return nil, rekeyErr+ }+ rollback := func(cause error) error {+ if previous == "" {+ return cause+ }+ return errors.Join(cause, s.rekeyRepositories(context.WithoutCancel(ctx), recoveryKey, previous))+ }+ runningVolumeBackups, countErr = s.store.CountRunningVolumeBackups(ctx)+ if countErr != nil {+ return nil, rollback(errors.Join(runningVolumeBackupsErr, countErr))+ }+ if runningVolumeBackups > 0 {+ return nil, rollback(runningVolumeBackupsErr)+ }+ if setErr := s.recoveryKeys.Set(ctx, recoveryKey); setErr != nil {+ return nil, rollback(setErr)}return &backuptypes.SystemBackupRecoveryKeyStatus{Configured: true}, nil}+func (s *Service) rekeyRepositories(ctx context.Context, previous, next string) error {+ systemRuns, err := s.store.CountRuns(ctx)+ if err != nil {+ return err+ }+ volumeRuns, err := s.store.CountVolumeRepositoryRuns(ctx)+ if err != nil {+ return err+ }+ if systemRuns == 0 && volumeRuns == 0 {+ return nil+ }+ var dockerClient *client.Client+ var systemRepositories []backup.Repository+ if systemRuns > 0 {+ dockerClient, err = s.dockerService.GetClient(ctx)+ if err != nil {+ return err+ }+ local, localErr := s.localRepository(ctx, dockerClient, false)+ if localErr != nil {+ return localErr+ }+ systemRepositories = append(systemRepositories, local)+ destinations, listErr := s.s3Destinations.ListAllS3Destinations(ctx)+ if listErr != nil {+ return fmt.Errorf("failed to list S3 destinations for re-key: %w", listErr)+ }+ for _, destination := range destinations {+ remote, remoteErr := s.remoteRepository(ctx, destination.ID)+ if remoteErr != nil {+ return remoteErr+ }+ systemRepositories = append(systemRepositories, remote)+ }+ if rekeyErr := s.engine.RekeyRepositories(ctx, dockerClient, systemRepositories, next, previous); rekeyErr != nil {+ return rekeyErr+ }+ }+ if volumeRuns == 0 {+ return nil+ }+ migrateErr := s.volumeService.MigrateRepositoryPasswords(ctx, previous, next)+ if migrateErr == nil || len(systemRepositories) == 0 || previous == "" {+ return migrateErr+ }+ return errors.Join(migrateErr, s.engine.RekeyRepositories(context.WithoutCancel(ctx), dockerClient, systemRepositories, previous, next))+}+func (s *Service) GetPolicies(ctx context.Context) (*backuptypes.SystemBackupPolicyCollection, error) {policies, err := s.store.Policies(ctx)if err != nil {@@ -1361,6 +1441,12 @@ func (s *Service) runScheduledBackup(ctx context.Context, policyID string) (sche}if runErr != nil {slog.ErrorContext(ctx, "Scheduled Arcane system backup failed", "policyId", policy.ID, "error", runErr)+ if policy.RetentionCount > 0 {+ if retentionErr := s.applyRetention(ctx, policy.ID, policy.RetentionCount, policy.S3Enabled); retentionErr != nil {+ slog.ErrorContext(ctx, "System backup retention failed", "policyId", policy.ID, "error", retentionErr)+ return scheduler.Outcome{}, errors.Join(runErr, retentionErr)+ }+ }return scheduler.Outcome{}, runErr}if policy.RetentionCount > 0 {@@ -1379,11 +1465,19 @@ func (s *Service) runScheduledBackup(ctx context.Context, policyID string) (sche// resumeScheduledBackup finishes the backup an interrupted scheduled run recorded, then applies retention.func (s *Service) resumeScheduledBackup(ctx context.Context, previous scheduler.Run, policyID string) (scheduler.Outcome, error) {outcome, err := s.reconcileBackup(ctx, previous, policyID, "")- if err != nil || outcome.Status != scheduler.Succeeded {- return outcome, err+ if err == nil && outcome.Status != scheduler.Succeeded {+ return outcome, nil+ }+ current, policyErr := s.store.Policy(ctx, policyID)+ if policyErr != nil {+ return outcome, errors.Join(err, policyErr)}- current, err := s.store.Policy(ctx, policyID)if err != nil {+ if current != nil && current.RetentionCount > 0 {+ if applyRetentionErr := s.applyRetention(ctx, current.ID, current.RetentionCount, current.S3Enabled); applyRetentionErr != nil {+ return outcome, errors.Join(err, applyRetentionErr)+ }+ }return outcome, err}if current != nil && current.RetentionCount > 0 {diff --git a/backend/internal/system/children/backup/service_test.go b/backend/internal/system/children/backup/service_test.goindex c03bebae2..644e8322a 100644--- a/backend/internal/system/children/backup/service_test.go+++ b/backend/internal/system/children/backup/service_test.go@@ -6,15 +6,22 @@ import ("github.com/getarcaneapp/arcane/types/v2/backup""github.com/libtnb/sqlite"+ "github.com/moby/moby/client""github.com/stretchr/testify/require"+ "go.getarcane.app/sys/crypto""gorm.io/gorm"+ backupengine "github.com/getarcaneapp/arcane/backend/v2/internal/backup""github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/database"+ "github.com/getarcaneapp/arcane/backend/v2/internal/docker""github.com/getarcaneapp/arcane/backend/v2/internal/s3""github.com/getarcaneapp/arcane/backend/v2/internal/settings"+ "github.com/getarcaneapp/arcane/backend/v2/internal/volume""github.com/getarcaneapp/arcane/backend/v2/pkg/pagination""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow/flowtest"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"+ francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing")func TestHistoryDestinationDecoration(t *testing.T) {@@ -80,3 +87,42 @@ func TestRecoveryEnvironmentIncludesRuntimeSecrets(t *testing.T) {require.Equal(t, "oidc-secret", environment["OIDC_CLIENT_SECRET"])require.Equal(t, "0640", environment["FILE_PERM"])}++func TestSetRecoveryKeyKeepsCurrentKeyWhenRekeyFails(t *testing.T) {+ gormDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})+ require.NoError(t, err)+ require.NoError(t, gormDB.AutoMigrate(&backupengine.SystemBackupRecoveryConfig{}, &s3.S3Destination{}))+ crypto.InitEncryption(&crypto.Config{EncryptionKey: "system-backup-rekey-test-key-32bytes", Environment: "test"})+ db := &database.DB{DB: gormDB}+ recoveryKeys := backupengine.NewRecoveryKeyStore(db)+ oldKey := "QWERTY-ABCDEF-234567-GHIJKL-MNOPQR-STUVWX-YZ2345-ZXCVBN"+ require.NoError(t, recoveryKeys.Set(t.Context(), oldKey))+ runtime := francistest.New(t)+ gate := runs.NewAdmission(runtime.Service(), t.Name())+ require.NoError(t, gate.Register(runtime))+ francistest.Start(t, runtime)+ engine := backupengine.NewEngine(gate, nil)+ t.Cleanup(func() { require.NoError(t, engine.Stop(context.WithoutCancel(t.Context()))) })+ dockerClient, err := client.New(client.WithHost("tcp://127.0.0.1:1"), client.WithAPIVersion("1.41"))+ require.NoError(t, err)+ dockerService := &docker.DockerClientService{Client: dockerClient}+ service := NewService(Dependencies{+ Store: Store{+ CountRuns: func(context.Context) (int64, error) { return 1, nil },+ CountVolumeRepositoryRuns: func(context.Context) (int64, error) { return 0, nil },+ CountRunningVolumeBackups: func(context.Context) (int64, error) { return 0, nil },+ },+ DB: db,+ Docker: dockerService,+ Volumes: volume.NewVolumeService(db, dockerService, nil, nil, nil, nil, nil, nil, nil, nil),+ Engine: engine,+ S3Destinations: s3.NewS3DestinationService(db, func(context.Context, string) error { return nil }),+ RecoveryKeys: recoveryKeys,+ })++ _, err = service.SetRecoveryKey(t.Context(), "ASDFGH-QWERTY-765432-LKJHGF-RQPONM-XWVUTS-5432ZY-NBVCXZ")+ require.Error(t, err)+ storedKey, err := recoveryKeys.Get(t.Context())+ require.NoError(t, err)+ require.Equal(t, oldKey, storedKey)+}diff --git a/backend/internal/system/children/prune/service.go b/backend/internal/system/children/prune/service.goindex adbc25456..c90abb725 100644--- a/backend/internal/system/children/prune/service.go+++ b/backend/internal/system/children/prune/service.go@@ -445,17 +445,24 @@ func (s *Service) pruneBuildCache(ctx context.Context, options system.PruneBuild}func (s *Service) pruneVolumes(ctx context.Context, options system.PruneVolumesOptions) (system.PruneAllResult, error) {- report, err := s.volumeService.PruneVolumesWithOptions(ctx, options.Mode == system.PruneVolumeModeAll)+ plan, err := s.preparePrunePlan(ctx, system.PruneAllRequest{Volumes: &options})if err != nil {return system.PruneAllResult{}, err}+ result, err := s.executePrunePlan(ctx, plan, scheduler.Run{}, "", false)+ if err != nil {+ return system.PruneAllResult{}, err+ }+ if !result.Success {+ return system.PruneAllResult{}, fmt.Errorf("failed to prune volumes: %s", strings.Join(result.Errors, "; "))+ }- slog.InfoContext(ctx, "Volume prune completed", "volumesDeleted", len(report.VolumesDeleted), "spaceReclaimed", report.SpaceReclaimed)+ slog.InfoContext(ctx, "Volume prune completed", "volumesDeleted", len(result.VolumesDeleted), "spaceReclaimed", result.VolumeSpaceReclaimed)return system.PruneAllResult{- VolumesDeleted: report.VolumesDeleted,- SpaceReclaimed: report.SpaceReclaimed,- VolumeSpaceReclaimed: report.SpaceReclaimed,+ VolumesDeleted: result.VolumesDeleted,+ SpaceReclaimed: result.VolumeSpaceReclaimed,+ VolumeSpaceReclaimed: result.VolumeSpaceReclaimed,}, nil}@@ -545,7 +552,9 @@ func (s *Service) PruneScheduled(ctx context.Context, environmentID string, reqfunc (s *Service) preparePrunePlan(ctx context.Context, req system.PruneAllRequest) (prunePlan, error) {if req.Volumes != nil {- s.volumeService.CleanupHelperContainers(ctx)+ if _, reapErr := s.volumeService.ReapIdleHelpers(ctx, 0); reapErr != nil {+ slog.WarnContext(ctx, "could not remove idle volume helpers before prune", "error", reapErr.Error())+ }}usage, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.DiskUsageResult, error) {return cli.DiskUsage(ctx, client.DiskUsageOptions{diff --git a/backend/internal/system/children/prune/service_test.go b/backend/internal/system/children/prune/service_test.gonew file mode 100644index 000000000..4e029427e--- /dev/null+++ b/backend/internal/system/children/prune/service_test.go@@ -0,0 +1,76 @@+package prune++import (+ "context"+ "fmt"+ "net"+ "net/http"+ "net/http/httptest"+ "path"+ "strings"+ "sync"+ "testing"+ "time"++ "github.com/getarcaneapp/arcane/types/v2/system"+ "github.com/libtnb/sqlite"+ "github.com/moby/moby/client"+ "github.com/stretchr/testify/require"+ "gorm.io/gorm"++ "github.com/getarcaneapp/arcane/backend/v2/internal/database"+ "github.com/getarcaneapp/arcane/backend/v2/internal/docker"+ "github.com/getarcaneapp/arcane/backend/v2/internal/event"+ "github.com/getarcaneapp/arcane/backend/v2/internal/volume"+)++func TestPruneVolumesSkipsBackupVolume(t *testing.T) {+ var mu sync.Mutex+ var deleted []string+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch {+ case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/system/df"):+ w.Header().Set("Content-Type", "application/json")+ _, _ = fmt.Fprint(w, `{"Volumes":[`++ `{"Name":"orphan","Driver":"local","UsageData":{"RefCount":0,"Size":100}},`++ `{"Name":"arcane-backups","Driver":"local","UsageData":{"RefCount":0,"Size":200}}]}`)+ case r.Method == http.MethodDelete && strings.Contains(r.URL.Path, "/volumes/"):+ mu.Lock()+ deleted = append(deleted, path.Base(r.URL.Path))+ mu.Unlock()+ w.WriteHeader(http.StatusNoContent)+ default:+ http.Error(w, "unexpected request "+r.Method+" "+r.URL.Path, http.StatusNotFound)+ }+ }))+ t.Cleanup(server.Close)++ dialer := &net.Dialer{}+ dockerClient, err := client.New(+ client.WithHost(server.URL),+ client.WithAPIVersion("1.41"),+ client.WithHTTPClient(server.Client()),+ client.WithDialContext(func(ctx context.Context, _, _ string) (net.Conn, error) {+ return dialer.DialContext(ctx, "tcp", server.Listener.Addr().String())+ }),+ )+ require.NoError(t, err)+ t.Cleanup(func() { _ = dockerClient.Close() })++ gormDB, err := gorm.Open(sqlite.Open(fmt.Sprintf("file:%s?mode=memory&cache=shared", t.Name())), &gorm.Config{})+ require.NoError(t, err)+ require.NoError(t, gormDB.AutoMigrate(&event.Event{}))+ db := &database.DB{DB: gormDB}++ dockerService := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(dockerClient)+ volumeService := volume.NewVolumeService(db, dockerService, event.NewEventService(db, nil, nil), nil, nil, nil, nil, nil, nil, nil)+ service := NewService(time.UTC, dockerService, nil, nil, volumeService, nil)++ result, err := service.pruneVolumes(t.Context(), system.PruneVolumesOptions{Mode: system.PruneVolumeModeAll})+ require.NoError(t, err)+ require.Equal(t, []string{"orphan"}, result.VolumesDeleted)+ require.EqualValues(t, 100, result.VolumeSpaceReclaimed)+ mu.Lock()+ defer mu.Unlock()+ require.Equal(t, []string{"orphan"}, deleted)+}diff --git a/backend/internal/system/service.go b/backend/internal/system/service.goindex 119c6c4c4..aba63acff 100644--- a/backend/internal/system/service.go+++ b/backend/internal/system/service.go@@ -607,6 +607,11 @@ func (s *SystemService) backupStore(s3Destinations *s3.S3DestinationService) syserr := s.db.WithContext(ctx).Model(&volume.VolumeBackup{}).Where("format = ?", volume.VolumeBackupFormatRustic).Count(&count).Errorreturn count, err},+ CountRunningVolumeBackups: func(ctx context.Context) (int64, error) {+ var count int64+ err := s.db.WithContext(ctx).Model(&volume.VolumeBackup{}).Where("status = ?", volume.VolumeBackupStatusRunning).Count(&count).Error+ return count, err+ },Policies: func(ctx context.Context) ([]backuptypes.SystemBackupPolicy, error) {var policies []SystemBackupPolicyif err := s.db.WithContext(ctx).Order("created_at ASC").Find(&policies).Error; err != nil {diff --git a/backend/internal/volume/children/backup/children/restore/service.go b/backend/internal/volume/children/backup/children/restore/service.goindex f61e9f8ae..6cc5244f9 100644--- a/backend/internal/volume/children/backup/children/restore/service.go+++ b/backend/internal/volume/children/backup/children/restore/service.go@@ -42,6 +42,23 @@ for member dodonetar -xzf "$archive" -C /volume -- "$@"`+const replaceVolumeScript = `set -e+: "${1:?}"+for f in /volume/* /volume/.[!.]* /volume/..?*; do+ case "$f" in "$1") ;; *) rm -rf -- "$f" ;; esac+done+for f in "$1"/* "$1"/.[!.]* "$1"/..?*; do+ if stat -c '%A' -- "$f" >/dev/null 2>&1; then mv -- "$f" /volume/; fi+done+rmdir -- "$1"`++const restoreArchiveScript = `set -e+tmp=$(mktemp -d /volume/.restore_tmp.XXXXXX)+tar -tzf "$1" >/dev/null+tar -xzf "$1" -C "$tmp"+set -- "$tmp"+` + replaceVolumeScript+// Dependencies are the backup runtime, storage, and helper-container operations restores run through.type Dependencies struct {Docker *docker.DockerClientService@@ -364,13 +381,9 @@ func (s *Service) restoreArchiveBackup(ctx context.Context, dockerClient *clientfullRestoreCmd := []string{"sh","-c",- fmt.Sprintf(- "set -e; tmp=$(mktemp -d /volume/.restore_tmp.XXXXXX); tar -tzf /backups/%s >/dev/null; tar -xzf "+- "/backups/%s -C \"$tmp\"; find /volume -mindepth 1 -maxdepth 1 -not -path \"$tmp\" -exec rm -rf -- {} +; "+- "find \"$tmp\" -mindepth 1 -maxdepth 1 -exec mv -- {} /volume/ \\;; rmdir \"$tmp\"",- filename,- filename,- ),+ restoreArchiveScript,+ "sh",+ path.Join("/backups", filename),}resp, err := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerCreateResult, error) {return dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{@@ -436,7 +449,7 @@ func (s *Service) RestoreArchiveMembers(ctx context.Context, containerID, filenareturn stderr, nil}-func (s *Service) UploadAndRestore(ctx context.Context, volumeName string, archive io.ReadSeeker, filename string, user usertypes.Actor) error {+func (s *Service) UploadAndRestore(ctx context.Context, volumeName string, archive io.ReadSeeker, filename string, user usertypes.Actor) (err error) {slog.DebugContext(ctx, "volume service: upload and restore", "volume", volumeName, "filename", filename, "user", user.ID)gzr, err := gzip.NewReader(archive)@@ -451,6 +464,17 @@ func (s *Service) UploadAndRestore(ctx context.Context, volumeName string, archiunlock := s.deps.Locks.Lock(volumeName)defer unlock()+ stopped, err := s.deps.StopContainers(ctx, volumeName, user, true)+ containersStopped := len(stopped) > 0+ defer func() {+ if containersStopped {+ _, restartErr := s.deps.StartContainers(context.WithoutCancel(ctx), stopped, user)+ err = errors.Join(err, restartErr)+ }+ }()+ if err != nil {+ return err+ }preBackupID, err := s.deps.SafetyBackup(ctx, volumeName, user)if err != nil {return fmt.Errorf("failed to create pre-restore backup: %w", err)@@ -472,6 +496,13 @@ func (s *Service) UploadAndRestore(ctx context.Context, volumeName string, archiif err != nil {return fmt.Errorf("failed to create temp restore dir: %w", err)}+ replaceStarted := false+ defer func() {+ if !replaceStarted {+ _, _, cleanupErr := s.deps.Exec(context.WithoutCancel(ctx), containerID, "", []string{"rm", "-rf", "--", tmpDir})+ err = errors.Join(err, cleanupErr)+ }+ }()if strings.TrimSpace(stderr) != "" {slog.DebugContext(ctx, "volume service: restore temp dir stderr", "volume", volumeName, "stderr", strings.TrimSpace(stderr))}@@ -487,7 +518,7 @@ func (s *Service) UploadAndRestore(ctx context.Context, volumeName string, archireturn fmt.Errorf("failed to restore from uploaded archive: %w", err)}- _, stderr, err = s.deps.Exec(ctx, containerID, "", []string{"sh", "-c", fmt.Sprintf("test -n \"$(find %s -mindepth 1 -maxdepth 1 -print -quit)\"", tmpDir)})+ _, stderr, err = s.deps.Exec(ctx, containerID, "", []string{"sh", "-c", `test -n "$(find "$1" -mindepth 1 -maxdepth 1)"`, "sh", tmpDir})if err != nil {return fmt.Errorf("uploaded archive appears empty or invalid: %w", err)}@@ -495,21 +526,21 @@ func (s *Service) UploadAndRestore(ctx context.Context, volumeName string, archislog.DebugContext(ctx, "volume service: restore validate stderr", "volume", volumeName, "stderr", strings.TrimSpace(stderr))}- _, stderr, err = s.deps.Exec(ctx, containerID, "", []string{"sh", "-c", "rm -rf /volume/* /volume/.[!.]* /volume/..?* 2>/dev/null || true"})+ replaceStarted = true+ _, stderr, err = s.deps.Exec(ctx, containerID, "", []string{"sh", "-c", replaceVolumeScript, "sh", tmpDir})if err != nil {- return fmt.Errorf("failed to clear volume before restore: %w", err)+ return fmt.Errorf("failed to replace volume contents: %w", err)}if strings.TrimSpace(stderr) != "" {- slog.DebugContext(ctx, "volume service: restore clear stderr", "volume", volumeName, "stderr", strings.TrimSpace(stderr))+ slog.DebugContext(ctx, "volume service: restore replace stderr", "volume", volumeName, "stderr", strings.TrimSpace(stderr))}- moveCmd := fmt.Sprintf("find %s -mindepth 1 -maxdepth 1 -exec mv -- {} /volume/ \\; && rmdir %s", tmpDir, tmpDir)- _, stderr, err = s.deps.Exec(ctx, containerID, "", []string{"sh", "-c", moveCmd})- if err != nil {- return fmt.Errorf("failed to move restored files into place: %w", err)- }- if strings.TrimSpace(stderr) != "" {- slog.DebugContext(ctx, "volume service: restore move stderr", "volume", volumeName, "stderr", strings.TrimSpace(stderr))+ if containersStopped {+ stopped, err = s.deps.StartContainers(context.WithoutCancel(ctx), stopped, user)+ containersStopped = len(stopped) > 0+ if err != nil {+ return err+ }}metadata := database.JSON{diff --git a/backend/internal/volume/children/backup/children/restore/service_test.go b/backend/internal/volume/children/backup/children/restore/service_test.goindex 936e85029..8577d3e2e 100644--- a/backend/internal/volume/children/backup/children/restore/service_test.go+++ b/backend/internal/volume/children/backup/children/restore/service_test.go@@ -19,27 +19,35 @@ func TestArchiveMembersForSelectionExpandsFolders(t *testing.T) {}func TestRestoreBackupFilesScriptUsesSupportedTooling(t *testing.T) {- for _, unsupported := range []string{- "if [",- "elif [",- "while [",- "test ",- "local ",- "find -P",- "-printf",- "sort -z",- "xargs",- "cat --",- "dirname",- "install ",- "--files-from",- "tar -r",- "$((",- } {- require.NotContains(t, restoreBackupFilesScript, unsupported)- }- if shellPath, err := exec.LookPath("sh"); err == nil {- output, syntaxErr := exec.Command(shellPath, "-n", "-c", restoreBackupFilesScript).CombinedOutput()- require.NoErrorf(t, syntaxErr, "restore script has invalid syntax: %s", output)+ for _, script := range []string{restoreBackupFilesScript, replaceVolumeScript, restoreArchiveScript} {+ for _, unsupported := range []string{+ "if [",+ "elif [",+ "while [",+ "test ",+ "local ",+ "find -P",+ "-printf",+ "sort -z",+ "xargs",+ "cat --",+ "dirname",+ "install ",+ "--files-from",+ "tar -r",+ "$((",+ "-not",+ "-exec",+ "-quit",+ } {+ require.NotContains(t, script, unsupported)+ }+ if shellPath, err := exec.LookPath("sh"); err == nil {+ output, syntaxErr := exec.Command(shellPath, "-n", "-c", script).CombinedOutput()+ require.NoErrorf(t, syntaxErr, "restore script has invalid syntax: %s", output)+ }+ require.NotContains(t, script, "rm -rf /volume/*")+ require.NotContains(t, script, "2>/dev/null || true")}+ require.Contains(t, replaceVolumeScript, `case "$f" in "$1")`)}diff --git a/backend/internal/volume/children/backup/helpers.go b/backend/internal/volume/children/backup/helpers.goindex 4f1ee4a90..4a6a25b1a 100644--- a/backend/internal/volume/children/backup/helpers.go+++ b/backend/internal/volume/children/backup/helpers.go@@ -101,7 +101,10 @@ func (s *Service) StorageMount(ctx context.Context, dockerClient *client.Client,return dockerClient.VolumeInspect(ctx, s.deps.BackupVolumeName, client.VolumeInspectOptions{})}); inspectErr != nil {if _, createErr := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.VolumeCreateResult, error) {- return dockerClient.VolumeCreate(ctx, client.VolumeCreateOptions{Name: s.deps.BackupVolumeName})+ return dockerClient.VolumeCreate(ctx, client.VolumeCreateOptions{+ Name: s.deps.BackupVolumeName,+ Labels: map[string]string{libarcane.InternalResourceLabel: "true"},+ })}); createErr != nil {return mount.Mount{}, fmt.Errorf("failed to create backup volume: %w", createErr)}@@ -264,10 +267,6 @@ func (s *Service) volumeBackupPassword(ctx context.Context, dockerClient *clientif done, _ := s.rekeyed.Load(repository.ID); done == key {continue}- if _, listErr := s.deps.Engine.ListSnapshots(ctx, dockerClient, repository, key); listErr == nil {- s.rekeyed.Store(repository.ID, key)- continue- }if repository.ID == localVolumeRepositoryID {writable, localErr := s.localRusticRepository(ctx, dockerClient, false)if localErr != nil {@@ -276,11 +275,14 @@ func (s *Service) volumeBackupPassword(ctx context.Context, dockerClient *client}repository = writable}- if changeErr := s.deps.Engine.ChangeRepositoryPassword(ctx, dockerClient, repository, legacyPassword, key); changeErr != nil {- slog.DebugContext(ctx, "volume backup repository was not re-keyed", "repository", repository.ID, "error", changeErr.Error())+ previous, rekeyErr := s.deps.Engine.RekeyRepository(ctx, dockerClient, repository, key, legacyPassword)+ if rekeyErr != nil {+ slog.DebugContext(ctx, "volume backup repository was not re-keyed", "repository", repository.ID, "error", rekeyErr.Error())continue}- slog.InfoContext(ctx, "Re-keyed volume backup repository to the recovery key", "repository", repository.ID)+ if previous != "" {+ slog.InfoContext(ctx, "Re-keyed volume backup repository to the recovery key", "repository", repository.ID)+ }s.rekeyed.Store(repository.ID, key)}return key, nildiff --git a/backend/internal/volume/children/backup/service.go b/backend/internal/volume/children/backup/service.goindex 37b14cfec..173ab38bf 100644--- a/backend/internal/volume/children/backup/service.go+++ b/backend/internal/volume/children/backup/service.go@@ -904,22 +904,15 @@ func discoveredVolumeBackup(destinationID, root string, snapshot backup.Discover}// MigrateRepositoryPasswords re-keys this instance's own volume backup repositories to the recovery key; other instances re-key their own roots.-func (s *Service) MigrateRepositoryPasswords(ctx context.Context) error {- if s.deps.RecoveryKeys == nil || s.deps.Engine == nil {+func (s *Service) MigrateRepositoryPasswords(ctx context.Context, previousKey, newKey string) error {+ if s.deps.Engine == nil {return nil}- _, err := s.deps.RecoveryKeys.Get(ctx)- if errors.Is(err, backup.ErrRecoveryKeyNotConfigured) {- return nil- }- if err != nil {- return err- }dockerClient, err := s.deps.Docker.GetClient(ctx)if err != nil {return err}- local, err := s.localRusticRepository(ctx, dockerClient, true)+ local, err := s.localRusticRepository(ctx, dockerClient, false)if err != nil {return err}@@ -937,8 +930,14 @@ func (s *Service) MigrateRepositoryPasswords(ctx context.Context) error {repositories = append(repositories, remote)}}- _, err = s.volumeBackupPassword(ctx, dockerClient, repositories...)- return err+ legacyPassword := kit.SHA256Hex(legacyVolumePasswordSalt + s.deps.EncryptionKey)+ if rekeyErr := s.deps.Engine.RekeyRepositories(ctx, dockerClient, repositories, newKey, previousKey, legacyPassword); rekeyErr != nil {+ return rekeyErr+ }+ for _, repository := range repositories {+ s.rekeyed.Store(repository.ID, newKey)+ }+ return nil}// StartBackup persists a running backup and submits its workflow.diff --git a/backend/internal/volume/children/backup/service_test.go b/backend/internal/volume/children/backup/service_test.goindex 3c70af66f..01bacaa4f 100644--- a/backend/internal/volume/children/backup/service_test.go+++ b/backend/internal/volume/children/backup/service_test.go@@ -59,7 +59,7 @@ func (s *volumeBackupPolicyScheduler) Submit(_ context.Context, request schedulereturn scheduler.Run{ID: request.RunID, JobID: request.JobID, EnvironmentID: request.EnvironmentID, Status: scheduler.Queued}, nil}-func TestVolumeBackupPolicy_RetentionIgnoresFailedRuns(t *testing.T) {+func TestVolumeBackupPolicy_RetentionCountsFailedRunsSeparately(t *testing.T) {gormDB, err := gorm.Open(sqlite.Open("file:volume-backup-retention-failed?mode=memory&cache=shared"), &gorm.Config{})require.NoError(t, err)applyVolumeBackupMigrations(t, gormDB)@@ -73,10 +73,18 @@ func TestVolumeBackupPolicy_RetentionIgnoresFailedRuns(t *testing.T) {"INSERT INTO volume_backups (id, volume_name, size, created_at, policy_id, status) VALUES (?, ?, ?, ?, ?, ?)","failed-run", "app-data", 0, time.Now(), policyID, backuptypes.VolumeBackupStatusFailed,).Error)+ require.NoError(t, gormDB.Exec(+ "INSERT INTO volume_backups (id, volume_name, size, created_at, policy_id, status, local_snapshot_id) VALUES (?, ?, ?, ?, ?, ?, ?)",+ "failed-snap-new", "app-data", 0, time.Now().Add(-2*time.Hour), policyID, backuptypes.VolumeBackupStatusFailed, "snapshot-2",+ ).Error)+ require.NoError(t, gormDB.Exec(+ "INSERT INTO volume_backups (id, volume_name, size, created_at, policy_id, status, local_snapshot_id) VALUES (?, ?, ?, ?, ?, ?, ?)",+ "failed-snap-old", "app-data", 0, time.Now().Add(-3*time.Hour), policyID, backuptypes.VolumeBackupStatusFailed, "snapshot-3",+ ).Error)expired, err := backup.ExpiredRunIDs(t.Context(), &database.DB{DB: gormDB}, "volume_backups", policyID, 1)require.NoError(t, err)- require.Empty(t, expired)+ require.Equal(t, []string{"failed-snap-old"}, expired)}func TestVolumeBackupPolicy_ScheduledRunCreatesActivity(t *testing.T) {diff --git a/backend/internal/volume/helpers.go b/backend/internal/volume/helpers.goindex 02303065d..79fb030aa 100644--- a/backend/internal/volume/helpers.go+++ b/backend/internal/volume/helpers.go@@ -330,7 +330,7 @@ func (s *VolumeService) CleanupHelperContainers(ctx context.Context) {// ReapIdleHelpers removes tracked helpers idle for idleTimeout; untracked orphans are left to the startup sweep.func (s *VolumeService) ReapIdleHelpers(ctx context.Context, idleTimeout time.Duration) (int, error) {- if idleTimeout <= 0 {+ if idleTimeout < 0 {return 0, nil}diff --git a/backend/internal/volume/service.go b/backend/internal/volume/service.goindex 1bd364666..34d7097da 100644--- a/backend/internal/volume/service.go+++ b/backend/internal/volume/service.go@@ -215,8 +215,8 @@ func (s *VolumeService) PruneLocalBackupRepository(ctx context.Context) error {}// MigrateRepositoryPasswords re-keys this instance's own volume backup repositories to the recovery key; other instances re-key their own roots.-func (s *VolumeService) MigrateRepositoryPasswords(ctx context.Context) error {- return s.backup.MigrateRepositoryPasswords(ctx)+func (s *VolumeService) MigrateRepositoryPasswords(ctx context.Context, previousKey, newKey string) error {+ return s.backup.MigrateRepositoryPasswords(ctx, previousKey, newKey)}func (s *VolumeService) backupStore() volumebackup.Store {@@ -491,27 +491,23 @@ func (s *VolumeService) DeleteVolume(ctx context.Context, name string, force boofunc (s *VolumeService) PruneVolumes(ctx context.Context) (*volumetypes.PruneReport, error) {slog.DebugContext(ctx, "volume service: prune volumes")- return s.PruneVolumesWithOptions(ctx, false)-}--func (s *VolumeService) PruneVolumesWithOptions(ctx context.Context, all bool) (*volumetypes.PruneReport, error) {- slog.DebugContext(ctx, "volume service: prune volumes with options", "all", all)dockerClient, err := s.dockerService.GetClient(ctx)if err != nil {return nil, fmt.Errorf("failed to connect to Docker: %w", err)}- // Stop all read-only browse helpers first; a helper mounting a volume marks it+ // Stop idle read-only browse helpers first; a helper mounting a volume marks it// "in use" and would prevent VolumePrune from reclaiming an otherwise-unused- // volumetypes. Helpers are re-created on demand on the next browse request.- s.CleanupHelperContainers(ctx)+ // volume. Helpers are re-created on demand on the next browse request.+ if _, reapErr := s.ReapIdleHelpers(ctx, 0); reapErr != nil {+ slog.WarnContext(ctx, "could not remove idle volume helpers before prune", "error", reapErr.Error())+ }// Docker's VolumesPrune behavior (API v1.42+):// - Without 'all' flag: Only removes anonymous (unnamed) volumes that are not in use// - With 'all=true' flag: Removes ALL unused volumes (both named and anonymous)// Note: Volumes are considered "in use" if referenced by any container (running or stopped)volumePruneResult, err := dockerClient.VolumePrune(ctx, client.VolumePruneOptions{- All: all,Filters: make(client.Filters).Add("label!", internalVolumePruneFilterValue),})if err != nil {@@ -520,7 +516,7 @@ func (s *VolumeService) PruneVolumesWithOptions(ctx context.Context, all bool) (metadata := database.JSON{"action": "prune",- "all": all,+ "all": false,"volumesDeleted": len(volumePruneResult.Report.VolumesDeleted),"spaceReclaimed": volumePruneResult.Report.SpaceReclaimed,"internalVolumeFilterLabel": internalVolumePruneFilterValue,