Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
feat(backups): discover S3 backups and re-key volume backups to the recovery key (#3885)
Neureka02208f5cd24e
diff --git a/backend/internal/backup/engine.go b/backend/internal/backup/engine.goindex 19d3654f9..e713dbb63 100644--- a/backend/internal/backup/engine.go+++ b/backend/internal/backup/engine.go@@ -5,11 +5,14 @@ package backupimport ("context"+ "crypto/rand"+ "encoding/base32""encoding/json/v2""fmt""io""log/slog""path"+ "regexp""slices""strings""time"@@ -17,6 +20,7 @@ import ("emperror.dev/errors""github.com/getarcaneapp/arcane/backend/v2/internal/actors""github.com/getarcaneapp/arcane/backend/v2/internal/common"+ "github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/image""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"rusticruntime "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/rustic"@@ -25,6 +29,8 @@ import ("github.com/moby/moby/api/types/container""github.com/moby/moby/api/types/mount""github.com/moby/moby/client"+ "go.getarcane.app/sys/crypto"+ "gorm.io/gorm")// Admission scopes shared by the backup engine and the per-policy job@@ -251,6 +257,7 @@ func (e *Engine) ReadSnapshotTextFile(ctx context.Context, dockerClient *client.type DiscoveredSnapshot struct {ID string `json:"id"`Time time.Time `json:"time"`+ Label string `json:"label"`Summary struct {TotalBytesProcessed int64 `json:"total_bytes_processed"`} `json:"summary"`@@ -293,6 +300,15 @@ func (e *Engine) ForgetSnapshots(ctx context.Context, dockerClient *client.Clienreturn err}+// ChangeRepositoryPassword re-keys the repository; the scratch Rustic image has no shell, so the new password travels as an argument.+func (e *Engine) ChangeRepositoryPassword(ctx context.Context, dockerClient *client.Client, repository Repository, currentPassword, newPassword string) error {+ if strings.TrimSpace(newPassword) == "" {+ return errors.New("new repository password is required")+ }+ _, err := e.runInternal(ctx, dockerClient, repository, currentPassword, []string{"key", "password", "--new-password", newPassword})+ return err+}+// Replicate copies one snapshot between repositories by restoring it into a// temporary volume and backing that volume up into the target repository. The// source is read once from the repository, never from the live data. Rustic's@@ -386,3 +402,93 @@ func (e *Engine) runContainerInternal(ctx context.Context, dockerClient *client.mounts = append(mounts, extraMounts...)return rusticruntime.Run(ctx, dockerClient, password, command, repository.Environment, mounts, arcaneNetworkModeInternal(ctx, dockerClient))}++// RecoveryKeyConfigID is the singleton row holding the instance-wide backup recovery key.+const RecoveryKeyConfigID = "system-recovery"++var (+ ErrRecoveryKeyNotConfigured = errors.New("recovery key is not configured")++ // RecoveryKeyFormat is 8 hyphenated groups of 6 base32 characters, used verbatim as the Rustic password.+ RecoveryKeyFormat = regexp.MustCompile(`^[A-Z2-7]{6}(-[A-Z2-7]{6}){7}$`)+)++// SystemBackupRecoveryConfig stores the encrypted recovery key; it must be reversible because Rustic needs the plaintext.+type SystemBackupRecoveryConfig struct {+ database.BaseModel++ EncryptedRecoveryKey string `gorm:"column:encrypted_recovery_key;type:text;not null"`+}++func (SystemBackupRecoveryConfig) TableName() string { return "system_backup_recovery_config" }++func ValidateRecoveryKey(key string) error {+ if !RecoveryKeyFormat.MatchString(strings.TrimSpace(key)) {+ return errors.New("enter the generated recovery key (8 groups of 6 characters)")+ }+ return nil+}++func GenerateRecoveryKey() (string, error) {+ raw := make([]byte, 30)+ if _, err := rand.Read(raw); err != nil {+ return "", fmt.Errorf("failed to generate recovery key: %w", err)+ }+ encoded := base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(raw)+ groups := make([]string, 0, len(encoded)/6)+ for i := 0; i < len(encoded); i += 6 {+ groups = append(groups, encoded[i:i+6])+ }+ return strings.Join(groups, "-"), nil+}++// RecoveryKeyStore persists the recovery key shared by the system and volume backup domains.+type RecoveryKeyStore struct {+ db *database.DB+}++func NewRecoveryKeyStore(db *database.DB) *RecoveryKeyStore {+ return &RecoveryKeyStore{db: db}+}++func (s *RecoveryKeyStore) Configured(ctx context.Context) (bool, error) {+ var count int64+ if err := s.db.WithContext(ctx).Model(&SystemBackupRecoveryConfig{}).+ Where("id = ? AND encrypted_recovery_key <> ''", RecoveryKeyConfigID).Count(&count).Error; err != nil {+ return false, fmt.Errorf("failed to load recovery key status: %w", err)+ }+ return count > 0, nil+}++// Get returns the decrypted recovery key or ErrRecoveryKeyNotConfigured.+func (s *RecoveryKeyStore) Get(ctx context.Context) (string, error) {+ var config SystemBackupRecoveryConfig+ if err := s.db.WithContext(ctx).+ Where("id = ? AND encrypted_recovery_key <> ''", RecoveryKeyConfigID).+ First(&config).Error; err != nil {+ if errors.Is(err, gorm.ErrRecordNotFound) {+ return "", ErrRecoveryKeyNotConfigured+ }+ return "", fmt.Errorf("failed to load recovery key: %w", err)+ }+ key, err := crypto.Decrypt(config.EncryptedRecoveryKey)+ if err != nil {+ return "", fmt.Errorf("failed to decrypt recovery key: %w", err)+ }+ return key, nil+}++func (s *RecoveryKeyStore) Set(ctx context.Context, recoveryKey string) error {+ if err := ValidateRecoveryKey(recoveryKey); err != nil {+ return err+ }+ encrypted, err := crypto.Encrypt(recoveryKey)+ if err != nil {+ return fmt.Errorf("failed to encrypt recovery key: %w", err)+ }+ config := SystemBackupRecoveryConfig{ID: RecoveryKeyConfigID, EncryptedRecoveryKey: encrypted}+ if err := s.db.WithContext(ctx).Save(&config).Error; err != nil {+ return fmt.Errorf("failed to save recovery key: %w", err)+ }+ return nil+}diff --git a/backend/internal/backup/engine_test.go b/backend/internal/backup/engine_test.goindex 678215148..1a4390067 100644--- a/backend/internal/backup/engine_test.go+++ b/backend/internal/backup/engine_test.go@@ -3,7 +3,11 @@ package backupimport ("testing"+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"+ "github.com/libtnb/sqlite""github.com/stretchr/testify/require"+ "go.getarcane.app/sys/crypto"+ "gorm.io/gorm")func TestMarkSnapshotDirectoriesInternal(t *testing.T) {@@ -66,3 +70,41 @@ func TestQualifySnapshotListingInternal(t *testing.T) {})}}++func TestRecoveryKeyStoreRoundTrip(t *testing.T) {+ gormDB, err := gorm.Open(sqlite.Open("file:recovery-key-store?mode=memory&cache=shared"), &gorm.Config{})+ require.NoError(t, err)+ require.NoError(t, gormDB.AutoMigrate(&SystemBackupRecoveryConfig{}))+ crypto.InitEncryption(&crypto.Config{EncryptionKey: "recovery-key-store-test-key-32bytes", Environment: "test"})+ store := NewRecoveryKeyStore(&database.DB{DB: gormDB})++ configured, err := store.Configured(t.Context())+ require.NoError(t, err)+ require.False(t, configured)++ _, err = store.Get(t.Context())+ require.ErrorIs(t, err, ErrRecoveryKeyNotConfigured)++ key, err := GenerateRecoveryKey()+ require.NoError(t, err)+ require.NoError(t, store.Set(t.Context(), key))++ configured, err = store.Configured(t.Context())+ require.NoError(t, err)+ require.True(t, configured)++ stored, err := store.Get(t.Context())+ require.NoError(t, err)+ require.Equal(t, key, stored)+}++func TestRecoveryKeyValidation(t *testing.T) {+ require.NoError(t, ValidateRecoveryKey("QWERTY-ABCDEF-234567-GHIJKL-MNOPQR-STUVWX-YZ2345-ZXCVBN"))+ require.Error(t, ValidateRecoveryKey("too-short"))+ require.Error(t, ValidateRecoveryKey(""))+ require.Error(t, ValidateRecoveryKey("QWERTY-ABCDEF-234567-GHIJKL-MNOPQR-STUVWX-YZ2345-ZXCVBN-EXTRA"))++ key, err := GenerateRecoveryKey()+ require.NoError(t, err)+ require.NoError(t, ValidateRecoveryKey(key))+}diff --git a/backend/internal/dashboard/service_test.go b/backend/internal/dashboard/service_test.goindex 401cdccf3..02ce3f876 100644--- a/backend/internal/dashboard/service_test.go+++ b/backend/internal/dashboard/service_test.go@@ -192,7 +192,7 @@ func TestDashboardService_GetSnapshot_ReturnsDashboardSnapshot(t *testing.T) {Status: project.ProjectStatusStopped,}).Error)projectSvc := project.NewProjectService(db, settingsSvc, nil, image.NewImageService(db, nil, nil, nil, nil, nil), nil, nil, nil, nil, config.Load())- svc := NewDashboardService(db, dockerSvc, nil, projectSvc, nil, settingsSvc, nil, nil, nil, volume.NewVolumeService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil))+ svc := NewDashboardService(db, dockerSvc, nil, projectSvc, nil, settingsSvc, nil, nil, nil, volume.NewVolumeService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil))snapshot, err := svc.GetSnapshot(context.Background(), DashboardActionItemsOptions{}, true)require.NoError(t, err)diff --git a/backend/internal/di/di.go b/backend/internal/di/di.goindex 3e7f14265..a830fa7f8 100644--- a/backend/internal/di/di.go+++ b/backend/internal/di/di.go@@ -8,6 +8,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/apns""github.com/getarcaneapp/arcane/backend/v2/internal/appimages""github.com/getarcaneapp/arcane/backend/v2/internal/auth"+ "github.com/getarcaneapp/arcane/backend/v2/internal/backup""github.com/getarcaneapp/arcane/backend/v2/internal/build""github.com/getarcaneapp/arcane/backend/v2/internal/diagnostics""github.com/getarcaneapp/arcane/backend/v2/internal/environment"@@ -106,6 +107,7 @@ var ServiceOptions = fx.Options(provideS3ModuleInternal,provideS3ServiceInternal,provideBackupEngineInternal,+ backup.NewRecoveryKeyStore,upload.NewUploadService,upload.New,diff --git a/backend/internal/di/providers.go b/backend/internal/di/providers.goindex f034f8dab..2a2e2b3bf 100644--- a/backend/internal/di/providers.go+++ b/backend/internal/di/providers.go@@ -252,20 +252,21 @@ func provideS3ServiceInternal(module *s3domain.Module) *s3domain.S3DestinationSereturn module.Service()}-func provideVolumeModuleInternal(lc fx.Lifecycle, db *database.DB, dockerService *docker.DockerClientService, eventService *event.EventService, settingsService *settings.SettingsService, imageService *image.ImageService, activityService *activity.ActivityService, containerModule *container.Module, engine *backup.Engine, s3Service *s3domain.S3DestinationService, environmentService *environment.EnvironmentService, cfg *config.Config, uploadService *upload.UploadService) *volume.Module {+func provideVolumeModuleInternal(lc fx.Lifecycle, db *database.DB, dockerService *docker.DockerClientService, eventService *event.EventService, settingsService *settings.SettingsService, imageService *image.ImageService, activityService *activity.ActivityService, containerModule *container.Module, engine *backup.Engine, s3Service *s3domain.S3DestinationService, environmentService *environment.EnvironmentService, cfg *config.Config, uploadService *upload.UploadService, recoveryKeys *backup.RecoveryKeyStore) *volume.Module {module := volume.New(volume.Dependencies{- DB: db,- Docker: dockerService,- Event: eventService,- Settings: settingsService,- Image: imageService,- Activity: activityService,- Environment: environmentService,- Container: containerModule.Service(),- Engine: engine,- S3: s3Service,- Config: cfg,- Upload: uploadService,+ DB: db,+ Docker: dockerService,+ Event: eventService,+ Settings: settingsService,+ Image: imageService,+ Activity: activityService,+ Environment: environmentService,+ Container: containerModule.Service(),+ Engine: engine,+ S3: s3Service,+ Config: cfg,+ Upload: uploadService,+ RecoveryKeys: recoveryKeys,})lc.Append(fx.Hook{OnStop: func(ctx context.Context) error {@@ -280,8 +281,8 @@ func provideVolumeServiceInternal(module *volume.Module) *volume.VolumeService {return module.Service()}-func provideSystemBackupServiceInternal(db *database.DB, dockerService *docker.DockerClientService, volumeModule *volume.Module, engine *backup.Engine, s3Service *s3domain.S3DestinationService, activityService *activity.ActivityService, settingsService *settings.SettingsService, cfg *config.Config) *systembackup.SystemBackupService {- return systembackup.NewSystemBackupService(db, dockerService, volumeModule.Service(), engine, s3Service, activityService, settingsService, cfg)+func provideSystemBackupServiceInternal(db *database.DB, dockerService *docker.DockerClientService, volumeModule *volume.Module, engine *backup.Engine, s3Service *s3domain.S3DestinationService, activityService *activity.ActivityService, settingsService *settings.SettingsService, cfg *config.Config, recoveryKeys *backup.RecoveryKeyStore) *systembackup.SystemBackupService {+ return systembackup.NewSystemBackupService(db, dockerService, volumeModule.Service(), engine, s3Service, activityService, settingsService, cfg, recoveryKeys)}func provideContainerModuleInternal(event *event.EventService, docker *docker.DockerClientService, image *image.ImageService, settings *settings.SettingsService, project *project.ProjectService, activity *activity.ActivityService) *container.Module {diff --git a/backend/internal/systembackup/model.go b/backend/internal/systembackup/model.goindex 810b98997..c94391276 100644--- a/backend/internal/systembackup/model.go+++ b/backend/internal/systembackup/model.go@@ -73,11 +73,3 @@ func (p SystemBackupPolicy) ToDTO(lastRun *SystemBackupRun) backuptypes.SystemBa}return dto}--type SystemBackupRecoveryConfig struct {- database.BaseModel-- EncryptedRecoveryKey string `gorm:"column:encrypted_recovery_key;type:text;not null"`-}--func (SystemBackupRecoveryConfig) TableName() string { return "system_backup_recovery_config" }diff --git a/backend/internal/systembackup/service.go b/backend/internal/systembackup/service.goindex 75e5fb9d6..a1856648a 100644--- a/backend/internal/systembackup/service.go+++ b/backend/internal/systembackup/service.go@@ -2,8 +2,6 @@ package systembackupimport ("context"- "crypto/rand"- "encoding/base32""encoding/json/jsontext""encoding/json/v2""fmt"@@ -13,7 +11,6 @@ import ("path""path/filepath""reflect"- "regexp""slices""strings""time"@@ -51,7 +48,6 @@ import ("github.com/moby/moby/client""go.getarcane.app/acfs""go.getarcane.app/sys/cgroup"- "go.getarcane.app/sys/crypto""gorm.io/gorm")@@ -74,14 +70,16 @@ type SystemBackupService struct {activityService *activity.ActivityServicesettingsService *settings.SettingsServiceconfig *config.Config+ recoveryKeys *backup.RecoveryKeyStorejobs *entityjobs.Registry}-func NewSystemBackupService(db *database.DB, dockerService *docker.DockerClientService, volumeService *volume.VolumeService, engine *backup.Engine, s3Destinations *s3domain.S3DestinationService, activityService *activity.ActivityService, settingsService *settings.SettingsService, cfg *config.Config) *SystemBackupService {+func NewSystemBackupService(db *database.DB, dockerService *docker.DockerClientService, volumeService *volume.VolumeService, engine *backup.Engine, s3Destinations *s3domain.S3DestinationService, activityService *activity.ActivityService, settingsService *settings.SettingsService, cfg *config.Config, recoveryKeys *backup.RecoveryKeyStore) *SystemBackupService {return &SystemBackupService{db: db, dockerService: dockerService, volumeService: volumeService, engine: engine,s3Destinations: s3Destinations, activityService: activityService, settingsService: settingsService, config: cfg,- jobs: entityjobs.New("system-backup:", backup.SystemAdmissionScope),+ recoveryKeys: recoveryKeys,+ jobs: entityjobs.New("system-backup:", backup.SystemAdmissionScope),}}@@ -91,19 +89,6 @@ func (s *SystemBackupService) SupportedDatabaseProvider() bool {return strings.HasPrefix(s.config.DatabaseURL, "file:")}-// recoveryKeyFormatInternal is the only accepted key shape, generated by-// GenerateRecoveryKey: 48 base32 characters (~240 bits from crypto/rand) in 8-// hyphenated groups for hand transcription. The key is the Rustic repository-// password and is used exactly as displayed, hyphens included.-var recoveryKeyFormatInternal = regexp.MustCompile(`^[A-Z2-7]{6}(-[A-Z2-7]{6}){7}$`)--func validateRecoveryKeyInternal(key string) error {- if !recoveryKeyFormatInternal.MatchString(strings.TrimSpace(key)) {- return errors.New("enter the generated recovery key (8 groups of 6 characters)")- }- return nil-}-func recoveryHelperExecutableInternal(mounts []containertypes.MountPoint, executablePath string) (string, *mount.Mount, error) {executablePath = strings.TrimSpace(executablePath)if executablePath == "" {@@ -272,24 +257,19 @@ func systemSnapshotPathFromFilesInternal(files []string) (string, error) {func (s *SystemBackupService) recoveryKeyInternal(ctx context.Context, supplied string) (string, error) {if strings.TrimSpace(supplied) != "" {- if err := validateRecoveryKeyInternal(supplied); err != nil {+ if err := backup.ValidateRecoveryKey(supplied); err != nil {return "", err}return supplied, nil}- var recoveryConfig SystemBackupRecoveryConfig- err := s.db.WithContext(ctx).Where("id = ?", systemRecoveryConfigID).First(&recoveryConfig).Error- if errors.Is(err, gorm.ErrRecordNotFound) {+ key, err := s.recoveryKeys.Get(ctx)+ if errors.Is(err, backup.ErrRecoveryKeyNotConfigured) {return "", errors.New("enter the recovery key or configure one in system backups")}if err != nil {- return "", fmt.Errorf("failed to load stored recovery key: %w", err)- }- key, err := crypto.Decrypt(recoveryConfig.EncryptedRecoveryKey)- if err != nil {- return "", fmt.Errorf("failed to decrypt stored recovery key: %w", err)+ return "", err}- return key, validateRecoveryKeyInternal(key)+ return key, nil}// resolveBackupPlanInternal resolves which destinations a run targets, from@@ -1391,8 +1371,6 @@ func (s *SystemBackupService) RestoreBackup(ctx context.Context, id, recoveryKeyreturn nil}-const systemRecoveryConfigID = "system-recovery"-func (s *SystemBackupService) loadPoliciesInternal(ctx context.Context) ([]SystemBackupPolicy, error) {var policies []SystemBackupPolicyif err := s.db.WithContext(ctx).Order("created_at ASC").Find(&policies).Error; err != nil {@@ -1432,36 +1410,22 @@ func rusticRestoreNetworkModeInternal(current *containertypes.InspectResponse) sreturn ""}-func (s *SystemBackupService) recoveryKeyConfiguredInternal(ctx context.Context) (bool, error) {- var count int64- if err := s.db.WithContext(ctx).Model(&SystemBackupRecoveryConfig{}).- Where("id = ? AND encrypted_recovery_key <> ''", systemRecoveryConfigID).Count(&count).Error; err != nil {- return false, fmt.Errorf("failed to load recovery key status: %w", err)- }- return count > 0, nil-}-func (s *SystemBackupService) GenerateRecoveryKey() (*backuptypes.SystemBackupRecoveryKey, error) {- raw := make([]byte, 30)- if _, err := rand.Read(raw); err != nil {- return nil, fmt.Errorf("failed to generate recovery key: %w", err)- }- encoded := base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(raw)- groups := make([]string, 0, len(encoded)/6)- for i := 0; i < len(encoded); i += 6 {- groups = append(groups, encoded[i:i+6])+ recoveryKey, err := backup.GenerateRecoveryKey()+ if err != nil {+ return nil, err}- return &backuptypes.SystemBackupRecoveryKey{RecoveryKey: strings.Join(groups, "-")}, nil+ return &backuptypes.SystemBackupRecoveryKey{RecoveryKey: recoveryKey}, nil}func (s *SystemBackupService) SetRecoveryKey(ctx context.Context, recoveryKey string) (*backuptypes.SystemBackupRecoveryKeyStatus, error) {- if err := validateRecoveryKeyInternal(recoveryKey); err != nil {+ if err := backup.ValidateRecoveryKey(recoveryKey); err != nil {return nil, err}// The repositories are encrypted with the key that initialized them and a// different key cannot open them, so a rotation would silently break every// scheduled backup and keyless restore. Refuse it while backups exist.- configured, err := s.recoveryKeyConfiguredInternal(ctx)+ configured, err := s.recoveryKeys.Configured(ctx)if err != nil {return nil, err}@@ -1474,15 +1438,22 @@ func (s *SystemBackupService) SetRecoveryKey(ctx context.Context, recoveryKey stif runs > 0 {return nil, errors.New("delete the existing system backups before replacing the recovery key; they can only be opened with the current key")}+ var volumeBackups int64+ if err := s.db.WithContext(ctx).Model(&volume.VolumeBackup{}).Where("format = ?", volume.VolumeBackupFormatRustic).Count(&volumeBackups).Error; err != nil {+ return nil, err+ }+ if volumeBackups > 0 {+ return nil, errors.New("delete the existing volume backups before replacing the recovery key; they can only be opened with the current key")+ }}}- encrypted, err := crypto.Encrypt(recoveryKey)- if err != nil {- return nil, fmt.Errorf("failed to encrypt recovery key: %w", err)+ if err := s.recoveryKeys.Set(ctx, recoveryKey); err != nil {+ return nil, err}- config := SystemBackupRecoveryConfig{ID: systemRecoveryConfigID, EncryptedRecoveryKey: encrypted}- if err := s.db.WithContext(ctx).Save(&config).Error; err != nil {- return nil, fmt.Errorf("failed to save recovery key: %w", err)+ if s.volumeService != nil {+ if err := s.volumeService.MigrateRepositoryPasswords(ctx); err != nil {+ slog.WarnContext(ctx, "failed to re-key volume backup repositories to the recovery key", "error", err.Error())+ }}return &backuptypes.SystemBackupRecoveryKeyStatus{Configured: true}, nil}@@ -1492,7 +1463,7 @@ func (s *SystemBackupService) GetPolicies(ctx context.Context) (*backuptypes.Sysif err != nil {return nil, err}- configured, err := s.recoveryKeyConfiguredInternal(ctx)+ configured, err := s.recoveryKeys.Configured(ctx)if err != nil {return nil, err}@@ -1522,7 +1493,7 @@ func (s *SystemBackupService) GetPolicies(ctx context.Context) (*backuptypes.Sys}func (s *SystemBackupService) UpdatePolicies(ctx context.Context, updates []backuptypes.UpdateSystemBackupPolicy) (*backuptypes.SystemBackupPolicyCollection, error) {- configured, err := s.recoveryKeyConfiguredInternal(ctx)+ configured, err := s.recoveryKeys.Configured(ctx)if err != nil {return nil, err}diff --git a/backend/internal/systembackup/service_test.go b/backend/internal/systembackup/service_test.goindex e6f007229..13c9c1688 100644--- a/backend/internal/systembackup/service_test.go+++ b/backend/internal/systembackup/service_test.go@@ -81,12 +81,13 @@ func TestRecoveryHelperExecutableInternal(t *testing.T) {func TestSystemBackupPoliciesRegisterIndependentJobs(t *testing.T) {gormDB, err := gorm.Open(sqlite.Open("file:system-backup-schedules?mode=memory&cache=shared"), &gorm.Config{})require.NoError(t, err)- require.NoError(t, gormDB.AutoMigrate(&SystemBackupPolicy{}, &SystemBackupRun{}, &SystemBackupRecoveryConfig{}))+ require.NoError(t, gormDB.AutoMigrate(&SystemBackupPolicy{}, &SystemBackupRun{}, &backup.SystemBackupRecoveryConfig{}))crypto.InitEncryption(&crypto.Config{EncryptionKey: "system-backup-policy-test-key-32bytes", Environment: "test"})service := &SystemBackupService{- db: &database.DB{DB: gormDB},- config: &config.Config{DatabaseURL: "file:system-backup-schedules-test.db"},- jobs: entityjobs.New("system-backup:", backup.SystemAdmissionScope),+ db: &database.DB{DB: gormDB},+ config: &config.Config{DatabaseURL: "file:system-backup-schedules-test.db"},+ recoveryKeys: backup.NewRecoveryKeyStore(&database.DB{DB: gormDB}),+ jobs: entityjobs.New("system-backup:", backup.SystemAdmissionScope),}scheduler := &systemBackupPolicySchedulerInternal{jobs: make(map[string]schedulertypes.Job)}require.NoError(t, service.SetScheduler(context.Background(), scheduler, newSystemBackupAdmissionGateForTestInternal(t)))@@ -148,10 +149,11 @@ func TestSystemBackupPoliciesRegisterIndependentJobs(t *testing.T) {func TestSystemBackupPolicyRequiresConfiguredRecoveryKeyWhenEnabled(t *testing.T) {gormDB, err := gorm.Open(sqlite.Open("file:system-backup-key-required?mode=memory&cache=shared"), &gorm.Config{})require.NoError(t, err)- require.NoError(t, gormDB.AutoMigrate(&SystemBackupPolicy{}, &SystemBackupRecoveryConfig{}))+ require.NoError(t, gormDB.AutoMigrate(&SystemBackupPolicy{}, &backup.SystemBackupRecoveryConfig{}))service := &SystemBackupService{- db: &database.DB{DB: gormDB},- config: &config.Config{DatabaseURL: "file:system-backup-key-required-test.db"},+ db: &database.DB{DB: gormDB},+ config: &config.Config{DatabaseURL: "file:system-backup-key-required-test.db"},+ recoveryKeys: backup.NewRecoveryKeyStore(&database.DB{DB: gormDB}),}_, err = service.UpdatePolicies(context.Background(), []backuptypes.UpdateSystemBackupPolicy{{diff --git a/backend/internal/volume/backup.go b/backend/internal/volume/backup.goindex b79b71058..986134337 100644--- a/backend/internal/volume/backup.go+++ b/backend/internal/volume/backup.go@@ -21,6 +21,7 @@ import ("emperror.dev/errors""gorm.io/gorm"+ cerrdefs "github.com/containerd/errdefs""github.com/getarcaneapp/arcane/backend/v2/internal/actors""github.com/getarcaneapp/arcane/backend/v2/internal/backup""github.com/getarcaneapp/arcane/backend/v2/internal/common"@@ -34,6 +35,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/projects""github.com/getarcaneapp/arcane/backend/v2/pkg/utils""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/backupbrowser"+ s3utils "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/s3"activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"volumetypes "github.com/getarcaneapp/arcane/types/v2/volume"@@ -471,11 +473,57 @@ const (localVolumeRepositoryID = "volumes:local")-func (s *VolumeService) rusticPasswordInternal() string {+func (s *VolumeService) legacyVolumePasswordInternal() string {sum := sha256.Sum256([]byte("arcane-volume-backups:" + s.encryptionKey))return hex.EncodeToString(sum[:])}+// volumeBackupPasswordInternal returns the recovery key once one is stored, re-keying the given repositories off the legacy derivation first.+func (s *VolumeService) volumeBackupPasswordInternal(ctx context.Context, dockerClient *client.Client, repositories ...backup.Repository) (string, error) {+ if s.recoveryKeys == nil {+ return s.legacyVolumePasswordInternal(), nil+ }+ key, err := s.recoveryKeys.Get(ctx)+ if errors.Is(err, backup.ErrRecoveryKeyNotConfigured) {+ return s.legacyVolumePasswordInternal(), nil+ }+ if err != nil {+ return "", err+ }+ for _, repository := range repositories {+ s.rekeyRepositoryInternal(ctx, dockerClient, repository, key)+ }+ return key, nil+}++// rekeyRepositoryInternal re-keys a legacy repository once per process; one that opens with neither password is left for the caller's operation to create or report.+func (s *VolumeService) rekeyRepositoryInternal(ctx context.Context, dockerClient *client.Client, repository backup.Repository, recoveryKey string) {+ if s.engine == nil {+ return+ }+ if done, _ := s.rekeyed.Load(repository.ID); done == recoveryKey {+ return+ }+ if _, err := s.engine.ListSnapshots(ctx, dockerClient, repository, recoveryKey); err == nil {+ s.rekeyed.Store(repository.ID, recoveryKey)+ return+ }+ if repository.ID == localVolumeRepositoryID {+ writable, err := s.localRusticRepositoryInternal(ctx, dockerClient, false)+ if err != nil {+ slog.WarnContext(ctx, "could not open the local volume backup repository for re-keying", "error", err.Error())+ return+ }+ repository = writable+ }+ if err := s.engine.ChangeRepositoryPassword(ctx, dockerClient, repository, s.legacyVolumePasswordInternal(), recoveryKey); err != nil {+ slog.DebugContext(ctx, "volume backup repository was not re-keyed", "repository", repository.ID, "error", err.Error())+ return+ }+ slog.InfoContext(ctx, "Re-keyed volume backup repository to the recovery key", "repository", repository.ID)+ s.rekeyed.Store(repository.ID, recoveryKey)+}+func (s *VolumeService) localRusticRepositoryInternal(ctx context.Context, dockerClient *client.Client, readOnly bool) (backup.Repository, error) {storage, err := s.resolveUsableBackupStorageMountInternal(ctx, dockerClient, "/repository", readOnly)if err != nil {@@ -489,19 +537,26 @@ func (s *VolumeService) localRusticRepositoryInternal(ctx context.Context, docke}func (s *VolumeService) remoteRusticRepositoryInternal(ctx context.Context, destinationID string) (backup.Repository, error) {+ return s.remoteRusticRepositoryForInstanceInternal(ctx, destinationID, "")+}++// remoteRusticRepositoryForInstanceInternal addresses one instance's root on the destination; empty means this instance.+func (s *VolumeService) remoteRusticRepositoryForInstanceInternal(ctx context.Context, destinationID, instanceID string) (backup.Repository, error) {if s.s3Destinations == nil {return backup.Repository{}, errors.New("S3 backup service is unavailable")}- configuration, err := s.s3Destinations.Configuration(ctx, destinationID)- if err != nil {- return backup.Repository{}, errors.New("the selected S3 backup destination is not configured")+ if strings.TrimSpace(instanceID) == "" {+ instanceID = strings.TrimSpace(s.settingsService.GetSettingsConfig().InstanceID.Value)}- instanceID := strings.TrimSpace(s.settingsService.GetSettingsConfig().InstanceID.Value)if instanceID == "" {return backup.Repository{}, errors.New("arcane instance ID is unavailable")}+ configuration, err := s.s3Destinations.Configuration(ctx, destinationID)+ if err != nil {+ return backup.Repository{}, errors.New("the selected S3 backup destination is not configured")+ }return backup.Repository{- ID: "volumes:s3:" + destinationID,+ ID: "volumes:s3:" + destinationID + ":" + instanceID,Environment: configuration.RusticEnvironment("arcane-volume-backups", instanceID),}, nil}@@ -512,7 +567,7 @@ func (s *VolumeService) rusticRepositoryForBackupInternal(ctx context.Context, dreturn repository, entry.LocalSnapshotID, err}if entry.RemoteSnapshotID != "" {- repository, err := s.remoteRusticRepositoryInternal(ctx, entry.S3DestinationID)+ repository, err := s.remoteRusticRepositoryForInstanceInternal(ctx, entry.S3DestinationID, entry.RemoteInstanceID)return repository, entry.RemoteSnapshotID, err}return backup.Repository{}, "", errors.New("volume backup has no Rustic snapshot")@@ -703,7 +758,11 @@ func (s *VolumeService) executeBackupInternal(ctx context.Context, entry *Volumeif repoErr != nil {return repoErr}- localSnapshot, err = s.engine.CreateSnapshot(ctx, dockerClient, repository, s.rusticPasswordInternal(), volumeName, volumeSourceMountInternal(volumeName))+ password, passwordErr := s.volumeBackupPasswordInternal(ctx, dockerClient, repository)+ if passwordErr != nil {+ return passwordErr+ }+ localSnapshot, err = s.engine.CreateSnapshot(ctx, dockerClient, repository, password, volumeName, volumeSourceMountInternal(volumeName))if err != nil {return fmt.Errorf("failed to create local Rustic snapshot: %w", err)}@@ -715,15 +774,19 @@ func (s *VolumeService) executeBackupInternal(ctx context.Context, entry *Volumeif repoErr != nil {return repoErr}+ password, passwordErr := s.volumeBackupPasswordInternal(ctx, dockerClient, remoteRepository)+ if passwordErr != nil {+ return passwordErr+ }var remoteSnapshot backup.Snapshotif plan.localEnabled {localRepository, localErr := s.localRusticRepositoryInternal(ctx, dockerClient, true)if localErr != nil {return localErr}- remoteSnapshot, err = s.engine.Replicate(ctx, dockerClient, localRepository, localSnapshot.ID, remoteRepository, s.rusticPasswordInternal(), volumeName)+ remoteSnapshot, err = s.engine.Replicate(ctx, dockerClient, localRepository, localSnapshot.ID, remoteRepository, password, volumeName)} else {- remoteSnapshot, err = s.engine.CreateSnapshot(ctx, dockerClient, remoteRepository, s.rusticPasswordInternal(), volumeName, volumeSourceMountInternal(volumeName))+ remoteSnapshot, err = s.engine.CreateSnapshot(ctx, dockerClient, remoteRepository, password, volumeName, volumeSourceMountInternal(volumeName))}if err != nil {return fmt.Errorf("failed to create S3 Rustic snapshot: %w", err)@@ -809,7 +872,11 @@ func (s *VolumeService) UploadBackup(ctx context.Context, backupID, s3Destinatioif err != nil {return nil, err}- snapshot, err := s.engine.Replicate(ctx, dockerClient, localRepository, entry.LocalSnapshotID, remoteRepository, s.rusticPasswordInternal(), entry.VolumeName)+ password, err := s.volumeBackupPasswordInternal(ctx, dockerClient, localRepository, remoteRepository)+ if err != nil {+ return nil, err+ }+ snapshot, err := s.engine.Replicate(ctx, dockerClient, localRepository, entry.LocalSnapshotID, remoteRepository, password, entry.VolumeName)if err != nil {return nil, fmt.Errorf("failed to upload Rustic snapshot to S3: %w", err)}@@ -859,18 +926,19 @@ func (s *VolumeService) deleteRusticBackupsInternal(ctx context.Context, entriesreturn err}var localEntries []*VolumeBackup- remoteGroups := make(map[string][]*VolumeBackup)+ remoteGroups := make(map[remoteRepositoryKeyInternal][]*VolumeBackup)for _, entry := range entries {if entry.LocalSnapshotID != "" {localEntries = append(localEntries, entry)}if includeRemote && entry.RemoteSnapshotID != "" {- remoteGroups[entry.S3DestinationID] = append(remoteGroups[entry.S3DestinationID], entry)+ key := remoteRepositoryKeyInternal{destinationID: entry.S3DestinationID, instanceID: entry.RemoteInstanceID}+ remoteGroups[key] = append(remoteGroups[key], entry)}}deleteErr := s.forgetLocalSnapshotsInternal(ctx, dockerClient, localEntries)- for destinationID, group := range remoteGroups {- deleteErr = errors.Combine(deleteErr, s.forgetRemoteSnapshotsInternal(ctx, dockerClient, destinationID, group))+ for key, group := range remoteGroups {+ deleteErr = errors.Combine(deleteErr, s.forgetRemoteSnapshotsInternal(ctx, dockerClient, key.destinationID, key.instanceID, group))}for _, entry := range entries {if entry.LocalSnapshotID == "" && entry.RemoteSnapshotID == "" {@@ -906,7 +974,7 @@ func (s *VolumeService) forgetLocalSnapshotsInternal(ctx context.Context, docker}repository, repoErr := s.localRusticRepositoryInternal(ctx, dockerClient, false)if repoErr == nil {- repoErr = s.engine.ForgetSnapshots(ctx, dockerClient, repository, s.rusticPasswordInternal(), snapshotIDs)+ repoErr = s.forgetSnapshotsInternal(ctx, dockerClient, repository, snapshotIDs)}if repoErr != nil {return fmt.Errorf("failed to delete local Rustic snapshots: %w", repoErr)@@ -917,14 +985,14 @@ func (s *VolumeService) forgetLocalSnapshotsInternal(ctx context.Context, dockerreturn nil}-func (s *VolumeService) forgetRemoteSnapshotsInternal(ctx context.Context, dockerClient *client.Client, destinationID string, entries []*VolumeBackup) error {+func (s *VolumeService) forgetRemoteSnapshotsInternal(ctx context.Context, dockerClient *client.Client, destinationID, instanceID string, entries []*VolumeBackup) error {snapshotIDs := make([]string, len(entries))for index, entry := range entries {snapshotIDs[index] = entry.RemoteSnapshotID}- repository, repoErr := s.remoteRusticRepositoryInternal(ctx, destinationID)+ repository, repoErr := s.remoteRusticRepositoryForInstanceInternal(ctx, destinationID, instanceID)if repoErr == nil {- repoErr = s.engine.ForgetSnapshots(ctx, dockerClient, repository, s.rusticPasswordInternal(), snapshotIDs)+ repoErr = s.forgetSnapshotsInternal(ctx, dockerClient, repository, snapshotIDs)}if repoErr != nil {return fmt.Errorf("failed to delete S3 Rustic snapshots: %w", repoErr)@@ -936,6 +1004,14 @@ func (s *VolumeService) forgetRemoteSnapshotsInternal(ctx context.Context, dockereturn nil}+func (s *VolumeService) forgetSnapshotsInternal(ctx context.Context, dockerClient *client.Client, repository backup.Repository, snapshotIDs []string) error {+ password, err := s.volumeBackupPasswordInternal(ctx, dockerClient, repository)+ if err != nil {+ return err+ }+ return s.engine.ForgetSnapshots(ctx, dockerClient, repository, password, snapshotIDs)+}+func (s *VolumeService) logBackupDeleteEventInternal(ctx context.Context, volumeName, backupID string, user *common.User) {actingUser := userif actingUser == nil {@@ -965,12 +1041,16 @@ func (s *VolumeService) RestoreBackup(ctx context.Context, volumeName, backupIDreturn err}var repository backup.Repository- var snapshotID string+ var snapshotID, password stringif entry.Format != VolumeBackupFormatArchive {repository, snapshotID, err = s.rusticRepositoryForBackupInternal(ctx, dockerClient, &entry)if err != nil {return err}+ password, err = s.volumeBackupPasswordInternal(ctx, dockerClient, repository)+ if err != nil {+ return err+ }}stopped, err := s.stopRunningContainersForBackupInternal(ctx, dockerClient, volumeName, user, true)containersStopped := len(stopped) > 0@@ -983,15 +1063,29 @@ func (s *VolumeService) RestoreBackup(ctx context.Context, volumeName, backupIDif err != nil {return err}- preBackup, err := s.CreateBackup(ctx, volumeName, user, VolumeBackupTriggerSafety, volumetypes.CreateBackupRequest{Destination: volumetypes.BackupDestinationLocal})- if err != nil {- return fmt.Errorf("failed to create pre-restore backup: %w", err)+ // A discovered backup may target a volume missing on this instance; create it and skip the safety backup of an empty volume.+ var preBackup *VolumeBackup+ var createdVolume bool+ if _, inspectErr := dockerClient.VolumeInspect(ctx, volumeName, client.VolumeInspectOptions{}); inspectErr != nil {+ if !cerrdefs.IsNotFound(inspectErr) {+ return fmt.Errorf("failed to inspect volume for restore: %w", inspectErr)+ }+ if _, createErr := dockerClient.VolumeCreate(ctx, client.VolumeCreateOptions{Name: volumeName}); createErr != nil {+ return fmt.Errorf("failed to create missing volume for restore: %w", createErr)+ }+ createdVolume = true+ slog.InfoContext(ctx, "Created missing volume for restore", "volume", volumeName)+ } else {+ preBackup, err = s.CreateBackup(ctx, volumeName, user, VolumeBackupTriggerSafety, volumetypes.CreateBackupRequest{Destination: volumetypes.BackupDestinationLocal})+ if err != nil {+ return fmt.Errorf("failed to create pre-restore backup: %w", err)+ }}if entry.Format == VolumeBackupFormatArchive {if err := s.restoreArchiveBackupInternal(ctx, dockerClient, volumeName, backupID); err != nil {return err}- } else if err := s.engine.RestoreSnapshot(ctx, dockerClient, repository, s.rusticPasswordInternal(), snapshotID, mount.Mount{Type: mount.TypeVolume, Source: volumeName, Target: "/volume"}, backup.RestoreOptions{DeleteExtra: true}); err != nil {+ } else if err := s.engine.RestoreSnapshot(ctx, dockerClient, repository, password, snapshotID, mount.Mount{Type: mount.TypeVolume, Source: volumeName, Target: "/volume"}, backup.RestoreOptions{DeleteExtra: true}); err != nil {return fmt.Errorf("failed to restore Rustic snapshot: %w", err)}if containersStopped {@@ -1001,7 +1095,10 @@ func (s *VolumeService) RestoreBackup(ctx context.Context, volumeName, backupIDreturn err}}- metadata := database.JSON{"action": "backup_restore", "backup_id": backupID, "pre_restore_backupId": preBackup.ID}+ metadata := database.JSON{"action": "backup_restore", "backup_id": backupID, "created_volume": createdVolume}+ if preBackup != nil {+ metadata["pre_restore_backupId"] = preBackup.ID+ }if logErr := s.eventService.LogVolumeEvent(ctx, event.EventTypeVolumeBackupRestore, volumeName, volumeName, user.ID, user.Username, "0", metadata); logErr != nil {slog.WarnContext(ctx, "could not log volume backup restore event", "volume", volumeName, "error", logErr)}@@ -1024,7 +1121,11 @@ func (s *VolumeService) ListBackupFiles(ctx context.Context, backupID string) ([if err != nil {return nil, err}- return s.engine.ListSnapshotFiles(ctx, dockerClient, repository, s.rusticPasswordInternal(), snapshotID, "", true)+ password, err := s.volumeBackupPasswordInternal(ctx, dockerClient, repository)+ if err != nil {+ return nil, err+ }+ return s.engine.ListSnapshotFiles(ctx, dockerClient, repository, password, snapshotID, "", true)}// BrowseBackupFiles returns one lazy-loaded page from a volume backup tree.@@ -1061,7 +1162,11 @@ func (s *VolumeService) backupFileEntriesInternal(ctx context.Context, entry *Voif err != nil {return nil, err}- listed, err := s.engine.ListSnapshotFiles(ctx, dockerClient, repository, s.rusticPasswordInternal(), snapshotID, browsePath+"/", recursive)+ password, err := s.volumeBackupPasswordInternal(ctx, dockerClient, repository)+ if err != nil {+ return nil, err+ }+ listed, err := s.engine.ListSnapshotFiles(ctx, dockerClient, repository, password, snapshotID, browsePath+"/", recursive)if err != nil {return nil, err}@@ -1106,7 +1211,11 @@ func (s *VolumeService) resolveVolumeBackupRestoreSelectionInternal(ctx context.eligible = backupbrowser.BuildEntries(resolved.archivePaths, "", true)} else {var listed []string- listed, err = s.engine.ListSnapshotFiles(ctx, dockerClient, repository, s.rusticPasswordInternal(), snapshotID, "", true)+ password, passwordErr := s.volumeBackupPasswordInternal(ctx, dockerClient, repository)+ if passwordErr != nil {+ return volumeBackupRestoreSelectionInternal{}, passwordErr+ }+ listed, err = s.engine.ListSnapshotFiles(ctx, dockerClient, repository, password, snapshotID, "", true)eligible = backupbrowser.BuildEntries(listed, "", true)}if err != nil {@@ -1127,9 +1236,13 @@ func (s *VolumeService) restoreVolumeBackupSelectionInternal(ctx context.Contextmembers := archiveMembersForSelectionInternal(selection.archivePaths, selection.entries)return s.restoreArchiveBackupFilesInternal(ctx, dockerClient, volumeName, backupID, members)}+ password, err := s.volumeBackupPasswordInternal(ctx, dockerClient, repository)+ if err != nil {+ return err+ }target := mount.Mount{Type: mount.TypeVolume, Source: volumeName, Target: "/volume"}if selection.globalRoot {- if err := s.engine.RestoreSnapshot(ctx, dockerClient, repository, s.rusticPasswordInternal(), snapshotID, target, backup.RestoreOptions{DeleteExtra: true}); err != nil {+ if err := s.engine.RestoreSnapshot(ctx, dockerClient, repository, password, snapshotID, target, backup.RestoreOptions{DeleteExtra: true}); err != nil {return fmt.Errorf("failed to restore Rustic snapshot: %w", err)}return nil@@ -1140,7 +1253,7 @@ func (s *VolumeService) restoreVolumeBackupSelectionInternal(ctx context.ContextsourcePath += "/"}options := backup.RestoreOptions{DeleteExtra: selectedEntry.IsDirectory, SourcePath: sourcePath, DestinationPath: path.Join(target.Target, selectedEntry.Path)}- if err := s.engine.RestoreSnapshot(ctx, dockerClient, repository, s.rusticPasswordInternal(), snapshotID, target, options); err != nil {+ if err := s.engine.RestoreSnapshot(ctx, dockerClient, repository, password, snapshotID, target, options); err != nil {return fmt.Errorf("failed to restore %s from Rustic snapshot: %w", selectedEntry.Path, err)}}@@ -1273,6 +1386,10 @@ func (s *VolumeService) downloadRusticBackupInternal(ctx context.Context, entryif err != nil {return nil, 0, err}+ password, err := s.volumeBackupPasswordInternal(ctx, dockerClient, repository)+ if err != nil {+ return nil, 0, err+ }scratchVolume := "arcane-rustic-download-" + uuid.New().String()if _, err := dockerClient.VolumeCreate(ctx, client.VolumeCreateOptions{Name: scratchVolume, Labels: volumehelper.Labels()}); err != nil {return nil, 0, fmt.Errorf("failed to create download scratch volume: %w", err)@@ -1280,7 +1397,7 @@ func (s *VolumeService) downloadRusticBackupInternal(ctx context.Context, entryremoveScratch := func() {_, _ = dockerClient.VolumeRemove(context.WithoutCancel(ctx), scratchVolume, client.VolumeRemoveOptions{Force: true})}- if err := s.engine.RestoreSnapshot(ctx, dockerClient, repository, s.rusticPasswordInternal(), entry.LocalSnapshotID, mount.Mount{Type: mount.TypeVolume, Source: scratchVolume, Target: "/volume"}, backup.RestoreOptions{DeleteExtra: true}); err != nil {+ if err := s.engine.RestoreSnapshot(ctx, dockerClient, repository, password, entry.LocalSnapshotID, mount.Mount{Type: mount.TypeVolume, Source: scratchVolume, Target: "/volume"}, backup.RestoreOptions{DeleteExtra: true}); err != nil {removeScratch()return nil, 0, fmt.Errorf("failed to restore Rustic snapshot for download: %w", err)}@@ -1926,3 +2043,138 @@ func (s *VolumeService) disableMissingS3Internal(ctx context.Context, policy *Vos.rescheduleVolumeBackupPolicyInternal(ctx, policy)return true, nil}++const systemRecoverySnapshotLabel = "arcane-system-recovery"++type remoteRepositoryKeyInternal struct {+ destinationID string+ instanceID string+}++// DiscoverRemoteBackups imports snapshots from every instance root on the destination; per-root failures are returned without blocking the others.+func (s *VolumeService) DiscoverRemoteBackups(ctx context.Context, destinationID string) (int, []string, error) {+ if s.engine == nil || s.s3Destinations == nil || s.recoveryKeys == nil {+ return 0, nil, errors.New("volume backup discovery is unavailable")+ }+ key, err := s.recoveryKeys.Get(ctx)+ if errors.Is(err, backup.ErrRecoveryKeyNotConfigured) {+ return 0, nil, errors.New("configure a recovery key in system backups before discovering volume backups")+ }+ if err != nil {+ return 0, nil, err+ }+ configuration, err := s.s3Destinations.Configuration(ctx, destinationID)+ if err != nil {+ return 0, nil, errors.New("the selected S3 backup destination is not configured")+ }+ roots, err := s3utils.ListRepositoryRoots(ctx, configuration, "arcane-volume-backups")+ if err != nil {+ return 0, nil, fmt.Errorf("failed to list volume backup repositories: %w", err)+ }+ var knownIDs []string+ if err := s.db.WithContext(ctx).Model(&VolumeBackup{}).+ Where("s3_destination_id = ? AND remote_snapshot_id <> ''", destinationID).+ Pluck("remote_snapshot_id", &knownIDs).Error; err != nil {+ return 0, nil, err+ }+ known := make(map[string]struct{}, len(knownIDs))+ for _, id := range knownIDs {+ known[id] = struct{}{}+ }+ dockerClient, err := s.dockerService.GetClient(ctx)+ if err != nil {+ return 0, nil, err+ }+ created := 0+ var failures []string+ for _, root := range roots {+ repository, repoErr := s.remoteRusticRepositoryForInstanceInternal(ctx, destinationID, root)+ if repoErr != nil {+ failures = append(failures, fmt.Sprintf("instance %s: %v", root, repoErr))+ continue+ }+ snapshots, snapErr := s.engine.ListSnapshots(ctx, dockerClient, repository, key)+ if snapErr != nil {+ failures = append(failures, fmt.Sprintf("instance %s: %v", root, snapErr))+ continue+ }+ for _, snapshot := range snapshots {+ if _, exists := known[snapshot.ID]; exists {+ continue+ }+ entry := discoveredVolumeBackupInternal(destinationID, root, snapshot)+ if entry == nil {+ continue+ }+ known[snapshot.ID] = struct{}{}+ if err := s.db.WithContext(ctx).Create(entry).Error; err != nil {+ return created, failures, fmt.Errorf("failed to save discovered volume backup: %w", err)+ }+ created+++ }+ }+ if len(failures) > 0 {+ slog.WarnContext(ctx, "volume backup discovery completed with failures", "destination", destinationID, "created", created, "failedRoots", len(failures))+ }+ return created, failures, nil+}++// discoveredVolumeBackupInternal maps a snapshot to a backup record via its volume label; unlabeled and system snapshots map to nil.+func discoveredVolumeBackupInternal(destinationID, root string, snapshot backup.DiscoveredSnapshot) *VolumeBackup {+ volumeName := strings.TrimSpace(snapshot.Label)+ if volumeName == "" || volumeName == systemRecoverySnapshotLabel {+ return nil+ }+ createdAt := snapshot.Time+ if createdAt.IsZero() {+ createdAt = time.Now().UTC()+ }+ entry := &VolumeBackup{+ VolumeName: volumeName, Size: snapshot.Summary.TotalBytesProcessed, CreatedAt: createdAt,+ Status: VolumeBackupStatusSucceeded, Trigger: VolumeBackupTriggerManual,+ Destination: volumetypes.BackupDestinationS3, Format: VolumeBackupFormatRustic,+ RemoteSnapshotID: snapshot.ID, S3DestinationID: destinationID, RemoteInstanceID: root,+ }+ entry.ID = fmt.Sprintf("remote-%s-%s-%s", destinationID, root, snapshot.ID)+ return entry+}++// MigrateRepositoryPasswords re-keys this instance's own volume backup repositories to the recovery key; other instances re-key their own roots.+func (s *VolumeService) MigrateRepositoryPasswords(ctx context.Context) error {+ if s.recoveryKeys == nil || s.engine == nil {+ return nil+ }+ key, err := s.recoveryKeys.Get(ctx)+ if errors.Is(err, backup.ErrRecoveryKeyNotConfigured) {+ return nil+ }+ if err != nil {+ return err+ }+ dockerClient, err := s.dockerService.GetClient(ctx)+ if err != nil {+ return err+ }+ local, err := s.localRusticRepositoryInternal(ctx, dockerClient, true)+ if err != nil {+ return err+ }+ repositories := []backup.Repository{local}+ if s.s3Destinations != nil {+ destinations, err := s.s3Destinations.ListS3DestinationsByID(ctx)+ if err != nil {+ return fmt.Errorf("failed to list S3 destinations for re-key: %w", err)+ }+ for destinationID := range destinations {+ remote, err := s.remoteRusticRepositoryInternal(ctx, destinationID)+ if err != nil {+ return err+ }+ repositories = append(repositories, remote)+ }+ }+ for _, repository := range repositories {+ s.rekeyRepositoryInternal(ctx, dockerClient, repository, key)+ }+ return nil+}diff --git a/backend/internal/volume/backup_management_test.go b/backend/internal/volume/backup_management_test.goindex 3754955f1..6de144a64 100644--- a/backend/internal/volume/backup_management_test.go+++ b/backend/internal/volume/backup_management_test.go@@ -2,14 +2,18 @@ package volumeimport ("context"+ "encoding/json/v2""testing"+ "time"+ "github.com/getarcaneapp/arcane/backend/v2/internal/backup""github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"volumetypes "github.com/getarcaneapp/arcane/types/v2/volume""github.com/libtnb/sqlite""github.com/stretchr/testify/require"+ "go.getarcane.app/sys/crypto""gorm.io/gorm")@@ -44,3 +48,65 @@ func TestListBackupsPaginatedByManagementTypeInternal(t *testing.T) {require.Equal(t, backuptypes.ManagementTypeVolume, entry.Type)}}+func TestVolumeBackupPasswordPrefersRecoveryKey(t *testing.T) {+ gormDB, err := gorm.Open(sqlite.Open("file:volume-backup-password?mode=memory&cache=shared"), &gorm.Config{})+ require.NoError(t, err)+ require.NoError(t, gormDB.AutoMigrate(&backup.SystemBackupRecoveryConfig{}))+ crypto.InitEncryption(&crypto.Config{EncryptionKey: "volume-backup-password-test-key-32bytes", Environment: "test"})+ recoveryKeys := backup.NewRecoveryKeyStore(&database.DB{DB: gormDB})+ service := &VolumeService{+ db: &database.DB{DB: gormDB},+ encryptionKey: "legacy-instance-secret",+ recoveryKeys: recoveryKeys,+ }++ password, err := service.volumeBackupPasswordInternal(t.Context(), nil)+ require.NoError(t, err)+ require.Equal(t, service.legacyVolumePasswordInternal(), password)++ recoveryKey := "QWERTY-ABCDEF-234567-GHIJKL-MNOPQR-STUVWX-YZ2345-ZXCVBN"+ require.NoError(t, recoveryKeys.Set(t.Context(), recoveryKey))+ password, err = service.volumeBackupPasswordInternal(t.Context(), nil)+ require.NoError(t, err)+ require.Equal(t, recoveryKey, password)+}++func TestDiscoveredVolumeBackupMapping(t *testing.T) {+ snapshot := backup.DiscoveredSnapshot{ID: "abc123", Label: "app-data", Time: time.Unix(1700000000, 0).UTC()}+ snapshot.Summary.TotalBytesProcessed = 2048++ entry := discoveredVolumeBackupInternal("destination-1", "instance-a", snapshot)+ require.NotNil(t, entry)+ require.Equal(t, "app-data", entry.VolumeName)+ require.Equal(t, "remote-destination-1-instance-a-abc123", entry.ID)+ require.Equal(t, VolumeBackupStatusSucceeded, entry.Status)+ require.Equal(t, VolumeBackupTriggerManual, entry.Trigger)+ require.Equal(t, volumetypes.BackupDestinationS3, entry.Destination)+ require.Equal(t, VolumeBackupFormatRustic, entry.Format)+ require.Equal(t, "instance-a", entry.RemoteInstanceID)+ require.Equal(t, "destination-1", entry.S3DestinationID)+ require.Equal(t, int64(2048), entry.Size)+ require.True(t, entry.CreatedAt.Equal(time.Unix(1700000000, 0).UTC()))++ // Snapshots without a volume label, and system recovery snapshots, never+ // map to a volume backup.+ require.Nil(t, discoveredVolumeBackupInternal("destination-1", "instance-a", backup.DiscoveredSnapshot{ID: "x"}))+ require.Nil(t, discoveredVolumeBackupInternal("destination-1", "instance-a", backup.DiscoveredSnapshot{ID: "x", Label: "arcane-system-recovery"}))++ // A missing snapshot timestamp falls back to now.+ fallback := discoveredVolumeBackupInternal("d", "r", backup.DiscoveredSnapshot{ID: "x", Label: "vol"})+ require.NotNil(t, fallback)+ require.False(t, fallback.CreatedAt.IsZero())+}++func TestDiscoveredSnapshotDecodesRusticLabel(t *testing.T) {+ // Rustic snapshot JSON carries the volume name in the label field written+ // by CreateSnapshot; discovery maps it back to the volume.+ payload := `{"id":"abc123","label":"app-data","time":"2026-09-08T10:00:00Z","summary":{"total_bytes_processed":2048}}`+ var snapshot backup.DiscoveredSnapshot+ require.NoError(t, json.Unmarshal([]byte(payload), &snapshot))+ require.Equal(t, "abc123", snapshot.ID)+ require.Equal(t, "app-data", snapshot.Label)+ require.Equal(t, int64(2048), snapshot.Summary.TotalBytesProcessed)+ require.False(t, snapshot.Time.IsZero())+}diff --git a/backend/internal/volume/handler.go b/backend/internal/volume/handler.goindex 3f3472829..09c855830 100644--- a/backend/internal/volume/handler.go+++ b/backend/internal/volume/handler.go@@ -184,6 +184,15 @@ type RestoreBackupFilesInput struct {Body backuptypes.RestoreSelection}+type DiscoverVolumeBackupsInput struct {+ EnvironmentID string `path:"id" doc:"Environment ID"`+ Body volumetypes.DiscoverBackupsRequest `doc:"Destination to scan"`+}++type DiscoverVolumeBackupsOutput struct {+ Body base.ApiResponse[volumetypes.DiscoverBackupsResponse]+}+type BackupHasPathInput struct {EnvironmentID string `path:"id" doc:"Environment ID"`BackupID string `path:"backupId" doc:"Backup ID"`@@ -400,6 +409,16 @@ func RegisterVolumes(api huma.API, dockerService *docker.DockerClientService, voSecurity: handlerutil.DefaultOperationSecurity(),}, authz.PermVolumesBackup, h.DeleteBackup)+ middleware.RegisterWithPermission(api, huma.Operation{+ OperationID: "discover-volume-backups",+ Method: http.MethodPost,+ Path: "/environments/{id}/volumes/backups/discover",+ Summary: "Discover volume backups on an S3 destination",+ Description: "Import existing volume backups stored on the destination by this or other Arcane instances",+ Tags: []string{"Volume Backup"},+ Security: handlerutil.DefaultOperationSecurity(),+ }, authz.PermVolumesBackup, h.DiscoverBackups)+middleware.RegisterWithPermission(api, huma.Operation{OperationID: "upload-retained-volume-backup-to-s3",Method: http.MethodPost,@@ -1079,6 +1098,25 @@ func (h *VolumeHandler) DeleteBackup(ctx context.Context, input *DeleteBackupInp}, nil}+func (h *VolumeHandler) DiscoverBackups(ctx context.Context, input *DiscoverVolumeBackupsInput) (*DiscoverVolumeBackupsOutput, error) {+ if _, err := handlerutil.RequireUser(ctx); err != nil {+ return nil, err+ }+ if strings.TrimSpace(input.Body.S3DestinationID) == "" {+ return nil, huma.Error400BadRequest("select an S3 destination to discover")+ }+ created, failures, err := h.volumeService.DiscoverRemoteBackups(ctx, input.Body.S3DestinationID)+ if err != nil {+ return nil, huma.Error400BadRequest(err.Error())+ }+ return &DiscoverVolumeBackupsOutput{+ Body: base.ApiResponse[volumetypes.DiscoverBackupsResponse]{+ Success: true,+ Data: volumetypes.DiscoverBackupsResponse{Count: created, Errors: failures},+ },+ }, nil+}+func (h *VolumeHandler) UploadBackup(ctx context.Context, input *UploadBackupInput) (*handlerutil.Out[*VolumeBackup], error) {if h.volumeService == nil {return nil, huma.Error500InternalServerError("service not available")diff --git a/backend/internal/volume/model.go b/backend/internal/volume/model.goindex d77cebc0e..3121f39c0 100644--- a/backend/internal/volume/model.go+++ b/backend/internal/volume/model.go@@ -44,6 +44,7 @@ type VolumeBackup struct {LocalSnapshotID string `json:"localSnapshotId,omitempty" gorm:"column:local_snapshot_id;type:text"`RemoteSnapshotID string `json:"remoteSnapshotId,omitempty" gorm:"column:remote_snapshot_id;type:text"`S3DestinationID string `json:"s3DestinationId,omitempty" gorm:"column:s3_destination_id;type:text;index"`+ RemoteInstanceID string `json:"remoteInstanceId,omitempty" gorm:"column:remote_instance_id;type:text"`S3DestinationName string `json:"s3DestinationName,omitempty" gorm:"-"`PolicyID string `json:"policyId,omitempty" gorm:"column:policy_id;type:text;index"`Error string `json:"error,omitempty" gorm:"column:error;type:text"`diff --git a/backend/internal/volume/module.go b/backend/internal/volume/module.goindex 2f0955301..fbe4863e9 100644--- a/backend/internal/volume/module.go+++ b/backend/internal/volume/module.go@@ -22,18 +22,19 @@ import (// Dependencies are the collaborators the volume domain needs.type Dependencies struct {- DB *database.DB- Docker *docker.DockerClientService- Event *event.EventService- Settings *settings.SettingsService- Image *image.ImageService- Activity *activity.ActivityService- Environment *environment.EnvironmentService- Container *container.ContainerService- Engine *backup.Engine- S3 *s3domain.S3DestinationService- Config *config.Config- Upload *upload.UploadService+ DB *database.DB+ Docker *docker.DockerClientService+ Event *event.EventService+ Settings *settings.SettingsService+ Image *image.ImageService+ Activity *activity.ActivityService+ Environment *environment.EnvironmentService+ Container *container.ContainerService+ Engine *backup.Engine+ S3 *s3domain.S3DestinationService+ Config *config.Config+ Upload *upload.UploadService+ RecoveryKeys *backup.RecoveryKeyStore}// Module wires the volume domain and mounts its routes.@@ -45,7 +46,7 @@ type Module struct {// New builds the volume domain from its dependencies.func New(deps Dependencies) *Module {return &Module{- service: NewVolumeService(deps.DB, deps.Docker, deps.Event, deps.Activity, deps.Settings, deps.Container, deps.Image, deps.Engine, deps.S3, deps.Config),+ service: NewVolumeService(deps.DB, deps.Docker, deps.Event, deps.Activity, deps.Settings, deps.Container, deps.Image, deps.Engine, deps.S3, deps.Config, deps.RecoveryKeys),deps: deps,}}diff --git a/backend/internal/volume/service.go b/backend/internal/volume/service.goindex 1c8a84802..d7f534033 100644--- a/backend/internal/volume/service.go+++ b/backend/internal/volume/service.go@@ -46,6 +46,8 @@ type VolumeService struct {s3Destinations *s3domain.S3DestinationServicebackupVolumeName stringencryptionKey string+ recoveryKeys *backup.RecoveryKeyStore+ rekeyed sync.MapworkspaceMaxDepth intworkspaceMaxEntries intworkspaceMaxFileSizeBytes int64@@ -75,7 +77,7 @@ type volumeWorkspaceLockContextInternal struct {const internalVolumePruneFilterValue = libarcane.InternalResourceLabel + "=true"-func NewVolumeService(db *database.DB, dockerService *docker.DockerClientService, eventService *event.EventService, activityService *activity.ActivityService, settingsService *settings.SettingsService, containerService *container.ContainerService, imageService *image.ImageService, engine *backup.Engine, s3Destinations *s3domain.S3DestinationService, cfg *config.Config) *VolumeService {+func NewVolumeService(db *database.DB, dockerService *docker.DockerClientService, eventService *event.EventService, activityService *activity.ActivityService, settingsService *settings.SettingsService, containerService *container.ContainerService, imageService *image.ImageService, engine *backup.Engine, s3Destinations *s3domain.S3DestinationService, cfg *config.Config, recoveryKeys *backup.RecoveryKeyStore) *VolumeService {slog.Debug("volume service: new")backupVolumeName := ""encryptionKey := ""@@ -104,6 +106,7 @@ func NewVolumeService(db *database.DB, dockerService *docker.DockerClientServices3Destinations: s3Destinations,backupVolumeName: backupVolumeName,encryptionKey: encryptionKey,+ recoveryKeys: recoveryKeys,workspaceMaxDepth: workspaceMaxDepth,workspaceMaxEntries: workspaceMaxEntries,workspaceMaxFileSizeBytes: workspacepkg.MaxFileSizeBytes(workspaceMaxFileSizeMB),diff --git a/backend/pkg/utils/s3/s3.go b/backend/pkg/utils/s3/s3.goindex f7dd099b3..09c83ccad 100644--- a/backend/pkg/utils/s3/s3.go+++ b/backend/pkg/utils/s3/s3.go@@ -255,6 +255,41 @@ func checkBackupObjectInternal(ctx context.Context, client *awss3.Client, bucketreturn object.Body.Close()}+// ListRepositoryRoots returns the first-level prefixes below root, one per Arcane instance.+func ListRepositoryRoots(ctx context.Context, configuration Configuration, root string) ([]string, error) {+ configuration = configuration.Normalized()+ if err := configuration.Validate(true); err != nil {+ return nil, err+ }+ client, err := newClientInternal(ctx, configuration)+ if err != nil {+ return nil, fmt.Errorf("failed to configure S3 repository listing: %w", err)+ }+ prefix := path.Join(configuration.Prefix, strings.Trim(root, "/")) + "/"+ var roots []string+ paginator := awss3.NewListObjectsV2Paginator(client, &awss3.ListObjectsV2Input{+ Bucket: aws.String(configuration.Bucket),+ Prefix: aws.String(prefix),+ Delimiter: aws.String("/"),+ })+ for paginator.HasMorePages() {+ page, err := paginator.NextPage(ctx)+ if err != nil {+ return nil, fmt.Errorf("failed to list S3 repository roots: %w", err)+ }+ for _, commonPrefix := range page.CommonPrefixes {+ if commonPrefix.Prefix == nil {+ continue+ }+ name := strings.Trim(strings.TrimPrefix(*commonPrefix.Prefix, prefix), "/")+ if name != "" {+ roots = append(roots, name)+ }+ }+ }+ return roots, nil+}+func isMissingResourceInternal(err error, codes ...string) bool {var apiErr smithy.APIErrorif !errors.As(err, &apiErr) {diff --git a/backend/resources/migrations/postgres/087_add_volume_backup_remote_instance.sql b/backend/resources/migrations/postgres/087_add_volume_backup_remote_instance.sqlnew file mode 100644index 000000000..0aa0bcdcc--- /dev/null+++ b/backend/resources/migrations/postgres/087_add_volume_backup_remote_instance.sql@@ -0,0 +1,5 @@+-- +goose Up+ALTER TABLE volume_backups ADD COLUMN remote_instance_id text NOT NULL DEFAULT '';++-- +goose Down+ALTER TABLE volume_backups DROP COLUMN remote_instance_id;diff --git a/backend/resources/migrations/sqlite/087_add_volume_backup_remote_instance.sql b/backend/resources/migrations/sqlite/087_add_volume_backup_remote_instance.sqlnew file mode 100644index 000000000..0aa0bcdcc--- /dev/null+++ b/backend/resources/migrations/sqlite/087_add_volume_backup_remote_instance.sql@@ -0,0 +1,5 @@+-- +goose Up+ALTER TABLE volume_backups ADD COLUMN remote_instance_id text NOT NULL DEFAULT '';++-- +goose Down+ALTER TABLE volume_backups DROP COLUMN remote_instance_id;diff --git a/frontend/messages/en.json b/frontend/messages/en.jsonindex b0ae4d941..897243c23 100644--- a/frontend/messages/en.json+++ b/frontend/messages/en.json@@ -965,6 +965,8 @@"volumes_backups_title": "Volume Backups","volume_backups_remove_selected_title": "Remove {count} Volume Backup(s)","volume_backups_remove_selected_message": "Are you sure you want to remove the selected volume backup(s)? Arcane will also attempt to remove their local and S3 copies. This action cannot be undone.",+ "volume_backups_discovered": "Found {count} existing volume backup(s).",+ "volume_backups_discover_failed": "Some volume backups could not be discovered.","volumes_backups_restore": "Restore","volumes_remove_in_use_warning": "Warning: This volume is currently in use by containers. Forcing removal may cause data loss or container issues.","volumes_containers_using_title": "Containers Using This Volume",@@ -3332,7 +3334,7 @@"s3_destination_delete_failed": "Failed to delete S3 destination","volume_backup_policy_title": "Automatic volume backups","volume_backup_policy_description": "Add independent schedules with their own destination, retention, and container shutdown settings.",- "volume_backup_encryption_note": "Volume backups are encrypted automatically with Arcane's internal encryption key; no separate recovery key is required. Keep the original encryption key to restore these backups on another Arcane instance.",+ "volume_backup_encryption_note": "Volume backups are encrypted with your recovery key. Configure the same recovery key on another Arcane instance to restore these backups there.","volume_backup_policy_configure": "Configure schedules","volume_backup_policy_saved": "Volume backup schedules saved","volume_backup_policy_save_failed": "Failed to save volume backup policy",diff --git a/frontend/src/lib/services/volume-backup-service.ts b/frontend/src/lib/services/volume-backup-service.tsindex 1441f9de3..5a4a6c3ab 100644--- a/frontend/src/lib/services/volume-backup-service.ts+++ b/frontend/src/lib/services/volume-backup-service.ts@@ -4,6 +4,7 @@ import { environmentStore } from '#lib/stores/environment.store.svelte.js';import type {BackupEntry,CreateVolumeBackupRequest,+ DiscoverVolumeBackupsResponse,UpdateVolumeBackupPolicy,VolumeBackupPolicyCollection} from '#lib/types/shared.js';@@ -41,6 +42,11 @@ class VolumeBackupService extends BaseAPIService {return res.data;}+ async discoverBackups(s3DestinationId: string): Promise<DiscoverVolumeBackupsResponse> {+ const envId = await environmentStore.getCurrentEnvironmentId();+ return this.handleResponse(this.api.post(`/environments/${envId}/volumes/backups/discover`, { s3DestinationId }));+ }+async restoreBackup(volumeName: string, backupId: string): Promise<unknown> {const envId = await environmentStore.getCurrentEnvironmentId();return this.handleResponse(this.api.post(`/environments/${envId}/volumes/${volumeName}/backups/${backupId}/restore`));diff --git a/frontend/src/lib/types/shared.ts b/frontend/src/lib/types/shared.tsindex c32e649e2..a7124afcb 100644--- a/frontend/src/lib/types/shared.ts+++ b/frontend/src/lib/types/shared.ts@@ -143,6 +143,11 @@ export interface VolumeBackupPolicyCollection {s3Available: boolean;}+export interface DiscoverVolumeBackupsResponse {+ count: number;+ errors?: string[];+}+// --- Customize search ---export interface CustomizationMeta {diff --git a/frontend/src/lib/utils/backups.ts b/frontend/src/lib/utils/backups.tsindex b0f8c75bb..db2d9776c 100644--- a/frontend/src/lib/utils/backups.ts+++ b/frontend/src/lib/utils/backups.ts@@ -1,4 +1,8 @@+import { toast } from 'svelte-sonner';import { m } from '#lib/paraglide/messages.js';+import { systemBackupService } from '#lib/services/system-backup-service.js';+import { volumeBackupService } from '#lib/services/volume-backup-service.js';+import { tryCatch } from '#lib/utils/try-catch.js';import type {BackupDestination,BackupManagementType,@@ -117,3 +121,39 @@ export function backupPolicyUpdateFromPolicy(policy: BackupPolicy, includeStopCo...(includeStopContainers ? { stopContainers: policy.stopContainers ?? false } : {})};}++export async function discoverDestinationBackups(destinationId: string): Promise<void> {+ const policiesResult = await tryCatch(systemBackupService.getPolicies());+ if (policiesResult.error !== null || !policiesResult.data.recoveryKeyStored) return;+ const systemResult = await tryCatch(systemBackupService.discover(destinationId, ''));+ if (systemResult.error === null && systemResult.data > 0) {+ toast.success(m.system_backups_discovered({ count: systemResult.data }));+ }+ const volumeResult = await tryCatch(volumeBackupService.discoverBackups(destinationId));+ if (volumeResult.error === null) {+ if (volumeResult.data.count > 0) {+ toast.success(m.volume_backups_discovered({ count: volumeResult.data.count }));+ }+ for (const failure of volumeResult.data.errors ?? []) {+ toast.warning(m.volume_backups_discover_failed(), { description: failure });+ }+ } else {+ toast.error(volumeResult.error instanceof Error ? volumeResult.error.message : m.volume_backups_discover_failed());+ }+}++export async function runAutomaticBackupDiscovery(destinations: { id: string }[]): Promise<boolean> {+ const results = await Promise.allSettled([+ ...destinations.map((item) => systemBackupService.discover(item.id, '')),+ ...destinations.map((item) => volumeBackupService.discoverBackups(item.id))+ ]);+ let discovered = 0;+ for (const result of results) {+ if (result.status === 'rejected') {+ console.warn('S3 backup discovery failed', result.reason);+ continue;+ }+ discovered += typeof result.value === 'number' ? result.value : result.value.count;+ }+ return discovered > 0;+}diff --git a/frontend/src/routes/(app)/settings/backups/+page.svelte b/frontend/src/routes/(app)/settings/backups/+page.svelteindex a25d79e86..94c20fe75 100644--- a/frontend/src/routes/(app)/settings/backups/+page.svelte+++ b/frontend/src/routes/(app)/settings/backups/+page.svelte@@ -20,8 +20,14 @@import SelectWithLabel from '#lib/components/form/select-with-label.svelte';import TextInputWithLabel from '#lib/components/form/text-input-with-label.svelte';import { systemBackupService } from '#lib/services/system-backup-service.js';+ import { volumeBackupService } from '#lib/services/volume-backup-service.js';import { hasPermission } from '#lib/utils/auth.js';- import { backupDestinationOptions, backupPolicyDestinationDisplay, s3DestinationOptions } from '#lib/utils/backups.js';+ import {+ backupDestinationOptions,+ backupPolicyDestinationDisplay,+ runAutomaticBackupDiscovery,+ s3DestinationOptions+ } from '#lib/utils/backups.js';import type { SearchPaginationSortRequest } from '#lib/types/shared.js';import type {BackupHistoryEntry,@@ -376,14 +382,8 @@if (!mounted || autoDiscovered || !policyCollection.recoveryKeyStored || !data.destinations.length) return;if (!hasPermission('system-backups:manage')) return;autoDiscovered = true;- const destinations = data.destinations;- const result = await tryCatch(Promise.all(destinations.map((item) => systemBackupService.discover(item.id, ''))));- if (!mounted) return;- if (result.error !== null) {- console.warn('S3 backup discovery failed', result.error);- return;- }- if (result.data.some((count) => count > 0)) await refresh();+ const found = await runAutomaticBackupDiscovery(data.destinations);+ if (mounted && found) await refresh();}onMount(() => {mounted = true;@@ -398,6 +398,29 @@void discoverStoredBackups();});+ function openVolumeRestore(backup: BackupHistoryEntry) {+ openConfirmDialog({+ title: m.volumes_backup_restore_title(),+ message: m.volumes_backup_restore_message({ volumeName: backup.resourceName }),+ confirm: {+ label: m.volumes_backups_restore(),+ action: async () => {+ const operationResult = await tryCatch(+ (async () => {+ const result = await volumeBackupService.restoreBackup(backup.resourceName, backup.id);+ toast.success(m.volumes_backup_restore_success(), activityToastOptions(extractActivityId(result)));+ })()+ );+ if (operationResult.error !== null) {+ const error = operationResult.error;++ toast.error(error instanceof Error ? error.message : m.common_failed());+ }+ }+ }+ });+ }+const backupActivity = useBackupActivity(() => '0',(activity) =>@@ -884,6 +907,7 @@onChanged={(options) => systemBackupService.listHistory(options)}onRestore={(item) => openAction('restore', item)}onRestoreFiles={openRestoreFiles}+ onRestoreVolume={openVolumeRestore}onUpload={(item) => openAction('upload', item)}onDelete={(item) => openAction('delete', item)}onOpenVolume={openVolumeBackups}diff --git a/frontend/src/routes/(app)/settings/backups/s3/+page.svelte b/frontend/src/routes/(app)/settings/backups/s3/+page.svelteindex 8b785618a..8af294960 100644--- a/frontend/src/routes/(app)/settings/backups/s3/+page.svelte+++ b/frontend/src/routes/(app)/settings/backups/s3/+page.svelte@@ -7,6 +7,7 @@import { RemoteEnvironmentIcon } from '#lib/icons/index.js';import { openConfirmDialog } from '#lib/components/confirm-dialog/index.js';import { s3DestinationService } from '#lib/services/s3-destination-service.js';+ import { discoverDestinationBackups } from '#lib/utils/backups.js';import type { CreateS3Destination, S3Destination } from '#lib/types/s3-destination.js';import type { SearchPaginationSortRequest } from '#lib/types/shared.js';import * as m from '#lib/paraglide/messages.js';@@ -37,17 +38,20 @@async function saveDestination(input: CreateS3Destination) {saving = true;try {+ let created: S3Destination | null = null;const operationResult = await tryCatch((async () => {if (selected) {await s3DestinationService.update(selected.id, input);toast.success(m.s3_destination_updated({ name: input.name }));} else {- await s3DestinationService.create(input);+ created = await s3DestinationService.create(input);toast.success(m.s3_destination_created({ name: input.name }));}destinations = await s3DestinationService.list(requestOptions);dialogOpen = false;+ selected = null;+ if (created) await discoverDestinationBackups(created.id);})());if (operationResult.error !== null) {diff --git a/frontend/src/routes/(app)/settings/backups/system-backup-table.svelte b/frontend/src/routes/(app)/settings/backups/system-backup-table.svelteindex e0ff35101..7d7b5deeb 100644--- a/frontend/src/routes/(app)/settings/backups/system-backup-table.svelte+++ b/frontend/src/routes/(app)/settings/backups/system-backup-table.svelte@@ -30,6 +30,7 @@onChanged,onRestore,onRestoreFiles,+ onRestoreVolume,onUpload,onDelete,onOpenVolume@@ -39,6 +40,7 @@onChanged: (options: SearchPaginationSortRequest) => Promise<Paginated<BackupHistoryEntry>>;onRestore: (backup: BackupHistoryEntry) => void;onRestoreFiles: (backup: BackupHistoryEntry) => void;+ onRestoreVolume?: (backup: BackupHistoryEntry) => void;onUpload: (backup: BackupHistoryEntry) => void;onDelete: (backup: BackupHistoryEntry) => void;onOpenVolume: (backup: BackupHistoryEntry) => void;@@ -88,6 +90,11 @@{/if}<RemoveMenuItem onclick={() => onDelete(item)} label={m.common_delete()} />{:else}+ {#if onRestoreVolume}+ <DropdownMenu.Item onclick={() => onRestoreVolume(item)} disabled={item.status !== 'succeeded'}+ ><RestartIcon class="size-4" />{m.volumes_backups_restore()}</DropdownMenu.Item+ >+ {/if}<DropdownMenu.Item onclick={() => onOpenVolume(item)}><VolumesIcon class="size-4" />{m.backups_open_volume()}</DropdownMenu.Item>diff --git a/types/volume/backup.go b/types/volume/backup.goindex dddcb24fd..7d9cc064f 100644--- a/types/volume/backup.go+++ b/types/volume/backup.go@@ -20,6 +20,15 @@ type UploadBackupRequest struct {S3DestinationID string `json:"s3DestinationId" doc:"S3 destination for the uploaded backup"`}+type DiscoverBackupsRequest struct {+ S3DestinationID string `json:"s3DestinationId" doc:"S3 destination to scan for existing volume backups"`+}++type DiscoverBackupsResponse struct {+ Count int `json:"count" doc:"Number of newly discovered volume backups"`+ Errors []string `json:"errors,omitempty" doc:"Per-repository failures encountered during discovery"`+}+type BackupFormat stringconst (