Skip to content

Modern Docker Management, Designed for Everyone

feat(backups): discover S3 backups and re-key volume backups to the recovery key (#3885)

Neureka02208f5cd24e
diff --git a/backend/internal/backup/engine.go b/backend/internal/backup/engine.go
index 19d3654f9..e713dbb63 100644
--- a/backend/internal/backup/engine.go
+++ b/backend/internal/backup/engine.go
@@ -5,11 +5,14 @@ package backup
import (
"context"
+ "crypto/rand"
+ "encoding/base32"
"encoding/json/v2"
"fmt"
"io"
"log/slog"
"path"
+ "regexp"
"slices"
"strings"
"time"
@@ -17,6 +20,7 @@ import (
"emperror.dev/errors"
"github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/image"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"
rusticruntime "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/rustic"
@@ -25,6 +29,8 @@ import (
"github.com/moby/moby/api/types/container"
"github.com/moby/moby/api/types/mount"
"github.com/moby/moby/client"
+ "go.getarcane.app/sys/crypto"
+ "gorm.io/gorm"
)
// Admission scopes shared by the backup engine and the per-policy job
@@ -251,6 +257,7 @@ func (e *Engine) ReadSnapshotTextFile(ctx context.Context, dockerClient *client.
type DiscoveredSnapshot struct {
ID string `json:"id"`
Time time.Time `json:"time"`
+ Label string `json:"label"`
Summary struct {
TotalBytesProcessed int64 `json:"total_bytes_processed"`
} `json:"summary"`
@@ -293,6 +300,15 @@ func (e *Engine) ForgetSnapshots(ctx context.Context, dockerClient *client.Clien
return err
}
+// ChangeRepositoryPassword re-keys the repository; the scratch Rustic image has no shell, so the new password travels as an argument.
+func (e *Engine) ChangeRepositoryPassword(ctx context.Context, dockerClient *client.Client, repository Repository, currentPassword, newPassword string) error {
+ if strings.TrimSpace(newPassword) == "" {
+ return errors.New("new repository password is required")
+ }
+ _, err := e.runInternal(ctx, dockerClient, repository, currentPassword, []string{"key", "password", "--new-password", newPassword})
+ return err
+}
+
// Replicate copies one snapshot between repositories by restoring it into a
// temporary volume and backing that volume up into the target repository. The
// source is read once from the repository, never from the live data. Rustic's
@@ -386,3 +402,93 @@ func (e *Engine) runContainerInternal(ctx context.Context, dockerClient *client.
mounts = append(mounts, extraMounts...)
return rusticruntime.Run(ctx, dockerClient, password, command, repository.Environment, mounts, arcaneNetworkModeInternal(ctx, dockerClient))
}
+
+// RecoveryKeyConfigID is the singleton row holding the instance-wide backup recovery key.
+const RecoveryKeyConfigID = "system-recovery"
+
+var (
+ ErrRecoveryKeyNotConfigured = errors.New("recovery key is not configured")
+
+ // RecoveryKeyFormat is 8 hyphenated groups of 6 base32 characters, used verbatim as the Rustic password.
+ RecoveryKeyFormat = regexp.MustCompile(`^[A-Z2-7]{6}(-[A-Z2-7]{6}){7}$`)
+)
+
+// SystemBackupRecoveryConfig stores the encrypted recovery key; it must be reversible because Rustic needs the plaintext.
+type SystemBackupRecoveryConfig struct {
+ database.BaseModel
+
+ EncryptedRecoveryKey string `gorm:"column:encrypted_recovery_key;type:text;not null"`
+}
+
+func (SystemBackupRecoveryConfig) TableName() string { return "system_backup_recovery_config" }
+
+func ValidateRecoveryKey(key string) error {
+ if !RecoveryKeyFormat.MatchString(strings.TrimSpace(key)) {
+ return errors.New("enter the generated recovery key (8 groups of 6 characters)")
+ }
+ return nil
+}
+
+func GenerateRecoveryKey() (string, error) {
+ raw := make([]byte, 30)
+ if _, err := rand.Read(raw); err != nil {
+ return "", fmt.Errorf("failed to generate recovery key: %w", err)
+ }
+ encoded := base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(raw)
+ groups := make([]string, 0, len(encoded)/6)
+ for i := 0; i < len(encoded); i += 6 {
+ groups = append(groups, encoded[i:i+6])
+ }
+ return strings.Join(groups, "-"), nil
+}
+
+// RecoveryKeyStore persists the recovery key shared by the system and volume backup domains.
+type RecoveryKeyStore struct {
+ db *database.DB
+}
+
+func NewRecoveryKeyStore(db *database.DB) *RecoveryKeyStore {
+ return &RecoveryKeyStore{db: db}
+}
+
+func (s *RecoveryKeyStore) Configured(ctx context.Context) (bool, error) {
+ var count int64
+ if err := s.db.WithContext(ctx).Model(&SystemBackupRecoveryConfig{}).
+ Where("id = ? AND encrypted_recovery_key <> ''", RecoveryKeyConfigID).Count(&count).Error; err != nil {
+ return false, fmt.Errorf("failed to load recovery key status: %w", err)
+ }
+ return count > 0, nil
+}
+
+// Get returns the decrypted recovery key or ErrRecoveryKeyNotConfigured.
+func (s *RecoveryKeyStore) Get(ctx context.Context) (string, error) {
+ var config SystemBackupRecoveryConfig
+ if err := s.db.WithContext(ctx).
+ Where("id = ? AND encrypted_recovery_key <> ''", RecoveryKeyConfigID).
+ First(&config).Error; err != nil {
+ if errors.Is(err, gorm.ErrRecordNotFound) {
+ return "", ErrRecoveryKeyNotConfigured
+ }
+ return "", fmt.Errorf("failed to load recovery key: %w", err)
+ }
+ key, err := crypto.Decrypt(config.EncryptedRecoveryKey)
+ if err != nil {
+ return "", fmt.Errorf("failed to decrypt recovery key: %w", err)
+ }
+ return key, nil
+}
+
+func (s *RecoveryKeyStore) Set(ctx context.Context, recoveryKey string) error {
+ if err := ValidateRecoveryKey(recoveryKey); err != nil {
+ return err
+ }
+ encrypted, err := crypto.Encrypt(recoveryKey)
+ if err != nil {
+ return fmt.Errorf("failed to encrypt recovery key: %w", err)
+ }
+ config := SystemBackupRecoveryConfig{ID: RecoveryKeyConfigID, EncryptedRecoveryKey: encrypted}
+ if err := s.db.WithContext(ctx).Save(&config).Error; err != nil {
+ return fmt.Errorf("failed to save recovery key: %w", err)
+ }
+ return nil
+}
diff --git a/backend/internal/backup/engine_test.go b/backend/internal/backup/engine_test.go
index 678215148..1a4390067 100644
--- a/backend/internal/backup/engine_test.go
+++ b/backend/internal/backup/engine_test.go
@@ -3,7 +3,11 @@ package backup
import (
"testing"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"
+ "github.com/libtnb/sqlite"
"github.com/stretchr/testify/require"
+ "go.getarcane.app/sys/crypto"
+ "gorm.io/gorm"
)
func TestMarkSnapshotDirectoriesInternal(t *testing.T) {
@@ -66,3 +70,41 @@ func TestQualifySnapshotListingInternal(t *testing.T) {
})
}
}
+
+func TestRecoveryKeyStoreRoundTrip(t *testing.T) {
+ gormDB, err := gorm.Open(sqlite.Open("file:recovery-key-store?mode=memory&cache=shared"), &gorm.Config{})
+ require.NoError(t, err)
+ require.NoError(t, gormDB.AutoMigrate(&SystemBackupRecoveryConfig{}))
+ crypto.InitEncryption(&crypto.Config{EncryptionKey: "recovery-key-store-test-key-32bytes", Environment: "test"})
+ store := NewRecoveryKeyStore(&database.DB{DB: gormDB})
+
+ configured, err := store.Configured(t.Context())
+ require.NoError(t, err)
+ require.False(t, configured)
+
+ _, err = store.Get(t.Context())
+ require.ErrorIs(t, err, ErrRecoveryKeyNotConfigured)
+
+ key, err := GenerateRecoveryKey()
+ require.NoError(t, err)
+ require.NoError(t, store.Set(t.Context(), key))
+
+ configured, err = store.Configured(t.Context())
+ require.NoError(t, err)
+ require.True(t, configured)
+
+ stored, err := store.Get(t.Context())
+ require.NoError(t, err)
+ require.Equal(t, key, stored)
+}
+
+func TestRecoveryKeyValidation(t *testing.T) {
+ require.NoError(t, ValidateRecoveryKey("QWERTY-ABCDEF-234567-GHIJKL-MNOPQR-STUVWX-YZ2345-ZXCVBN"))
+ require.Error(t, ValidateRecoveryKey("too-short"))
+ require.Error(t, ValidateRecoveryKey(""))
+ require.Error(t, ValidateRecoveryKey("QWERTY-ABCDEF-234567-GHIJKL-MNOPQR-STUVWX-YZ2345-ZXCVBN-EXTRA"))
+
+ key, err := GenerateRecoveryKey()
+ require.NoError(t, err)
+ require.NoError(t, ValidateRecoveryKey(key))
+}
diff --git a/backend/internal/dashboard/service_test.go b/backend/internal/dashboard/service_test.go
index 401cdccf3..02ce3f876 100644
--- a/backend/internal/dashboard/service_test.go
+++ b/backend/internal/dashboard/service_test.go
@@ -192,7 +192,7 @@ func TestDashboardService_GetSnapshot_ReturnsDashboardSnapshot(t *testing.T) {
Status: project.ProjectStatusStopped,
}).Error)
projectSvc := project.NewProjectService(db, settingsSvc, nil, image.NewImageService(db, nil, nil, nil, nil, nil), nil, nil, nil, nil, config.Load())
- svc := NewDashboardService(db, dockerSvc, nil, projectSvc, nil, settingsSvc, nil, nil, nil, volume.NewVolumeService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil))
+ svc := NewDashboardService(db, dockerSvc, nil, projectSvc, nil, settingsSvc, nil, nil, nil, volume.NewVolumeService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil))
snapshot, err := svc.GetSnapshot(context.Background(), DashboardActionItemsOptions{}, true)
require.NoError(t, err)
diff --git a/backend/internal/di/di.go b/backend/internal/di/di.go
index 3e7f14265..a830fa7f8 100644
--- a/backend/internal/di/di.go
+++ b/backend/internal/di/di.go
@@ -8,6 +8,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/apns"
"github.com/getarcaneapp/arcane/backend/v2/internal/appimages"
"github.com/getarcaneapp/arcane/backend/v2/internal/auth"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/backup"
"github.com/getarcaneapp/arcane/backend/v2/internal/build"
"github.com/getarcaneapp/arcane/backend/v2/internal/diagnostics"
"github.com/getarcaneapp/arcane/backend/v2/internal/environment"
@@ -106,6 +107,7 @@ var ServiceOptions = fx.Options(
provideS3ModuleInternal,
provideS3ServiceInternal,
provideBackupEngineInternal,
+ backup.NewRecoveryKeyStore,
upload.NewUploadService,
upload.New,
diff --git a/backend/internal/di/providers.go b/backend/internal/di/providers.go
index f034f8dab..2a2e2b3bf 100644
--- a/backend/internal/di/providers.go
+++ b/backend/internal/di/providers.go
@@ -252,20 +252,21 @@ func provideS3ServiceInternal(module *s3domain.Module) *s3domain.S3DestinationSe
return module.Service()
}
-func provideVolumeModuleInternal(lc fx.Lifecycle, db *database.DB, dockerService *docker.DockerClientService, eventService *event.EventService, settingsService *settings.SettingsService, imageService *image.ImageService, activityService *activity.ActivityService, containerModule *container.Module, engine *backup.Engine, s3Service *s3domain.S3DestinationService, environmentService *environment.EnvironmentService, cfg *config.Config, uploadService *upload.UploadService) *volume.Module {
+func provideVolumeModuleInternal(lc fx.Lifecycle, db *database.DB, dockerService *docker.DockerClientService, eventService *event.EventService, settingsService *settings.SettingsService, imageService *image.ImageService, activityService *activity.ActivityService, containerModule *container.Module, engine *backup.Engine, s3Service *s3domain.S3DestinationService, environmentService *environment.EnvironmentService, cfg *config.Config, uploadService *upload.UploadService, recoveryKeys *backup.RecoveryKeyStore) *volume.Module {
module := volume.New(volume.Dependencies{
- DB: db,
- Docker: dockerService,
- Event: eventService,
- Settings: settingsService,
- Image: imageService,
- Activity: activityService,
- Environment: environmentService,
- Container: containerModule.Service(),
- Engine: engine,
- S3: s3Service,
- Config: cfg,
- Upload: uploadService,
+ DB: db,
+ Docker: dockerService,
+ Event: eventService,
+ Settings: settingsService,
+ Image: imageService,
+ Activity: activityService,
+ Environment: environmentService,
+ Container: containerModule.Service(),
+ Engine: engine,
+ S3: s3Service,
+ Config: cfg,
+ Upload: uploadService,
+ RecoveryKeys: recoveryKeys,
})
lc.Append(fx.Hook{
OnStop: func(ctx context.Context) error {
@@ -280,8 +281,8 @@ func provideVolumeServiceInternal(module *volume.Module) *volume.VolumeService {
return module.Service()
}
-func provideSystemBackupServiceInternal(db *database.DB, dockerService *docker.DockerClientService, volumeModule *volume.Module, engine *backup.Engine, s3Service *s3domain.S3DestinationService, activityService *activity.ActivityService, settingsService *settings.SettingsService, cfg *config.Config) *systembackup.SystemBackupService {
- return systembackup.NewSystemBackupService(db, dockerService, volumeModule.Service(), engine, s3Service, activityService, settingsService, cfg)
+func provideSystemBackupServiceInternal(db *database.DB, dockerService *docker.DockerClientService, volumeModule *volume.Module, engine *backup.Engine, s3Service *s3domain.S3DestinationService, activityService *activity.ActivityService, settingsService *settings.SettingsService, cfg *config.Config, recoveryKeys *backup.RecoveryKeyStore) *systembackup.SystemBackupService {
+ return systembackup.NewSystemBackupService(db, dockerService, volumeModule.Service(), engine, s3Service, activityService, settingsService, cfg, recoveryKeys)
}
func provideContainerModuleInternal(event *event.EventService, docker *docker.DockerClientService, image *image.ImageService, settings *settings.SettingsService, project *project.ProjectService, activity *activity.ActivityService) *container.Module {
diff --git a/backend/internal/systembackup/model.go b/backend/internal/systembackup/model.go
index 810b98997..c94391276 100644
--- a/backend/internal/systembackup/model.go
+++ b/backend/internal/systembackup/model.go
@@ -73,11 +73,3 @@ func (p SystemBackupPolicy) ToDTO(lastRun *SystemBackupRun) backuptypes.SystemBa
}
return dto
}
-
-type SystemBackupRecoveryConfig struct {
- database.BaseModel
-
- EncryptedRecoveryKey string `gorm:"column:encrypted_recovery_key;type:text;not null"`
-}
-
-func (SystemBackupRecoveryConfig) TableName() string { return "system_backup_recovery_config" }
diff --git a/backend/internal/systembackup/service.go b/backend/internal/systembackup/service.go
index 75e5fb9d6..a1856648a 100644
--- a/backend/internal/systembackup/service.go
+++ b/backend/internal/systembackup/service.go
@@ -2,8 +2,6 @@ package systembackup
import (
"context"
- "crypto/rand"
- "encoding/base32"
"encoding/json/jsontext"
"encoding/json/v2"
"fmt"
@@ -13,7 +11,6 @@ import (
"path"
"path/filepath"
"reflect"
- "regexp"
"slices"
"strings"
"time"
@@ -51,7 +48,6 @@ import (
"github.com/moby/moby/client"
"go.getarcane.app/acfs"
"go.getarcane.app/sys/cgroup"
- "go.getarcane.app/sys/crypto"
"gorm.io/gorm"
)
@@ -74,14 +70,16 @@ type SystemBackupService struct {
activityService *activity.ActivityService
settingsService *settings.SettingsService
config *config.Config
+ recoveryKeys *backup.RecoveryKeyStore
jobs *entityjobs.Registry
}
-func NewSystemBackupService(db *database.DB, dockerService *docker.DockerClientService, volumeService *volume.VolumeService, engine *backup.Engine, s3Destinations *s3domain.S3DestinationService, activityService *activity.ActivityService, settingsService *settings.SettingsService, cfg *config.Config) *SystemBackupService {
+func NewSystemBackupService(db *database.DB, dockerService *docker.DockerClientService, volumeService *volume.VolumeService, engine *backup.Engine, s3Destinations *s3domain.S3DestinationService, activityService *activity.ActivityService, settingsService *settings.SettingsService, cfg *config.Config, recoveryKeys *backup.RecoveryKeyStore) *SystemBackupService {
return &SystemBackupService{
db: db, dockerService: dockerService, volumeService: volumeService, engine: engine,
s3Destinations: s3Destinations, activityService: activityService, settingsService: settingsService, config: cfg,
- jobs: entityjobs.New("system-backup:", backup.SystemAdmissionScope),
+ recoveryKeys: recoveryKeys,
+ jobs: entityjobs.New("system-backup:", backup.SystemAdmissionScope),
}
}
@@ -91,19 +89,6 @@ func (s *SystemBackupService) SupportedDatabaseProvider() bool {
return strings.HasPrefix(s.config.DatabaseURL, "file:")
}
-// recoveryKeyFormatInternal is the only accepted key shape, generated by
-// GenerateRecoveryKey: 48 base32 characters (~240 bits from crypto/rand) in 8
-// hyphenated groups for hand transcription. The key is the Rustic repository
-// password and is used exactly as displayed, hyphens included.
-var recoveryKeyFormatInternal = regexp.MustCompile(`^[A-Z2-7]{6}(-[A-Z2-7]{6}){7}$`)
-
-func validateRecoveryKeyInternal(key string) error {
- if !recoveryKeyFormatInternal.MatchString(strings.TrimSpace(key)) {
- return errors.New("enter the generated recovery key (8 groups of 6 characters)")
- }
- return nil
-}
-
func recoveryHelperExecutableInternal(mounts []containertypes.MountPoint, executablePath string) (string, *mount.Mount, error) {
executablePath = strings.TrimSpace(executablePath)
if executablePath == "" {
@@ -272,24 +257,19 @@ func systemSnapshotPathFromFilesInternal(files []string) (string, error) {
func (s *SystemBackupService) recoveryKeyInternal(ctx context.Context, supplied string) (string, error) {
if strings.TrimSpace(supplied) != "" {
- if err := validateRecoveryKeyInternal(supplied); err != nil {
+ if err := backup.ValidateRecoveryKey(supplied); err != nil {
return "", err
}
return supplied, nil
}
- var recoveryConfig SystemBackupRecoveryConfig
- err := s.db.WithContext(ctx).Where("id = ?", systemRecoveryConfigID).First(&recoveryConfig).Error
- if errors.Is(err, gorm.ErrRecordNotFound) {
+ key, err := s.recoveryKeys.Get(ctx)
+ if errors.Is(err, backup.ErrRecoveryKeyNotConfigured) {
return "", errors.New("enter the recovery key or configure one in system backups")
}
if err != nil {
- return "", fmt.Errorf("failed to load stored recovery key: %w", err)
- }
- key, err := crypto.Decrypt(recoveryConfig.EncryptedRecoveryKey)
- if err != nil {
- return "", fmt.Errorf("failed to decrypt stored recovery key: %w", err)
+ return "", err
}
- return key, validateRecoveryKeyInternal(key)
+ return key, nil
}
// resolveBackupPlanInternal resolves which destinations a run targets, from
@@ -1391,8 +1371,6 @@ func (s *SystemBackupService) RestoreBackup(ctx context.Context, id, recoveryKey
return nil
}
-const systemRecoveryConfigID = "system-recovery"
-
func (s *SystemBackupService) loadPoliciesInternal(ctx context.Context) ([]SystemBackupPolicy, error) {
var policies []SystemBackupPolicy
if err := s.db.WithContext(ctx).Order("created_at ASC").Find(&policies).Error; err != nil {
@@ -1432,36 +1410,22 @@ func rusticRestoreNetworkModeInternal(current *containertypes.InspectResponse) s
return ""
}
-func (s *SystemBackupService) recoveryKeyConfiguredInternal(ctx context.Context) (bool, error) {
- var count int64
- if err := s.db.WithContext(ctx).Model(&SystemBackupRecoveryConfig{}).
- Where("id = ? AND encrypted_recovery_key <> ''", systemRecoveryConfigID).Count(&count).Error; err != nil {
- return false, fmt.Errorf("failed to load recovery key status: %w", err)
- }
- return count > 0, nil
-}
-
func (s *SystemBackupService) GenerateRecoveryKey() (*backuptypes.SystemBackupRecoveryKey, error) {
- raw := make([]byte, 30)
- if _, err := rand.Read(raw); err != nil {
- return nil, fmt.Errorf("failed to generate recovery key: %w", err)
- }
- encoded := base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(raw)
- groups := make([]string, 0, len(encoded)/6)
- for i := 0; i < len(encoded); i += 6 {
- groups = append(groups, encoded[i:i+6])
+ recoveryKey, err := backup.GenerateRecoveryKey()
+ if err != nil {
+ return nil, err
}
- return &backuptypes.SystemBackupRecoveryKey{RecoveryKey: strings.Join(groups, "-")}, nil
+ return &backuptypes.SystemBackupRecoveryKey{RecoveryKey: recoveryKey}, nil
}
func (s *SystemBackupService) SetRecoveryKey(ctx context.Context, recoveryKey string) (*backuptypes.SystemBackupRecoveryKeyStatus, error) {
- if err := validateRecoveryKeyInternal(recoveryKey); err != nil {
+ if err := backup.ValidateRecoveryKey(recoveryKey); err != nil {
return nil, err
}
// The repositories are encrypted with the key that initialized them and a
// different key cannot open them, so a rotation would silently break every
// scheduled backup and keyless restore. Refuse it while backups exist.
- configured, err := s.recoveryKeyConfiguredInternal(ctx)
+ configured, err := s.recoveryKeys.Configured(ctx)
if err != nil {
return nil, err
}
@@ -1474,15 +1438,22 @@ func (s *SystemBackupService) SetRecoveryKey(ctx context.Context, recoveryKey st
if runs > 0 {
return nil, errors.New("delete the existing system backups before replacing the recovery key; they can only be opened with the current key")
}
+ var volumeBackups int64
+ if err := s.db.WithContext(ctx).Model(&volume.VolumeBackup{}).Where("format = ?", volume.VolumeBackupFormatRustic).Count(&volumeBackups).Error; err != nil {
+ return nil, err
+ }
+ if volumeBackups > 0 {
+ return nil, errors.New("delete the existing volume backups before replacing the recovery key; they can only be opened with the current key")
+ }
}
}
- encrypted, err := crypto.Encrypt(recoveryKey)
- if err != nil {
- return nil, fmt.Errorf("failed to encrypt recovery key: %w", err)
+ if err := s.recoveryKeys.Set(ctx, recoveryKey); err != nil {
+ return nil, err
}
- config := SystemBackupRecoveryConfig{ID: systemRecoveryConfigID, EncryptedRecoveryKey: encrypted}
- if err := s.db.WithContext(ctx).Save(&config).Error; err != nil {
- return nil, fmt.Errorf("failed to save recovery key: %w", err)
+ if s.volumeService != nil {
+ if err := s.volumeService.MigrateRepositoryPasswords(ctx); err != nil {
+ slog.WarnContext(ctx, "failed to re-key volume backup repositories to the recovery key", "error", err.Error())
+ }
}
return &backuptypes.SystemBackupRecoveryKeyStatus{Configured: true}, nil
}
@@ -1492,7 +1463,7 @@ func (s *SystemBackupService) GetPolicies(ctx context.Context) (*backuptypes.Sys
if err != nil {
return nil, err
}
- configured, err := s.recoveryKeyConfiguredInternal(ctx)
+ configured, err := s.recoveryKeys.Configured(ctx)
if err != nil {
return nil, err
}
@@ -1522,7 +1493,7 @@ func (s *SystemBackupService) GetPolicies(ctx context.Context) (*backuptypes.Sys
}
func (s *SystemBackupService) UpdatePolicies(ctx context.Context, updates []backuptypes.UpdateSystemBackupPolicy) (*backuptypes.SystemBackupPolicyCollection, error) {
- configured, err := s.recoveryKeyConfiguredInternal(ctx)
+ configured, err := s.recoveryKeys.Configured(ctx)
if err != nil {
return nil, err
}
diff --git a/backend/internal/systembackup/service_test.go b/backend/internal/systembackup/service_test.go
index e6f007229..13c9c1688 100644
--- a/backend/internal/systembackup/service_test.go
+++ b/backend/internal/systembackup/service_test.go
@@ -81,12 +81,13 @@ func TestRecoveryHelperExecutableInternal(t *testing.T) {
func TestSystemBackupPoliciesRegisterIndependentJobs(t *testing.T) {
gormDB, err := gorm.Open(sqlite.Open("file:system-backup-schedules?mode=memory&cache=shared"), &gorm.Config{})
require.NoError(t, err)
- require.NoError(t, gormDB.AutoMigrate(&SystemBackupPolicy{}, &SystemBackupRun{}, &SystemBackupRecoveryConfig{}))
+ require.NoError(t, gormDB.AutoMigrate(&SystemBackupPolicy{}, &SystemBackupRun{}, &backup.SystemBackupRecoveryConfig{}))
crypto.InitEncryption(&crypto.Config{EncryptionKey: "system-backup-policy-test-key-32bytes", Environment: "test"})
service := &SystemBackupService{
- db: &database.DB{DB: gormDB},
- config: &config.Config{DatabaseURL: "file:system-backup-schedules-test.db"},
- jobs: entityjobs.New("system-backup:", backup.SystemAdmissionScope),
+ db: &database.DB{DB: gormDB},
+ config: &config.Config{DatabaseURL: "file:system-backup-schedules-test.db"},
+ recoveryKeys: backup.NewRecoveryKeyStore(&database.DB{DB: gormDB}),
+ jobs: entityjobs.New("system-backup:", backup.SystemAdmissionScope),
}
scheduler := &systemBackupPolicySchedulerInternal{jobs: make(map[string]schedulertypes.Job)}
require.NoError(t, service.SetScheduler(context.Background(), scheduler, newSystemBackupAdmissionGateForTestInternal(t)))
@@ -148,10 +149,11 @@ func TestSystemBackupPoliciesRegisterIndependentJobs(t *testing.T) {
func TestSystemBackupPolicyRequiresConfiguredRecoveryKeyWhenEnabled(t *testing.T) {
gormDB, err := gorm.Open(sqlite.Open("file:system-backup-key-required?mode=memory&cache=shared"), &gorm.Config{})
require.NoError(t, err)
- require.NoError(t, gormDB.AutoMigrate(&SystemBackupPolicy{}, &SystemBackupRecoveryConfig{}))
+ require.NoError(t, gormDB.AutoMigrate(&SystemBackupPolicy{}, &backup.SystemBackupRecoveryConfig{}))
service := &SystemBackupService{
- db: &database.DB{DB: gormDB},
- config: &config.Config{DatabaseURL: "file:system-backup-key-required-test.db"},
+ db: &database.DB{DB: gormDB},
+ config: &config.Config{DatabaseURL: "file:system-backup-key-required-test.db"},
+ recoveryKeys: backup.NewRecoveryKeyStore(&database.DB{DB: gormDB}),
}
_, err = service.UpdatePolicies(context.Background(), []backuptypes.UpdateSystemBackupPolicy{{
diff --git a/backend/internal/volume/backup.go b/backend/internal/volume/backup.go
index b79b71058..986134337 100644
--- a/backend/internal/volume/backup.go
+++ b/backend/internal/volume/backup.go
@@ -21,6 +21,7 @@ import (
"emperror.dev/errors"
"gorm.io/gorm"
+ cerrdefs "github.com/containerd/errdefs"
"github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/backup"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
@@ -34,6 +35,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/projects"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/backupbrowser"
+ s3utils "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/s3"
activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"
volumetypes "github.com/getarcaneapp/arcane/types/v2/volume"
@@ -471,11 +473,57 @@ const (
localVolumeRepositoryID = "volumes:local"
)
-func (s *VolumeService) rusticPasswordInternal() string {
+func (s *VolumeService) legacyVolumePasswordInternal() string {
sum := sha256.Sum256([]byte("arcane-volume-backups:" + s.encryptionKey))
return hex.EncodeToString(sum[:])
}
+// volumeBackupPasswordInternal returns the recovery key once one is stored, re-keying the given repositories off the legacy derivation first.
+func (s *VolumeService) volumeBackupPasswordInternal(ctx context.Context, dockerClient *client.Client, repositories ...backup.Repository) (string, error) {
+ if s.recoveryKeys == nil {
+ return s.legacyVolumePasswordInternal(), nil
+ }
+ key, err := s.recoveryKeys.Get(ctx)
+ if errors.Is(err, backup.ErrRecoveryKeyNotConfigured) {
+ return s.legacyVolumePasswordInternal(), nil
+ }
+ if err != nil {
+ return "", err
+ }
+ for _, repository := range repositories {
+ s.rekeyRepositoryInternal(ctx, dockerClient, repository, key)
+ }
+ return key, nil
+}
+
+// rekeyRepositoryInternal re-keys a legacy repository once per process; one that opens with neither password is left for the caller's operation to create or report.
+func (s *VolumeService) rekeyRepositoryInternal(ctx context.Context, dockerClient *client.Client, repository backup.Repository, recoveryKey string) {
+ if s.engine == nil {
+ return
+ }
+ if done, _ := s.rekeyed.Load(repository.ID); done == recoveryKey {
+ return
+ }
+ if _, err := s.engine.ListSnapshots(ctx, dockerClient, repository, recoveryKey); err == nil {
+ s.rekeyed.Store(repository.ID, recoveryKey)
+ return
+ }
+ if repository.ID == localVolumeRepositoryID {
+ writable, err := s.localRusticRepositoryInternal(ctx, dockerClient, false)
+ if err != nil {
+ slog.WarnContext(ctx, "could not open the local volume backup repository for re-keying", "error", err.Error())
+ return
+ }
+ repository = writable
+ }
+ if err := s.engine.ChangeRepositoryPassword(ctx, dockerClient, repository, s.legacyVolumePasswordInternal(), recoveryKey); err != nil {
+ slog.DebugContext(ctx, "volume backup repository was not re-keyed", "repository", repository.ID, "error", err.Error())
+ return
+ }
+ slog.InfoContext(ctx, "Re-keyed volume backup repository to the recovery key", "repository", repository.ID)
+ s.rekeyed.Store(repository.ID, recoveryKey)
+}
+
func (s *VolumeService) localRusticRepositoryInternal(ctx context.Context, dockerClient *client.Client, readOnly bool) (backup.Repository, error) {
storage, err := s.resolveUsableBackupStorageMountInternal(ctx, dockerClient, "/repository", readOnly)
if err != nil {
@@ -489,19 +537,26 @@ func (s *VolumeService) localRusticRepositoryInternal(ctx context.Context, docke
}
func (s *VolumeService) remoteRusticRepositoryInternal(ctx context.Context, destinationID string) (backup.Repository, error) {
+ return s.remoteRusticRepositoryForInstanceInternal(ctx, destinationID, "")
+}
+
+// remoteRusticRepositoryForInstanceInternal addresses one instance's root on the destination; empty means this instance.
+func (s *VolumeService) remoteRusticRepositoryForInstanceInternal(ctx context.Context, destinationID, instanceID string) (backup.Repository, error) {
if s.s3Destinations == nil {
return backup.Repository{}, errors.New("S3 backup service is unavailable")
}
- configuration, err := s.s3Destinations.Configuration(ctx, destinationID)
- if err != nil {
- return backup.Repository{}, errors.New("the selected S3 backup destination is not configured")
+ if strings.TrimSpace(instanceID) == "" {
+ instanceID = strings.TrimSpace(s.settingsService.GetSettingsConfig().InstanceID.Value)
}
- instanceID := strings.TrimSpace(s.settingsService.GetSettingsConfig().InstanceID.Value)
if instanceID == "" {
return backup.Repository{}, errors.New("arcane instance ID is unavailable")
}
+ configuration, err := s.s3Destinations.Configuration(ctx, destinationID)
+ if err != nil {
+ return backup.Repository{}, errors.New("the selected S3 backup destination is not configured")
+ }
return backup.Repository{
- ID: "volumes:s3:" + destinationID,
+ ID: "volumes:s3:" + destinationID + ":" + instanceID,
Environment: configuration.RusticEnvironment("arcane-volume-backups", instanceID),
}, nil
}
@@ -512,7 +567,7 @@ func (s *VolumeService) rusticRepositoryForBackupInternal(ctx context.Context, d
return repository, entry.LocalSnapshotID, err
}
if entry.RemoteSnapshotID != "" {
- repository, err := s.remoteRusticRepositoryInternal(ctx, entry.S3DestinationID)
+ repository, err := s.remoteRusticRepositoryForInstanceInternal(ctx, entry.S3DestinationID, entry.RemoteInstanceID)
return repository, entry.RemoteSnapshotID, err
}
return backup.Repository{}, "", errors.New("volume backup has no Rustic snapshot")
@@ -703,7 +758,11 @@ func (s *VolumeService) executeBackupInternal(ctx context.Context, entry *Volume
if repoErr != nil {
return repoErr
}
- localSnapshot, err = s.engine.CreateSnapshot(ctx, dockerClient, repository, s.rusticPasswordInternal(), volumeName, volumeSourceMountInternal(volumeName))
+ password, passwordErr := s.volumeBackupPasswordInternal(ctx, dockerClient, repository)
+ if passwordErr != nil {
+ return passwordErr
+ }
+ localSnapshot, err = s.engine.CreateSnapshot(ctx, dockerClient, repository, password, volumeName, volumeSourceMountInternal(volumeName))
if err != nil {
return fmt.Errorf("failed to create local Rustic snapshot: %w", err)
}
@@ -715,15 +774,19 @@ func (s *VolumeService) executeBackupInternal(ctx context.Context, entry *Volume
if repoErr != nil {
return repoErr
}
+ password, passwordErr := s.volumeBackupPasswordInternal(ctx, dockerClient, remoteRepository)
+ if passwordErr != nil {
+ return passwordErr
+ }
var remoteSnapshot backup.Snapshot
if plan.localEnabled {
localRepository, localErr := s.localRusticRepositoryInternal(ctx, dockerClient, true)
if localErr != nil {
return localErr
}
- remoteSnapshot, err = s.engine.Replicate(ctx, dockerClient, localRepository, localSnapshot.ID, remoteRepository, s.rusticPasswordInternal(), volumeName)
+ remoteSnapshot, err = s.engine.Replicate(ctx, dockerClient, localRepository, localSnapshot.ID, remoteRepository, password, volumeName)
} else {
- remoteSnapshot, err = s.engine.CreateSnapshot(ctx, dockerClient, remoteRepository, s.rusticPasswordInternal(), volumeName, volumeSourceMountInternal(volumeName))
+ remoteSnapshot, err = s.engine.CreateSnapshot(ctx, dockerClient, remoteRepository, password, volumeName, volumeSourceMountInternal(volumeName))
}
if err != nil {
return fmt.Errorf("failed to create S3 Rustic snapshot: %w", err)
@@ -809,7 +872,11 @@ func (s *VolumeService) UploadBackup(ctx context.Context, backupID, s3Destinatio
if err != nil {
return nil, err
}
- snapshot, err := s.engine.Replicate(ctx, dockerClient, localRepository, entry.LocalSnapshotID, remoteRepository, s.rusticPasswordInternal(), entry.VolumeName)
+ password, err := s.volumeBackupPasswordInternal(ctx, dockerClient, localRepository, remoteRepository)
+ if err != nil {
+ return nil, err
+ }
+ snapshot, err := s.engine.Replicate(ctx, dockerClient, localRepository, entry.LocalSnapshotID, remoteRepository, password, entry.VolumeName)
if err != nil {
return nil, fmt.Errorf("failed to upload Rustic snapshot to S3: %w", err)
}
@@ -859,18 +926,19 @@ func (s *VolumeService) deleteRusticBackupsInternal(ctx context.Context, entries
return err
}
var localEntries []*VolumeBackup
- remoteGroups := make(map[string][]*VolumeBackup)
+ remoteGroups := make(map[remoteRepositoryKeyInternal][]*VolumeBackup)
for _, entry := range entries {
if entry.LocalSnapshotID != "" {
localEntries = append(localEntries, entry)
}
if includeRemote && entry.RemoteSnapshotID != "" {
- remoteGroups[entry.S3DestinationID] = append(remoteGroups[entry.S3DestinationID], entry)
+ key := remoteRepositoryKeyInternal{destinationID: entry.S3DestinationID, instanceID: entry.RemoteInstanceID}
+ remoteGroups[key] = append(remoteGroups[key], entry)
}
}
deleteErr := s.forgetLocalSnapshotsInternal(ctx, dockerClient, localEntries)
- for destinationID, group := range remoteGroups {
- deleteErr = errors.Combine(deleteErr, s.forgetRemoteSnapshotsInternal(ctx, dockerClient, destinationID, group))
+ for key, group := range remoteGroups {
+ deleteErr = errors.Combine(deleteErr, s.forgetRemoteSnapshotsInternal(ctx, dockerClient, key.destinationID, key.instanceID, group))
}
for _, entry := range entries {
if entry.LocalSnapshotID == "" && entry.RemoteSnapshotID == "" {
@@ -906,7 +974,7 @@ func (s *VolumeService) forgetLocalSnapshotsInternal(ctx context.Context, docker
}
repository, repoErr := s.localRusticRepositoryInternal(ctx, dockerClient, false)
if repoErr == nil {
- repoErr = s.engine.ForgetSnapshots(ctx, dockerClient, repository, s.rusticPasswordInternal(), snapshotIDs)
+ repoErr = s.forgetSnapshotsInternal(ctx, dockerClient, repository, snapshotIDs)
}
if repoErr != nil {
return fmt.Errorf("failed to delete local Rustic snapshots: %w", repoErr)
@@ -917,14 +985,14 @@ func (s *VolumeService) forgetLocalSnapshotsInternal(ctx context.Context, docker
return nil
}
-func (s *VolumeService) forgetRemoteSnapshotsInternal(ctx context.Context, dockerClient *client.Client, destinationID string, entries []*VolumeBackup) error {
+func (s *VolumeService) forgetRemoteSnapshotsInternal(ctx context.Context, dockerClient *client.Client, destinationID, instanceID string, entries []*VolumeBackup) error {
snapshotIDs := make([]string, len(entries))
for index, entry := range entries {
snapshotIDs[index] = entry.RemoteSnapshotID
}
- repository, repoErr := s.remoteRusticRepositoryInternal(ctx, destinationID)
+ repository, repoErr := s.remoteRusticRepositoryForInstanceInternal(ctx, destinationID, instanceID)
if repoErr == nil {
- repoErr = s.engine.ForgetSnapshots(ctx, dockerClient, repository, s.rusticPasswordInternal(), snapshotIDs)
+ repoErr = s.forgetSnapshotsInternal(ctx, dockerClient, repository, snapshotIDs)
}
if repoErr != nil {
return fmt.Errorf("failed to delete S3 Rustic snapshots: %w", repoErr)
@@ -936,6 +1004,14 @@ func (s *VolumeService) forgetRemoteSnapshotsInternal(ctx context.Context, docke
return nil
}
+func (s *VolumeService) forgetSnapshotsInternal(ctx context.Context, dockerClient *client.Client, repository backup.Repository, snapshotIDs []string) error {
+ password, err := s.volumeBackupPasswordInternal(ctx, dockerClient, repository)
+ if err != nil {
+ return err
+ }
+ return s.engine.ForgetSnapshots(ctx, dockerClient, repository, password, snapshotIDs)
+}
+
func (s *VolumeService) logBackupDeleteEventInternal(ctx context.Context, volumeName, backupID string, user *common.User) {
actingUser := user
if actingUser == nil {
@@ -965,12 +1041,16 @@ func (s *VolumeService) RestoreBackup(ctx context.Context, volumeName, backupID
return err
}
var repository backup.Repository
- var snapshotID string
+ var snapshotID, password string
if entry.Format != VolumeBackupFormatArchive {
repository, snapshotID, err = s.rusticRepositoryForBackupInternal(ctx, dockerClient, &entry)
if err != nil {
return err
}
+ password, err = s.volumeBackupPasswordInternal(ctx, dockerClient, repository)
+ if err != nil {
+ return err
+ }
}
stopped, err := s.stopRunningContainersForBackupInternal(ctx, dockerClient, volumeName, user, true)
containersStopped := len(stopped) > 0
@@ -983,15 +1063,29 @@ func (s *VolumeService) RestoreBackup(ctx context.Context, volumeName, backupID
if err != nil {
return err
}
- preBackup, err := s.CreateBackup(ctx, volumeName, user, VolumeBackupTriggerSafety, volumetypes.CreateBackupRequest{Destination: volumetypes.BackupDestinationLocal})
- if err != nil {
- return fmt.Errorf("failed to create pre-restore backup: %w", err)
+ // A discovered backup may target a volume missing on this instance; create it and skip the safety backup of an empty volume.
+ var preBackup *VolumeBackup
+ var createdVolume bool
+ if _, inspectErr := dockerClient.VolumeInspect(ctx, volumeName, client.VolumeInspectOptions{}); inspectErr != nil {
+ if !cerrdefs.IsNotFound(inspectErr) {
+ return fmt.Errorf("failed to inspect volume for restore: %w", inspectErr)
+ }
+ if _, createErr := dockerClient.VolumeCreate(ctx, client.VolumeCreateOptions{Name: volumeName}); createErr != nil {
+ return fmt.Errorf("failed to create missing volume for restore: %w", createErr)
+ }
+ createdVolume = true
+ slog.InfoContext(ctx, "Created missing volume for restore", "volume", volumeName)
+ } else {
+ preBackup, err = s.CreateBackup(ctx, volumeName, user, VolumeBackupTriggerSafety, volumetypes.CreateBackupRequest{Destination: volumetypes.BackupDestinationLocal})
+ if err != nil {
+ return fmt.Errorf("failed to create pre-restore backup: %w", err)
+ }
}
if entry.Format == VolumeBackupFormatArchive {
if err := s.restoreArchiveBackupInternal(ctx, dockerClient, volumeName, backupID); err != nil {
return err
}
- } else if err := s.engine.RestoreSnapshot(ctx, dockerClient, repository, s.rusticPasswordInternal(), snapshotID, mount.Mount{Type: mount.TypeVolume, Source: volumeName, Target: "/volume"}, backup.RestoreOptions{DeleteExtra: true}); err != nil {
+ } else if err := s.engine.RestoreSnapshot(ctx, dockerClient, repository, password, snapshotID, mount.Mount{Type: mount.TypeVolume, Source: volumeName, Target: "/volume"}, backup.RestoreOptions{DeleteExtra: true}); err != nil {
return fmt.Errorf("failed to restore Rustic snapshot: %w", err)
}
if containersStopped {
@@ -1001,7 +1095,10 @@ func (s *VolumeService) RestoreBackup(ctx context.Context, volumeName, backupID
return err
}
}
- metadata := database.JSON{"action": "backup_restore", "backup_id": backupID, "pre_restore_backupId": preBackup.ID}
+ metadata := database.JSON{"action": "backup_restore", "backup_id": backupID, "created_volume": createdVolume}
+ if preBackup != nil {
+ metadata["pre_restore_backupId"] = preBackup.ID
+ }
if logErr := s.eventService.LogVolumeEvent(ctx, event.EventTypeVolumeBackupRestore, volumeName, volumeName, user.ID, user.Username, "0", metadata); logErr != nil {
slog.WarnContext(ctx, "could not log volume backup restore event", "volume", volumeName, "error", logErr)
}
@@ -1024,7 +1121,11 @@ func (s *VolumeService) ListBackupFiles(ctx context.Context, backupID string) ([
if err != nil {
return nil, err
}
- return s.engine.ListSnapshotFiles(ctx, dockerClient, repository, s.rusticPasswordInternal(), snapshotID, "", true)
+ password, err := s.volumeBackupPasswordInternal(ctx, dockerClient, repository)
+ if err != nil {
+ return nil, err
+ }
+ return s.engine.ListSnapshotFiles(ctx, dockerClient, repository, password, snapshotID, "", true)
}
// BrowseBackupFiles returns one lazy-loaded page from a volume backup tree.
@@ -1061,7 +1162,11 @@ func (s *VolumeService) backupFileEntriesInternal(ctx context.Context, entry *Vo
if err != nil {
return nil, err
}
- listed, err := s.engine.ListSnapshotFiles(ctx, dockerClient, repository, s.rusticPasswordInternal(), snapshotID, browsePath+"/", recursive)
+ password, err := s.volumeBackupPasswordInternal(ctx, dockerClient, repository)
+ if err != nil {
+ return nil, err
+ }
+ listed, err := s.engine.ListSnapshotFiles(ctx, dockerClient, repository, password, snapshotID, browsePath+"/", recursive)
if err != nil {
return nil, err
}
@@ -1106,7 +1211,11 @@ func (s *VolumeService) resolveVolumeBackupRestoreSelectionInternal(ctx context.
eligible = backupbrowser.BuildEntries(resolved.archivePaths, "", true)
} else {
var listed []string
- listed, err = s.engine.ListSnapshotFiles(ctx, dockerClient, repository, s.rusticPasswordInternal(), snapshotID, "", true)
+ password, passwordErr := s.volumeBackupPasswordInternal(ctx, dockerClient, repository)
+ if passwordErr != nil {
+ return volumeBackupRestoreSelectionInternal{}, passwordErr
+ }
+ listed, err = s.engine.ListSnapshotFiles(ctx, dockerClient, repository, password, snapshotID, "", true)
eligible = backupbrowser.BuildEntries(listed, "", true)
}
if err != nil {
@@ -1127,9 +1236,13 @@ func (s *VolumeService) restoreVolumeBackupSelectionInternal(ctx context.Context
members := archiveMembersForSelectionInternal(selection.archivePaths, selection.entries)
return s.restoreArchiveBackupFilesInternal(ctx, dockerClient, volumeName, backupID, members)
}
+ password, err := s.volumeBackupPasswordInternal(ctx, dockerClient, repository)
+ if err != nil {
+ return err
+ }
target := mount.Mount{Type: mount.TypeVolume, Source: volumeName, Target: "/volume"}
if selection.globalRoot {
- if err := s.engine.RestoreSnapshot(ctx, dockerClient, repository, s.rusticPasswordInternal(), snapshotID, target, backup.RestoreOptions{DeleteExtra: true}); err != nil {
+ if err := s.engine.RestoreSnapshot(ctx, dockerClient, repository, password, snapshotID, target, backup.RestoreOptions{DeleteExtra: true}); err != nil {
return fmt.Errorf("failed to restore Rustic snapshot: %w", err)
}
return nil
@@ -1140,7 +1253,7 @@ func (s *VolumeService) restoreVolumeBackupSelectionInternal(ctx context.Context
sourcePath += "/"
}
options := backup.RestoreOptions{DeleteExtra: selectedEntry.IsDirectory, SourcePath: sourcePath, DestinationPath: path.Join(target.Target, selectedEntry.Path)}
- if err := s.engine.RestoreSnapshot(ctx, dockerClient, repository, s.rusticPasswordInternal(), snapshotID, target, options); err != nil {
+ if err := s.engine.RestoreSnapshot(ctx, dockerClient, repository, password, snapshotID, target, options); err != nil {
return fmt.Errorf("failed to restore %s from Rustic snapshot: %w", selectedEntry.Path, err)
}
}
@@ -1273,6 +1386,10 @@ func (s *VolumeService) downloadRusticBackupInternal(ctx context.Context, entry
if err != nil {
return nil, 0, err
}
+ password, err := s.volumeBackupPasswordInternal(ctx, dockerClient, repository)
+ if err != nil {
+ return nil, 0, err
+ }
scratchVolume := "arcane-rustic-download-" + uuid.New().String()
if _, err := dockerClient.VolumeCreate(ctx, client.VolumeCreateOptions{Name: scratchVolume, Labels: volumehelper.Labels()}); err != nil {
return nil, 0, fmt.Errorf("failed to create download scratch volume: %w", err)
@@ -1280,7 +1397,7 @@ func (s *VolumeService) downloadRusticBackupInternal(ctx context.Context, entry
removeScratch := func() {
_, _ = dockerClient.VolumeRemove(context.WithoutCancel(ctx), scratchVolume, client.VolumeRemoveOptions{Force: true})
}
- if err := s.engine.RestoreSnapshot(ctx, dockerClient, repository, s.rusticPasswordInternal(), entry.LocalSnapshotID, mount.Mount{Type: mount.TypeVolume, Source: scratchVolume, Target: "/volume"}, backup.RestoreOptions{DeleteExtra: true}); err != nil {
+ if err := s.engine.RestoreSnapshot(ctx, dockerClient, repository, password, entry.LocalSnapshotID, mount.Mount{Type: mount.TypeVolume, Source: scratchVolume, Target: "/volume"}, backup.RestoreOptions{DeleteExtra: true}); err != nil {
removeScratch()
return nil, 0, fmt.Errorf("failed to restore Rustic snapshot for download: %w", err)
}
@@ -1926,3 +2043,138 @@ func (s *VolumeService) disableMissingS3Internal(ctx context.Context, policy *Vo
s.rescheduleVolumeBackupPolicyInternal(ctx, policy)
return true, nil
}
+
+const systemRecoverySnapshotLabel = "arcane-system-recovery"
+
+type remoteRepositoryKeyInternal struct {
+ destinationID string
+ instanceID string
+}
+
+// DiscoverRemoteBackups imports snapshots from every instance root on the destination; per-root failures are returned without blocking the others.
+func (s *VolumeService) DiscoverRemoteBackups(ctx context.Context, destinationID string) (int, []string, error) {
+ if s.engine == nil || s.s3Destinations == nil || s.recoveryKeys == nil {
+ return 0, nil, errors.New("volume backup discovery is unavailable")
+ }
+ key, err := s.recoveryKeys.Get(ctx)
+ if errors.Is(err, backup.ErrRecoveryKeyNotConfigured) {
+ return 0, nil, errors.New("configure a recovery key in system backups before discovering volume backups")
+ }
+ if err != nil {
+ return 0, nil, err
+ }
+ configuration, err := s.s3Destinations.Configuration(ctx, destinationID)
+ if err != nil {
+ return 0, nil, errors.New("the selected S3 backup destination is not configured")
+ }
+ roots, err := s3utils.ListRepositoryRoots(ctx, configuration, "arcane-volume-backups")
+ if err != nil {
+ return 0, nil, fmt.Errorf("failed to list volume backup repositories: %w", err)
+ }
+ var knownIDs []string
+ if err := s.db.WithContext(ctx).Model(&VolumeBackup{}).
+ Where("s3_destination_id = ? AND remote_snapshot_id <> ''", destinationID).
+ Pluck("remote_snapshot_id", &knownIDs).Error; err != nil {
+ return 0, nil, err
+ }
+ known := make(map[string]struct{}, len(knownIDs))
+ for _, id := range knownIDs {
+ known[id] = struct{}{}
+ }
+ dockerClient, err := s.dockerService.GetClient(ctx)
+ if err != nil {
+ return 0, nil, err
+ }
+ created := 0
+ var failures []string
+ for _, root := range roots {
+ repository, repoErr := s.remoteRusticRepositoryForInstanceInternal(ctx, destinationID, root)
+ if repoErr != nil {
+ failures = append(failures, fmt.Sprintf("instance %s: %v", root, repoErr))
+ continue
+ }
+ snapshots, snapErr := s.engine.ListSnapshots(ctx, dockerClient, repository, key)
+ if snapErr != nil {
+ failures = append(failures, fmt.Sprintf("instance %s: %v", root, snapErr))
+ continue
+ }
+ for _, snapshot := range snapshots {
+ if _, exists := known[snapshot.ID]; exists {
+ continue
+ }
+ entry := discoveredVolumeBackupInternal(destinationID, root, snapshot)
+ if entry == nil {
+ continue
+ }
+ known[snapshot.ID] = struct{}{}
+ if err := s.db.WithContext(ctx).Create(entry).Error; err != nil {
+ return created, failures, fmt.Errorf("failed to save discovered volume backup: %w", err)
+ }
+ created++
+ }
+ }
+ if len(failures) > 0 {
+ slog.WarnContext(ctx, "volume backup discovery completed with failures", "destination", destinationID, "created", created, "failedRoots", len(failures))
+ }
+ return created, failures, nil
+}
+
+// discoveredVolumeBackupInternal maps a snapshot to a backup record via its volume label; unlabeled and system snapshots map to nil.
+func discoveredVolumeBackupInternal(destinationID, root string, snapshot backup.DiscoveredSnapshot) *VolumeBackup {
+ volumeName := strings.TrimSpace(snapshot.Label)
+ if volumeName == "" || volumeName == systemRecoverySnapshotLabel {
+ return nil
+ }
+ createdAt := snapshot.Time
+ if createdAt.IsZero() {
+ createdAt = time.Now().UTC()
+ }
+ entry := &VolumeBackup{
+ VolumeName: volumeName, Size: snapshot.Summary.TotalBytesProcessed, CreatedAt: createdAt,
+ Status: VolumeBackupStatusSucceeded, Trigger: VolumeBackupTriggerManual,
+ Destination: volumetypes.BackupDestinationS3, Format: VolumeBackupFormatRustic,
+ RemoteSnapshotID: snapshot.ID, S3DestinationID: destinationID, RemoteInstanceID: root,
+ }
+ entry.ID = fmt.Sprintf("remote-%s-%s-%s", destinationID, root, snapshot.ID)
+ return entry
+}
+
+// MigrateRepositoryPasswords re-keys this instance's own volume backup repositories to the recovery key; other instances re-key their own roots.
+func (s *VolumeService) MigrateRepositoryPasswords(ctx context.Context) error {
+ if s.recoveryKeys == nil || s.engine == nil {
+ return nil
+ }
+ key, err := s.recoveryKeys.Get(ctx)
+ if errors.Is(err, backup.ErrRecoveryKeyNotConfigured) {
+ return nil
+ }
+ if err != nil {
+ return err
+ }
+ dockerClient, err := s.dockerService.GetClient(ctx)
+ if err != nil {
+ return err
+ }
+ local, err := s.localRusticRepositoryInternal(ctx, dockerClient, true)
+ if err != nil {
+ return err
+ }
+ repositories := []backup.Repository{local}
+ if s.s3Destinations != nil {
+ destinations, err := s.s3Destinations.ListS3DestinationsByID(ctx)
+ if err != nil {
+ return fmt.Errorf("failed to list S3 destinations for re-key: %w", err)
+ }
+ for destinationID := range destinations {
+ remote, err := s.remoteRusticRepositoryInternal(ctx, destinationID)
+ if err != nil {
+ return err
+ }
+ repositories = append(repositories, remote)
+ }
+ }
+ for _, repository := range repositories {
+ s.rekeyRepositoryInternal(ctx, dockerClient, repository, key)
+ }
+ return nil
+}
diff --git a/backend/internal/volume/backup_management_test.go b/backend/internal/volume/backup_management_test.go
index 3754955f1..6de144a64 100644
--- a/backend/internal/volume/backup_management_test.go
+++ b/backend/internal/volume/backup_management_test.go
@@ -2,14 +2,18 @@ package volume
import (
"context"
+ "encoding/json/v2"
"testing"
+ "time"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/backup"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"
volumetypes "github.com/getarcaneapp/arcane/types/v2/volume"
"github.com/libtnb/sqlite"
"github.com/stretchr/testify/require"
+ "go.getarcane.app/sys/crypto"
"gorm.io/gorm"
)
@@ -44,3 +48,65 @@ func TestListBackupsPaginatedByManagementTypeInternal(t *testing.T) {
require.Equal(t, backuptypes.ManagementTypeVolume, entry.Type)
}
}
+func TestVolumeBackupPasswordPrefersRecoveryKey(t *testing.T) {
+ gormDB, err := gorm.Open(sqlite.Open("file:volume-backup-password?mode=memory&cache=shared"), &gorm.Config{})
+ require.NoError(t, err)
+ require.NoError(t, gormDB.AutoMigrate(&backup.SystemBackupRecoveryConfig{}))
+ crypto.InitEncryption(&crypto.Config{EncryptionKey: "volume-backup-password-test-key-32bytes", Environment: "test"})
+ recoveryKeys := backup.NewRecoveryKeyStore(&database.DB{DB: gormDB})
+ service := &VolumeService{
+ db: &database.DB{DB: gormDB},
+ encryptionKey: "legacy-instance-secret",
+ recoveryKeys: recoveryKeys,
+ }
+
+ password, err := service.volumeBackupPasswordInternal(t.Context(), nil)
+ require.NoError(t, err)
+ require.Equal(t, service.legacyVolumePasswordInternal(), password)
+
+ recoveryKey := "QWERTY-ABCDEF-234567-GHIJKL-MNOPQR-STUVWX-YZ2345-ZXCVBN"
+ require.NoError(t, recoveryKeys.Set(t.Context(), recoveryKey))
+ password, err = service.volumeBackupPasswordInternal(t.Context(), nil)
+ require.NoError(t, err)
+ require.Equal(t, recoveryKey, password)
+}
+
+func TestDiscoveredVolumeBackupMapping(t *testing.T) {
+ snapshot := backup.DiscoveredSnapshot{ID: "abc123", Label: "app-data", Time: time.Unix(1700000000, 0).UTC()}
+ snapshot.Summary.TotalBytesProcessed = 2048
+
+ entry := discoveredVolumeBackupInternal("destination-1", "instance-a", snapshot)
+ require.NotNil(t, entry)
+ require.Equal(t, "app-data", entry.VolumeName)
+ require.Equal(t, "remote-destination-1-instance-a-abc123", entry.ID)
+ require.Equal(t, VolumeBackupStatusSucceeded, entry.Status)
+ require.Equal(t, VolumeBackupTriggerManual, entry.Trigger)
+ require.Equal(t, volumetypes.BackupDestinationS3, entry.Destination)
+ require.Equal(t, VolumeBackupFormatRustic, entry.Format)
+ require.Equal(t, "instance-a", entry.RemoteInstanceID)
+ require.Equal(t, "destination-1", entry.S3DestinationID)
+ require.Equal(t, int64(2048), entry.Size)
+ require.True(t, entry.CreatedAt.Equal(time.Unix(1700000000, 0).UTC()))
+
+ // Snapshots without a volume label, and system recovery snapshots, never
+ // map to a volume backup.
+ require.Nil(t, discoveredVolumeBackupInternal("destination-1", "instance-a", backup.DiscoveredSnapshot{ID: "x"}))
+ require.Nil(t, discoveredVolumeBackupInternal("destination-1", "instance-a", backup.DiscoveredSnapshot{ID: "x", Label: "arcane-system-recovery"}))
+
+ // A missing snapshot timestamp falls back to now.
+ fallback := discoveredVolumeBackupInternal("d", "r", backup.DiscoveredSnapshot{ID: "x", Label: "vol"})
+ require.NotNil(t, fallback)
+ require.False(t, fallback.CreatedAt.IsZero())
+}
+
+func TestDiscoveredSnapshotDecodesRusticLabel(t *testing.T) {
+ // Rustic snapshot JSON carries the volume name in the label field written
+ // by CreateSnapshot; discovery maps it back to the volume.
+ payload := `{"id":"abc123","label":"app-data","time":"2026-09-08T10:00:00Z","summary":{"total_bytes_processed":2048}}`
+ var snapshot backup.DiscoveredSnapshot
+ require.NoError(t, json.Unmarshal([]byte(payload), &snapshot))
+ require.Equal(t, "abc123", snapshot.ID)
+ require.Equal(t, "app-data", snapshot.Label)
+ require.Equal(t, int64(2048), snapshot.Summary.TotalBytesProcessed)
+ require.False(t, snapshot.Time.IsZero())
+}
diff --git a/backend/internal/volume/handler.go b/backend/internal/volume/handler.go
index 3f3472829..09c855830 100644
--- a/backend/internal/volume/handler.go
+++ b/backend/internal/volume/handler.go
@@ -184,6 +184,15 @@ type RestoreBackupFilesInput struct {
Body backuptypes.RestoreSelection
}
+type DiscoverVolumeBackupsInput struct {
+ EnvironmentID string `path:"id" doc:"Environment ID"`
+ Body volumetypes.DiscoverBackupsRequest `doc:"Destination to scan"`
+}
+
+type DiscoverVolumeBackupsOutput struct {
+ Body base.ApiResponse[volumetypes.DiscoverBackupsResponse]
+}
+
type BackupHasPathInput struct {
EnvironmentID string `path:"id" doc:"Environment ID"`
BackupID string `path:"backupId" doc:"Backup ID"`
@@ -400,6 +409,16 @@ func RegisterVolumes(api huma.API, dockerService *docker.DockerClientService, vo
Security: handlerutil.DefaultOperationSecurity(),
}, authz.PermVolumesBackup, h.DeleteBackup)
+ middleware.RegisterWithPermission(api, huma.Operation{
+ OperationID: "discover-volume-backups",
+ Method: http.MethodPost,
+ Path: "/environments/{id}/volumes/backups/discover",
+ Summary: "Discover volume backups on an S3 destination",
+ Description: "Import existing volume backups stored on the destination by this or other Arcane instances",
+ Tags: []string{"Volume Backup"},
+ Security: handlerutil.DefaultOperationSecurity(),
+ }, authz.PermVolumesBackup, h.DiscoverBackups)
+
middleware.RegisterWithPermission(api, huma.Operation{
OperationID: "upload-retained-volume-backup-to-s3",
Method: http.MethodPost,
@@ -1079,6 +1098,25 @@ func (h *VolumeHandler) DeleteBackup(ctx context.Context, input *DeleteBackupInp
}, nil
}
+func (h *VolumeHandler) DiscoverBackups(ctx context.Context, input *DiscoverVolumeBackupsInput) (*DiscoverVolumeBackupsOutput, error) {
+ if _, err := handlerutil.RequireUser(ctx); err != nil {
+ return nil, err
+ }
+ if strings.TrimSpace(input.Body.S3DestinationID) == "" {
+ return nil, huma.Error400BadRequest("select an S3 destination to discover")
+ }
+ created, failures, err := h.volumeService.DiscoverRemoteBackups(ctx, input.Body.S3DestinationID)
+ if err != nil {
+ return nil, huma.Error400BadRequest(err.Error())
+ }
+ return &DiscoverVolumeBackupsOutput{
+ Body: base.ApiResponse[volumetypes.DiscoverBackupsResponse]{
+ Success: true,
+ Data: volumetypes.DiscoverBackupsResponse{Count: created, Errors: failures},
+ },
+ }, nil
+}
+
func (h *VolumeHandler) UploadBackup(ctx context.Context, input *UploadBackupInput) (*handlerutil.Out[*VolumeBackup], error) {
if h.volumeService == nil {
return nil, huma.Error500InternalServerError("service not available")
diff --git a/backend/internal/volume/model.go b/backend/internal/volume/model.go
index d77cebc0e..3121f39c0 100644
--- a/backend/internal/volume/model.go
+++ b/backend/internal/volume/model.go
@@ -44,6 +44,7 @@ type VolumeBackup struct {
LocalSnapshotID string `json:"localSnapshotId,omitempty" gorm:"column:local_snapshot_id;type:text"`
RemoteSnapshotID string `json:"remoteSnapshotId,omitempty" gorm:"column:remote_snapshot_id;type:text"`
S3DestinationID string `json:"s3DestinationId,omitempty" gorm:"column:s3_destination_id;type:text;index"`
+ RemoteInstanceID string `json:"remoteInstanceId,omitempty" gorm:"column:remote_instance_id;type:text"`
S3DestinationName string `json:"s3DestinationName,omitempty" gorm:"-"`
PolicyID string `json:"policyId,omitempty" gorm:"column:policy_id;type:text;index"`
Error string `json:"error,omitempty" gorm:"column:error;type:text"`
diff --git a/backend/internal/volume/module.go b/backend/internal/volume/module.go
index 2f0955301..fbe4863e9 100644
--- a/backend/internal/volume/module.go
+++ b/backend/internal/volume/module.go
@@ -22,18 +22,19 @@ import (
// Dependencies are the collaborators the volume domain needs.
type Dependencies struct {
- DB *database.DB
- Docker *docker.DockerClientService
- Event *event.EventService
- Settings *settings.SettingsService
- Image *image.ImageService
- Activity *activity.ActivityService
- Environment *environment.EnvironmentService
- Container *container.ContainerService
- Engine *backup.Engine
- S3 *s3domain.S3DestinationService
- Config *config.Config
- Upload *upload.UploadService
+ DB *database.DB
+ Docker *docker.DockerClientService
+ Event *event.EventService
+ Settings *settings.SettingsService
+ Image *image.ImageService
+ Activity *activity.ActivityService
+ Environment *environment.EnvironmentService
+ Container *container.ContainerService
+ Engine *backup.Engine
+ S3 *s3domain.S3DestinationService
+ Config *config.Config
+ Upload *upload.UploadService
+ RecoveryKeys *backup.RecoveryKeyStore
}
// Module wires the volume domain and mounts its routes.
@@ -45,7 +46,7 @@ type Module struct {
// New builds the volume domain from its dependencies.
func New(deps Dependencies) *Module {
return &Module{
- service: NewVolumeService(deps.DB, deps.Docker, deps.Event, deps.Activity, deps.Settings, deps.Container, deps.Image, deps.Engine, deps.S3, deps.Config),
+ service: NewVolumeService(deps.DB, deps.Docker, deps.Event, deps.Activity, deps.Settings, deps.Container, deps.Image, deps.Engine, deps.S3, deps.Config, deps.RecoveryKeys),
deps: deps,
}
}
diff --git a/backend/internal/volume/service.go b/backend/internal/volume/service.go
index 1c8a84802..d7f534033 100644
--- a/backend/internal/volume/service.go
+++ b/backend/internal/volume/service.go
@@ -46,6 +46,8 @@ type VolumeService struct {
s3Destinations *s3domain.S3DestinationService
backupVolumeName string
encryptionKey string
+ recoveryKeys *backup.RecoveryKeyStore
+ rekeyed sync.Map
workspaceMaxDepth int
workspaceMaxEntries int
workspaceMaxFileSizeBytes int64
@@ -75,7 +77,7 @@ type volumeWorkspaceLockContextInternal struct {
const internalVolumePruneFilterValue = libarcane.InternalResourceLabel + "=true"
-func NewVolumeService(db *database.DB, dockerService *docker.DockerClientService, eventService *event.EventService, activityService *activity.ActivityService, settingsService *settings.SettingsService, containerService *container.ContainerService, imageService *image.ImageService, engine *backup.Engine, s3Destinations *s3domain.S3DestinationService, cfg *config.Config) *VolumeService {
+func NewVolumeService(db *database.DB, dockerService *docker.DockerClientService, eventService *event.EventService, activityService *activity.ActivityService, settingsService *settings.SettingsService, containerService *container.ContainerService, imageService *image.ImageService, engine *backup.Engine, s3Destinations *s3domain.S3DestinationService, cfg *config.Config, recoveryKeys *backup.RecoveryKeyStore) *VolumeService {
slog.Debug("volume service: new")
backupVolumeName := ""
encryptionKey := ""
@@ -104,6 +106,7 @@ func NewVolumeService(db *database.DB, dockerService *docker.DockerClientService
s3Destinations: s3Destinations,
backupVolumeName: backupVolumeName,
encryptionKey: encryptionKey,
+ recoveryKeys: recoveryKeys,
workspaceMaxDepth: workspaceMaxDepth,
workspaceMaxEntries: workspaceMaxEntries,
workspaceMaxFileSizeBytes: workspacepkg.MaxFileSizeBytes(workspaceMaxFileSizeMB),
diff --git a/backend/pkg/utils/s3/s3.go b/backend/pkg/utils/s3/s3.go
index f7dd099b3..09c83ccad 100644
--- a/backend/pkg/utils/s3/s3.go
+++ b/backend/pkg/utils/s3/s3.go
@@ -255,6 +255,41 @@ func checkBackupObjectInternal(ctx context.Context, client *awss3.Client, bucket
return object.Body.Close()
}
+// ListRepositoryRoots returns the first-level prefixes below root, one per Arcane instance.
+func ListRepositoryRoots(ctx context.Context, configuration Configuration, root string) ([]string, error) {
+ configuration = configuration.Normalized()
+ if err := configuration.Validate(true); err != nil {
+ return nil, err
+ }
+ client, err := newClientInternal(ctx, configuration)
+ if err != nil {
+ return nil, fmt.Errorf("failed to configure S3 repository listing: %w", err)
+ }
+ prefix := path.Join(configuration.Prefix, strings.Trim(root, "/")) + "/"
+ var roots []string
+ paginator := awss3.NewListObjectsV2Paginator(client, &awss3.ListObjectsV2Input{
+ Bucket: aws.String(configuration.Bucket),
+ Prefix: aws.String(prefix),
+ Delimiter: aws.String("/"),
+ })
+ for paginator.HasMorePages() {
+ page, err := paginator.NextPage(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to list S3 repository roots: %w", err)
+ }
+ for _, commonPrefix := range page.CommonPrefixes {
+ if commonPrefix.Prefix == nil {
+ continue
+ }
+ name := strings.Trim(strings.TrimPrefix(*commonPrefix.Prefix, prefix), "/")
+ if name != "" {
+ roots = append(roots, name)
+ }
+ }
+ }
+ return roots, nil
+}
+
func isMissingResourceInternal(err error, codes ...string) bool {
var apiErr smithy.APIError
if !errors.As(err, &apiErr) {
diff --git a/backend/resources/migrations/postgres/087_add_volume_backup_remote_instance.sql b/backend/resources/migrations/postgres/087_add_volume_backup_remote_instance.sql
new file mode 100644
index 000000000..0aa0bcdcc
--- /dev/null
+++ b/backend/resources/migrations/postgres/087_add_volume_backup_remote_instance.sql
@@ -0,0 +1,5 @@
+-- +goose Up
+ALTER TABLE volume_backups ADD COLUMN remote_instance_id text NOT NULL DEFAULT '';
+
+-- +goose Down
+ALTER TABLE volume_backups DROP COLUMN remote_instance_id;
diff --git a/backend/resources/migrations/sqlite/087_add_volume_backup_remote_instance.sql b/backend/resources/migrations/sqlite/087_add_volume_backup_remote_instance.sql
new file mode 100644
index 000000000..0aa0bcdcc
--- /dev/null
+++ b/backend/resources/migrations/sqlite/087_add_volume_backup_remote_instance.sql
@@ -0,0 +1,5 @@
+-- +goose Up
+ALTER TABLE volume_backups ADD COLUMN remote_instance_id text NOT NULL DEFAULT '';
+
+-- +goose Down
+ALTER TABLE volume_backups DROP COLUMN remote_instance_id;
diff --git a/frontend/messages/en.json b/frontend/messages/en.json
index b0ae4d941..897243c23 100644
--- a/frontend/messages/en.json
+++ b/frontend/messages/en.json
@@ -965,6 +965,8 @@
"volumes_backups_title": "Volume Backups",
"volume_backups_remove_selected_title": "Remove {count} Volume Backup(s)",
"volume_backups_remove_selected_message": "Are you sure you want to remove the selected volume backup(s)? Arcane will also attempt to remove their local and S3 copies. This action cannot be undone.",
+ "volume_backups_discovered": "Found {count} existing volume backup(s).",
+ "volume_backups_discover_failed": "Some volume backups could not be discovered.",
"volumes_backups_restore": "Restore",
"volumes_remove_in_use_warning": "Warning: This volume is currently in use by containers. Forcing removal may cause data loss or container issues.",
"volumes_containers_using_title": "Containers Using This Volume",
@@ -3332,7 +3334,7 @@
"s3_destination_delete_failed": "Failed to delete S3 destination",
"volume_backup_policy_title": "Automatic volume backups",
"volume_backup_policy_description": "Add independent schedules with their own destination, retention, and container shutdown settings.",
- "volume_backup_encryption_note": "Volume backups are encrypted automatically with Arcane's internal encryption key; no separate recovery key is required. Keep the original encryption key to restore these backups on another Arcane instance.",
+ "volume_backup_encryption_note": "Volume backups are encrypted with your recovery key. Configure the same recovery key on another Arcane instance to restore these backups there.",
"volume_backup_policy_configure": "Configure schedules",
"volume_backup_policy_saved": "Volume backup schedules saved",
"volume_backup_policy_save_failed": "Failed to save volume backup policy",
diff --git a/frontend/src/lib/services/volume-backup-service.ts b/frontend/src/lib/services/volume-backup-service.ts
index 1441f9de3..5a4a6c3ab 100644
--- a/frontend/src/lib/services/volume-backup-service.ts
+++ b/frontend/src/lib/services/volume-backup-service.ts
@@ -4,6 +4,7 @@ import { environmentStore } from '#lib/stores/environment.store.svelte.js';
import type {
BackupEntry,
CreateVolumeBackupRequest,
+ DiscoverVolumeBackupsResponse,
UpdateVolumeBackupPolicy,
VolumeBackupPolicyCollection
} from '#lib/types/shared.js';
@@ -41,6 +42,11 @@ class VolumeBackupService extends BaseAPIService {
return res.data;
}
+ async discoverBackups(s3DestinationId: string): Promise<DiscoverVolumeBackupsResponse> {
+ const envId = await environmentStore.getCurrentEnvironmentId();
+ return this.handleResponse(this.api.post(`/environments/${envId}/volumes/backups/discover`, { s3DestinationId }));
+ }
+
async restoreBackup(volumeName: string, backupId: string): Promise<unknown> {
const envId = await environmentStore.getCurrentEnvironmentId();
return this.handleResponse(this.api.post(`/environments/${envId}/volumes/${volumeName}/backups/${backupId}/restore`));
diff --git a/frontend/src/lib/types/shared.ts b/frontend/src/lib/types/shared.ts
index c32e649e2..a7124afcb 100644
--- a/frontend/src/lib/types/shared.ts
+++ b/frontend/src/lib/types/shared.ts
@@ -143,6 +143,11 @@ export interface VolumeBackupPolicyCollection {
s3Available: boolean;
}
+export interface DiscoverVolumeBackupsResponse {
+ count: number;
+ errors?: string[];
+}
+
// --- Customize search ---
export interface CustomizationMeta {
diff --git a/frontend/src/lib/utils/backups.ts b/frontend/src/lib/utils/backups.ts
index b0f8c75bb..db2d9776c 100644
--- a/frontend/src/lib/utils/backups.ts
+++ b/frontend/src/lib/utils/backups.ts
@@ -1,4 +1,8 @@
+import { toast } from 'svelte-sonner';
import { m } from '#lib/paraglide/messages.js';
+import { systemBackupService } from '#lib/services/system-backup-service.js';
+import { volumeBackupService } from '#lib/services/volume-backup-service.js';
+import { tryCatch } from '#lib/utils/try-catch.js';
import type {
BackupDestination,
BackupManagementType,
@@ -117,3 +121,39 @@ export function backupPolicyUpdateFromPolicy(policy: BackupPolicy, includeStopCo
...(includeStopContainers ? { stopContainers: policy.stopContainers ?? false } : {})
};
}
+
+export async function discoverDestinationBackups(destinationId: string): Promise<void> {
+ const policiesResult = await tryCatch(systemBackupService.getPolicies());
+ if (policiesResult.error !== null || !policiesResult.data.recoveryKeyStored) return;
+ const systemResult = await tryCatch(systemBackupService.discover(destinationId, ''));
+ if (systemResult.error === null && systemResult.data > 0) {
+ toast.success(m.system_backups_discovered({ count: systemResult.data }));
+ }
+ const volumeResult = await tryCatch(volumeBackupService.discoverBackups(destinationId));
+ if (volumeResult.error === null) {
+ if (volumeResult.data.count > 0) {
+ toast.success(m.volume_backups_discovered({ count: volumeResult.data.count }));
+ }
+ for (const failure of volumeResult.data.errors ?? []) {
+ toast.warning(m.volume_backups_discover_failed(), { description: failure });
+ }
+ } else {
+ toast.error(volumeResult.error instanceof Error ? volumeResult.error.message : m.volume_backups_discover_failed());
+ }
+}
+
+export async function runAutomaticBackupDiscovery(destinations: { id: string }[]): Promise<boolean> {
+ const results = await Promise.allSettled([
+ ...destinations.map((item) => systemBackupService.discover(item.id, '')),
+ ...destinations.map((item) => volumeBackupService.discoverBackups(item.id))
+ ]);
+ let discovered = 0;
+ for (const result of results) {
+ if (result.status === 'rejected') {
+ console.warn('S3 backup discovery failed', result.reason);
+ continue;
+ }
+ discovered += typeof result.value === 'number' ? result.value : result.value.count;
+ }
+ return discovered > 0;
+}
diff --git a/frontend/src/routes/(app)/settings/backups/+page.svelte b/frontend/src/routes/(app)/settings/backups/+page.svelte
index a25d79e86..94c20fe75 100644
--- a/frontend/src/routes/(app)/settings/backups/+page.svelte
+++ b/frontend/src/routes/(app)/settings/backups/+page.svelte
@@ -20,8 +20,14 @@
import SelectWithLabel from '#lib/components/form/select-with-label.svelte';
import TextInputWithLabel from '#lib/components/form/text-input-with-label.svelte';
import { systemBackupService } from '#lib/services/system-backup-service.js';
+ import { volumeBackupService } from '#lib/services/volume-backup-service.js';
import { hasPermission } from '#lib/utils/auth.js';
- import { backupDestinationOptions, backupPolicyDestinationDisplay, s3DestinationOptions } from '#lib/utils/backups.js';
+ import {
+ backupDestinationOptions,
+ backupPolicyDestinationDisplay,
+ runAutomaticBackupDiscovery,
+ s3DestinationOptions
+ } from '#lib/utils/backups.js';
import type { SearchPaginationSortRequest } from '#lib/types/shared.js';
import type {
BackupHistoryEntry,
@@ -376,14 +382,8 @@
if (!mounted || autoDiscovered || !policyCollection.recoveryKeyStored || !data.destinations.length) return;
if (!hasPermission('system-backups:manage')) return;
autoDiscovered = true;
- const destinations = data.destinations;
- const result = await tryCatch(Promise.all(destinations.map((item) => systemBackupService.discover(item.id, ''))));
- if (!mounted) return;
- if (result.error !== null) {
- console.warn('S3 backup discovery failed', result.error);
- return;
- }
- if (result.data.some((count) => count > 0)) await refresh();
+ const found = await runAutomaticBackupDiscovery(data.destinations);
+ if (mounted && found) await refresh();
}
onMount(() => {
mounted = true;
@@ -398,6 +398,29 @@
void discoverStoredBackups();
});
+ function openVolumeRestore(backup: BackupHistoryEntry) {
+ openConfirmDialog({
+ title: m.volumes_backup_restore_title(),
+ message: m.volumes_backup_restore_message({ volumeName: backup.resourceName }),
+ confirm: {
+ label: m.volumes_backups_restore(),
+ action: async () => {
+ const operationResult = await tryCatch(
+ (async () => {
+ const result = await volumeBackupService.restoreBackup(backup.resourceName, backup.id);
+ toast.success(m.volumes_backup_restore_success(), activityToastOptions(extractActivityId(result)));
+ })()
+ );
+ if (operationResult.error !== null) {
+ const error = operationResult.error;
+
+ toast.error(error instanceof Error ? error.message : m.common_failed());
+ }
+ }
+ }
+ });
+ }
+
const backupActivity = useBackupActivity(
() => '0',
(activity) =>
@@ -884,6 +907,7 @@
onChanged={(options) => systemBackupService.listHistory(options)}
onRestore={(item) => openAction('restore', item)}
onRestoreFiles={openRestoreFiles}
+ onRestoreVolume={openVolumeRestore}
onUpload={(item) => openAction('upload', item)}
onDelete={(item) => openAction('delete', item)}
onOpenVolume={openVolumeBackups}
diff --git a/frontend/src/routes/(app)/settings/backups/s3/+page.svelte b/frontend/src/routes/(app)/settings/backups/s3/+page.svelte
index 8b785618a..8af294960 100644
--- a/frontend/src/routes/(app)/settings/backups/s3/+page.svelte
+++ b/frontend/src/routes/(app)/settings/backups/s3/+page.svelte
@@ -7,6 +7,7 @@
import { RemoteEnvironmentIcon } from '#lib/icons/index.js';
import { openConfirmDialog } from '#lib/components/confirm-dialog/index.js';
import { s3DestinationService } from '#lib/services/s3-destination-service.js';
+ import { discoverDestinationBackups } from '#lib/utils/backups.js';
import type { CreateS3Destination, S3Destination } from '#lib/types/s3-destination.js';
import type { SearchPaginationSortRequest } from '#lib/types/shared.js';
import * as m from '#lib/paraglide/messages.js';
@@ -37,17 +38,20 @@
async function saveDestination(input: CreateS3Destination) {
saving = true;
try {
+ let created: S3Destination | null = null;
const operationResult = await tryCatch(
(async () => {
if (selected) {
await s3DestinationService.update(selected.id, input);
toast.success(m.s3_destination_updated({ name: input.name }));
} else {
- await s3DestinationService.create(input);
+ created = await s3DestinationService.create(input);
toast.success(m.s3_destination_created({ name: input.name }));
}
destinations = await s3DestinationService.list(requestOptions);
dialogOpen = false;
+ selected = null;
+ if (created) await discoverDestinationBackups(created.id);
})()
);
if (operationResult.error !== null) {
diff --git a/frontend/src/routes/(app)/settings/backups/system-backup-table.svelte b/frontend/src/routes/(app)/settings/backups/system-backup-table.svelte
index e0ff35101..7d7b5deeb 100644
--- a/frontend/src/routes/(app)/settings/backups/system-backup-table.svelte
+++ b/frontend/src/routes/(app)/settings/backups/system-backup-table.svelte
@@ -30,6 +30,7 @@
onChanged,
onRestore,
onRestoreFiles,
+ onRestoreVolume,
onUpload,
onDelete,
onOpenVolume
@@ -39,6 +40,7 @@
onChanged: (options: SearchPaginationSortRequest) => Promise<Paginated<BackupHistoryEntry>>;
onRestore: (backup: BackupHistoryEntry) => void;
onRestoreFiles: (backup: BackupHistoryEntry) => void;
+ onRestoreVolume?: (backup: BackupHistoryEntry) => void;
onUpload: (backup: BackupHistoryEntry) => void;
onDelete: (backup: BackupHistoryEntry) => void;
onOpenVolume: (backup: BackupHistoryEntry) => void;
@@ -88,6 +90,11 @@
{/if}
<RemoveMenuItem onclick={() => onDelete(item)} label={m.common_delete()} />
{:else}
+ {#if onRestoreVolume}
+ <DropdownMenu.Item onclick={() => onRestoreVolume(item)} disabled={item.status !== 'succeeded'}
+ ><RestartIcon class="size-4" />{m.volumes_backups_restore()}</DropdownMenu.Item
+ >
+ {/if}
<DropdownMenu.Item onclick={() => onOpenVolume(item)}
><VolumesIcon class="size-4" />{m.backups_open_volume()}</DropdownMenu.Item
>
diff --git a/types/volume/backup.go b/types/volume/backup.go
index dddcb24fd..7d9cc064f 100644
--- a/types/volume/backup.go
+++ b/types/volume/backup.go
@@ -20,6 +20,15 @@ type UploadBackupRequest struct {
S3DestinationID string `json:"s3DestinationId" doc:"S3 destination for the uploaded backup"`
}
+type DiscoverBackupsRequest struct {
+ S3DestinationID string `json:"s3DestinationId" doc:"S3 destination to scan for existing volume backups"`
+}
+
+type DiscoverBackupsResponse struct {
+ Count int `json:"count" doc:"Number of newly discovered volume backups"`
+ Errors []string `json:"errors,omitempty" doc:"Per-repository failures encountered during discovery"`
+}
+
type BackupFormat string
const (