Skip to content

Modern Docker Management, Designed for Everyone

fix: make backup restores and recovery key changes data-safe (#4487)

Kyle Mendellc2d8e8d6cab0
diff --git a/backend/cli/recovery/recovery.go b/backend/cli/recovery/recovery.go
index 5c35a45fb..2fd755108 100644
--- a/backend/cli/recovery/recovery.go
+++ b/backend/cli/recovery/recovery.go
@@ -21,11 +21,13 @@ import (
"github.com/spf13/cobra"
"go.getarcane.app/docker"
"go.getarcane.app/docker/compat"
+ "go.getarcane.app/sys/crypto"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"github.com/getarcaneapp/arcane/backend/v2/cli/upgrade"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/backup"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/internal/system"
@@ -131,6 +133,11 @@ func runRestoreInternal(cmd *cobra.Command, _ []string) error {
// Projects were restored into the current directory, so the recovered
// configuration must keep pointing there rather than at the backup-time path.
manifest.Environment["PROJECTS_DIRECTORY"] = cmp.Or(request.ProjectsSetting, manifest.Environment["PROJECTS_DIRECTORY"])
+ crypto.InitEncryption(&crypto.Config{
+ EncryptionKey: manifest.Environment["ENCRYPTION_KEY"],
+ Environment: manifest.Environment["ENVIRONMENT"],
+ AgentMode: manifest.Environment["AGENT_MODE"] == "true",
+ })
if finalizeRestoredBackupErr := finalizeRestoredBackupInternal(ctx, manifest.Environment["DATABASE_URL"], manifest.BackupID, manifest.ActivityID, request); finalizeRestoredBackupErr != nil {
return fmt.Errorf("finalize restored system backup: %w", finalizeRestoredBackupErr)
}
@@ -161,6 +168,11 @@ func finalizeRestoredBackupInternal(ctx context.Context, databaseURL, manifestBa
if finalizeRestoredRunErr := finalizeRestoredRunInternal(db, manifestBackupID, request); finalizeRestoredRunErr != nil {
return finalizeRestoredRunErr
}
+ if strings.TrimSpace(request.RecoveryKey) != "" {
+ if setRecoveryKeyErr := backup.NewRecoveryKeyStore(&database.DB{DB: db}).Set(ctx, request.RecoveryKey); setRecoveryKeyErr != nil {
+ return setRecoveryKeyErr
+ }
+ }
if preserveSafetyBackupErr := preserveSafetyBackupInternal(db, request.SafetyBackup); preserveSafetyBackupErr != nil {
return preserveSafetyBackupErr
}
@@ -223,23 +235,23 @@ func finalizeRestoredRunInternal(db *gorm.DB, manifestBackupID string, request r
return db.Model(&system.SystemBackupRun{}).Where("id = ?", run.ID).Updates(values).Error
}
-func preserveSafetyBackupInternal(db *gorm.DB, backup *recoverytypes.SafetyBackup) error {
- if backup == nil || strings.TrimSpace(backup.ID) == "" || strings.TrimSpace(backup.LocalSnapshotID) == "" {
+func preserveSafetyBackupInternal(db *gorm.DB, safetyBackup *recoverytypes.SafetyBackup) error {
+ if safetyBackup == nil || strings.TrimSpace(safetyBackup.ID) == "" || strings.TrimSpace(safetyBackup.LocalSnapshotID) == "" {
return nil
}
onConflict := map[string]any{
- "size": backup.Size, "updated_at": time.Now().UTC(), "status": system.SystemBackupStatusSucceeded,
+ "size": safetyBackup.Size, "updated_at": time.Now().UTC(), "status": system.SystemBackupStatusSucceeded,
"trigger": system.SystemBackupTriggerSafety, "destination": backuptypes.SystemBackupDestinationLocal,
- "local_snapshot_id": backup.LocalSnapshotID, "error": "",
+ "local_snapshot_id": safetyBackup.LocalSnapshotID, "error": "",
}
return db.Model(&system.SystemBackupRun{}).
Clauses(clause.OnConflict{Columns: []clause.Column{{Name: "id"}}, DoUpdates: clause.Assignments(onConflict)}).
Create(map[string]any{
- "id": backup.ID, "size": backup.Size, "created_at": backup.CreatedAt, "updated_at": time.Now().UTC(),
+ "id": safetyBackup.ID, "size": safetyBackup.Size, "created_at": safetyBackup.CreatedAt, "updated_at": time.Now().UTC(),
"status": system.SystemBackupStatusSucceeded,
"trigger": system.SystemBackupTriggerSafety,
"destination": backuptypes.SystemBackupDestinationLocal,
- "local_snapshot_id": backup.LocalSnapshotID, "remote_snapshot_id": "", "s3_destination_id": "",
+ "local_snapshot_id": safetyBackup.LocalSnapshotID, "remote_snapshot_id": "", "s3_destination_id": "",
"policy_id": "", "error": "",
}).Error
}
diff --git a/backend/cli/recovery/recovery_test.go b/backend/cli/recovery/recovery_test.go
index 64f58f03a..e6e459f45 100644
--- a/backend/cli/recovery/recovery_test.go
+++ b/backend/cli/recovery/recovery_test.go
@@ -9,9 +9,11 @@ import (
recoverytypes "github.com/getarcaneapp/arcane/types/v2/recovery"
"github.com/libtnb/sqlite"
"github.com/stretchr/testify/require"
+ "go.getarcane.app/sys/crypto"
"gorm.io/gorm"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/backup"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/internal/system"
@@ -24,7 +26,7 @@ func newRestoredDatabaseForTestInternal(t *testing.T, runIDs ...string) (string,
require.NoError(t, err)
db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{})
require.NoError(t, err)
- require.NoError(t, db.AutoMigrate(&system.SystemBackupRun{}, &activity.Activity{}, &settings.SettingVariable{}))
+ require.NoError(t, db.AutoMigrate(&system.SystemBackupRun{}, &activity.Activity{}, &settings.SettingVariable{}, &backup.SystemBackupRecoveryConfig{}))
require.NoError(t, db.Create(&settings.SettingVariable{Key: "projectsDirectory", Value: "/app/data/projects"}).Error)
for i, id := range runIDs {
require.NoError(t, db.Create(&system.SystemBackupRun{
@@ -46,9 +48,13 @@ func newRestoredDatabaseForTestInternal(t *testing.T, runIDs ...string) (string,
func TestFinalizeRestoredBackupInternal(t *testing.T) {
databaseURL, dsn := newRestoredDatabaseForTestInternal(t, "older", "selected")
+ crypto.InitEncryption(&crypto.Config{EncryptionKey: "recovery-finalize-test-key-32bytes", Environment: "test"})
+ recoveryKey, err := backup.GenerateRecoveryKey()
+ require.NoError(t, err)
require.NoError(t, finalizeRestoredBackupInternal(t.Context(), databaseURL, "selected", "activity-1", recoverytypes.RestoreRequest{
BackupID: "request-id", RemoteSnapshotID: "snapshot-1", S3DestinationID: "destination-1", Size: 1234,
+ RecoveryKey: recoveryKey,
ProjectsSetting: "/srv/projects",
SafetyBackup: &recoverytypes.SafetyBackup{
ID: "safety", LocalSnapshotID: "safety-snapshot", Size: 4321,
@@ -77,6 +83,9 @@ func TestFinalizeRestoredBackupInternal(t *testing.T) {
var projectsSetting settings.SettingVariable
require.NoError(t, db.First(&projectsSetting, "key = ?", "projectsDirectory").Error)
require.Equal(t, "/srv/projects", projectsSetting.Value)
+ storedKey, err := backup.NewRecoveryKeyStore(&database.DB{DB: db}).Get(t.Context())
+ require.NoError(t, err)
+ require.Equal(t, recoveryKey, storedKey)
}
func TestFinalizeRestoredBackupInternalFallsBackForLegacyManifest(t *testing.T) {
diff --git a/backend/internal/backup/service.go b/backend/internal/backup/service.go
index 830492928..386d2988a 100644
--- a/backend/internal/backup/service.go
+++ b/backend/internal/backup/service.go
@@ -497,7 +497,7 @@ func (e *Engine) ForgetSnapshots(ctx context.Context, dockerClient *client.Clien
}
defer unlock()
output, err := e.runContainer(ctx, dockerClient, repository, password, []string{"snapshots", "--json"})
- if len(snapshotIDs) == 0 && err != nil && strings.Contains(err.Error(), rusticRepositoryMissingMessage) {
+ if err != nil && strings.Contains(err.Error(), rusticRepositoryMissingMessage) {
return nil
}
if err != nil {
@@ -540,6 +540,62 @@ func (e *Engine) ChangeRepositoryPassword(ctx context.Context, dockerClient *cli
return err
}
+func (e *Engine) RekeyRepository(ctx context.Context, dockerClient *client.Client, repository Repository, newPassword string, currentPasswords ...string) (string, error) {
+ _, err := e.ListSnapshots(ctx, dockerClient, repository, newPassword)
+ if err == nil || strings.Contains(err.Error(), rusticRepositoryMissingMessage) {
+ return "", nil
+ }
+ for _, candidate := range currentPasswords {
+ if candidate == "" || candidate == newPassword {
+ continue
+ }
+ if changeErr := e.ChangeRepositoryPassword(ctx, dockerClient, repository, candidate, newPassword); changeErr == nil {
+ return candidate, nil
+ }
+ }
+ return "", fmt.Errorf("backup repository %s does not open with the recovery key: %w", repository.ID, err)
+}
+
+func (e *Engine) RekeyRepositories(ctx context.Context, dockerClient *client.Client, repositories []Repository, newPassword string, currentPasswords ...string) error {
+ type rekeyed struct {
+ repository Repository
+ previous string
+ }
+ changed := make([]rekeyed, 0, len(repositories))
+ for _, repository := range repositories {
+ previous, err := e.RekeyRepository(ctx, dockerClient, repository, newPassword, currentPasswords...)
+ if err != nil {
+ errs := []error{err}
+ var failedRollbackErrs []error
+ for _, candidate := range currentPasswords {
+ if candidate == "" || candidate == newPassword {
+ continue
+ }
+ _, rollbackErr := e.RekeyRepository(context.WithoutCancel(ctx), dockerClient, repository, candidate, newPassword)
+ if rollbackErr == nil {
+ failedRollbackErrs = nil
+ break
+ }
+ failedRollbackErrs = append(failedRollbackErrs, fmt.Errorf("failed to restore the previous key on backup repository %s: %w", repository.ID, rollbackErr))
+ }
+ errs = append(errs, failedRollbackErrs...)
+ for _, done := range changed {
+ if _, rollbackErr := e.RekeyRepository(context.WithoutCancel(ctx), dockerClient, done.repository, done.previous, newPassword); rollbackErr != nil {
+ errs = append(errs, fmt.Errorf("failed to restore the previous key on backup repository %s: %w", done.repository.ID, rollbackErr))
+ }
+ }
+ return errors.Join(errs...)
+ }
+ if previous != "" {
+ changed = append(changed, rekeyed{
+ repository: repository,
+ previous: previous,
+ })
+ }
+ }
+ return nil
+}
+
// Replicate copies one snapshot into another repository through a temporary volume, never the live data.
// Rustic's native `copy` needs a TOML profile for the target, which the env-only Repository cannot express.
func (e *Engine) Replicate(ctx context.Context, dockerClient *client.Client, from Repository, fromSnapshotID string, to Repository, password, label string, tags ...string) (Snapshot, error) {
@@ -970,20 +1026,27 @@ func CheckScheduledRemote(ctx context.Context, db *database.DB, destinations *s3
return fmt.Errorf("%w: %s", ErrRemoteRepositoryMissing, result.Reason)
}
-// ExpiredRunIDs returns the IDs of succeeded runs with snapshots that fall
-// outside the newest keep entries for one policy, oldest last. Callers delete
-// each run through their own delete path so snapshots are forgotten too.
+// ExpiredRunIDs returns the IDs of succeeded and failed runs with snapshots that
+// fall outside the newest keep entries of their status for one policy, oldest
+// last within each status. Callers delete each run through their own delete
+// path so snapshots are forgotten too.
func ExpiredRunIDs(ctx context.Context, db *database.DB, table, policyID string, keep int) ([]string, error) {
- var ids []string
- err := db.WithContext(ctx).
- Table(table).
- Where(
- "policy_id = ? AND status = ? AND (COALESCE(local_snapshot_id, '') <> '' OR COALESCE(remote_snapshot_id, '') <> '')",
- policyID,
- "succeeded",
- ).
- Order("created_at DESC").
- Offset(keep).
- Pluck("id", &ids).Error
- return ids, err
+ var expired []string
+ for _, status := range []string{"succeeded", "failed"} {
+ var ids []string
+ if err := db.WithContext(ctx).
+ Table(table).
+ Where(
+ "policy_id = ? AND status = ? AND (COALESCE(local_snapshot_id, '') <> '' OR COALESCE(remote_snapshot_id, '') <> '')",
+ policyID,
+ status,
+ ).
+ Order("created_at DESC").
+ Offset(keep).
+ Pluck("id", &ids).Error; err != nil {
+ return nil, err
+ }
+ expired = append(expired, ids...)
+ }
+ return expired, nil
}
diff --git a/backend/internal/s3/service.go b/backend/internal/s3/service.go
index 6a9fa4c30..44c49090f 100644
--- a/backend/internal/s3/service.go
+++ b/backend/internal/s3/service.go
@@ -164,7 +164,7 @@ func (s *S3DestinationService) UpdateS3Destination(ctx context.Context, id strin
}
// The persisted configuration without the
// decrypted secret, for comparing connection fields against an update.
- if configuration.SecretAccessKey == "" && !s3.ConnectionFieldsEqual(s3.Configuration{
+ stored := s3.Configuration{
ID: destination.ID,
Name: destination.Name,
Endpoint: destination.Endpoint,
@@ -174,9 +174,15 @@ func (s *S3DestinationService) UpdateS3Destination(ctx context.Context, id strin
Prefix: destination.Prefix,
UseSSL: destination.UseSSL,
ForcePathStyle: destination.ForcePathStyle,
- }.Normalized(), configuration) {
+ }.Normalized()
+ if configuration.SecretAccessKey == "" && !s3.ConnectionFieldsEqual(stored, configuration) {
return nil, errS3SecretRequired
}
+ if stored.Endpoint != configuration.Endpoint || stored.Bucket != configuration.Bucket || stored.Region != configuration.Region || stored.Prefix != configuration.Prefix {
+ if ensureUnusedErr := s.ensureUnused(ctx, id); ensureUnusedErr != nil {
+ return nil, fmt.Errorf("the bucket, prefix, endpoint or region cannot change while backups use the destination: %w", ensureUnusedErr)
+ }
+ }
encryptedSecret, err := func() (string, error) {
if configuration.SecretAccessKey == "" {
return destination.SecretAccessKey, nil
@@ -210,6 +216,16 @@ func (s *S3DestinationService) DeleteS3Destination(ctx context.Context, id strin
if err != nil {
return err
}
+ if ensureUnusedErr := s.ensureUnused(ctx, id); ensureUnusedErr != nil {
+ return ensureUnusedErr
+ }
+ if deleteDestinationErr := s.db.WithContext(ctx).Delete(destination).Error; deleteDestinationErr != nil {
+ return fmt.Errorf("failed to delete S3 destination: %w", deleteDestinationErr)
+ }
+ return nil
+}
+
+func (s *S3DestinationService) ensureUnused(ctx context.Context, id string) error {
inUse, err := s.DestinationInUse(ctx, id)
if err != nil {
return err
@@ -220,9 +236,6 @@ func (s *S3DestinationService) DeleteS3Destination(ctx context.Context, id strin
if checkRemoteReferencesErr := s.checkRemoteReferences(ctx, id); checkRemoteReferencesErr != nil {
return fmt.Errorf("%w: %s", ErrS3DestinationInUse, checkRemoteReferencesErr.Error())
}
- if deleteDestinationErr := s.db.WithContext(ctx).Delete(destination).Error; deleteDestinationErr != nil {
- return fmt.Errorf("failed to delete S3 destination: %w", deleteDestinationErr)
- }
return nil
}
diff --git a/backend/internal/s3/service_test.go b/backend/internal/s3/service_test.go
index acb492c31..7c9597f93 100644
--- a/backend/internal/s3/service_test.go
+++ b/backend/internal/s3/service_test.go
@@ -117,6 +117,41 @@ func TestS3DestinationService_DeleteRejectsConfiguredDestination(t *testing.T) {
require.ErrorIs(t, err, ErrS3DestinationInUse)
}
+func TestS3DestinationService_UpdateRejectsLocationChangeWhileInUse(t *testing.T) {
+ service, gormDB := setupS3DestinationServiceTest(t)
+ ctx := t.Context()
+ destination, err := service.CreateS3Destination(ctx, backup.CreateS3Destination{
+ Name: "In use",
+ Bucket: "arcane-backups",
+ Region: "us-east-1",
+ AccessKeyID: "access-key",
+ SecretAccessKey: "secret-key",
+ UseSSL: true,
+ })
+ require.NoError(t, err)
+ require.NoError(t, gormDB.Exec("INSERT INTO volume_backups (s3_destination_id, remote_snapshot_id) VALUES (?, ?)", destination.ID, "snapshot-1").Error)
+
+ _, err = service.UpdateS3Destination(ctx, destination.ID, backup.UpdateS3Destination{
+ Name: destination.Name,
+ Bucket: "other-bucket",
+ Region: destination.Region,
+ AccessKeyID: destination.AccessKeyID,
+ SecretAccessKey: "secret-key",
+ UseSSL: destination.UseSSL,
+ })
+ require.ErrorIs(t, err, ErrS3DestinationInUse)
+
+ updated, err := service.UpdateS3Destination(ctx, destination.ID, backup.UpdateS3Destination{
+ Name: "Renamed",
+ Bucket: destination.Bucket,
+ Region: destination.Region,
+ AccessKeyID: destination.AccessKeyID,
+ UseSSL: destination.UseSSL,
+ })
+ require.NoError(t, err)
+ require.Equal(t, "Renamed", updated.Name)
+}
+
func TestS3DestinationService_SyncS3Destinations(t *testing.T) {
service, gormDB := setupS3DestinationServiceTest(t)
ctx := t.Context()
diff --git a/backend/internal/system/children/backup/handler.go b/backend/internal/system/children/backup/handler.go
index e20567497..75e3894d0 100644
--- a/backend/internal/system/children/backup/handler.go
+++ b/backend/internal/system/children/backup/handler.go
@@ -238,7 +238,7 @@ func (h *Handler) Restore(ctx context.Context, input *RestoreSystemBackupInput)
Step: "Preparing system restore", Message: "Preparing Arcane system restore", SuccessMessage: "Arcane system restore started",
Metadata: database.JSON{"action": "restore_system_backup", "backupId": input.ID},
}, func(activityCtx context.Context) error {
- return h.service.RestoreBackup(activityCtx, input.ID, input.Body.RecoveryKey, *user)
+ return h.service.RestoreBackup(activityCtx, input.ID, input.Body.RecoveryKey)
})
if err != nil {
return nil, huma.Error500InternalServerError(err.Error())
diff --git a/backend/internal/system/children/backup/service.go b/backend/internal/system/children/backup/service.go
index 1403bdd09..6570013d5 100644
--- a/backend/internal/system/children/backup/service.go
+++ b/backend/internal/system/children/backup/service.go
@@ -83,6 +83,7 @@ type Store struct {
RemoteSnapshotIDs func(ctx context.Context, destinationID string) ([]string, error)
CountRuns func(ctx context.Context) (int64, error)
CountVolumeRepositoryRuns func(ctx context.Context) (int64, error)
+ CountRunningVolumeBackups func(ctx context.Context) (int64, error)
Policies func(ctx context.Context) ([]backuptypes.SystemBackupPolicy, error)
Policy func(ctx context.Context, id string) (*backuptypes.SystemBackupPolicy, error)
ReconcilePolicies func(
@@ -919,6 +920,11 @@ func (s *Service) PruneLocalRepository(ctx context.Context) error {
}
func (s *Service) DiscoverRemoteBackups(ctx context.Context, request backuptypes.DiscoverSystemBackupsRequest) (int, error) {
+ lease, err := s.acquireRun(ctx)
+ if err != nil {
+ return 0, err
+ }
+ defer lease.Release(ctx)
recoveryKey, err := s.recoveryKey(ctx, request.RecoveryKey)
if err != nil {
return 0, err
@@ -1019,9 +1025,14 @@ func (s *Service) UploadBackup(ctx context.Context, id string, request backuptyp
return run, nil
}
-func (s *Service) RestoreBackup(ctx context.Context, id, recoveryKey string, user usertypes.Actor) (opErr error) {
+func (s *Service) RestoreBackup(ctx context.Context, id, recoveryKey string) (opErr error) {
ctx, finish := s.startOperation(ctx, "restore", attribute.String("arcane.backup.id", id))
defer func() { finish(opErr) }()
+ lease, err := s.acquireRun(ctx)
+ if err != nil {
+ return err
+ }
+ defer lease.Release(ctx)
run, err := s.store.Run(ctx, id)
if err != nil {
return err
@@ -1064,10 +1075,14 @@ func (s *Service) RestoreBackup(ctx context.Context, id, recoveryKey string, use
return err
}
// A local safety snapshot is created before the detached helper is allowed to stop Arcane and replace its data.
- safetyBackup, err := s.CreateBackup(ctx, user, backuptypes.SystemBackupTriggerSafety, backuptypes.CreateSystemBackupRequest{
+ prepared, prepareErr := s.prepareBackup(ctx, backuptypes.SystemBackupTriggerSafety, backuptypes.CreateSystemBackupRequest{
Destination: backuptypes.SystemBackupDestinationLocal,
RecoveryKey: key,
})
+ if prepareErr != nil {
+ return fmt.Errorf("failed to create pre-restore system backup: %w", prepareErr)
+ }
+ safetyBackup, err := s.executeBackup(ctx, prepared)
if err != nil {
return fmt.Errorf("failed to create pre-restore system backup: %w", err)
}
@@ -1201,38 +1216,103 @@ func (s *Service) SetRecoveryKey(ctx context.Context, recoveryKey string) (*back
if err := backup.ValidateRecoveryKey(recoveryKey); err != nil {
return nil, err
}
- // Repositories only open with the key that initialized them, so refuse a rotation while backups exist.
- configured, err := s.recoveryKeys.Configured(ctx)
+ lease, err := s.acquireRun(ctx)
if err != nil {
return nil, err
}
- if configured {
- if current, keyErr := s.recoveryKeys.Get(ctx); keyErr == nil && current != recoveryKey {
- localRuns, countLocalBackupsErr := s.store.CountRuns(ctx)
- if countLocalBackupsErr != nil {
- return nil, countLocalBackupsErr
- }
- if localRuns > 0 {
- return nil, errors.New("delete the existing system backups before replacing the recovery key; they can only be opened with the current key")
- }
- volumeBackups, countVolumeBackupsErr := s.store.CountVolumeRepositoryRuns(ctx)
- if countVolumeBackupsErr != nil {
- return nil, countVolumeBackupsErr
- }
- if volumeBackups > 0 {
- return nil, errors.New("delete the existing volume backups before replacing the recovery key; they can only be opened with the current key")
- }
+ defer lease.Release(ctx)
+ previous, err := s.recoveryKeys.Get(ctx)
+ if err != nil {
+ if !errors.Is(err, backup.ErrRecoveryKeyNotConfigured) {
+ slog.WarnContext(ctx, "could not read the current recovery key; existing backups must already open with the new key", "error", err.Error())
}
+ previous = ""
}
- if setErr := s.recoveryKeys.Set(ctx, recoveryKey); setErr != nil {
- return nil, setErr
+ if previous == recoveryKey {
+ if setErr := s.recoveryKeys.Set(ctx, recoveryKey); setErr != nil {
+ return nil, setErr
+ }
+ return &backuptypes.SystemBackupRecoveryKeyStatus{Configured: true}, nil
+ }
+ runningVolumeBackupsErr := common.Classify(common.ErrConflict, errors.New("wait for the running volume backups to finish before replacing the recovery key"))
+ runningVolumeBackups, countErr := s.store.CountRunningVolumeBackups(ctx)
+ if countErr != nil {
+ return nil, countErr
}
- if migrateRepositoryPasswordsErr := s.volumeService.MigrateRepositoryPasswords(ctx); migrateRepositoryPasswordsErr != nil {
- slog.WarnContext(ctx, "failed to re-key volume backup repositories to the recovery key", "error", migrateRepositoryPasswordsErr.Error())
+ if runningVolumeBackups > 0 {
+ return nil, runningVolumeBackupsErr
+ }
+ if rekeyErr := s.rekeyRepositories(ctx, previous, recoveryKey); rekeyErr != nil {
+ return nil, rekeyErr
+ }
+ rollback := func(cause error) error {
+ if previous == "" {
+ return cause
+ }
+ return errors.Join(cause, s.rekeyRepositories(context.WithoutCancel(ctx), recoveryKey, previous))
+ }
+ runningVolumeBackups, countErr = s.store.CountRunningVolumeBackups(ctx)
+ if countErr != nil {
+ return nil, rollback(errors.Join(runningVolumeBackupsErr, countErr))
+ }
+ if runningVolumeBackups > 0 {
+ return nil, rollback(runningVolumeBackupsErr)
+ }
+ if setErr := s.recoveryKeys.Set(ctx, recoveryKey); setErr != nil {
+ return nil, rollback(setErr)
}
return &backuptypes.SystemBackupRecoveryKeyStatus{Configured: true}, nil
}
+func (s *Service) rekeyRepositories(ctx context.Context, previous, next string) error {
+ systemRuns, err := s.store.CountRuns(ctx)
+ if err != nil {
+ return err
+ }
+ volumeRuns, err := s.store.CountVolumeRepositoryRuns(ctx)
+ if err != nil {
+ return err
+ }
+ if systemRuns == 0 && volumeRuns == 0 {
+ return nil
+ }
+ var dockerClient *client.Client
+ var systemRepositories []backup.Repository
+ if systemRuns > 0 {
+ dockerClient, err = s.dockerService.GetClient(ctx)
+ if err != nil {
+ return err
+ }
+ local, localErr := s.localRepository(ctx, dockerClient, false)
+ if localErr != nil {
+ return localErr
+ }
+ systemRepositories = append(systemRepositories, local)
+ destinations, listErr := s.s3Destinations.ListAllS3Destinations(ctx)
+ if listErr != nil {
+ return fmt.Errorf("failed to list S3 destinations for re-key: %w", listErr)
+ }
+ for _, destination := range destinations {
+ remote, remoteErr := s.remoteRepository(ctx, destination.ID)
+ if remoteErr != nil {
+ return remoteErr
+ }
+ systemRepositories = append(systemRepositories, remote)
+ }
+ if rekeyErr := s.engine.RekeyRepositories(ctx, dockerClient, systemRepositories, next, previous); rekeyErr != nil {
+ return rekeyErr
+ }
+ }
+ if volumeRuns == 0 {
+ return nil
+ }
+ migrateErr := s.volumeService.MigrateRepositoryPasswords(ctx, previous, next)
+ if migrateErr == nil || len(systemRepositories) == 0 || previous == "" {
+ return migrateErr
+ }
+ return errors.Join(migrateErr, s.engine.RekeyRepositories(context.WithoutCancel(ctx), dockerClient, systemRepositories, previous, next))
+}
+
func (s *Service) GetPolicies(ctx context.Context) (*backuptypes.SystemBackupPolicyCollection, error) {
policies, err := s.store.Policies(ctx)
if err != nil {
@@ -1361,6 +1441,12 @@ func (s *Service) runScheduledBackup(ctx context.Context, policyID string) (sche
}
if runErr != nil {
slog.ErrorContext(ctx, "Scheduled Arcane system backup failed", "policyId", policy.ID, "error", runErr)
+ if policy.RetentionCount > 0 {
+ if retentionErr := s.applyRetention(ctx, policy.ID, policy.RetentionCount, policy.S3Enabled); retentionErr != nil {
+ slog.ErrorContext(ctx, "System backup retention failed", "policyId", policy.ID, "error", retentionErr)
+ return scheduler.Outcome{}, errors.Join(runErr, retentionErr)
+ }
+ }
return scheduler.Outcome{}, runErr
}
if policy.RetentionCount > 0 {
@@ -1379,11 +1465,19 @@ func (s *Service) runScheduledBackup(ctx context.Context, policyID string) (sche
// resumeScheduledBackup finishes the backup an interrupted scheduled run recorded, then applies retention.
func (s *Service) resumeScheduledBackup(ctx context.Context, previous scheduler.Run, policyID string) (scheduler.Outcome, error) {
outcome, err := s.reconcileBackup(ctx, previous, policyID, "")
- if err != nil || outcome.Status != scheduler.Succeeded {
- return outcome, err
+ if err == nil && outcome.Status != scheduler.Succeeded {
+ return outcome, nil
+ }
+ current, policyErr := s.store.Policy(ctx, policyID)
+ if policyErr != nil {
+ return outcome, errors.Join(err, policyErr)
}
- current, err := s.store.Policy(ctx, policyID)
if err != nil {
+ if current != nil && current.RetentionCount > 0 {
+ if applyRetentionErr := s.applyRetention(ctx, current.ID, current.RetentionCount, current.S3Enabled); applyRetentionErr != nil {
+ return outcome, errors.Join(err, applyRetentionErr)
+ }
+ }
return outcome, err
}
if current != nil && current.RetentionCount > 0 {
diff --git a/backend/internal/system/children/backup/service_test.go b/backend/internal/system/children/backup/service_test.go
index c03bebae2..644e8322a 100644
--- a/backend/internal/system/children/backup/service_test.go
+++ b/backend/internal/system/children/backup/service_test.go
@@ -6,15 +6,22 @@ import (
"github.com/getarcaneapp/arcane/types/v2/backup"
"github.com/libtnb/sqlite"
+ "github.com/moby/moby/client"
"github.com/stretchr/testify/require"
+ "go.getarcane.app/sys/crypto"
"gorm.io/gorm"
+ backupengine "github.com/getarcaneapp/arcane/backend/v2/internal/backup"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/docker"
"github.com/getarcaneapp/arcane/backend/v2/internal/s3"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/volume"
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow/flowtest"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
+ francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing"
)
func TestHistoryDestinationDecoration(t *testing.T) {
@@ -80,3 +87,42 @@ func TestRecoveryEnvironmentIncludesRuntimeSecrets(t *testing.T) {
require.Equal(t, "oidc-secret", environment["OIDC_CLIENT_SECRET"])
require.Equal(t, "0640", environment["FILE_PERM"])
}
+
+func TestSetRecoveryKeyKeepsCurrentKeyWhenRekeyFails(t *testing.T) {
+ gormDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
+ require.NoError(t, err)
+ require.NoError(t, gormDB.AutoMigrate(&backupengine.SystemBackupRecoveryConfig{}, &s3.S3Destination{}))
+ crypto.InitEncryption(&crypto.Config{EncryptionKey: "system-backup-rekey-test-key-32bytes", Environment: "test"})
+ db := &database.DB{DB: gormDB}
+ recoveryKeys := backupengine.NewRecoveryKeyStore(db)
+ oldKey := "QWERTY-ABCDEF-234567-GHIJKL-MNOPQR-STUVWX-YZ2345-ZXCVBN"
+ require.NoError(t, recoveryKeys.Set(t.Context(), oldKey))
+ runtime := francistest.New(t)
+ gate := runs.NewAdmission(runtime.Service(), t.Name())
+ require.NoError(t, gate.Register(runtime))
+ francistest.Start(t, runtime)
+ engine := backupengine.NewEngine(gate, nil)
+ t.Cleanup(func() { require.NoError(t, engine.Stop(context.WithoutCancel(t.Context()))) })
+ dockerClient, err := client.New(client.WithHost("tcp://127.0.0.1:1"), client.WithAPIVersion("1.41"))
+ require.NoError(t, err)
+ dockerService := &docker.DockerClientService{Client: dockerClient}
+ service := NewService(Dependencies{
+ Store: Store{
+ CountRuns: func(context.Context) (int64, error) { return 1, nil },
+ CountVolumeRepositoryRuns: func(context.Context) (int64, error) { return 0, nil },
+ CountRunningVolumeBackups: func(context.Context) (int64, error) { return 0, nil },
+ },
+ DB: db,
+ Docker: dockerService,
+ Volumes: volume.NewVolumeService(db, dockerService, nil, nil, nil, nil, nil, nil, nil, nil),
+ Engine: engine,
+ S3Destinations: s3.NewS3DestinationService(db, func(context.Context, string) error { return nil }),
+ RecoveryKeys: recoveryKeys,
+ })
+
+ _, err = service.SetRecoveryKey(t.Context(), "ASDFGH-QWERTY-765432-LKJHGF-RQPONM-XWVUTS-5432ZY-NBVCXZ")
+ require.Error(t, err)
+ storedKey, err := recoveryKeys.Get(t.Context())
+ require.NoError(t, err)
+ require.Equal(t, oldKey, storedKey)
+}
diff --git a/backend/internal/system/children/prune/service.go b/backend/internal/system/children/prune/service.go
index adbc25456..c90abb725 100644
--- a/backend/internal/system/children/prune/service.go
+++ b/backend/internal/system/children/prune/service.go
@@ -445,17 +445,24 @@ func (s *Service) pruneBuildCache(ctx context.Context, options system.PruneBuild
}
func (s *Service) pruneVolumes(ctx context.Context, options system.PruneVolumesOptions) (system.PruneAllResult, error) {
- report, err := s.volumeService.PruneVolumesWithOptions(ctx, options.Mode == system.PruneVolumeModeAll)
+ plan, err := s.preparePrunePlan(ctx, system.PruneAllRequest{Volumes: &options})
if err != nil {
return system.PruneAllResult{}, err
}
+ result, err := s.executePrunePlan(ctx, plan, scheduler.Run{}, "", false)
+ if err != nil {
+ return system.PruneAllResult{}, err
+ }
+ if !result.Success {
+ return system.PruneAllResult{}, fmt.Errorf("failed to prune volumes: %s", strings.Join(result.Errors, "; "))
+ }
- slog.InfoContext(ctx, "Volume prune completed", "volumesDeleted", len(report.VolumesDeleted), "spaceReclaimed", report.SpaceReclaimed)
+ slog.InfoContext(ctx, "Volume prune completed", "volumesDeleted", len(result.VolumesDeleted), "spaceReclaimed", result.VolumeSpaceReclaimed)
return system.PruneAllResult{
- VolumesDeleted: report.VolumesDeleted,
- SpaceReclaimed: report.SpaceReclaimed,
- VolumeSpaceReclaimed: report.SpaceReclaimed,
+ VolumesDeleted: result.VolumesDeleted,
+ SpaceReclaimed: result.VolumeSpaceReclaimed,
+ VolumeSpaceReclaimed: result.VolumeSpaceReclaimed,
}, nil
}
@@ -545,7 +552,9 @@ func (s *Service) PruneScheduled(ctx context.Context, environmentID string, req
func (s *Service) preparePrunePlan(ctx context.Context, req system.PruneAllRequest) (prunePlan, error) {
if req.Volumes != nil {
- s.volumeService.CleanupHelperContainers(ctx)
+ if _, reapErr := s.volumeService.ReapIdleHelpers(ctx, 0); reapErr != nil {
+ slog.WarnContext(ctx, "could not remove idle volume helpers before prune", "error", reapErr.Error())
+ }
}
usage, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.DiskUsageResult, error) {
return cli.DiskUsage(ctx, client.DiskUsageOptions{
diff --git a/backend/internal/system/children/prune/service_test.go b/backend/internal/system/children/prune/service_test.go
new file mode 100644
index 000000000..4e029427e
--- /dev/null
+++ b/backend/internal/system/children/prune/service_test.go
@@ -0,0 +1,76 @@
+package prune
+
+import (
+ "context"
+ "fmt"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "path"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/getarcaneapp/arcane/types/v2/system"
+ "github.com/libtnb/sqlite"
+ "github.com/moby/moby/client"
+ "github.com/stretchr/testify/require"
+ "gorm.io/gorm"
+
+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/docker"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/event"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/volume"
+)
+
+func TestPruneVolumesSkipsBackupVolume(t *testing.T) {
+ var mu sync.Mutex
+ var deleted []string
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch {
+ case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/system/df"):
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = fmt.Fprint(w, `{"Volumes":[`+
+ `{"Name":"orphan","Driver":"local","UsageData":{"RefCount":0,"Size":100}},`+
+ `{"Name":"arcane-backups","Driver":"local","UsageData":{"RefCount":0,"Size":200}}]}`)
+ case r.Method == http.MethodDelete && strings.Contains(r.URL.Path, "/volumes/"):
+ mu.Lock()
+ deleted = append(deleted, path.Base(r.URL.Path))
+ mu.Unlock()
+ w.WriteHeader(http.StatusNoContent)
+ default:
+ http.Error(w, "unexpected request "+r.Method+" "+r.URL.Path, http.StatusNotFound)
+ }
+ }))
+ t.Cleanup(server.Close)
+
+ dialer := &net.Dialer{}
+ dockerClient, err := client.New(
+ client.WithHost(server.URL),
+ client.WithAPIVersion("1.41"),
+ client.WithHTTPClient(server.Client()),
+ client.WithDialContext(func(ctx context.Context, _, _ string) (net.Conn, error) {
+ return dialer.DialContext(ctx, "tcp", server.Listener.Addr().String())
+ }),
+ )
+ require.NoError(t, err)
+ t.Cleanup(func() { _ = dockerClient.Close() })
+
+ gormDB, err := gorm.Open(sqlite.Open(fmt.Sprintf("file:%s?mode=memory&cache=shared", t.Name())), &gorm.Config{})
+ require.NoError(t, err)
+ require.NoError(t, gormDB.AutoMigrate(&event.Event{}))
+ db := &database.DB{DB: gormDB}
+
+ dockerService := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(dockerClient)
+ volumeService := volume.NewVolumeService(db, dockerService, event.NewEventService(db, nil, nil), nil, nil, nil, nil, nil, nil, nil)
+ service := NewService(time.UTC, dockerService, nil, nil, volumeService, nil)
+
+ result, err := service.pruneVolumes(t.Context(), system.PruneVolumesOptions{Mode: system.PruneVolumeModeAll})
+ require.NoError(t, err)
+ require.Equal(t, []string{"orphan"}, result.VolumesDeleted)
+ require.EqualValues(t, 100, result.VolumeSpaceReclaimed)
+ mu.Lock()
+ defer mu.Unlock()
+ require.Equal(t, []string{"orphan"}, deleted)
+}
diff --git a/backend/internal/system/service.go b/backend/internal/system/service.go
index 119c6c4c4..aba63acff 100644
--- a/backend/internal/system/service.go
+++ b/backend/internal/system/service.go
@@ -607,6 +607,11 @@ func (s *SystemService) backupStore(s3Destinations *s3.S3DestinationService) sys
err := s.db.WithContext(ctx).Model(&volume.VolumeBackup{}).Where("format = ?", volume.VolumeBackupFormatRustic).Count(&count).Error
return count, err
},
+ CountRunningVolumeBackups: func(ctx context.Context) (int64, error) {
+ var count int64
+ err := s.db.WithContext(ctx).Model(&volume.VolumeBackup{}).Where("status = ?", volume.VolumeBackupStatusRunning).Count(&count).Error
+ return count, err
+ },
Policies: func(ctx context.Context) ([]backuptypes.SystemBackupPolicy, error) {
var policies []SystemBackupPolicy
if err := s.db.WithContext(ctx).Order("created_at ASC").Find(&policies).Error; err != nil {
diff --git a/backend/internal/volume/children/backup/children/restore/service.go b/backend/internal/volume/children/backup/children/restore/service.go
index f61e9f8ae..6cc5244f9 100644
--- a/backend/internal/volume/children/backup/children/restore/service.go
+++ b/backend/internal/volume/children/backup/children/restore/service.go
@@ -42,6 +42,23 @@ for member do
done
tar -xzf "$archive" -C /volume -- "$@"`
+const replaceVolumeScript = `set -e
+: "${1:?}"
+for f in /volume/* /volume/.[!.]* /volume/..?*; do
+ case "$f" in "$1") ;; *) rm -rf -- "$f" ;; esac
+done
+for f in "$1"/* "$1"/.[!.]* "$1"/..?*; do
+ if stat -c '%A' -- "$f" >/dev/null 2>&1; then mv -- "$f" /volume/; fi
+done
+rmdir -- "$1"`
+
+const restoreArchiveScript = `set -e
+tmp=$(mktemp -d /volume/.restore_tmp.XXXXXX)
+tar -tzf "$1" >/dev/null
+tar -xzf "$1" -C "$tmp"
+set -- "$tmp"
+` + replaceVolumeScript
+
// Dependencies are the backup runtime, storage, and helper-container operations restores run through.
type Dependencies struct {
Docker *docker.DockerClientService
@@ -364,13 +381,9 @@ func (s *Service) restoreArchiveBackup(ctx context.Context, dockerClient *client
fullRestoreCmd := []string{
"sh",
"-c",
- fmt.Sprintf(
- "set -e; tmp=$(mktemp -d /volume/.restore_tmp.XXXXXX); tar -tzf /backups/%s >/dev/null; tar -xzf "+
- "/backups/%s -C \"$tmp\"; find /volume -mindepth 1 -maxdepth 1 -not -path \"$tmp\" -exec rm -rf -- {} +; "+
- "find \"$tmp\" -mindepth 1 -maxdepth 1 -exec mv -- {} /volume/ \\;; rmdir \"$tmp\"",
- filename,
- filename,
- ),
+ restoreArchiveScript,
+ "sh",
+ path.Join("/backups", filename),
}
resp, err := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerCreateResult, error) {
return dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{
@@ -436,7 +449,7 @@ func (s *Service) RestoreArchiveMembers(ctx context.Context, containerID, filena
return stderr, nil
}
-func (s *Service) UploadAndRestore(ctx context.Context, volumeName string, archive io.ReadSeeker, filename string, user usertypes.Actor) error {
+func (s *Service) UploadAndRestore(ctx context.Context, volumeName string, archive io.ReadSeeker, filename string, user usertypes.Actor) (err error) {
slog.DebugContext(ctx, "volume service: upload and restore", "volume", volumeName, "filename", filename, "user", user.ID)
gzr, err := gzip.NewReader(archive)
@@ -451,6 +464,17 @@ func (s *Service) UploadAndRestore(ctx context.Context, volumeName string, archi
unlock := s.deps.Locks.Lock(volumeName)
defer unlock()
+ stopped, err := s.deps.StopContainers(ctx, volumeName, user, true)
+ containersStopped := len(stopped) > 0
+ defer func() {
+ if containersStopped {
+ _, restartErr := s.deps.StartContainers(context.WithoutCancel(ctx), stopped, user)
+ err = errors.Join(err, restartErr)
+ }
+ }()
+ if err != nil {
+ return err
+ }
preBackupID, err := s.deps.SafetyBackup(ctx, volumeName, user)
if err != nil {
return fmt.Errorf("failed to create pre-restore backup: %w", err)
@@ -472,6 +496,13 @@ func (s *Service) UploadAndRestore(ctx context.Context, volumeName string, archi
if err != nil {
return fmt.Errorf("failed to create temp restore dir: %w", err)
}
+ replaceStarted := false
+ defer func() {
+ if !replaceStarted {
+ _, _, cleanupErr := s.deps.Exec(context.WithoutCancel(ctx), containerID, "", []string{"rm", "-rf", "--", tmpDir})
+ err = errors.Join(err, cleanupErr)
+ }
+ }()
if strings.TrimSpace(stderr) != "" {
slog.DebugContext(ctx, "volume service: restore temp dir stderr", "volume", volumeName, "stderr", strings.TrimSpace(stderr))
}
@@ -487,7 +518,7 @@ func (s *Service) UploadAndRestore(ctx context.Context, volumeName string, archi
return fmt.Errorf("failed to restore from uploaded archive: %w", err)
}
- _, stderr, err = s.deps.Exec(ctx, containerID, "", []string{"sh", "-c", fmt.Sprintf("test -n \"$(find %s -mindepth 1 -maxdepth 1 -print -quit)\"", tmpDir)})
+ _, stderr, err = s.deps.Exec(ctx, containerID, "", []string{"sh", "-c", `test -n "$(find "$1" -mindepth 1 -maxdepth 1)"`, "sh", tmpDir})
if err != nil {
return fmt.Errorf("uploaded archive appears empty or invalid: %w", err)
}
@@ -495,21 +526,21 @@ func (s *Service) UploadAndRestore(ctx context.Context, volumeName string, archi
slog.DebugContext(ctx, "volume service: restore validate stderr", "volume", volumeName, "stderr", strings.TrimSpace(stderr))
}
- _, stderr, err = s.deps.Exec(ctx, containerID, "", []string{"sh", "-c", "rm -rf /volume/* /volume/.[!.]* /volume/..?* 2>/dev/null || true"})
+ replaceStarted = true
+ _, stderr, err = s.deps.Exec(ctx, containerID, "", []string{"sh", "-c", replaceVolumeScript, "sh", tmpDir})
if err != nil {
- return fmt.Errorf("failed to clear volume before restore: %w", err)
+ return fmt.Errorf("failed to replace volume contents: %w", err)
}
if strings.TrimSpace(stderr) != "" {
- slog.DebugContext(ctx, "volume service: restore clear stderr", "volume", volumeName, "stderr", strings.TrimSpace(stderr))
+ slog.DebugContext(ctx, "volume service: restore replace stderr", "volume", volumeName, "stderr", strings.TrimSpace(stderr))
}
- moveCmd := fmt.Sprintf("find %s -mindepth 1 -maxdepth 1 -exec mv -- {} /volume/ \\; && rmdir %s", tmpDir, tmpDir)
- _, stderr, err = s.deps.Exec(ctx, containerID, "", []string{"sh", "-c", moveCmd})
- if err != nil {
- return fmt.Errorf("failed to move restored files into place: %w", err)
- }
- if strings.TrimSpace(stderr) != "" {
- slog.DebugContext(ctx, "volume service: restore move stderr", "volume", volumeName, "stderr", strings.TrimSpace(stderr))
+ if containersStopped {
+ stopped, err = s.deps.StartContainers(context.WithoutCancel(ctx), stopped, user)
+ containersStopped = len(stopped) > 0
+ if err != nil {
+ return err
+ }
}
metadata := database.JSON{
diff --git a/backend/internal/volume/children/backup/children/restore/service_test.go b/backend/internal/volume/children/backup/children/restore/service_test.go
index 936e85029..8577d3e2e 100644
--- a/backend/internal/volume/children/backup/children/restore/service_test.go
+++ b/backend/internal/volume/children/backup/children/restore/service_test.go
@@ -19,27 +19,35 @@ func TestArchiveMembersForSelectionExpandsFolders(t *testing.T) {
}
func TestRestoreBackupFilesScriptUsesSupportedTooling(t *testing.T) {
- for _, unsupported := range []string{
- "if [",
- "elif [",
- "while [",
- "test ",
- "local ",
- "find -P",
- "-printf",
- "sort -z",
- "xargs",
- "cat --",
- "dirname",
- "install ",
- "--files-from",
- "tar -r",
- "$((",
- } {
- require.NotContains(t, restoreBackupFilesScript, unsupported)
- }
- if shellPath, err := exec.LookPath("sh"); err == nil {
- output, syntaxErr := exec.Command(shellPath, "-n", "-c", restoreBackupFilesScript).CombinedOutput()
- require.NoErrorf(t, syntaxErr, "restore script has invalid syntax: %s", output)
+ for _, script := range []string{restoreBackupFilesScript, replaceVolumeScript, restoreArchiveScript} {
+ for _, unsupported := range []string{
+ "if [",
+ "elif [",
+ "while [",
+ "test ",
+ "local ",
+ "find -P",
+ "-printf",
+ "sort -z",
+ "xargs",
+ "cat --",
+ "dirname",
+ "install ",
+ "--files-from",
+ "tar -r",
+ "$((",
+ "-not",
+ "-exec",
+ "-quit",
+ } {
+ require.NotContains(t, script, unsupported)
+ }
+ if shellPath, err := exec.LookPath("sh"); err == nil {
+ output, syntaxErr := exec.Command(shellPath, "-n", "-c", script).CombinedOutput()
+ require.NoErrorf(t, syntaxErr, "restore script has invalid syntax: %s", output)
+ }
+ require.NotContains(t, script, "rm -rf /volume/*")
+ require.NotContains(t, script, "2>/dev/null || true")
}
+ require.Contains(t, replaceVolumeScript, `case "$f" in "$1")`)
}
diff --git a/backend/internal/volume/children/backup/helpers.go b/backend/internal/volume/children/backup/helpers.go
index 4f1ee4a90..4a6a25b1a 100644
--- a/backend/internal/volume/children/backup/helpers.go
+++ b/backend/internal/volume/children/backup/helpers.go
@@ -101,7 +101,10 @@ func (s *Service) StorageMount(ctx context.Context, dockerClient *client.Client,
return dockerClient.VolumeInspect(ctx, s.deps.BackupVolumeName, client.VolumeInspectOptions{})
}); inspectErr != nil {
if _, createErr := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.VolumeCreateResult, error) {
- return dockerClient.VolumeCreate(ctx, client.VolumeCreateOptions{Name: s.deps.BackupVolumeName})
+ return dockerClient.VolumeCreate(ctx, client.VolumeCreateOptions{
+ Name: s.deps.BackupVolumeName,
+ Labels: map[string]string{libarcane.InternalResourceLabel: "true"},
+ })
}); createErr != nil {
return mount.Mount{}, fmt.Errorf("failed to create backup volume: %w", createErr)
}
@@ -264,10 +267,6 @@ func (s *Service) volumeBackupPassword(ctx context.Context, dockerClient *client
if done, _ := s.rekeyed.Load(repository.ID); done == key {
continue
}
- if _, listErr := s.deps.Engine.ListSnapshots(ctx, dockerClient, repository, key); listErr == nil {
- s.rekeyed.Store(repository.ID, key)
- continue
- }
if repository.ID == localVolumeRepositoryID {
writable, localErr := s.localRusticRepository(ctx, dockerClient, false)
if localErr != nil {
@@ -276,11 +275,14 @@ func (s *Service) volumeBackupPassword(ctx context.Context, dockerClient *client
}
repository = writable
}
- if changeErr := s.deps.Engine.ChangeRepositoryPassword(ctx, dockerClient, repository, legacyPassword, key); changeErr != nil {
- slog.DebugContext(ctx, "volume backup repository was not re-keyed", "repository", repository.ID, "error", changeErr.Error())
+ previous, rekeyErr := s.deps.Engine.RekeyRepository(ctx, dockerClient, repository, key, legacyPassword)
+ if rekeyErr != nil {
+ slog.DebugContext(ctx, "volume backup repository was not re-keyed", "repository", repository.ID, "error", rekeyErr.Error())
continue
}
- slog.InfoContext(ctx, "Re-keyed volume backup repository to the recovery key", "repository", repository.ID)
+ if previous != "" {
+ slog.InfoContext(ctx, "Re-keyed volume backup repository to the recovery key", "repository", repository.ID)
+ }
s.rekeyed.Store(repository.ID, key)
}
return key, nil
diff --git a/backend/internal/volume/children/backup/service.go b/backend/internal/volume/children/backup/service.go
index 37b14cfec..173ab38bf 100644
--- a/backend/internal/volume/children/backup/service.go
+++ b/backend/internal/volume/children/backup/service.go
@@ -904,22 +904,15 @@ func discoveredVolumeBackup(destinationID, root string, snapshot backup.Discover
}
// MigrateRepositoryPasswords re-keys this instance's own volume backup repositories to the recovery key; other instances re-key their own roots.
-func (s *Service) MigrateRepositoryPasswords(ctx context.Context) error {
- if s.deps.RecoveryKeys == nil || s.deps.Engine == nil {
+func (s *Service) MigrateRepositoryPasswords(ctx context.Context, previousKey, newKey string) error {
+ if s.deps.Engine == nil {
return nil
}
- _, err := s.deps.RecoveryKeys.Get(ctx)
- if errors.Is(err, backup.ErrRecoveryKeyNotConfigured) {
- return nil
- }
- if err != nil {
- return err
- }
dockerClient, err := s.deps.Docker.GetClient(ctx)
if err != nil {
return err
}
- local, err := s.localRusticRepository(ctx, dockerClient, true)
+ local, err := s.localRusticRepository(ctx, dockerClient, false)
if err != nil {
return err
}
@@ -937,8 +930,14 @@ func (s *Service) MigrateRepositoryPasswords(ctx context.Context) error {
repositories = append(repositories, remote)
}
}
- _, err = s.volumeBackupPassword(ctx, dockerClient, repositories...)
- return err
+ legacyPassword := kit.SHA256Hex(legacyVolumePasswordSalt + s.deps.EncryptionKey)
+ if rekeyErr := s.deps.Engine.RekeyRepositories(ctx, dockerClient, repositories, newKey, previousKey, legacyPassword); rekeyErr != nil {
+ return rekeyErr
+ }
+ for _, repository := range repositories {
+ s.rekeyed.Store(repository.ID, newKey)
+ }
+ return nil
}
// StartBackup persists a running backup and submits its workflow.
diff --git a/backend/internal/volume/children/backup/service_test.go b/backend/internal/volume/children/backup/service_test.go
index 3c70af66f..01bacaa4f 100644
--- a/backend/internal/volume/children/backup/service_test.go
+++ b/backend/internal/volume/children/backup/service_test.go
@@ -59,7 +59,7 @@ func (s *volumeBackupPolicyScheduler) Submit(_ context.Context, request schedule
return scheduler.Run{ID: request.RunID, JobID: request.JobID, EnvironmentID: request.EnvironmentID, Status: scheduler.Queued}, nil
}
-func TestVolumeBackupPolicy_RetentionIgnoresFailedRuns(t *testing.T) {
+func TestVolumeBackupPolicy_RetentionCountsFailedRunsSeparately(t *testing.T) {
gormDB, err := gorm.Open(sqlite.Open("file:volume-backup-retention-failed?mode=memory&cache=shared"), &gorm.Config{})
require.NoError(t, err)
applyVolumeBackupMigrations(t, gormDB)
@@ -73,10 +73,18 @@ func TestVolumeBackupPolicy_RetentionIgnoresFailedRuns(t *testing.T) {
"INSERT INTO volume_backups (id, volume_name, size, created_at, policy_id, status) VALUES (?, ?, ?, ?, ?, ?)",
"failed-run", "app-data", 0, time.Now(), policyID, backuptypes.VolumeBackupStatusFailed,
).Error)
+ require.NoError(t, gormDB.Exec(
+ "INSERT INTO volume_backups (id, volume_name, size, created_at, policy_id, status, local_snapshot_id) VALUES (?, ?, ?, ?, ?, ?, ?)",
+ "failed-snap-new", "app-data", 0, time.Now().Add(-2*time.Hour), policyID, backuptypes.VolumeBackupStatusFailed, "snapshot-2",
+ ).Error)
+ require.NoError(t, gormDB.Exec(
+ "INSERT INTO volume_backups (id, volume_name, size, created_at, policy_id, status, local_snapshot_id) VALUES (?, ?, ?, ?, ?, ?, ?)",
+ "failed-snap-old", "app-data", 0, time.Now().Add(-3*time.Hour), policyID, backuptypes.VolumeBackupStatusFailed, "snapshot-3",
+ ).Error)
expired, err := backup.ExpiredRunIDs(t.Context(), &database.DB{DB: gormDB}, "volume_backups", policyID, 1)
require.NoError(t, err)
- require.Empty(t, expired)
+ require.Equal(t, []string{"failed-snap-old"}, expired)
}
func TestVolumeBackupPolicy_ScheduledRunCreatesActivity(t *testing.T) {
diff --git a/backend/internal/volume/helpers.go b/backend/internal/volume/helpers.go
index 02303065d..79fb030aa 100644
--- a/backend/internal/volume/helpers.go
+++ b/backend/internal/volume/helpers.go
@@ -330,7 +330,7 @@ func (s *VolumeService) CleanupHelperContainers(ctx context.Context) {
// ReapIdleHelpers removes tracked helpers idle for idleTimeout; untracked orphans are left to the startup sweep.
func (s *VolumeService) ReapIdleHelpers(ctx context.Context, idleTimeout time.Duration) (int, error) {
- if idleTimeout <= 0 {
+ if idleTimeout < 0 {
return 0, nil
}
diff --git a/backend/internal/volume/service.go b/backend/internal/volume/service.go
index 1bd364666..34d7097da 100644
--- a/backend/internal/volume/service.go
+++ b/backend/internal/volume/service.go
@@ -215,8 +215,8 @@ func (s *VolumeService) PruneLocalBackupRepository(ctx context.Context) error {
}
// MigrateRepositoryPasswords re-keys this instance's own volume backup repositories to the recovery key; other instances re-key their own roots.
-func (s *VolumeService) MigrateRepositoryPasswords(ctx context.Context) error {
- return s.backup.MigrateRepositoryPasswords(ctx)
+func (s *VolumeService) MigrateRepositoryPasswords(ctx context.Context, previousKey, newKey string) error {
+ return s.backup.MigrateRepositoryPasswords(ctx, previousKey, newKey)
}
func (s *VolumeService) backupStore() volumebackup.Store {
@@ -491,27 +491,23 @@ func (s *VolumeService) DeleteVolume(ctx context.Context, name string, force boo
func (s *VolumeService) PruneVolumes(ctx context.Context) (*volumetypes.PruneReport, error) {
slog.DebugContext(ctx, "volume service: prune volumes")
- return s.PruneVolumesWithOptions(ctx, false)
-}
-
-func (s *VolumeService) PruneVolumesWithOptions(ctx context.Context, all bool) (*volumetypes.PruneReport, error) {
- slog.DebugContext(ctx, "volume service: prune volumes with options", "all", all)
dockerClient, err := s.dockerService.GetClient(ctx)
if err != nil {
return nil, fmt.Errorf("failed to connect to Docker: %w", err)
}
- // Stop all read-only browse helpers first; a helper mounting a volume marks it
+ // Stop idle read-only browse helpers first; a helper mounting a volume marks it
// "in use" and would prevent VolumePrune from reclaiming an otherwise-unused
- // volumetypes. Helpers are re-created on demand on the next browse request.
- s.CleanupHelperContainers(ctx)
+ // volume. Helpers are re-created on demand on the next browse request.
+ if _, reapErr := s.ReapIdleHelpers(ctx, 0); reapErr != nil {
+ slog.WarnContext(ctx, "could not remove idle volume helpers before prune", "error", reapErr.Error())
+ }
// Docker's VolumesPrune behavior (API v1.42+):
// - Without 'all' flag: Only removes anonymous (unnamed) volumes that are not in use
// - With 'all=true' flag: Removes ALL unused volumes (both named and anonymous)
// Note: Volumes are considered "in use" if referenced by any container (running or stopped)
volumePruneResult, err := dockerClient.VolumePrune(ctx, client.VolumePruneOptions{
- All: all,
Filters: make(client.Filters).Add("label!", internalVolumePruneFilterValue),
})
if err != nil {
@@ -520,7 +516,7 @@ func (s *VolumeService) PruneVolumesWithOptions(ctx context.Context, all bool) (
metadata := database.JSON{
"action": "prune",
- "all": all,
+ "all": false,
"volumesDeleted": len(volumePruneResult.Report.VolumesDeleted),
"spaceReclaimed": volumePruneResult.Report.SpaceReclaimed,
"internalVolumeFilterLabel": internalVolumePruneFilterValue,