Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix: detect missing S3 repositories and pause remote backups (#3869)
Kyle Mendell4ff83971e1a2
diff --git a/backend/internal/backup/remote.go b/backend/internal/backup/remote.gonew file mode 100644index 000000000..f641ea771--- /dev/null+++ b/backend/internal/backup/remote.go@@ -0,0 +1,77 @@+package backup++import (+ "context"+ "fmt"+ "time"++ "emperror.dev/errors"+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"+ s3domain "github.com/getarcaneapp/arcane/backend/v2/internal/s3"+ s3util "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/s3"+ backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"+)++var ErrRemoteRepositoryMissing = errors.New("S3 backup storage is missing")++const RemoteDisabledMessage = "S3 backup storage is missing. Remote backups were disabled; edit the policy to resume."++// RemoteSnapshotChecker memoizes targeted checks within one response budget.+// A nil result means the remote copy could not be verified.+func RemoteSnapshotChecker(ctx context.Context, destinations *s3domain.S3DestinationService, root string) func(string, string) *bool {+ checked := make(map[string]*backuptypes.RepositoryObservation)+ deadline := time.Now().Add(10 * time.Second)+ return func(destinationID, snapshotID string) *bool {+ if destinations == nil || root == "" || destinationID == "" || snapshotID == "" {+ return nil+ }+ key := destinationID + ":" + snapshotID+ observation, seen := checked[key]+ if !seen {+ checked[key] = nil+ checkCtx, cancel := context.WithDeadline(ctx, deadline)+ defer cancel()+ configuration, err := destinations.Configuration(checkCtx, destinationID)+ if err != nil {+ return nil+ }+ result, err := s3util.CheckRepository(checkCtx, configuration, root, snapshotID)+ if err != nil {+ return nil+ }+ observation = &result+ checked[key] = observation+ }+ if observation == nil {+ return nil+ }+ return new(observation.Available && observation.SnapshotAvailable)+ }+}++// CheckScheduledRemote permits first-use initialization, but detects lost repositories.+func CheckScheduledRemote(ctx context.Context, db *database.DB, destinations *s3domain.S3DestinationService, table, destinationID, root string) error {+ if destinations == nil {+ return errors.New("S3 destinations are unavailable")+ }+ configuration, err := destinations.Configuration(ctx, destinationID)+ if err != nil {+ return err+ }+ checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second)+ defer cancel()+ result, err := s3util.CheckRepository(checkCtx, configuration, root, "")+ if err != nil || result.Available {+ return err+ }+ if result.Reason == s3util.RepositoryReasonMissingRepository {+ var retained int64+ if err := db.WithContext(ctx).Table(table).Where("s3_destination_id = ? AND COALESCE(remote_snapshot_id, '') <> ''", destinationID).Count(&retained).Error; err != nil {+ return err+ }+ if retained == 0 {+ return nil+ }+ }+ return fmt.Errorf("%w: %s", ErrRemoteRepositoryMissing, result.Reason)+}diff --git a/backend/internal/s3/service_test.go b/backend/internal/s3/service_test.goindex 64caa72ae..fca44df8c 100644--- a/backend/internal/s3/service_test.go+++ b/backend/internal/s3/service_test.go@@ -13,6 +13,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"+ s3config "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/s3"backuptypes "github.com/getarcaneapp/arcane/types/v2/backup""github.com/libtnb/sqlite""github.com/stretchr/testify/require"@@ -272,6 +273,69 @@ func TestS3DestinationService_TestS3DestinationRoundTrip(t *testing.T) {require.Equal(t, "arcane-backups", persisted.Bucket)require.Equal(t, "production", persisted.Prefix)require.Empty(t, persisted.Region)++ for _, scenario := range []struct {+ name string+ code string+ status int+ reason string+ wantError bool+ snapshotError bool+ configOnly bool+ }{+ {name: "only the requested snapshot is read"},+ {name: "repository check reads only config", configOnly: true},+ {name: "confirmed bucket loss", code: "NoSuchBucket", status: http.StatusNotFound, reason: "missing_bucket"},+ {name: "confirmed repository loss", code: "NoSuchKey", status: http.StatusNotFound, reason: "missing_repository"},+ {name: "access denied is not deletion", code: "AccessDenied", status: http.StatusForbidden, wantError: true},+ {name: "generic not found is inconclusive", code: "NotFound", status: http.StatusNotFound, wantError: true},+ {name: "missing snapshot leaves repository available", code: "NoSuchKey", status: http.StatusNotFound, snapshotError: true},+ {name: "snapshot access denied is inconclusive", code: "AccessDenied", status: http.StatusForbidden, snapshotError: true, wantError: true},+ } {+ t.Run(scenario.name, func(t *testing.T) {+ snapshotID := strings.Repeat("a", 64)+ metadataServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ isConfig := r.URL.Path == "/arcane-backups/production/repository/config"+ isSnapshot := !scenario.configOnly && r.URL.Path == "/arcane-backups/production/repository/snapshots/"+snapshotID+ if r.Method != http.MethodGet || (!isConfig && !isSnapshot) || r.URL.Query().Get("list-type") != "" {+ t.Errorf("unexpected metadata request: %s %s", r.Method, r.URL)+ w.WriteHeader(http.StatusBadRequest)+ return+ }+ if r.Header.Get("Range") != "bytes=0-0" {+ t.Errorf("metadata request must be limited to one byte")+ }+ if scenario.code != "" && (!scenario.snapshotError || !isConfig) {+ w.Header().Set("Content-Type", "application/xml")+ w.WriteHeader(scenario.status)+ _, _ = io.WriteString(w, "<Error><Code>"+scenario.code+"</Code><Message>test response</Message></Error>")+ return+ }+ w.Header().Set("Content-Range", "bytes 0-0/100")+ w.WriteHeader(http.StatusPartialContent)+ _, _ = io.WriteString(w, "x")+ }))+ defer metadataServer.Close()+ configuration := s3config.Configuration{+ Name: "Metadata destination", Endpoint: metadataServer.URL, Bucket: "arcane-backups",+ AccessKeyID: "test-access", SecretAccessKey: "test-secret", Prefix: "production", ForcePathStyle: true,+ }+ requestedID := snapshotID+ if scenario.configOnly {+ requestedID = ""+ }+ observation, checkErr := s3config.CheckRepository(t.Context(), configuration, "repository", requestedID)+ if scenario.wantError {+ require.Error(t, checkErr)+ require.Empty(t, observation.Reason)+ return+ }+ require.NoError(t, checkErr)+ require.Equal(t, scenario.reason, observation.Reason)+ require.Equal(t, scenario.reason == "", observation.Available)+ require.Equal(t, scenario.reason == "" && !scenario.configOnly && !scenario.snapshotError, observation.SnapshotAvailable)+ })+ }}func TestListS3DestinationsByIDInternal(t *testing.T) {diff --git a/backend/internal/systembackup/service.go b/backend/internal/systembackup/service.goindex 55cbf0bbc..75e5fb9d6 100644--- a/backend/internal/systembackup/service.go+++ b/backend/internal/systembackup/service.go@@ -525,10 +525,12 @@ func (s *SystemBackupService) ListBackups(ctx context.Context, params paginationnames = available}}+ remoteAvailable := backup.RemoteSnapshotChecker(ctx, s.s3Destinations, "arcane-system-recovery")result := make([]backuptypes.SystemBackupRun, len(runs))for i := range runs {runs[i].S3DestinationName = names[runs[i].S3DestinationID].Nameresult[i] = runs[i].ToDTO()+ result[i].RemoteAvailable = remoteAvailable(runs[i].S3DestinationID, runs[i].RemoteSnapshotID)}return result, response, nil}@@ -1084,18 +1086,18 @@ func (s *SystemBackupService) DeleteBackup(ctx context.Context, id, recoveryKeyif err != nil {return err}- return s.deleteRunsInternal(ctx, []*SystemBackupRun{run}, recoveryKey)+ return s.deleteRunsInternal(ctx, []*SystemBackupRun{run}, recoveryKey, true)}// deleteRunsInternal forgets snapshots grouped by repository under the caller's run lease.-func (s *SystemBackupService) deleteRunsInternal(ctx context.Context, runs []*SystemBackupRun, recoveryKey string) error {+func (s *SystemBackupService) deleteRunsInternal(ctx context.Context, runs []*SystemBackupRun, recoveryKey string, includeRemote bool) error {var localRuns []*SystemBackupRunremoteGroups := make(map[string][]*SystemBackupRun)for _, run := range runs {if run.LocalSnapshotID != "" {localRuns = append(localRuns, run)}- if run.RemoteSnapshotID != "" {+ if includeRemote && run.RemoteSnapshotID != "" {remoteGroups[run.S3DestinationID] = append(remoteGroups[run.S3DestinationID], run)}}@@ -1123,7 +1125,6 @@ func (s *SystemBackupService) deleteRunsInternal(ctx context.Context, runs []*Sy}continue}- run.Error = deleteErr.Error()if saveErr := s.db.WithContext(ctx).Save(run).Error; saveErr != nil {deleteErr = errors.Combine(deleteErr, saveErr)}@@ -1579,7 +1580,7 @@ func (s *SystemBackupService) runScheduledBackupInternal(ctx context.Context, pooutcome := jobcontext.ConfirmedTarget(previous, policy.ID)if outcome.Status == schedulertypes.Succeeded {if policy.RetentionCount > 0 {- if err := s.applyRetentionInternal(ctx, policy.ID, policy.RetentionCount); err != nil {+ if err := s.applyRetentionInternal(ctx, policy.ID, policy.RetentionCount, policy.S3Enabled); err != nil {outcome.Status = schedulertypes.Partialreturn outcome, err}@@ -1587,6 +1588,13 @@ func (s *SystemBackupService) runScheduledBackupInternal(ctx context.Context, poreturn outcome, nil}}+ remoteDisabled, checkErr := s.disableMissingS3Internal(ctx, policy)+ if checkErr != nil {+ return schedulertypes.Outcome{}, checkErr+ }+ if remoteDisabled && !policy.LocalEnabled {+ return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: backup.RemoteDisabledMessage}, nil+ }var run *SystemBackupRunactivityID, runErr := activitylib.RunHandlerActivity(ctx, s.activityService, activitylib.HandlerOptions{EnvironmentID: "0", Type: activitytypes.TypeResourceAction, ResourceType: "system_backup",@@ -1611,12 +1619,15 @@ func (s *SystemBackupService) runScheduledBackupInternal(ctx context.Context, poreturn schedulertypes.Outcome{ActivityID: activityID}, runErr}if policy.RetentionCount > 0 {- if retentionErr := s.applyRetentionInternal(ctx, policy.ID, policy.RetentionCount); retentionErr != nil {+ if retentionErr := s.applyRetentionInternal(ctx, policy.ID, policy.RetentionCount, policy.S3Enabled); retentionErr != nil {slog.ErrorContext(ctx, "System backup retention failed", "policyId", policy.ID, "error", retentionErr)return schedulertypes.Outcome{Status: schedulertypes.Partial, ActivityID: activityID, Message: "Backup completed but retention failed"}, retentionErr}}slog.InfoContext(ctx, "Scheduled Arcane system backup completed", "backupId", run.ID, "policyId", policy.ID)+ if remoteDisabled {+ return schedulertypes.Outcome{Status: schedulertypes.Partial, ActivityID: activityID, Message: backup.RemoteDisabledMessage}, nil+ }return schedulertypes.Outcome{Status: schedulertypes.Succeeded, ActivityID: activityID}, nil}@@ -1648,7 +1659,7 @@ func (s *SystemBackupService) rescheduleSystemBackupPolicyInternal(ctx context.Creturn outcome, err}if current != nil && current.RetentionCount > 0 {- if err := s.applyRetentionInternal(ctx, policyID, current.RetentionCount); err != nil {+ if err := s.applyRetentionInternal(ctx, policyID, current.RetentionCount, current.S3Enabled); err != nil {outcome.Status = schedulertypes.Partialoutcome.Message = "Backup completed but retention failed"return outcome, err@@ -1682,7 +1693,7 @@ func (s *SystemBackupService) RegisterBackupJobOnStartup(ctx context.Context) {slog.InfoContext(ctx, "Registered backup schedules", "systemPolicies", len(policies), "volumePolicies", volumePolicyCount)}-func (s *SystemBackupService) applyRetentionInternal(ctx context.Context, policyID string, keep int) error {+func (s *SystemBackupService) applyRetentionInternal(ctx context.Context, policyID string, keep int, includeRemote bool) error {expired, err := backup.ExpiredRunIDs(ctx, s.db, "system_backup_runs", policyID, keep)if err != nil || len(expired) == 0 {return err@@ -1699,5 +1710,32 @@ func (s *SystemBackupService) applyRetentionInternal(ctx context.Context, policyif err := s.db.WithContext(ctx).Where("id IN ?", expired).Find(&runs).Error; err != nil {return err}- return s.deleteRunsInternal(ctx, runs, "")+ return s.deleteRunsInternal(ctx, runs, "", includeRemote)+}++func (s *SystemBackupService) disableMissingS3Internal(ctx context.Context, policy *SystemBackupPolicy) (bool, error) {+ if !policy.S3Enabled {+ return false, nil+ }+ err := backup.CheckScheduledRemote(ctx, s.db, s.s3Destinations, "system_backup_runs", policy.S3DestinationID, "arcane-system-recovery")+ if !errors.Is(err, backup.ErrRemoteRepositoryMissing) {+ return false, nil+ }+ column := "enabled"+ if policy.LocalEnabled {+ column = "s3_enabled"+ }+ result := s.db.WithContext(ctx).Model(&SystemBackupPolicy{}).+ Where("id = ? AND s3_destination_id = ? AND enabled = ? AND s3_enabled = ? AND local_enabled = ?", policy.ID, policy.S3DestinationID, true, true, policy.LocalEnabled).+ Update(column, false)+ if result.Error != nil || result.RowsAffected == 0 {+ return false, result.Error+ }+ if policy.LocalEnabled {+ policy.S3Enabled = false+ } else {+ policy.Enabled = false+ }+ s.rescheduleSystemBackupPolicyInternal(ctx, policy)+ return true, nil}diff --git a/backend/internal/systembackup/service_test.go b/backend/internal/systembackup/service_test.goindex cf08d592b..e6f007229 100644--- a/backend/internal/systembackup/service_test.go+++ b/backend/internal/systembackup/service_test.go@@ -2,6 +2,8 @@ package systembackupimport ("context"+ "net/http"+ "net/http/httptest""slices""testing""time"@@ -11,6 +13,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/database"recoverytypes "github.com/getarcaneapp/arcane/backend/v2/internal/recovery"+ s3domain "github.com/getarcaneapp/arcane/backend/v2/internal/s3""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/entityjobs"backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"@@ -113,6 +116,33 @@ func TestSystemBackupPoliciesRegisterIndependentJobs(t *testing.T) {require.Equal(t, firstID, collection.Policies[0].ID)require.Equal(t, 9, collection.Policies[0].RetentionCount)require.Len(t, scheduler.jobs, 1)++ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ w.Header().Set("Content-Type", "application/xml")+ w.WriteHeader(http.StatusNotFound)+ _, _ = w.Write([]byte("<Error><Code>NoSuchBucket</Code></Error>"))+ }))+ defer server.Close()+ require.NoError(t, gormDB.AutoMigrate(&s3domain.S3Destination{}))+ service.s3Destinations = s3domain.NewS3DestinationService(service.db)+ destination, err := service.s3Destinations.CreateS3Destination(t.Context(), backuptypes.CreateS3Destination{+ Name: "Missing storage", Endpoint: server.URL, Bucket: "backups", AccessKeyID: "test", SecretAccessKey: "test", ForcePathStyle: true,+ })+ require.NoError(t, err)+ for _, local := range []bool{false, true} {+ policy := &SystemBackupPolicy{Enabled: true, LocalEnabled: local, S3Enabled: true, S3DestinationID: destination.ID, Schedule: "0 0 2 * * *"}+ require.NoError(t, gormDB.Create(policy).Error)+ require.NoError(t, gormDB.Model(policy).Update("local_enabled", local).Error)+ service.rescheduleSystemBackupPolicyInternal(t.Context(), policy)+ disabled, err := service.disableMissingS3Internal(t.Context(), policy)+ require.NoError(t, err)+ require.True(t, disabled)+ require.NoError(t, gormDB.First(policy, "id = ?", policy.ID).Error)+ require.Equal(t, local, policy.Enabled)+ require.Equal(t, !local, policy.S3Enabled)+ require.Equal(t, local, scheduler.HasJob(service.jobs.JobName(policy.ID)))+ }+}func TestSystemBackupPolicyRequiresConfiguredRecoveryKeyWhenEnabled(t *testing.T) {diff --git a/backend/internal/systembackup/system_volume.go b/backend/internal/systembackup/system_volume.goindex c29db4397..b84d680f7 100644--- a/backend/internal/systembackup/system_volume.go+++ b/backend/internal/systembackup/system_volume.go@@ -154,13 +154,7 @@ func (s *SystemBackupService) UpdateSystemVolumeBackupConfig(ctx context.Context*policy = normalizedreturn nil},- Persist: func(ctx context.Context, policies []backuptypes.SystemVolumeBackupPolicy) error {- encoded, encodeErr := json.Marshal(backuptypes.SystemVolumeBackupPolicyCollection{Policies: policies})- if encodeErr != nil {- return fmt.Errorf("encode policies: %w", encodeErr)- }- return s.settingsService.UpdateSetting(ctx, systemVolumeBackupConfigKey, string(encoded))- },+ Persist: s.saveSystemVolumeBackupPoliciesInternal,Unregister: func(ctx context.Context, policyID string) {s.jobs.Unregister(ctx, systemVolumeBackupJobPrefix+policyID)},@@ -385,6 +379,13 @@ func (s *SystemBackupService) runScheduledSystemVolumeBackupInternal(ctx contextif policy == nil || !policy.Enabled {return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil}+ remoteDisabled, checkErr := s.disableMissingVolumeS3Internal(ctx, policy)+ if checkErr != nil {+ return schedulertypes.Outcome{}, checkErr+ }+ if remoteDisabled && !policy.LocalEnabled {+ return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: backup.RemoteDisabledMessage}, nil+ }result, err := s.runSystemVolumeBackupsInternal(ctx, backuptypes.RunSystemVolumeBackupsRequest{PolicyID: policyID}, volume.VolumeBackupTriggerScheduled)if errors.Is(err, ErrSystemBackupAlreadyRunning) {slog.InfoContext(ctx, "Scheduled system-managed volume backups skipped; a system backup is running", "policyId", policyID)@@ -396,7 +397,10 @@ func (s *SystemBackupService) runScheduledSystemVolumeBackupInternal(ctx context}slog.InfoContext(ctx, "Scheduled system-managed volume backups completed", "policyId", policyID, "matched", result.Matched, "succeeded", result.Succeeded, "failed", result.Failed, "skipped", result.Skipped)outcome := schedulertypes.Outcome{Status: schedulertypes.Succeeded}- if result.Failed > 0 {+ if remoteDisabled {+ outcome.Message = backup.RemoteDisabledMessage+ }+ if result.Failed > 0 || remoteDisabled {outcome.Status = schedulertypes.Partial}for _, failure := range result.Failures {@@ -453,6 +457,20 @@ func (s *SystemBackupService) ListBackupHistory(ctx context.Context, params pagireturn nil, pagination.Response{}, fmt.Errorf("list backup history: %w", err)}decorateHistoryDestinationsInternal(ctx, s.s3Destinations, history)+ systemAvailable := backup.RemoteSnapshotChecker(ctx, s.s3Destinations, "arcane-system-recovery")+ volumeRoot := ""+ if s.settingsService != nil {+ volumeRoot = "arcane-volume-backups/" + s.settingsService.GetSettingsConfig().InstanceID.Value+ }+ volumeAvailable := backup.RemoteSnapshotChecker(ctx, s.s3Destinations, volumeRoot)+ for i := range history {+ check := systemAvailable+ if history[i].ResourceType == "volume" {+ check = volumeAvailable+ }+ history[i].RemoteAvailable = check(history[i].S3DestinationID, history[i].RemoteSnapshotID)+ }+page.GrandTotalItems = page.TotalItemsreturn history, page, nil}@@ -469,3 +487,47 @@ func decorateHistoryDestinationsInternal(ctx context.Context, service *s3domain.history[i].S3DestinationName = destinations[history[i].S3DestinationID].Name}}++func (s *SystemBackupService) saveSystemVolumeBackupPoliciesInternal(ctx context.Context, policies []backuptypes.SystemVolumeBackupPolicy) error {+ encoded, err := json.Marshal(backuptypes.SystemVolumeBackupPolicyCollection{Policies: policies})+ if err != nil {+ return fmt.Errorf("encode policies: %w", err)+ }+ return s.settingsService.UpdateSetting(ctx, systemVolumeBackupConfigKey, string(encoded))+}++func (s *SystemBackupService) disableMissingVolumeS3Internal(ctx context.Context, policy *backuptypes.SystemVolumeBackupPolicy) (bool, error) {+ if !policy.S3Enabled {+ return false, nil+ }+ root := "arcane-volume-backups/" + s.settingsService.GetSettingsConfig().InstanceID.Value+ err := backup.CheckScheduledRemote(ctx, s.db, s.s3Destinations, "volume_backups", policy.S3DestinationID, root)+ if !errors.Is(err, backup.ErrRemoteRepositoryMissing) {+ return false, nil+ }+ collection, err := s.loadSystemVolumeBackupPoliciesInternal()+ if err != nil {+ return false, err+ }+ for i := range collection.Policies {+ current := &collection.Policies[i]+ if current.ID != policy.ID {+ continue+ }+ if current.S3DestinationID != policy.S3DestinationID || current.S3Enabled != policy.S3Enabled || current.Enabled != policy.Enabled || current.LocalEnabled != policy.LocalEnabled {+ return false, nil+ }+ if current.LocalEnabled {+ current.S3Enabled = false+ } else {+ current.Enabled = false+ }+ if err := s.saveSystemVolumeBackupPoliciesInternal(ctx, collection.Policies); err != nil {+ return false, err+ }+ *policy = *current+ s.rescheduleSystemVolumeBackupInternal(ctx, policy)+ return true, nil+ }+ return false, nil+}diff --git a/backend/internal/volume/backup.go b/backend/internal/volume/backup.goindex b12487fd5..b79b71058 100644--- a/backend/internal/volume/backup.go+++ b/backend/internal/volume/backup.go@@ -382,8 +382,14 @@ func (s *VolumeService) ListBackupsPaginated(ctx context.Context, volumeName str}}}+ root := ""+ if s.settingsService != nil {+ root = "arcane-volume-backups/" + s.settingsService.GetSettingsConfig().InstanceID.Value+ }+ remoteAvailable := backup.RemoteSnapshotChecker(ctx, s.s3Destinations, root)for i := range backups {backups[i].Type = volumeBackupManagementTypeInternal(backups[i].PolicyID)+ backups[i].RemoteAvailable = remoteAvailable(backups[i].S3DestinationID, backups[i].RemoteSnapshotID)}return backups, pagination.BuildResponse(totalItems, totalItems, params), nil@@ -738,7 +744,7 @@ func (s *VolumeService) executeBackupInternal(ctx context.Context, entry *Volume// and the deferred status-save would otherwise mark it failed and hide it// from ExpiredRunIDs forever.if trigger != VolumeBackupTriggerSafety && plan.policy != nil && plan.policy.RetentionCount > 0 {- if retentionErr := s.applyVolumeBackupRetentionInternal(ctx, plan.policy.ID, plan.policy.RetentionCount); retentionErr != nil {+ if retentionErr := s.applyVolumeBackupRetentionInternal(ctx, plan.policy.ID, plan.policy.RetentionCount, plan.s3Enabled); retentionErr != nil {slog.ErrorContext(ctx, "Volume backup retention failed", "volume", volumeName, "policy", plan.policy.ID, "error", retentionErr)}}@@ -843,11 +849,11 @@ func (s *VolumeService) deleteBackupInternal(ctx context.Context, backupID strinif entry.Format == VolumeBackupFormatArchive {return s.deleteArchiveBackupInternal(ctx, &entry, user)}- return s.deleteRusticBackupsInternal(ctx, []*VolumeBackup{&entry}, user)+ return s.deleteRusticBackupsInternal(ctx, []*VolumeBackup{&entry}, user, true)}// deleteRusticBackupsInternal forgets snapshots grouped by repository so each repository is pruned once.-func (s *VolumeService) deleteRusticBackupsInternal(ctx context.Context, entries []*VolumeBackup, user *common.User) error {+func (s *VolumeService) deleteRusticBackupsInternal(ctx context.Context, entries []*VolumeBackup, user *common.User, includeRemote bool) error {dockerClient, err := s.dockerService.GetClient(ctx)if err != nil {return err@@ -858,7 +864,7 @@ func (s *VolumeService) deleteRusticBackupsInternal(ctx context.Context, entriesif entry.LocalSnapshotID != "" {localEntries = append(localEntries, entry)}- if entry.RemoteSnapshotID != "" {+ if includeRemote && entry.RemoteSnapshotID != "" {remoteGroups[entry.S3DestinationID] = append(remoteGroups[entry.S3DestinationID], entry)}}@@ -883,7 +889,6 @@ func (s *VolumeService) deleteRusticBackupsInternal(ctx context.Context, entriesdefault:entry.Destination = volumetypes.BackupDestinationS3}- entry.Error = deleteErr.Error()if saveErr := s.db.WithContext(ctx).Save(entry).Error; saveErr != nil {deleteErr = errors.Combine(deleteErr, saveErr)}@@ -1766,7 +1771,7 @@ func (s *VolumeService) UpdateBackupPolicies(ctx context.Context, volumeName strreturn s.GetBackupPolicies(ctx, volumeName)}-func (s *VolumeService) applyVolumeBackupRetentionInternal(ctx context.Context, policyID string, retentionCount int) error {+func (s *VolumeService) applyVolumeBackupRetentionInternal(ctx context.Context, policyID string, retentionCount int, includeRemote bool) error {expired, err := backup.ExpiredRunIDs(ctx, s.db, "volume_backups", policyID, retentionCount)if err != nil || len(expired) == 0 {return err@@ -1775,7 +1780,7 @@ func (s *VolumeService) applyVolumeBackupRetentionInternal(ctx context.Context,if err := s.db.WithContext(ctx).Where("id IN ?", expired).Find(&entries).Error; err != nil {return err}- return s.deleteRusticBackupsInternal(ctx, entries, nil)+ return s.deleteRusticBackupsInternal(ctx, entries, nil, includeRemote)}func (s *VolumeService) runScheduledBackupInternal(ctx context.Context, policyID string) (schedulertypes.Outcome, error) {@@ -1792,6 +1797,13 @@ func (s *VolumeService) runScheduledBackupInternal(ctx context.Context, policyIDreturn outcome, nil}}+ remoteDisabled, checkErr := s.disableMissingS3Internal(ctx, &policy)+ if checkErr != nil {+ return schedulertypes.Outcome{}, checkErr+ }+ if remoteDisabled && !policy.LocalEnabled {+ return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: backup.RemoteDisabledMessage}, nil+ }var entry *VolumeBackupactivityID, err := activitylib.RunHandlerActivity(ctx, s.activityService, activitylib.HandlerOptions{EnvironmentID: "0",@@ -1828,6 +1840,9 @@ func (s *VolumeService) runScheduledBackupInternal(ctx context.Context, policyIDreturn schedulertypes.Outcome{}, err}slog.InfoContext(ctx, "Scheduled volume backup completed", "volume", policy.VolumeName, "backup_id", entry.ID, "remote_snapshot_id", entry.RemoteSnapshotID)+ if remoteDisabled {+ return schedulertypes.Outcome{Status: schedulertypes.Partial, ActivityID: activityID, Message: backup.RemoteDisabledMessage}, nil+ }return schedulertypes.Outcome{Status: schedulertypes.Succeeded, ActivityID: activityID}, nil}@@ -1884,3 +1899,30 @@ func (s *VolumeService) removeVolumeBackupPolicyInternal(ctx context.Context, voslog.WarnContext(ctx, "Failed to delete volume backup policy", "volume", volumeName, "error", err)}}++func (s *VolumeService) disableMissingS3Internal(ctx context.Context, policy *VolumeBackupPolicy) (bool, error) {+ if !policy.S3Enabled {+ return false, nil+ }+ err := backup.CheckScheduledRemote(ctx, s.db, s.s3Destinations, "volume_backups", policy.S3DestinationID, "arcane-volume-backups/"+s.settingsService.GetSettingsConfig().InstanceID.Value)+ if !errors.Is(err, backup.ErrRemoteRepositoryMissing) {+ return false, nil+ }+ column := "enabled"+ if policy.LocalEnabled {+ column = "s3_enabled"+ }+ result := s.db.WithContext(ctx).Model(&VolumeBackupPolicy{}).+ Where("id = ? AND s3_destination_id = ? AND enabled = ? AND s3_enabled = ? AND local_enabled = ?", policy.ID, policy.S3DestinationID, true, true, policy.LocalEnabled).+ Update(column, false)+ if result.Error != nil || result.RowsAffected == 0 {+ return false, result.Error+ }+ if policy.LocalEnabled {+ policy.S3Enabled = false+ } else {+ policy.Enabled = false+ }+ s.rescheduleVolumeBackupPolicyInternal(ctx, policy)+ return true, nil+}diff --git a/backend/internal/volume/model.go b/backend/internal/volume/model.goindex 205d8cf57..d77cebc0e 100644--- a/backend/internal/volume/model.go+++ b/backend/internal/volume/model.go@@ -49,6 +49,7 @@ type VolumeBackup struct {Error string `json:"error,omitempty" gorm:"column:error;type:text"`ActivityID *string `json:"activityId,omitempty" gorm:"-"`Type backuptypes.ManagementType `json:"type" gorm:"-"`+ RemoteAvailable *bool `json:"remoteAvailable,omitempty" gorm:"-"`}func (*VolumeBackup) TableName() string {@@ -57,6 +58,7 @@ func (*VolumeBackup) TableName() string {func (b *VolumeBackup) ToDTO() volume.BackupEntry {return volume.BackupEntry{+ RemoteAvailable: b.RemoteAvailable,ActivityID: b.ActivityID,ID: b.ID,VolumeName: b.VolumeName,diff --git a/backend/internal/volume/service_test.go b/backend/internal/volume/service_test.goindex 8dc58399e..924812316 100644--- a/backend/internal/volume/service_test.go+++ b/backend/internal/volume/service_test.go@@ -719,7 +719,7 @@ func TestVolumeBackupPolicy_RetentionIgnoresFailedRuns(t *testing.T) {}).Error)service := &VolumeService{db: &database.DB{DB: gormDB}}- require.NoError(t, service.applyVolumeBackupRetentionInternal(context.Background(), policyID, 1))+ require.NoError(t, service.applyVolumeBackupRetentionInternal(context.Background(), policyID, 1, true))var backups []VolumeBackuprequire.NoError(t, gormDB.Order("created_at ASC").Find(&backups).Error)diff --git a/backend/pkg/utils/s3/s3.go b/backend/pkg/utils/s3/s3.goindex 3983212ae..f7dd099b3 100644--- a/backend/pkg/utils/s3/s3.go+++ b/backend/pkg/utils/s3/s3.go@@ -10,6 +10,7 @@ import ("io""net/url""path"+ "slices""strings""emperror.dev/errors"@@ -18,9 +19,16 @@ import (awsconfig "github.com/aws/aws-sdk-go-v2/config""github.com/aws/aws-sdk-go-v2/credentials"awss3 "github.com/aws/aws-sdk-go-v2/service/s3"+ "github.com/aws/smithy-go"+ backuptypes "github.com/getarcaneapp/arcane/types/v2/backup""github.com/google/uuid")+const (+ RepositoryReasonMissingBucket = "missing_bucket"+ RepositoryReasonMissingRepository = "missing_repository"+)+const connectionTestPayload = "arcane-s3-connection-test"// Configuration contains the credentials and addressing options for an S3 destination.@@ -200,6 +208,61 @@ func TestConnection(ctx context.Context, configuration Configuration) (err errorreturn nil}+// CheckRepository reads only the config and, when supplied, one snapshot object.+func CheckRepository(ctx context.Context, configuration Configuration, root, snapshotID string) (backuptypes.RepositoryObservation, error) {+ configuration = configuration.Normalized()+ if err := configuration.Validate(true); err != nil {+ return backuptypes.RepositoryObservation{}, err+ }+ client, err := newClientInternal(ctx, configuration)+ if err != nil {+ return backuptypes.RepositoryObservation{}, fmt.Errorf("failed to configure S3 repository check: %w", err)+ }+ configKey := path.Join(configuration.Prefix, root, "config")+ if err := checkBackupObjectInternal(ctx, client, configuration.Bucket, configKey); err != nil {+ if isMissingResourceInternal(err, "NoSuchBucket") {+ return backuptypes.RepositoryObservation{Reason: RepositoryReasonMissingBucket}, nil+ }+ if isMissingResourceInternal(err, "NoSuchKey") {+ return backuptypes.RepositoryObservation{Reason: RepositoryReasonMissingRepository}, nil+ }+ return backuptypes.RepositoryObservation{}, err+ }+ result := backuptypes.RepositoryObservation{Available: true}+ if snapshotID == "" {+ return result, nil+ }+ snapshotKey := path.Join(configuration.Prefix, root, "snapshots", snapshotID)+ if err := checkBackupObjectInternal(ctx, client, configuration.Bucket, snapshotKey); err != nil {+ if isMissingResourceInternal(err, "NoSuchKey") {+ return result, nil+ }+ if isMissingResourceInternal(err, "NoSuchBucket") {+ return backuptypes.RepositoryObservation{Reason: RepositoryReasonMissingBucket}, nil+ }+ return backuptypes.RepositoryObservation{}, err+ }+ result.SnapshotAvailable = true+ return result, nil+}++// A one-byte GET preserves the named S3 errors that generic HEAD responses omit.+func checkBackupObjectInternal(ctx context.Context, client *awss3.Client, bucket, key string) error {+ object, err := client.GetObject(ctx, &awss3.GetObjectInput{Bucket: aws.String(bucket), Key: aws.String(key), Range: aws.String("bytes=0-0")})+ if err != nil {+ return err+ }+ return object.Body.Close()+}++func isMissingResourceInternal(err error, codes ...string) bool {+ var apiErr smithy.APIError+ if !errors.As(err, &apiErr) {+ return false+ }+ return slices.Contains(codes, apiErr.ErrorCode())+}+func newClientInternal(ctx context.Context, configuration Configuration) (*awss3.Client, error) {region := configuration.Regionif region == "" {diff --git a/frontend/messages/en.json b/frontend/messages/en.jsonindex 31d6caf9a..f38f95b93 100644--- a/frontend/messages/en.json+++ b/frontend/messages/en.json@@ -3272,6 +3272,7 @@"backups_trigger_scheduled": "Scheduled","backups_trigger_safety": "Pre-restore safety","backups_destination_s3": "S3",+ "backups_remote_unavailable": "Remote copy unavailable","backups_destination_local_s3": "Local + S3","s3_destinations_title": "S3 Destinations","s3_destinations_description": "Create and manage reusable S3-compatible storage destinations for Arcane and volume backups.",diff --git a/frontend/src/lib/components/arcane-table/cells/backup-destination-cell.svelte b/frontend/src/lib/components/arcane-table/cells/backup-destination-cell.svelteindex d07b6d945..d2abe7083 100644--- a/frontend/src/lib/components/arcane-table/cells/backup-destination-cell.svelte+++ b/frontend/src/lib/components/arcane-table/cells/backup-destination-cell.svelte@@ -1,9 +1,10 @@<script lang="ts">+ import * as m from '#lib/paraglide/messages.js';import { Badge } from '#lib/components/ui/badge/index.js';import type { BackupRun } from '#lib/types/backup.js';import { backupDestinationLabel, backupDestinationName } from '#lib/utils/backups.js';- let { item }: { item: Pick<BackupRun, 'destination' | 's3DestinationId' | 's3DestinationName'> } = $props();+ let { item }: { item: Pick<BackupRun, 'destination' | 's3DestinationId' | 's3DestinationName' | 'remoteAvailable'> } = $props();</script><div class="flex items-center gap-2">@@ -11,4 +12,7 @@{#if item.destination !== 'local' && backupDestinationName(item)}<span class="max-w-48 truncate text-xs text-muted-foreground">{backupDestinationName(item)}</span>{/if}+ {#if item.remoteAvailable === false}+ <span class="text-xs text-destructive">{m.backups_remote_unavailable()}</span>+ {/if}</div>diff --git a/frontend/src/lib/types/backup.ts b/frontend/src/lib/types/backup.tsindex 06be6ff3a..f37e4cdb8 100644--- a/frontend/src/lib/types/backup.ts+++ b/frontend/src/lib/types/backup.ts@@ -6,6 +6,7 @@ export type BackupTrigger = 'manual' | 'scheduled' | 'safety';export type BackupManagementType = 'system' | 'volume';export type BackupRun = {+ remoteAvailable?: boolean;activityId?: string;id: string;size: number;diff --git a/types/backup/repository.go b/types/backup/repository.gonew file mode 100644index 000000000..a1436c1de--- /dev/null+++ b/types/backup/repository.go@@ -0,0 +1,8 @@+package backup++// RepositoryObservation describes a read-only S3 metadata check.+type RepositoryObservation struct {+ Available bool+ Reason string+ SnapshotAvailable bool+}diff --git a/types/backup/system_backup.go b/types/backup/system_backup.goindex 58bf2926d..c17f4a256 100644--- a/types/backup/system_backup.go+++ b/types/backup/system_backup.go@@ -11,6 +11,7 @@ const ()type SystemBackupRun struct {+ RemoteAvailable *bool `json:"remoteAvailable,omitempty"`ActivityID string `json:"activityId,omitempty"`ID string `json:"id"`Size int64 `json:"size"`diff --git a/types/backup/system_volume_backup.go b/types/backup/system_volume_backup.goindex 78ee13ad7..d0879d10a 100644--- a/types/backup/system_volume_backup.go+++ b/types/backup/system_volume_backup.go@@ -96,6 +96,7 @@ type SystemVolumeBackupRunResult struct {// HistoryEntry is a common view over Arcane recovery and volume backup records.type HistoryEntry struct {+ RemoteAvailable *bool `json:"remoteAvailable,omitempty" gorm:"-"`ID string `json:"id" sortable:"true"`Size int64 `json:"size" sortable:"true"`CreatedAt time.Time `json:"createdAt" sortable:"true"`diff --git a/types/volume/backup.go b/types/volume/backup.goindex 7b6feb799..dddcb24fd 100644--- a/types/volume/backup.go+++ b/types/volume/backup.go@@ -28,6 +28,7 @@ const ()type BackupEntry struct {+ RemoteAvailable *bool `json:"remoteAvailable,omitempty"`ActivityID *string `json:"activityId,omitempty"`ID string `json:"id" doc:"Unique identifier of the backup"`VolumeName string `json:"volumeName" doc:"Name of the volume"`