Skip to content

Modern Docker Management, Designed for Everyone

fix: detect missing S3 repositories and pause remote backups (#3869)

Kyle Mendell4ff83971e1a2
diff --git a/backend/internal/backup/remote.go b/backend/internal/backup/remote.go
new file mode 100644
index 000000000..f641ea771
--- /dev/null
+++ b/backend/internal/backup/remote.go
@@ -0,0 +1,77 @@
+package backup
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "emperror.dev/errors"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"
+ s3domain "github.com/getarcaneapp/arcane/backend/v2/internal/s3"
+ s3util "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/s3"
+ backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"
+)
+
+var ErrRemoteRepositoryMissing = errors.New("S3 backup storage is missing")
+
+const RemoteDisabledMessage = "S3 backup storage is missing. Remote backups were disabled; edit the policy to resume."
+
+// RemoteSnapshotChecker memoizes targeted checks within one response budget.
+// A nil result means the remote copy could not be verified.
+func RemoteSnapshotChecker(ctx context.Context, destinations *s3domain.S3DestinationService, root string) func(string, string) *bool {
+ checked := make(map[string]*backuptypes.RepositoryObservation)
+ deadline := time.Now().Add(10 * time.Second)
+ return func(destinationID, snapshotID string) *bool {
+ if destinations == nil || root == "" || destinationID == "" || snapshotID == "" {
+ return nil
+ }
+ key := destinationID + ":" + snapshotID
+ observation, seen := checked[key]
+ if !seen {
+ checked[key] = nil
+ checkCtx, cancel := context.WithDeadline(ctx, deadline)
+ defer cancel()
+ configuration, err := destinations.Configuration(checkCtx, destinationID)
+ if err != nil {
+ return nil
+ }
+ result, err := s3util.CheckRepository(checkCtx, configuration, root, snapshotID)
+ if err != nil {
+ return nil
+ }
+ observation = &result
+ checked[key] = observation
+ }
+ if observation == nil {
+ return nil
+ }
+ return new(observation.Available && observation.SnapshotAvailable)
+ }
+}
+
+// CheckScheduledRemote permits first-use initialization, but detects lost repositories.
+func CheckScheduledRemote(ctx context.Context, db *database.DB, destinations *s3domain.S3DestinationService, table, destinationID, root string) error {
+ if destinations == nil {
+ return errors.New("S3 destinations are unavailable")
+ }
+ configuration, err := destinations.Configuration(ctx, destinationID)
+ if err != nil {
+ return err
+ }
+ checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
+ defer cancel()
+ result, err := s3util.CheckRepository(checkCtx, configuration, root, "")
+ if err != nil || result.Available {
+ return err
+ }
+ if result.Reason == s3util.RepositoryReasonMissingRepository {
+ var retained int64
+ if err := db.WithContext(ctx).Table(table).Where("s3_destination_id = ? AND COALESCE(remote_snapshot_id, '') <> ''", destinationID).Count(&retained).Error; err != nil {
+ return err
+ }
+ if retained == 0 {
+ return nil
+ }
+ }
+ return fmt.Errorf("%w: %s", ErrRemoteRepositoryMissing, result.Reason)
+}
diff --git a/backend/internal/s3/service_test.go b/backend/internal/s3/service_test.go
index 64caa72ae..fca44df8c 100644
--- a/backend/internal/s3/service_test.go
+++ b/backend/internal/s3/service_test.go
@@ -13,6 +13,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
+ s3config "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/s3"
backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"
"github.com/libtnb/sqlite"
"github.com/stretchr/testify/require"
@@ -272,6 +273,69 @@ func TestS3DestinationService_TestS3DestinationRoundTrip(t *testing.T) {
require.Equal(t, "arcane-backups", persisted.Bucket)
require.Equal(t, "production", persisted.Prefix)
require.Empty(t, persisted.Region)
+
+ for _, scenario := range []struct {
+ name string
+ code string
+ status int
+ reason string
+ wantError bool
+ snapshotError bool
+ configOnly bool
+ }{
+ {name: "only the requested snapshot is read"},
+ {name: "repository check reads only config", configOnly: true},
+ {name: "confirmed bucket loss", code: "NoSuchBucket", status: http.StatusNotFound, reason: "missing_bucket"},
+ {name: "confirmed repository loss", code: "NoSuchKey", status: http.StatusNotFound, reason: "missing_repository"},
+ {name: "access denied is not deletion", code: "AccessDenied", status: http.StatusForbidden, wantError: true},
+ {name: "generic not found is inconclusive", code: "NotFound", status: http.StatusNotFound, wantError: true},
+ {name: "missing snapshot leaves repository available", code: "NoSuchKey", status: http.StatusNotFound, snapshotError: true},
+ {name: "snapshot access denied is inconclusive", code: "AccessDenied", status: http.StatusForbidden, snapshotError: true, wantError: true},
+ } {
+ t.Run(scenario.name, func(t *testing.T) {
+ snapshotID := strings.Repeat("a", 64)
+ metadataServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ isConfig := r.URL.Path == "/arcane-backups/production/repository/config"
+ isSnapshot := !scenario.configOnly && r.URL.Path == "/arcane-backups/production/repository/snapshots/"+snapshotID
+ if r.Method != http.MethodGet || (!isConfig && !isSnapshot) || r.URL.Query().Get("list-type") != "" {
+ t.Errorf("unexpected metadata request: %s %s", r.Method, r.URL)
+ w.WriteHeader(http.StatusBadRequest)
+ return
+ }
+ if r.Header.Get("Range") != "bytes=0-0" {
+ t.Errorf("metadata request must be limited to one byte")
+ }
+ if scenario.code != "" && (!scenario.snapshotError || !isConfig) {
+ w.Header().Set("Content-Type", "application/xml")
+ w.WriteHeader(scenario.status)
+ _, _ = io.WriteString(w, "<Error><Code>"+scenario.code+"</Code><Message>test response</Message></Error>")
+ return
+ }
+ w.Header().Set("Content-Range", "bytes 0-0/100")
+ w.WriteHeader(http.StatusPartialContent)
+ _, _ = io.WriteString(w, "x")
+ }))
+ defer metadataServer.Close()
+ configuration := s3config.Configuration{
+ Name: "Metadata destination", Endpoint: metadataServer.URL, Bucket: "arcane-backups",
+ AccessKeyID: "test-access", SecretAccessKey: "test-secret", Prefix: "production", ForcePathStyle: true,
+ }
+ requestedID := snapshotID
+ if scenario.configOnly {
+ requestedID = ""
+ }
+ observation, checkErr := s3config.CheckRepository(t.Context(), configuration, "repository", requestedID)
+ if scenario.wantError {
+ require.Error(t, checkErr)
+ require.Empty(t, observation.Reason)
+ return
+ }
+ require.NoError(t, checkErr)
+ require.Equal(t, scenario.reason, observation.Reason)
+ require.Equal(t, scenario.reason == "", observation.Available)
+ require.Equal(t, scenario.reason == "" && !scenario.configOnly && !scenario.snapshotError, observation.SnapshotAvailable)
+ })
+ }
}
func TestListS3DestinationsByIDInternal(t *testing.T) {
diff --git a/backend/internal/systembackup/service.go b/backend/internal/systembackup/service.go
index 55cbf0bbc..75e5fb9d6 100644
--- a/backend/internal/systembackup/service.go
+++ b/backend/internal/systembackup/service.go
@@ -525,10 +525,12 @@ func (s *SystemBackupService) ListBackups(ctx context.Context, params pagination
names = available
}
}
+ remoteAvailable := backup.RemoteSnapshotChecker(ctx, s.s3Destinations, "arcane-system-recovery")
result := make([]backuptypes.SystemBackupRun, len(runs))
for i := range runs {
runs[i].S3DestinationName = names[runs[i].S3DestinationID].Name
result[i] = runs[i].ToDTO()
+ result[i].RemoteAvailable = remoteAvailable(runs[i].S3DestinationID, runs[i].RemoteSnapshotID)
}
return result, response, nil
}
@@ -1084,18 +1086,18 @@ func (s *SystemBackupService) DeleteBackup(ctx context.Context, id, recoveryKey
if err != nil {
return err
}
- return s.deleteRunsInternal(ctx, []*SystemBackupRun{run}, recoveryKey)
+ return s.deleteRunsInternal(ctx, []*SystemBackupRun{run}, recoveryKey, true)
}
// deleteRunsInternal forgets snapshots grouped by repository under the caller's run lease.
-func (s *SystemBackupService) deleteRunsInternal(ctx context.Context, runs []*SystemBackupRun, recoveryKey string) error {
+func (s *SystemBackupService) deleteRunsInternal(ctx context.Context, runs []*SystemBackupRun, recoveryKey string, includeRemote bool) error {
var localRuns []*SystemBackupRun
remoteGroups := make(map[string][]*SystemBackupRun)
for _, run := range runs {
if run.LocalSnapshotID != "" {
localRuns = append(localRuns, run)
}
- if run.RemoteSnapshotID != "" {
+ if includeRemote && run.RemoteSnapshotID != "" {
remoteGroups[run.S3DestinationID] = append(remoteGroups[run.S3DestinationID], run)
}
}
@@ -1123,7 +1125,6 @@ func (s *SystemBackupService) deleteRunsInternal(ctx context.Context, runs []*Sy
}
continue
}
- run.Error = deleteErr.Error()
if saveErr := s.db.WithContext(ctx).Save(run).Error; saveErr != nil {
deleteErr = errors.Combine(deleteErr, saveErr)
}
@@ -1579,7 +1580,7 @@ func (s *SystemBackupService) runScheduledBackupInternal(ctx context.Context, po
outcome := jobcontext.ConfirmedTarget(previous, policy.ID)
if outcome.Status == schedulertypes.Succeeded {
if policy.RetentionCount > 0 {
- if err := s.applyRetentionInternal(ctx, policy.ID, policy.RetentionCount); err != nil {
+ if err := s.applyRetentionInternal(ctx, policy.ID, policy.RetentionCount, policy.S3Enabled); err != nil {
outcome.Status = schedulertypes.Partial
return outcome, err
}
@@ -1587,6 +1588,13 @@ func (s *SystemBackupService) runScheduledBackupInternal(ctx context.Context, po
return outcome, nil
}
}
+ remoteDisabled, checkErr := s.disableMissingS3Internal(ctx, policy)
+ if checkErr != nil {
+ return schedulertypes.Outcome{}, checkErr
+ }
+ if remoteDisabled && !policy.LocalEnabled {
+ return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: backup.RemoteDisabledMessage}, nil
+ }
var run *SystemBackupRun
activityID, runErr := activitylib.RunHandlerActivity(ctx, s.activityService, activitylib.HandlerOptions{
EnvironmentID: "0", Type: activitytypes.TypeResourceAction, ResourceType: "system_backup",
@@ -1611,12 +1619,15 @@ func (s *SystemBackupService) runScheduledBackupInternal(ctx context.Context, po
return schedulertypes.Outcome{ActivityID: activityID}, runErr
}
if policy.RetentionCount > 0 {
- if retentionErr := s.applyRetentionInternal(ctx, policy.ID, policy.RetentionCount); retentionErr != nil {
+ if retentionErr := s.applyRetentionInternal(ctx, policy.ID, policy.RetentionCount, policy.S3Enabled); retentionErr != nil {
slog.ErrorContext(ctx, "System backup retention failed", "policyId", policy.ID, "error", retentionErr)
return schedulertypes.Outcome{Status: schedulertypes.Partial, ActivityID: activityID, Message: "Backup completed but retention failed"}, retentionErr
}
}
slog.InfoContext(ctx, "Scheduled Arcane system backup completed", "backupId", run.ID, "policyId", policy.ID)
+ if remoteDisabled {
+ return schedulertypes.Outcome{Status: schedulertypes.Partial, ActivityID: activityID, Message: backup.RemoteDisabledMessage}, nil
+ }
return schedulertypes.Outcome{Status: schedulertypes.Succeeded, ActivityID: activityID}, nil
}
@@ -1648,7 +1659,7 @@ func (s *SystemBackupService) rescheduleSystemBackupPolicyInternal(ctx context.C
return outcome, err
}
if current != nil && current.RetentionCount > 0 {
- if err := s.applyRetentionInternal(ctx, policyID, current.RetentionCount); err != nil {
+ if err := s.applyRetentionInternal(ctx, policyID, current.RetentionCount, current.S3Enabled); err != nil {
outcome.Status = schedulertypes.Partial
outcome.Message = "Backup completed but retention failed"
return outcome, err
@@ -1682,7 +1693,7 @@ func (s *SystemBackupService) RegisterBackupJobOnStartup(ctx context.Context) {
slog.InfoContext(ctx, "Registered backup schedules", "systemPolicies", len(policies), "volumePolicies", volumePolicyCount)
}
-func (s *SystemBackupService) applyRetentionInternal(ctx context.Context, policyID string, keep int) error {
+func (s *SystemBackupService) applyRetentionInternal(ctx context.Context, policyID string, keep int, includeRemote bool) error {
expired, err := backup.ExpiredRunIDs(ctx, s.db, "system_backup_runs", policyID, keep)
if err != nil || len(expired) == 0 {
return err
@@ -1699,5 +1710,32 @@ func (s *SystemBackupService) applyRetentionInternal(ctx context.Context, policy
if err := s.db.WithContext(ctx).Where("id IN ?", expired).Find(&runs).Error; err != nil {
return err
}
- return s.deleteRunsInternal(ctx, runs, "")
+ return s.deleteRunsInternal(ctx, runs, "", includeRemote)
+}
+
+func (s *SystemBackupService) disableMissingS3Internal(ctx context.Context, policy *SystemBackupPolicy) (bool, error) {
+ if !policy.S3Enabled {
+ return false, nil
+ }
+ err := backup.CheckScheduledRemote(ctx, s.db, s.s3Destinations, "system_backup_runs", policy.S3DestinationID, "arcane-system-recovery")
+ if !errors.Is(err, backup.ErrRemoteRepositoryMissing) {
+ return false, nil
+ }
+ column := "enabled"
+ if policy.LocalEnabled {
+ column = "s3_enabled"
+ }
+ result := s.db.WithContext(ctx).Model(&SystemBackupPolicy{}).
+ Where("id = ? AND s3_destination_id = ? AND enabled = ? AND s3_enabled = ? AND local_enabled = ?", policy.ID, policy.S3DestinationID, true, true, policy.LocalEnabled).
+ Update(column, false)
+ if result.Error != nil || result.RowsAffected == 0 {
+ return false, result.Error
+ }
+ if policy.LocalEnabled {
+ policy.S3Enabled = false
+ } else {
+ policy.Enabled = false
+ }
+ s.rescheduleSystemBackupPolicyInternal(ctx, policy)
+ return true, nil
}
diff --git a/backend/internal/systembackup/service_test.go b/backend/internal/systembackup/service_test.go
index cf08d592b..e6f007229 100644
--- a/backend/internal/systembackup/service_test.go
+++ b/backend/internal/systembackup/service_test.go
@@ -2,6 +2,8 @@ package systembackup
import (
"context"
+ "net/http"
+ "net/http/httptest"
"slices"
"testing"
"time"
@@ -11,6 +13,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
recoverytypes "github.com/getarcaneapp/arcane/backend/v2/internal/recovery"
+ s3domain "github.com/getarcaneapp/arcane/backend/v2/internal/s3"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/entityjobs"
backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"
schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
@@ -113,6 +116,33 @@ func TestSystemBackupPoliciesRegisterIndependentJobs(t *testing.T) {
require.Equal(t, firstID, collection.Policies[0].ID)
require.Equal(t, 9, collection.Policies[0].RetentionCount)
require.Len(t, scheduler.jobs, 1)
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", "application/xml")
+ w.WriteHeader(http.StatusNotFound)
+ _, _ = w.Write([]byte("<Error><Code>NoSuchBucket</Code></Error>"))
+ }))
+ defer server.Close()
+ require.NoError(t, gormDB.AutoMigrate(&s3domain.S3Destination{}))
+ service.s3Destinations = s3domain.NewS3DestinationService(service.db)
+ destination, err := service.s3Destinations.CreateS3Destination(t.Context(), backuptypes.CreateS3Destination{
+ Name: "Missing storage", Endpoint: server.URL, Bucket: "backups", AccessKeyID: "test", SecretAccessKey: "test", ForcePathStyle: true,
+ })
+ require.NoError(t, err)
+ for _, local := range []bool{false, true} {
+ policy := &SystemBackupPolicy{Enabled: true, LocalEnabled: local, S3Enabled: true, S3DestinationID: destination.ID, Schedule: "0 0 2 * * *"}
+ require.NoError(t, gormDB.Create(policy).Error)
+ require.NoError(t, gormDB.Model(policy).Update("local_enabled", local).Error)
+ service.rescheduleSystemBackupPolicyInternal(t.Context(), policy)
+ disabled, err := service.disableMissingS3Internal(t.Context(), policy)
+ require.NoError(t, err)
+ require.True(t, disabled)
+ require.NoError(t, gormDB.First(policy, "id = ?", policy.ID).Error)
+ require.Equal(t, local, policy.Enabled)
+ require.Equal(t, !local, policy.S3Enabled)
+ require.Equal(t, local, scheduler.HasJob(service.jobs.JobName(policy.ID)))
+ }
+
}
func TestSystemBackupPolicyRequiresConfiguredRecoveryKeyWhenEnabled(t *testing.T) {
diff --git a/backend/internal/systembackup/system_volume.go b/backend/internal/systembackup/system_volume.go
index c29db4397..b84d680f7 100644
--- a/backend/internal/systembackup/system_volume.go
+++ b/backend/internal/systembackup/system_volume.go
@@ -154,13 +154,7 @@ func (s *SystemBackupService) UpdateSystemVolumeBackupConfig(ctx context.Context
*policy = normalized
return nil
},
- Persist: func(ctx context.Context, policies []backuptypes.SystemVolumeBackupPolicy) error {
- encoded, encodeErr := json.Marshal(backuptypes.SystemVolumeBackupPolicyCollection{Policies: policies})
- if encodeErr != nil {
- return fmt.Errorf("encode policies: %w", encodeErr)
- }
- return s.settingsService.UpdateSetting(ctx, systemVolumeBackupConfigKey, string(encoded))
- },
+ Persist: s.saveSystemVolumeBackupPoliciesInternal,
Unregister: func(ctx context.Context, policyID string) {
s.jobs.Unregister(ctx, systemVolumeBackupJobPrefix+policyID)
},
@@ -385,6 +379,13 @@ func (s *SystemBackupService) runScheduledSystemVolumeBackupInternal(ctx context
if policy == nil || !policy.Enabled {
return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil
}
+ remoteDisabled, checkErr := s.disableMissingVolumeS3Internal(ctx, policy)
+ if checkErr != nil {
+ return schedulertypes.Outcome{}, checkErr
+ }
+ if remoteDisabled && !policy.LocalEnabled {
+ return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: backup.RemoteDisabledMessage}, nil
+ }
result, err := s.runSystemVolumeBackupsInternal(ctx, backuptypes.RunSystemVolumeBackupsRequest{PolicyID: policyID}, volume.VolumeBackupTriggerScheduled)
if errors.Is(err, ErrSystemBackupAlreadyRunning) {
slog.InfoContext(ctx, "Scheduled system-managed volume backups skipped; a system backup is running", "policyId", policyID)
@@ -396,7 +397,10 @@ func (s *SystemBackupService) runScheduledSystemVolumeBackupInternal(ctx context
}
slog.InfoContext(ctx, "Scheduled system-managed volume backups completed", "policyId", policyID, "matched", result.Matched, "succeeded", result.Succeeded, "failed", result.Failed, "skipped", result.Skipped)
outcome := schedulertypes.Outcome{Status: schedulertypes.Succeeded}
- if result.Failed > 0 {
+ if remoteDisabled {
+ outcome.Message = backup.RemoteDisabledMessage
+ }
+ if result.Failed > 0 || remoteDisabled {
outcome.Status = schedulertypes.Partial
}
for _, failure := range result.Failures {
@@ -453,6 +457,20 @@ func (s *SystemBackupService) ListBackupHistory(ctx context.Context, params pagi
return nil, pagination.Response{}, fmt.Errorf("list backup history: %w", err)
}
decorateHistoryDestinationsInternal(ctx, s.s3Destinations, history)
+ systemAvailable := backup.RemoteSnapshotChecker(ctx, s.s3Destinations, "arcane-system-recovery")
+ volumeRoot := ""
+ if s.settingsService != nil {
+ volumeRoot = "arcane-volume-backups/" + s.settingsService.GetSettingsConfig().InstanceID.Value
+ }
+ volumeAvailable := backup.RemoteSnapshotChecker(ctx, s.s3Destinations, volumeRoot)
+ for i := range history {
+ check := systemAvailable
+ if history[i].ResourceType == "volume" {
+ check = volumeAvailable
+ }
+ history[i].RemoteAvailable = check(history[i].S3DestinationID, history[i].RemoteSnapshotID)
+ }
+
page.GrandTotalItems = page.TotalItems
return history, page, nil
}
@@ -469,3 +487,47 @@ func decorateHistoryDestinationsInternal(ctx context.Context, service *s3domain.
history[i].S3DestinationName = destinations[history[i].S3DestinationID].Name
}
}
+
+func (s *SystemBackupService) saveSystemVolumeBackupPoliciesInternal(ctx context.Context, policies []backuptypes.SystemVolumeBackupPolicy) error {
+ encoded, err := json.Marshal(backuptypes.SystemVolumeBackupPolicyCollection{Policies: policies})
+ if err != nil {
+ return fmt.Errorf("encode policies: %w", err)
+ }
+ return s.settingsService.UpdateSetting(ctx, systemVolumeBackupConfigKey, string(encoded))
+}
+
+func (s *SystemBackupService) disableMissingVolumeS3Internal(ctx context.Context, policy *backuptypes.SystemVolumeBackupPolicy) (bool, error) {
+ if !policy.S3Enabled {
+ return false, nil
+ }
+ root := "arcane-volume-backups/" + s.settingsService.GetSettingsConfig().InstanceID.Value
+ err := backup.CheckScheduledRemote(ctx, s.db, s.s3Destinations, "volume_backups", policy.S3DestinationID, root)
+ if !errors.Is(err, backup.ErrRemoteRepositoryMissing) {
+ return false, nil
+ }
+ collection, err := s.loadSystemVolumeBackupPoliciesInternal()
+ if err != nil {
+ return false, err
+ }
+ for i := range collection.Policies {
+ current := &collection.Policies[i]
+ if current.ID != policy.ID {
+ continue
+ }
+ if current.S3DestinationID != policy.S3DestinationID || current.S3Enabled != policy.S3Enabled || current.Enabled != policy.Enabled || current.LocalEnabled != policy.LocalEnabled {
+ return false, nil
+ }
+ if current.LocalEnabled {
+ current.S3Enabled = false
+ } else {
+ current.Enabled = false
+ }
+ if err := s.saveSystemVolumeBackupPoliciesInternal(ctx, collection.Policies); err != nil {
+ return false, err
+ }
+ *policy = *current
+ s.rescheduleSystemVolumeBackupInternal(ctx, policy)
+ return true, nil
+ }
+ return false, nil
+}
diff --git a/backend/internal/volume/backup.go b/backend/internal/volume/backup.go
index b12487fd5..b79b71058 100644
--- a/backend/internal/volume/backup.go
+++ b/backend/internal/volume/backup.go
@@ -382,8 +382,14 @@ func (s *VolumeService) ListBackupsPaginated(ctx context.Context, volumeName str
}
}
}
+ root := ""
+ if s.settingsService != nil {
+ root = "arcane-volume-backups/" + s.settingsService.GetSettingsConfig().InstanceID.Value
+ }
+ remoteAvailable := backup.RemoteSnapshotChecker(ctx, s.s3Destinations, root)
for i := range backups {
backups[i].Type = volumeBackupManagementTypeInternal(backups[i].PolicyID)
+ backups[i].RemoteAvailable = remoteAvailable(backups[i].S3DestinationID, backups[i].RemoteSnapshotID)
}
return backups, pagination.BuildResponse(totalItems, totalItems, params), nil
@@ -738,7 +744,7 @@ func (s *VolumeService) executeBackupInternal(ctx context.Context, entry *Volume
// and the deferred status-save would otherwise mark it failed and hide it
// from ExpiredRunIDs forever.
if trigger != VolumeBackupTriggerSafety && plan.policy != nil && plan.policy.RetentionCount > 0 {
- if retentionErr := s.applyVolumeBackupRetentionInternal(ctx, plan.policy.ID, plan.policy.RetentionCount); retentionErr != nil {
+ if retentionErr := s.applyVolumeBackupRetentionInternal(ctx, plan.policy.ID, plan.policy.RetentionCount, plan.s3Enabled); retentionErr != nil {
slog.ErrorContext(ctx, "Volume backup retention failed", "volume", volumeName, "policy", plan.policy.ID, "error", retentionErr)
}
}
@@ -843,11 +849,11 @@ func (s *VolumeService) deleteBackupInternal(ctx context.Context, backupID strin
if entry.Format == VolumeBackupFormatArchive {
return s.deleteArchiveBackupInternal(ctx, &entry, user)
}
- return s.deleteRusticBackupsInternal(ctx, []*VolumeBackup{&entry}, user)
+ return s.deleteRusticBackupsInternal(ctx, []*VolumeBackup{&entry}, user, true)
}
// deleteRusticBackupsInternal forgets snapshots grouped by repository so each repository is pruned once.
-func (s *VolumeService) deleteRusticBackupsInternal(ctx context.Context, entries []*VolumeBackup, user *common.User) error {
+func (s *VolumeService) deleteRusticBackupsInternal(ctx context.Context, entries []*VolumeBackup, user *common.User, includeRemote bool) error {
dockerClient, err := s.dockerService.GetClient(ctx)
if err != nil {
return err
@@ -858,7 +864,7 @@ func (s *VolumeService) deleteRusticBackupsInternal(ctx context.Context, entries
if entry.LocalSnapshotID != "" {
localEntries = append(localEntries, entry)
}
- if entry.RemoteSnapshotID != "" {
+ if includeRemote && entry.RemoteSnapshotID != "" {
remoteGroups[entry.S3DestinationID] = append(remoteGroups[entry.S3DestinationID], entry)
}
}
@@ -883,7 +889,6 @@ func (s *VolumeService) deleteRusticBackupsInternal(ctx context.Context, entries
default:
entry.Destination = volumetypes.BackupDestinationS3
}
- entry.Error = deleteErr.Error()
if saveErr := s.db.WithContext(ctx).Save(entry).Error; saveErr != nil {
deleteErr = errors.Combine(deleteErr, saveErr)
}
@@ -1766,7 +1771,7 @@ func (s *VolumeService) UpdateBackupPolicies(ctx context.Context, volumeName str
return s.GetBackupPolicies(ctx, volumeName)
}
-func (s *VolumeService) applyVolumeBackupRetentionInternal(ctx context.Context, policyID string, retentionCount int) error {
+func (s *VolumeService) applyVolumeBackupRetentionInternal(ctx context.Context, policyID string, retentionCount int, includeRemote bool) error {
expired, err := backup.ExpiredRunIDs(ctx, s.db, "volume_backups", policyID, retentionCount)
if err != nil || len(expired) == 0 {
return err
@@ -1775,7 +1780,7 @@ func (s *VolumeService) applyVolumeBackupRetentionInternal(ctx context.Context,
if err := s.db.WithContext(ctx).Where("id IN ?", expired).Find(&entries).Error; err != nil {
return err
}
- return s.deleteRusticBackupsInternal(ctx, entries, nil)
+ return s.deleteRusticBackupsInternal(ctx, entries, nil, includeRemote)
}
func (s *VolumeService) runScheduledBackupInternal(ctx context.Context, policyID string) (schedulertypes.Outcome, error) {
@@ -1792,6 +1797,13 @@ func (s *VolumeService) runScheduledBackupInternal(ctx context.Context, policyID
return outcome, nil
}
}
+ remoteDisabled, checkErr := s.disableMissingS3Internal(ctx, &policy)
+ if checkErr != nil {
+ return schedulertypes.Outcome{}, checkErr
+ }
+ if remoteDisabled && !policy.LocalEnabled {
+ return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: backup.RemoteDisabledMessage}, nil
+ }
var entry *VolumeBackup
activityID, err := activitylib.RunHandlerActivity(ctx, s.activityService, activitylib.HandlerOptions{
EnvironmentID: "0",
@@ -1828,6 +1840,9 @@ func (s *VolumeService) runScheduledBackupInternal(ctx context.Context, policyID
return schedulertypes.Outcome{}, err
}
slog.InfoContext(ctx, "Scheduled volume backup completed", "volume", policy.VolumeName, "backup_id", entry.ID, "remote_snapshot_id", entry.RemoteSnapshotID)
+ if remoteDisabled {
+ return schedulertypes.Outcome{Status: schedulertypes.Partial, ActivityID: activityID, Message: backup.RemoteDisabledMessage}, nil
+ }
return schedulertypes.Outcome{Status: schedulertypes.Succeeded, ActivityID: activityID}, nil
}
@@ -1884,3 +1899,30 @@ func (s *VolumeService) removeVolumeBackupPolicyInternal(ctx context.Context, vo
slog.WarnContext(ctx, "Failed to delete volume backup policy", "volume", volumeName, "error", err)
}
}
+
+func (s *VolumeService) disableMissingS3Internal(ctx context.Context, policy *VolumeBackupPolicy) (bool, error) {
+ if !policy.S3Enabled {
+ return false, nil
+ }
+ err := backup.CheckScheduledRemote(ctx, s.db, s.s3Destinations, "volume_backups", policy.S3DestinationID, "arcane-volume-backups/"+s.settingsService.GetSettingsConfig().InstanceID.Value)
+ if !errors.Is(err, backup.ErrRemoteRepositoryMissing) {
+ return false, nil
+ }
+ column := "enabled"
+ if policy.LocalEnabled {
+ column = "s3_enabled"
+ }
+ result := s.db.WithContext(ctx).Model(&VolumeBackupPolicy{}).
+ Where("id = ? AND s3_destination_id = ? AND enabled = ? AND s3_enabled = ? AND local_enabled = ?", policy.ID, policy.S3DestinationID, true, true, policy.LocalEnabled).
+ Update(column, false)
+ if result.Error != nil || result.RowsAffected == 0 {
+ return false, result.Error
+ }
+ if policy.LocalEnabled {
+ policy.S3Enabled = false
+ } else {
+ policy.Enabled = false
+ }
+ s.rescheduleVolumeBackupPolicyInternal(ctx, policy)
+ return true, nil
+}
diff --git a/backend/internal/volume/model.go b/backend/internal/volume/model.go
index 205d8cf57..d77cebc0e 100644
--- a/backend/internal/volume/model.go
+++ b/backend/internal/volume/model.go
@@ -49,6 +49,7 @@ type VolumeBackup struct {
Error string `json:"error,omitempty" gorm:"column:error;type:text"`
ActivityID *string `json:"activityId,omitempty" gorm:"-"`
Type backuptypes.ManagementType `json:"type" gorm:"-"`
+ RemoteAvailable *bool `json:"remoteAvailable,omitempty" gorm:"-"`
}
func (*VolumeBackup) TableName() string {
@@ -57,6 +58,7 @@ func (*VolumeBackup) TableName() string {
func (b *VolumeBackup) ToDTO() volume.BackupEntry {
return volume.BackupEntry{
+ RemoteAvailable: b.RemoteAvailable,
ActivityID: b.ActivityID,
ID: b.ID,
VolumeName: b.VolumeName,
diff --git a/backend/internal/volume/service_test.go b/backend/internal/volume/service_test.go
index 8dc58399e..924812316 100644
--- a/backend/internal/volume/service_test.go
+++ b/backend/internal/volume/service_test.go
@@ -719,7 +719,7 @@ func TestVolumeBackupPolicy_RetentionIgnoresFailedRuns(t *testing.T) {
}).Error)
service := &VolumeService{db: &database.DB{DB: gormDB}}
- require.NoError(t, service.applyVolumeBackupRetentionInternal(context.Background(), policyID, 1))
+ require.NoError(t, service.applyVolumeBackupRetentionInternal(context.Background(), policyID, 1, true))
var backups []VolumeBackup
require.NoError(t, gormDB.Order("created_at ASC").Find(&backups).Error)
diff --git a/backend/pkg/utils/s3/s3.go b/backend/pkg/utils/s3/s3.go
index 3983212ae..f7dd099b3 100644
--- a/backend/pkg/utils/s3/s3.go
+++ b/backend/pkg/utils/s3/s3.go
@@ -10,6 +10,7 @@ import (
"io"
"net/url"
"path"
+ "slices"
"strings"
"emperror.dev/errors"
@@ -18,9 +19,16 @@ import (
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
awss3 "github.com/aws/aws-sdk-go-v2/service/s3"
+ "github.com/aws/smithy-go"
+ backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"
"github.com/google/uuid"
)
+const (
+ RepositoryReasonMissingBucket = "missing_bucket"
+ RepositoryReasonMissingRepository = "missing_repository"
+)
+
const connectionTestPayload = "arcane-s3-connection-test"
// Configuration contains the credentials and addressing options for an S3 destination.
@@ -200,6 +208,61 @@ func TestConnection(ctx context.Context, configuration Configuration) (err error
return nil
}
+// CheckRepository reads only the config and, when supplied, one snapshot object.
+func CheckRepository(ctx context.Context, configuration Configuration, root, snapshotID string) (backuptypes.RepositoryObservation, error) {
+ configuration = configuration.Normalized()
+ if err := configuration.Validate(true); err != nil {
+ return backuptypes.RepositoryObservation{}, err
+ }
+ client, err := newClientInternal(ctx, configuration)
+ if err != nil {
+ return backuptypes.RepositoryObservation{}, fmt.Errorf("failed to configure S3 repository check: %w", err)
+ }
+ configKey := path.Join(configuration.Prefix, root, "config")
+ if err := checkBackupObjectInternal(ctx, client, configuration.Bucket, configKey); err != nil {
+ if isMissingResourceInternal(err, "NoSuchBucket") {
+ return backuptypes.RepositoryObservation{Reason: RepositoryReasonMissingBucket}, nil
+ }
+ if isMissingResourceInternal(err, "NoSuchKey") {
+ return backuptypes.RepositoryObservation{Reason: RepositoryReasonMissingRepository}, nil
+ }
+ return backuptypes.RepositoryObservation{}, err
+ }
+ result := backuptypes.RepositoryObservation{Available: true}
+ if snapshotID == "" {
+ return result, nil
+ }
+ snapshotKey := path.Join(configuration.Prefix, root, "snapshots", snapshotID)
+ if err := checkBackupObjectInternal(ctx, client, configuration.Bucket, snapshotKey); err != nil {
+ if isMissingResourceInternal(err, "NoSuchKey") {
+ return result, nil
+ }
+ if isMissingResourceInternal(err, "NoSuchBucket") {
+ return backuptypes.RepositoryObservation{Reason: RepositoryReasonMissingBucket}, nil
+ }
+ return backuptypes.RepositoryObservation{}, err
+ }
+ result.SnapshotAvailable = true
+ return result, nil
+}
+
+// A one-byte GET preserves the named S3 errors that generic HEAD responses omit.
+func checkBackupObjectInternal(ctx context.Context, client *awss3.Client, bucket, key string) error {
+ object, err := client.GetObject(ctx, &awss3.GetObjectInput{Bucket: aws.String(bucket), Key: aws.String(key), Range: aws.String("bytes=0-0")})
+ if err != nil {
+ return err
+ }
+ return object.Body.Close()
+}
+
+func isMissingResourceInternal(err error, codes ...string) bool {
+ var apiErr smithy.APIError
+ if !errors.As(err, &apiErr) {
+ return false
+ }
+ return slices.Contains(codes, apiErr.ErrorCode())
+}
+
func newClientInternal(ctx context.Context, configuration Configuration) (*awss3.Client, error) {
region := configuration.Region
if region == "" {
diff --git a/frontend/messages/en.json b/frontend/messages/en.json
index 31d6caf9a..f38f95b93 100644
--- a/frontend/messages/en.json
+++ b/frontend/messages/en.json
@@ -3272,6 +3272,7 @@
"backups_trigger_scheduled": "Scheduled",
"backups_trigger_safety": "Pre-restore safety",
"backups_destination_s3": "S3",
+ "backups_remote_unavailable": "Remote copy unavailable",
"backups_destination_local_s3": "Local + S3",
"s3_destinations_title": "S3 Destinations",
"s3_destinations_description": "Create and manage reusable S3-compatible storage destinations for Arcane and volume backups.",
diff --git a/frontend/src/lib/components/arcane-table/cells/backup-destination-cell.svelte b/frontend/src/lib/components/arcane-table/cells/backup-destination-cell.svelte
index d07b6d945..d2abe7083 100644
--- a/frontend/src/lib/components/arcane-table/cells/backup-destination-cell.svelte
+++ b/frontend/src/lib/components/arcane-table/cells/backup-destination-cell.svelte
@@ -1,9 +1,10 @@
<script lang="ts">
+ import * as m from '#lib/paraglide/messages.js';
import { Badge } from '#lib/components/ui/badge/index.js';
import type { BackupRun } from '#lib/types/backup.js';
import { backupDestinationLabel, backupDestinationName } from '#lib/utils/backups.js';
- let { item }: { item: Pick<BackupRun, 'destination' | 's3DestinationId' | 's3DestinationName'> } = $props();
+ let { item }: { item: Pick<BackupRun, 'destination' | 's3DestinationId' | 's3DestinationName' | 'remoteAvailable'> } = $props();
</script>
<div class="flex items-center gap-2">
@@ -11,4 +12,7 @@
{#if item.destination !== 'local' && backupDestinationName(item)}
<span class="max-w-48 truncate text-xs text-muted-foreground">{backupDestinationName(item)}</span>
{/if}
+ {#if item.remoteAvailable === false}
+ <span class="text-xs text-destructive">{m.backups_remote_unavailable()}</span>
+ {/if}
</div>
diff --git a/frontend/src/lib/types/backup.ts b/frontend/src/lib/types/backup.ts
index 06be6ff3a..f37e4cdb8 100644
--- a/frontend/src/lib/types/backup.ts
+++ b/frontend/src/lib/types/backup.ts
@@ -6,6 +6,7 @@ export type BackupTrigger = 'manual' | 'scheduled' | 'safety';
export type BackupManagementType = 'system' | 'volume';
export type BackupRun = {
+ remoteAvailable?: boolean;
activityId?: string;
id: string;
size: number;
diff --git a/types/backup/repository.go b/types/backup/repository.go
new file mode 100644
index 000000000..a1436c1de
--- /dev/null
+++ b/types/backup/repository.go
@@ -0,0 +1,8 @@
+package backup
+
+// RepositoryObservation describes a read-only S3 metadata check.
+type RepositoryObservation struct {
+ Available bool
+ Reason string
+ SnapshotAvailable bool
+}
diff --git a/types/backup/system_backup.go b/types/backup/system_backup.go
index 58bf2926d..c17f4a256 100644
--- a/types/backup/system_backup.go
+++ b/types/backup/system_backup.go
@@ -11,6 +11,7 @@ const (
)
type SystemBackupRun struct {
+ RemoteAvailable *bool `json:"remoteAvailable,omitempty"`
ActivityID string `json:"activityId,omitempty"`
ID string `json:"id"`
Size int64 `json:"size"`
diff --git a/types/backup/system_volume_backup.go b/types/backup/system_volume_backup.go
index 78ee13ad7..d0879d10a 100644
--- a/types/backup/system_volume_backup.go
+++ b/types/backup/system_volume_backup.go
@@ -96,6 +96,7 @@ type SystemVolumeBackupRunResult struct {
// HistoryEntry is a common view over Arcane recovery and volume backup records.
type HistoryEntry struct {
+ RemoteAvailable *bool `json:"remoteAvailable,omitempty" gorm:"-"`
ID string `json:"id" sortable:"true"`
Size int64 `json:"size" sortable:"true"`
CreatedAt time.Time `json:"createdAt" sortable:"true"`
diff --git a/types/volume/backup.go b/types/volume/backup.go
index 7b6feb799..dddcb24fd 100644
--- a/types/volume/backup.go
+++ b/types/volume/backup.go
@@ -28,6 +28,7 @@ const (
)
type BackupEntry struct {
+ RemoteAvailable *bool `json:"remoteAvailable,omitempty"`
ActivityID *string `json:"activityId,omitempty"`
ID string `json:"id" doc:"Unique identifier of the backup"`
VolumeName string `json:"volumeName" doc:"Name of the volume"`