Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
feat: automated s3 backup (#3459)
affeldt28555c5dfbac47
diff --git a/.devcontainer/devcontainer-lock.json b/.devcontainer/devcontainer-lock.jsonnew file mode 100644index 000000000..dab65ac66--- /dev/null+++ b/.devcontainer/devcontainer-lock.json@@ -0,0 +1,24 @@+{+ "features": {+ "ghcr.io/devcontainers/features/common-utils:2": {+ "version": "2.5.9",+ "resolved": "ghcr.io/devcontainers/features/common-utils@sha256:cb0c4d3c276f157eed17935747e364178d75fee17f55c4e129966f64633deb3a",+ "integrity": "sha256:cb0c4d3c276f157eed17935747e364178d75fee17f55c4e129966f64633deb3a"+ },+ "ghcr.io/devcontainers/features/docker-outside-of-docker:1": {+ "version": "1.10.0",+ "resolved": "ghcr.io/devcontainers/features/docker-outside-of-docker@sha256:c2c2cf829505ead8e4892c88c31b6594ae94a2bbb209e16e1fac456c1a3a624e",+ "integrity": "sha256:c2c2cf829505ead8e4892c88c31b6594ae94a2bbb209e16e1fac456c1a3a624e"+ },+ "ghcr.io/devcontainers/features/go:1": {+ "version": "1.3.4",+ "resolved": "ghcr.io/devcontainers/features/go@sha256:d85e921f91b41340055bb12b325d9d551170ed04b3b832e33530bf42f167c032",+ "integrity": "sha256:d85e921f91b41340055bb12b325d9d551170ed04b3b832e33530bf42f167c032"+ },+ "ghcr.io/devcontainers/features/node:2": {+ "version": "2.1.0",+ "resolved": "ghcr.io/devcontainers/features/node@sha256:586c9a6f7dd40bd3ba2cd41e7f2f88dcc31fbe5d1442afcbf07ffbc66b686857",+ "integrity": "sha256:586c9a6f7dd40bd3ba2cd41e7f2f88dcc31fbe5d1442afcbf07ffbc66b686857"+ }+ }+}diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 4b3c997c3..c41bea684 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -271,6 +271,8 @@ jobs:run: |docker pull public.ecr.aws/nginx/nginx:stable-alpinedocker pull ghcr.io/linuxserver/radarr:nightly+ docker pull quay.io/minio/minio:RELEASE.2025-09-07T16-13-09Z+ docker pull ghcr.io/rustic-rs/rustic:v0.11.2docker save public.ecr.aws/nginx/nginx:stable-alpine ghcr.io/linuxserver/radarr:nightly > /tmp/test-images.tar- name: Setup and Install Playwright Dependenciesdiff --git a/backend/api/api.go b/backend/api/api.goindex 440e3ac23..b80e8d564 100644--- a/backend/api/api.go+++ b/backend/api/api.go@@ -33,9 +33,11 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/project""github.com/getarcaneapp/arcane/backend/v2/internal/registry""github.com/getarcaneapp/arcane/backend/v2/internal/role"+ s3domain "github.com/getarcaneapp/arcane/backend/v2/internal/s3""github.com/getarcaneapp/arcane/backend/v2/internal/search""github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/backend/v2/internal/swarm"+ "github.com/getarcaneapp/arcane/backend/v2/internal/systembackup""github.com/getarcaneapp/arcane/backend/v2/internal/template""github.com/getarcaneapp/arcane/backend/v2/internal/user""github.com/getarcaneapp/arcane/backend/v2/internal/variable"@@ -227,6 +229,8 @@ type HandlerDeps struct {Build *build.BuildServiceBuildWorkspace *build.BuildWorkspaceServiceVolume *volume.Module+ S3Destination *s3domain.Module+ SystemBackup *systembackup.ModuleNetwork *network.NetworkServicePort *port.PortServiceSwarm *swarm.Module@@ -419,6 +423,8 @@ func registerHandlersInternal(api huma.API, deps HandlerDeps, handlerAppCtx handbuild.RegisterBuildWorkspaces(api, deps.BuildWorkspace, deps.Upload.Service())deps.ImageUpdate.RegisterRoutes(api, handlerAppCtx)deps.Settings.RegisterRoutes(api)+ deps.S3Destination.RegisterRoutes(api)+ deps.SystemBackup.RegisterRoutes(api, handlerAppCtx)deps.JobSchedule.RegisterRoutes(api)deps.Volume.RegisterRoutes(api, handlerAppCtx)deps.Container.RegisterRoutes(api, handlerAppCtx)diff --git a/backend/cli/recovery/recovery.go b/backend/cli/recovery/recovery.gonew file mode 100644index 000000000..06d3d9a14--- /dev/null+++ b/backend/cli/recovery/recovery.go@@ -0,0 +1,241 @@+package recovery++import (+ "context"+ "encoding/json"+ "fmt"+ "log/slog"+ "os"+ "strings"+ "time"++ "emperror.dev/errors"+ "gorm.io/gorm"+ "gorm.io/gorm/clause"++ "github.com/getarcaneapp/arcane/backend/v2/cli/upgrade"+ "github.com/getarcaneapp/arcane/backend/v2/internal/activity"+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"+ recoverytypes "github.com/getarcaneapp/arcane/backend/v2/internal/recovery"+ "github.com/getarcaneapp/arcane/backend/v2/internal/systembackup"+ dockerutil "github.com/getarcaneapp/arcane/backend/v2/pkg/dockerutil"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"+ rusticruntime "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/rustic"+ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"+ backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"+ "github.com/libtnb/sqlite"+ "github.com/moby/moby/api/types/container"+ "github.com/moby/moby/api/types/mount"+ "github.com/moby/moby/client"+ "github.com/spf13/cobra"+)++const rusticImage = rusticruntime.DefaultImage++var requestPath string++var RestoreCmd = &cobra.Command{+ Use: "recovery-restore",+ Short: "Apply a prepared Arcane recovery snapshot",+ Hidden: true,+ RunE: runRestoreInternal,+}++func init() {+ RestoreCmd.Flags().StringVar(&requestPath, "request", "/app/data/.arcane-recovery-request.json", "Prepared recovery request")+}++func runRestoreInternal(_ *cobra.Command, _ []string) error {+ ctx := context.Background()+ data, err := os.ReadFile(requestPath)+ if err != nil {+ return fmt.Errorf("read recovery request: %w", err)+ }+ var request recoverytypes.RestoreRequest+ if err := json.Unmarshal(data, &request); err != nil {+ return fmt.Errorf("decode recovery request: %w", err)+ }+ _ = os.Remove(requestPath)+ dockerClient, err := client.New(client.FromEnv)+ if err != nil {+ return fmt.Errorf("connect to Docker: %w", err)+ }+ defer func() { _ = dockerClient.Close() }()+ inspect, err := libarcane.ContainerInspectWithCompatibility(ctx, dockerClient, request.ContainerID, client.ContainerInspectOptions{})+ if err != nil {+ return fmt.Errorf("inspect Arcane container: %w", err)+ }+ if _, err := dockerClient.ContainerStop(ctx, request.ContainerID, client.ContainerStopOptions{Timeout: new(30)}); err != nil {+ return fmt.Errorf("stop Arcane container: %w", err)+ }+ if err := restoreSnapshotInternal(ctx, dockerClient, request); err != nil {+ _, _ = dockerClient.ContainerStart(ctx, request.ContainerID, client.ContainerStartOptions{})+ return err+ }+ manifestData, err := os.ReadFile("/app/data/.arcane-recovery.json")+ if err != nil {+ _, _ = dockerClient.ContainerStart(ctx, request.ContainerID, client.ContainerStartOptions{})+ return fmt.Errorf("read restored recovery manifest: %w", err)+ }+ var manifest recoverytypes.Manifest+ if err := json.Unmarshal(manifestData, &manifest); err != nil {+ _, _ = dockerClient.ContainerStart(ctx, request.ContainerID, client.ContainerStartOptions{})+ return fmt.Errorf("decode restored recovery manifest: %w", err)+ }+ _ = os.Remove("/app/data/.arcane-recovery.json")+ if manifest.FormatVersion != 1 || len(manifest.Environment) == 0 {+ _, _ = dockerClient.ContainerStart(ctx, request.ContainerID, client.ContainerStartOptions{})+ return errors.New("unsupported or incomplete Arcane recovery manifest")+ }+ if err := finalizeRestoredBackupInternal(ctx, manifest.Environment["DATABASE_URL"], manifest.BackupID, manifest.ActivityID, request); err != nil {+ _, _ = dockerClient.ContainerStart(ctx, request.ContainerID, client.ContainerStartOptions{})+ return fmt.Errorf("finalize restored system backup: %w", err)+ }+ if err := upgrade.UpgradeContainer(ctx, dockerClient, inspect.Container, request.ContainerImage, manifest.Environment); err != nil {+ return fmt.Errorf("recreate Arcane container with recovered configuration: %w", err)+ }+ return nil+}++func finalizeRestoredBackupInternal(ctx context.Context, databaseURL, manifestBackupID, manifestActivityID string, request recoverytypes.RestoreRequest) error {+ if !strings.HasPrefix(databaseURL, "file:") {+ return errors.New("restored Arcane database is not SQLite")+ }+ dsn, err := database.ParseSQLiteConnectionString(databaseURL)+ if err != nil {+ return err+ }+ db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{})+ if err != nil {+ return err+ }+ defer func() {+ if sqlDB, dbErr := db.DB(); dbErr == nil {+ _ = sqlDB.Close()+ }+ }()+ db = db.WithContext(ctx)+ if err := finalizeRestoredRunInternal(db, manifestBackupID, request); err != nil {+ return err+ }+ if err := preserveSafetyBackupInternal(db, request.SafetyBackup); err != nil {+ return err+ }+ return finalizeRestoredActivityInternal(db, manifestActivityID)+}++// finalizeRestoredRunInternal marks the restored run succeeded, filling only+// the fields the snapshot's database copy did not already carry. Updates use+// explicit column maps because the restored schema may predate the binary's.+func finalizeRestoredRunInternal(db *gorm.DB, manifestBackupID string, request recoverytypes.RestoreRequest) error {+ var run systembackup.SystemBackupRun+ found := false+ for _, backupID := range []string{manifestBackupID, request.BackupID} {+ if strings.TrimSpace(backupID) == "" {+ continue+ }+ err := db.Where("id = ? AND status = ?", backupID, systembackup.SystemBackupStatusRunning).First(&run).Error+ if err == nil {+ found = true+ break+ }+ if !errors.Is(err, gorm.ErrRecordNotFound) {+ return err+ }+ }+ if !found {+ err := db.Where("status = ?", systembackup.SystemBackupStatusRunning).Order("created_at DESC").First(&run).Error+ if errors.Is(err, gorm.ErrRecordNotFound) {+ return nil+ }+ if err != nil {+ return err+ }+ }+ values := map[string]any{"status": systembackup.SystemBackupStatusSucceeded, "error": ""}+ if run.Size == 0 {+ values["size"] = request.Size+ }+ if run.LocalSnapshotID == "" {+ values["local_snapshot_id"] = request.LocalSnapshotID+ }+ if run.RemoteSnapshotID == "" {+ values["remote_snapshot_id"] = request.RemoteSnapshotID+ }+ if run.S3DestinationID == "" {+ values["s3_destination_id"] = request.S3DestinationID+ }+ return db.Model(&systembackup.SystemBackupRun{}).Where("id = ?", run.ID).Updates(values).Error+}++func preserveSafetyBackupInternal(db *gorm.DB, backup *recoverytypes.SafetyBackup) error {+ if backup == nil || strings.TrimSpace(backup.ID) == "" || strings.TrimSpace(backup.LocalSnapshotID) == "" {+ return nil+ }+ onConflict := map[string]any{+ "size": backup.Size, "updated_at": time.Now().UTC(), "status": systembackup.SystemBackupStatusSucceeded,+ "trigger": systembackup.SystemBackupTriggerSafety, "destination": backuptypes.SystemBackupDestinationLocal,+ "local_snapshot_id": backup.LocalSnapshotID, "error": "",+ }+ return db.Model(&systembackup.SystemBackupRun{}).+ Clauses(clause.OnConflict{Columns: []clause.Column{{Name: "id"}}, DoUpdates: clause.Assignments(onConflict)}).+ Create(map[string]any{+ "id": backup.ID, "size": backup.Size, "created_at": backup.CreatedAt, "updated_at": time.Now().UTC(),+ "status": systembackup.SystemBackupStatusSucceeded,+ "trigger": systembackup.SystemBackupTriggerSafety,+ "destination": backuptypes.SystemBackupDestinationLocal,+ "local_snapshot_id": backup.LocalSnapshotID, "remote_snapshot_id": "", "s3_destination_id": "",+ "policy_id": "", "error": "",+ }).Error+}++func finalizeRestoredActivityInternal(db *gorm.DB, activityID string) error {+ var entry activity.Activity+ query := db.Where("status = ?", activitytypes.StatusRunning)+ if strings.TrimSpace(activityID) != "" {+ query = query.Where("id = ?", activityID)+ } else {+ query = query.Where("resource_type = ?", "system_backup").Order("started_at DESC")+ }+ err := query.First(&entry).Error+ if errors.Is(err, gorm.ErrRecordNotFound) {+ return nil+ }+ if err != nil {+ return err+ }+ now := time.Now().UTC()+ return db.Model(&activity.Activity{}).Where("id = ?", entry.ID).Updates(map[string]any{+ "status": activitytypes.StatusSuccess, "progress": 100, "step": "System backup completed",+ "latest_message": "Arcane system backup created successfully", "error": nil,+ "ended_at": now, "duration_ms": now.Sub(entry.StartedAt).Milliseconds(),+ }).Error+}++func restoreSnapshotInternal(ctx context.Context, dockerClient *client.Client, request recoverytypes.RestoreRequest) error {+ if _, err := dockerClient.ImageInspect(ctx, rusticImage); err != nil {+ reader, pullErr := dockerClient.ImagePull(ctx, rusticImage, client.ImagePullOptions{})+ if pullErr != nil {+ return fmt.Errorf("pull Arcane tools image for Rustic: %w", pullErr)+ }+ if pullErr = dockerutil.RenderJSONMessageStream(reader, nil); pullErr != nil {+ _ = reader.Close()+ return fmt.Errorf("pull Arcane tools image for Rustic: %w", pullErr)+ }+ _ = reader.Close()+ }+ mounts := append([]mount.Mount{}, request.RepositoryMounts...)+ request.AppDataMount.Target = "/restore"+ request.AppDataMount.ReadOnly = false+ mounts = append(mounts, request.AppDataMount)+ snapshotPath := strings.TrimSpace(request.SnapshotPath)+ if snapshotPath == "" {+ snapshotPath = "/"+ }+ command := []string{"restore", "--delete", request.SnapshotID + ":" + snapshotPath, "/restore"}+ if _, err := rusticruntime.Run(ctx, dockerClient, request.RecoveryKey, command, request.RepositoryEnvironment, mounts, container.NetworkMode(request.NetworkMode)); err != nil {+ slog.Error("Rustic system restore failed", "error", err)+ return fmt.Errorf("rustic system restore failed: %w", err)+ }+ return nil+}diff --git a/backend/cli/recovery/recovery_test.go b/backend/cli/recovery/recovery_test.gonew file mode 100644index 000000000..ca9756524--- /dev/null+++ b/backend/cli/recovery/recovery_test.go@@ -0,0 +1,89 @@+package recovery++import (+ "context"+ "path/filepath"+ "testing"+ "time"++ "github.com/getarcaneapp/arcane/backend/v2/internal/activity"+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"+ recoverytypes "github.com/getarcaneapp/arcane/backend/v2/internal/recovery"+ "github.com/getarcaneapp/arcane/backend/v2/internal/systembackup"+ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"+ "github.com/libtnb/sqlite"+ "github.com/stretchr/testify/require"+ "gorm.io/gorm"+)++func newRestoredDatabaseForTestInternal(t *testing.T, runIDs ...string) (string, string) {+ t.Helper()+ databaseURL := "file:" + filepath.Join(t.TempDir(), "arcane.db")+ dsn, err := database.ParseSQLiteConnectionString(databaseURL)+ require.NoError(t, err)+ db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{})+ require.NoError(t, err)+ require.NoError(t, db.AutoMigrate(&systembackup.SystemBackupRun{}, &activity.Activity{}))+ for i, id := range runIDs {+ require.NoError(t, db.Create(&systembackup.SystemBackupRun{+ ID: id, CreatedAt: time.Date(2026, 1, 1+i, 0, 0, 0, 0, time.UTC),+ Status: systembackup.SystemBackupStatusRunning,+ }).Error)+ }+ resourceType := "system_backup"+ require.NoError(t, db.Create(&activity.Activity{+ ID: "activity-1", EnvironmentID: "0", Type: activitytypes.TypeResourceAction,+ Status: activitytypes.StatusRunning, ResourceType: &resourceType,+ StartedAt: time.Date(2026, 1, 2, 0, 0, 0, 0, time.UTC),+ }).Error)+ sqlDB, err := db.DB()+ require.NoError(t, err)+ require.NoError(t, sqlDB.Close())+ return databaseURL, dsn+}++func TestFinalizeRestoredBackupInternal(t *testing.T) {+ databaseURL, dsn := newRestoredDatabaseForTestInternal(t, "older", "selected")++ require.NoError(t, finalizeRestoredBackupInternal(context.Background(), databaseURL, "selected", "activity-1", recoverytypes.RestoreRequest{+ BackupID: "request-id", RemoteSnapshotID: "snapshot-1", S3DestinationID: "destination-1", Size: 1234,+ SafetyBackup: &recoverytypes.SafetyBackup{+ ID: "safety", LocalSnapshotID: "safety-snapshot", Size: 4321,+ CreatedAt: time.Date(2026, 1, 3, 0, 0, 0, 0, time.UTC),+ },+ }))++ db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{})+ require.NoError(t, err)+ var selected, older, safety systembackup.SystemBackupRun+ require.NoError(t, db.First(&selected, "id = ?", "selected").Error)+ require.Equal(t, systembackup.SystemBackupStatusSucceeded, selected.Status)+ require.EqualValues(t, 1234, selected.Size)+ require.Equal(t, "snapshot-1", selected.RemoteSnapshotID)+ require.Equal(t, "destination-1", selected.S3DestinationID)+ require.NoError(t, db.First(&older, "id = ?", "older").Error)+ require.Equal(t, systembackup.SystemBackupStatusRunning, older.Status)+ require.NoError(t, db.First(&safety, "id = ?", "safety").Error)+ require.Equal(t, systembackup.SystemBackupStatusSucceeded, safety.Status)+ require.Equal(t, systembackup.SystemBackupTriggerSafety, safety.Trigger)+ require.Equal(t, "safety-snapshot", safety.LocalSnapshotID)+ require.EqualValues(t, 4321, safety.Size)+ var entry activity.Activity+ require.NoError(t, db.First(&entry, "id = ?", "activity-1").Error)+ require.Equal(t, activitytypes.StatusSuccess, entry.Status)+}++func TestFinalizeRestoredBackupInternalFallsBackForLegacyManifest(t *testing.T) {+ databaseURL, dsn := newRestoredDatabaseForTestInternal(t, "older", "newest")++ require.NoError(t, finalizeRestoredBackupInternal(context.Background(), databaseURL, "", "", recoverytypes.RestoreRequest{BackupID: "missing-discovery-id"}))++ db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{})+ require.NoError(t, err)+ var newest systembackup.SystemBackupRun+ require.NoError(t, db.First(&newest, "id = ?", "newest").Error)+ require.Equal(t, systembackup.SystemBackupStatusSucceeded, newest.Status)+ var entry activity.Activity+ require.NoError(t, db.First(&entry, "id = ?", "activity-1").Error)+ require.Equal(t, activitytypes.StatusSuccess, entry.Status)+}diff --git a/backend/cli/root.go b/backend/cli/root.goindex 9cc851084..fa8083cc7 100644--- a/backend/cli/root.go+++ b/backend/cli/root.go@@ -9,6 +9,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/cli/admin""github.com/getarcaneapp/arcane/backend/v2/cli/generate"+ "github.com/getarcaneapp/arcane/backend/v2/cli/recovery""github.com/getarcaneapp/arcane/backend/v2/cli/upgrade""github.com/getarcaneapp/arcane/backend/v2/internal/bootstrap""github.com/getarcaneapp/arcane/backend/v2/internal/config"@@ -41,6 +42,7 @@ func Execute() {func init() {rootCmd.AddCommand(admin.AdminCmd)rootCmd.AddCommand(upgrade.UpgradeCmd)+ rootCmd.AddCommand(recovery.RestoreCmd)rootCmd.AddCommand(generate.GenerateCmd)}diff --git a/backend/cli/upgrade/recovery_environment_test.go b/backend/cli/upgrade/recovery_environment_test.gonew file mode 100644index 000000000..1689c4c7f--- /dev/null+++ b/backend/cli/upgrade/recovery_environment_test.go@@ -0,0 +1,22 @@+package upgrade++import (+ "slices"+ "testing"++ "github.com/stretchr/testify/require"+)++func TestApplyRecoveredEnvironmentInternal(t *testing.T) {+ result := applyRecoveredEnvironmentInternal([]string{+ "JWT_SECRET=fresh", "JWT_SECRET_FILE=/run/secrets/jwt", "ENCRYPTION_KEY__FILE=/run/secrets/key", "KEEP=value",+ }, map[string]string{"JWT_SECRET": "recovered-jwt", "ENCRYPTION_KEY": "recovered-key", "NEW_SETTING": "new"})++ require.Contains(t, result, "JWT_SECRET=recovered-jwt")+ require.Contains(t, result, "ENCRYPTION_KEY=recovered-key")+ require.Contains(t, result, "NEW_SETTING=new")+ require.Contains(t, result, "KEEP=value")+ require.False(t, slices.ContainsFunc(result, func(value string) bool {+ return value == "JWT_SECRET_FILE=/run/secrets/jwt" || value == "ENCRYPTION_KEY__FILE=/run/secrets/key"+ }))+}diff --git a/backend/cli/upgrade/upgrade.go b/backend/cli/upgrade/upgrade.goindex 186a99cf1..b401902dc 100644--- a/backend/cli/upgrade/upgrade.go+++ b/backend/cli/upgrade/upgrade.go@@ -117,7 +117,7 @@ func runUpgrade(cmd *cobra.Command, args []string) error {// Perform the upgradeslog.Info("Starting container upgrade", "container", containerName)- if err := upgradeContainer(ctx, dockerClient, targetContainer, imageToPull); err != nil {+ if err := UpgradeContainer(ctx, dockerClient, targetContainer, imageToPull, nil); err != nil {return errors.WrapIf(err, "failed to upgrade container")}@@ -488,7 +488,10 @@ func pullImage(ctx context.Context, dockerClient *client.Client, imageName strinreturn docker.RenderJSONMessageStream(reader, os.Stdout)}-func upgradeContainer(ctx context.Context, dockerClient *client.Client, oldContainer container.InspectResponse, newImage string) error {+// UpgradeContainer recreates an Arcane container with the supplied image and+// optional environment replacements. It is invoked from detached maintenance+// containers so stopping the target cannot interrupt the operation.+func UpgradeContainer(ctx context.Context, dockerClient *client.Client, oldContainer container.InspectResponse, newImage string, environment map[string]string) error {originalName := strings.TrimPrefix(oldContainer.Name, "/")oldName := fmt.Sprintf("%s-old-%d", originalName, time.Now().UnixNano())@@ -496,6 +499,9 @@ func upgradeContainer(ctx context.Context, dockerClient *client.Client, oldContaconfig := *oldContainer.Configconfig.Image = newImageconfig.Labels = normalizeRecreatedArcaneLabelsInternal(config.Labels)+ if len(environment) > 0 {+ config.Env = applyRecoveredEnvironmentInternal(config.Env, environment)+ }config.Labels = refreshRecreatedContainerLabelsInternal(ctx, dockerClient, config.Labels, oldContainer.Image, newImage)@@ -619,6 +625,40 @@ func upgradeContainer(ctx context.Context, dockerClient *client.Client, oldContareturn nil}+func applyRecoveredEnvironmentInternal(current []string, recovered map[string]string) []string {+ seen := make(map[string]struct{}, len(recovered))+ updated := make([]string, 0, len(current)+len(recovered))+ for _, entry := range current {+ name, _, found := strings.Cut(entry, "=")+ if !found {+ updated = append(updated, entry)+ continue+ }+ if baseName, ok := strings.CutSuffix(name, "__FILE"); ok {+ if _, ok := recovered[baseName]; ok {+ continue+ }+ }+ if baseName, ok := strings.CutSuffix(name, "_FILE"); ok {+ if _, ok := recovered[baseName]; ok {+ continue+ }+ }+ if replacement, ok := recovered[name]; ok {+ updated = append(updated, name+"="+replacement)+ seen[name] = struct{}{}+ } else {+ updated = append(updated, entry)+ }+ }+ for name, value := range recovered {+ if _, ok := seen[name]; !ok {+ updated = append(updated, name+"="+value)+ }+ }+ return updated+}+// looksLikeContainerID checks if a string looks like a Docker container ID// (12 or 64 lowercase hex characters, which Docker auto-generates as hostnames)func looksLikeContainerID(s string) bool {diff --git a/backend/go.mod b/backend/go.modindex 09d2d3457..88cbc4f4a 100644--- a/backend/go.mod+++ b/backend/go.mod@@ -11,9 +11,11 @@ require (emperror.dev/emperror v0.33.0emperror.dev/errors v0.8.1github.com/anthdm/hollywood v1.0.5+ github.com/aws/aws-sdk-go-v2 v1.43.6github.com/aws/aws-sdk-go-v2/config v1.32.37github.com/aws/aws-sdk-go-v2/credentials v1.19.36github.com/aws/aws-sdk-go-v2/service/ecr v1.60.6+ github.com/aws/aws-sdk-go-v2/service/s3 v1.106.0github.com/cenkalti/backoff/v5 v5.0.3github.com/coder/websocket v1.8.15github.com/compose-spec/compose-go/v2 v2.14.0@@ -33,6 +35,7 @@ require (github.com/gofrs/flock v0.13.0github.com/golang-jwt/jwt/v5 v5.3.1github.com/google/go-containerregistry v0.21.9+ github.com/google/uuid v1.6.0github.com/jinzhu/copier v0.4.0github.com/klauspost/compress v1.19.2github.com/labstack/echo/v5 v5.3.1@@ -91,13 +94,15 @@ require (github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirectgithub.com/agnivade/levenshtein v1.2.1 // indirectgithub.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect- github.com/aws/aws-sdk-go-v2 v1.43.6 // indirect+ github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 // indirectgithub.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.37 // indirectgithub.com/aws/aws-sdk-go-v2/internal/configsources v1.4.37 // indirectgithub.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.37 // indirectgithub.com/aws/aws-sdk-go-v2/internal/v4a v1.4.38 // indirectgithub.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.17 // indirect+ github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.24 // indirectgithub.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.37 // indirect+ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.32 // indirectgithub.com/aws/aws-sdk-go-v2/service/signin v1.5.6 // indirectgithub.com/aws/aws-sdk-go-v2/service/sso v1.33.6 // indirectgithub.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.6 // indirect@@ -175,7 +180,6 @@ require (github.com/google/go-tpm v0.9.8 // indirectgithub.com/google/go-tpm-tools v0.4.8 // indirectgithub.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect- github.com/google/uuid v1.6.0 // indirectgithub.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirectgithub.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirectgithub.com/hashicorp/errwrap v1.1.0 // indirectdiff --git a/backend/go.sum b/backend/go.sumindex 36a1b844b..3c1fd298c 100644--- a/backend/go.sum+++ b/backend/go.sum@@ -81,6 +81,8 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dgithub.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=github.com/aws/aws-sdk-go-v2 v1.43.6 h1:RrmFcqCBxkJuf7g1axVo5krB4jM/AO8r5e5oujrgdoQ=github.com/aws/aws-sdk-go-v2 v1.43.6/go.mod h1:tXpPM+v0D1lndmga+HqqLDIzUFJlEeR21aspVklHF00=+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 h1:3IZY0XAJquT3aHzbkHfPzy4ACPcEjVG0x87KOwtpqGY=+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14/go.mod h1:zwM6veDkhGgQFqkBy+uT28AAYpLu+uFMlPl+rCg/73E=github.com/aws/aws-sdk-go-v2/config v1.32.37 h1:Ljl7LOJB6ym0liuEl0+TZ3d7f5I8MEZN1Cj9PINlj/g=github.com/aws/aws-sdk-go-v2/config v1.32.37/go.mod h1:WJ7pe7ZPpmG8Q5kKS53zeypIV4FBGACxmte8Uc6SgUc=github.com/aws/aws-sdk-go-v2/credentials v1.19.36 h1:84s5xMme6ENYEdKG8rsbSFFg/8+lbHBeM9QYSO0gnDk=@@ -97,10 +99,16 @@ github.com/aws/aws-sdk-go-v2/service/ecr v1.60.6 h1:RbjO6G1wu+q43r0322sDABXi9vq/github.com/aws/aws-sdk-go-v2/service/ecr v1.60.6/go.mod h1:snsosIuclt9tpFKzldCnu1ykT5SYEND/bK9qTJmoJ+o=github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.17 h1:OvYZOB3qA6zvfdRFiRFRzVSiElMYrz3GdntkXZxlp1o=github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.17/go.mod h1:JgR/2Ew50ACfIWau1oeMRX59tMtC0kM+PYQGEaT04cY=+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.24 h1:mdPwDQPqxlw9Sc62Nt15yjEcARaDbPXkjRYtXsUripo=+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.24/go.mod h1:ls5ytnwLTcQaUu32fMYXFI3MjpKuTwL840PAm9iqyEg=github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.37 h1:a3D4AjrOrTrP8+d9ILBthqrElf0z1JNol09Xvnwcys8=github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.37/go.mod h1:ky0gTu+ukvUTuUKFIpp6Wid4oninrkCyvbFkVs0kpHM=+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.32 h1:jWXtZdCnhXa9sGFixRaU2AxT4DIVse9HS4E2f+/KwV0=+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.32/go.mod h1:9JS1UpfVvyD/ZPX8GsKb/Pq8scEM+7GP5fqh9SwH7po=github.com/aws/aws-sdk-go-v2/service/kms v1.52.0 h1:QNtg+Mtj1zmepk568+UKBD5DFfqh+ESTUUqQT27JkQc=github.com/aws/aws-sdk-go-v2/service/kms v1.52.0/go.mod h1:Y0+uxvxz6ib4KktRdK0V4X45Vcs/JyYoz8H71pO8xeI=+github.com/aws/aws-sdk-go-v2/service/s3 v1.106.0 h1:7QZWVJZWzHivHWIa+5TELLaBBkbuoj0GPwQtMlJ0sqk=+github.com/aws/aws-sdk-go-v2/service/s3 v1.106.0/go.mod h1:fcvq5L7dK+5cQFicEJwpI6e6Wn8NY2i6yT5wRLYVc7s=github.com/aws/aws-sdk-go-v2/service/signin v1.5.6 h1:i68sFvXidKlkiSvI7d7Ilc1/UvW4CtBOaivH7jhG4fs=github.com/aws/aws-sdk-go-v2/service/signin v1.5.6/go.mod h1:/h7Obr9WTtzbjTHGASRQwLN7Bupw+TC3x8x7fyx39hE=github.com/aws/aws-sdk-go-v2/service/sso v1.33.6 h1:tpfGChmjUmv3W9WlRvy+stwKDTbFFdq8Zk9DbFPrfMU=diff --git a/backend/internal/backup/engine.go b/backend/internal/backup/engine.gonew file mode 100644index 000000000..046fb7329--- /dev/null+++ b/backend/internal/backup/engine.go@@ -0,0 +1,296 @@+// Package backup owns the shared Rustic backup engine used by volume and+// system backups: typed repository operations, per-repository serialization+// through the actor runtime, and run admission.+package backup++import (+ "context"+ "encoding/json/v2"+ "fmt"+ "io"+ "log/slog"+ "strings"+ "time"++ "emperror.dev/errors"+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"+ "github.com/getarcaneapp/arcane/backend/v2/internal/common"+ "github.com/getarcaneapp/arcane/backend/v2/internal/image"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"+ rusticruntime "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/rustic"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/volumehelper"+ "github.com/google/uuid"+ "github.com/moby/moby/api/types/container"+ "github.com/moby/moby/api/types/mount"+ "github.com/moby/moby/client"+)++// Admission scopes shared by the backup engine and the per-policy job+// registries so scheduled and manual runs contend on the same leases.+const (+ VolumeAdmissionScope = "volume-backup"+ SystemAdmissionScope = "system-backup"+)++// Repository addresses one Rustic repository. ID is the stable serialization+// key: operations against the same ID run serially, different IDs concurrently.+type Repository struct {+ ID string+ Environment []string+ Mounts []mount.Mount+}++// Snapshot is the result of a completed Rustic backup operation.+type Snapshot struct {+ ID string `json:"id"`+ Size int64 `json:"-"`+}++type rusticSnapshotOutputInternal struct {+ ID string `json:"id"`+ Summary struct {+ TotalBytesProcessed int64 `json:"total_bytes_processed"`+ } `json:"summary"`+}++// RestoreOptions selects what a snapshot restore writes and where.+type RestoreOptions struct {+ // DeleteExtra removes files in the target that are absent from the snapshot.+ DeleteExtra bool+ // SourcePath restores only this path inside the snapshot when non-empty.+ SourcePath string+ // DestinationPath overrides the restore target path inside the helper+ // container; the target mount's path is used when empty.+ DestinationPath string+}++// Engine executes typed Rustic operations through the official Rustic image.+// Concurrency is actor-owned: one executor per repository ID plus the shared+// application admission gate for run exclusivity.+type Engine struct {+ imageService *image.ImageService+ runtime *actors.Runtime+ admission *actors.Gate[actors.AdmissionKey]+ lifecycleCtx context.Context+ executors *actors.StateMap[string, *actors.Executor]+}++// NewEngine creates the shared backup engine on the actor runtime. ctx is the+// application lifecycle context repository executors are spawned on.+func NewEngine(ctx context.Context, runtime *actors.Runtime, admission *actors.Gate[actors.AdmissionKey], imageService *image.ImageService) *Engine {+ return &Engine{+ imageService: imageService,+ runtime: runtime,+ admission: admission,+ lifecycleCtx: ctx,+ executors: actors.NewStateMap[string, *actors.Executor](),+ }+}++// TryAcquireRun admits at most one in-flight backup run per (scope, id),+// shared between scheduled jobs and manual API triggers.+func (e *Engine) TryAcquireRun(ctx context.Context, scope, id string) (*actors.Lease[actors.AdmissionKey], bool, error) {+ if e == nil {+ return nil, false, errors.New("backup engine is unavailable")+ }+ return e.admission.TryAcquire(ctx, actors.AdmissionKey{Scope: scope, ID: id})+}++// Stop joins every repository executor.+func (e *Engine) Stop(ctx context.Context) error {+ if e == nil {+ return nil+ }+ executors, err := e.executors.Drain(ctx, "stop backup repository executors")+ if err != nil {+ return err+ }+ var stopErr error+ for _, executor := range executors {+ stopErr = errors.Combine(stopErr, executor.Stop(ctx))+ }+ return stopErr+}++// CreateSnapshot backs the source mount up into the repository and returns the+// created snapshot. The repository is initialized on first use.+func (e *Engine) CreateSnapshot(ctx context.Context, dockerClient *client.Client, repository Repository, password, label string, source mount.Mount) (Snapshot, error) {+ output, err := e.runInternal(ctx, dockerClient, repository, password,+ []string{"backup", "--init", "--json", "--as-path", "/", "--host", "arcane", "--label", label, "--", source.Target},+ source,+ )+ if err != nil {+ return Snapshot{}, err+ }+ var decoded rusticSnapshotOutputInternal+ if err := json.Unmarshal([]byte(output), &decoded); err != nil {+ return Snapshot{}, fmt.Errorf("failed to decode Rustic snapshot: %w", err)+ }+ if decoded.ID == "" {+ return Snapshot{}, errors.New("rustic did not return a snapshot ID")+ }+ return Snapshot{ID: decoded.ID, Size: decoded.Summary.TotalBytesProcessed}, nil+}++// RestoreSnapshot restores a snapshot (or one path inside it) onto the target mount.+func (e *Engine) RestoreSnapshot(ctx context.Context, dockerClient *client.Client, repository Repository, password, snapshotID string, target mount.Mount, options RestoreOptions) error {+ command := []string{"restore"}+ if options.DeleteExtra {+ command = append(command, "--delete")+ }+ source := snapshotID + ":/"+ if options.SourcePath != "" {+ source = snapshotID + ":/" + strings.TrimPrefix(options.SourcePath, "/")+ }+ destination := options.DestinationPath+ if destination == "" {+ destination = target.Target+ }+ command = append(command, "--", source, destination)+ _, err := e.runInternal(ctx, dockerClient, repository, password, command, target)+ return err+}++// ListSnapshotFiles returns every path recorded in the snapshot.+func (e *Engine) ListSnapshotFiles(ctx context.Context, dockerClient *client.Client, repository Repository, password, snapshotID string) ([]string, error) {+ output, err := e.runInternal(ctx, dockerClient, repository, password, []string{"ls", "--json", "--recursive", "--", snapshotID + ":/"})+ if err != nil {+ return nil, fmt.Errorf("failed to list Rustic snapshot: %w", err)+ }+ var files []string+ if err := json.Unmarshal([]byte(output), &files); err != nil {+ return nil, fmt.Errorf("failed to decode Rustic file list: %w", err)+ }+ return files, nil+}++// DiscoveredSnapshot describes one snapshot found in a repository.+type DiscoveredSnapshot struct {+ ID string `json:"id"`+ Time time.Time `json:"time"`+ Summary struct {+ TotalBytesProcessed int64 `json:"total_bytes_processed"`+ } `json:"summary"`+}++// ListSnapshots enumerates every snapshot in the repository.+func (e *Engine) ListSnapshots(ctx context.Context, dockerClient *client.Client, repository Repository, password string) ([]DiscoveredSnapshot, error) {+ output, err := e.runInternal(ctx, dockerClient, repository, password, []string{"snapshots", "--json"})+ if err != nil {+ return nil, err+ }+ var snapshots []DiscoveredSnapshot+ if err := json.Unmarshal([]byte(output), &snapshots); err != nil {+ return nil, fmt.Errorf("failed to decode Rustic snapshots: %w", err)+ }+ if len(snapshots) == 0 || snapshots[0].ID != "" {+ return snapshots, nil+ }+ // Newer Rustic versions group snapshots by (host, label).+ var groups []struct {+ Snapshots []DiscoveredSnapshot `json:"snapshots"`+ }+ if err := json.Unmarshal([]byte(output), &groups); err != nil {+ return nil, fmt.Errorf("failed to decode grouped Rustic snapshots: %w", err)+ }+ snapshots = snapshots[:0]+ for _, group := range groups {+ snapshots = append(snapshots, group.Snapshots...)+ }+ return snapshots, nil+}++// ForgetSnapshot removes the snapshot from the repository and prunes its data.+func (e *Engine) ForgetSnapshot(ctx context.Context, dockerClient *client.Client, repository Repository, password, snapshotID string) error {+ _, err := e.runInternal(ctx, dockerClient, repository, password, []string{"forget", "--prune", "--", snapshotID})+ return err+}++// Replicate copies one snapshot between repositories by restoring it into a+// temporary volume and backing that volume up into the target repository. The+// source is read once from the repository, never from the live data. Rustic's+// native `copy` would move only missing packs, but it addresses the target via+// a TOML config profile, which the env-only Repository cannot express yet —+// the materialize-and-rebackup here trades disk and I/O for that simplicity.+func (e *Engine) Replicate(ctx context.Context, dockerClient *client.Client, from Repository, fromSnapshotID string, to Repository, password, label string) (Snapshot, error) {+ temporaryVolume := "arcane-rustic-copy-" + uuid.NewString()+ if _, err := dockerClient.VolumeCreate(ctx, client.VolumeCreateOptions{Name: temporaryVolume, Labels: volumehelper.Labels()}); err != nil {+ return Snapshot{}, fmt.Errorf("failed to create temporary Rustic copy volume: %w", err)+ }+ defer func() {+ _, _ = dockerClient.VolumeRemove(context.WithoutCancel(ctx), temporaryVolume, client.VolumeRemoveOptions{Force: true})+ }()+ copyMount := mount.Mount{Type: mount.TypeVolume, Source: temporaryVolume, Target: "/volume"}+ if err := e.RestoreSnapshot(ctx, dockerClient, from, password, fromSnapshotID, copyMount, RestoreOptions{DeleteExtra: true}); err != nil {+ return Snapshot{}, fmt.Errorf("failed to load Rustic snapshot for replication: %w", err)+ }+ copyMount.ReadOnly = true+ snapshot, err := e.CreateSnapshot(ctx, dockerClient, to, password, label, copyMount)+ if err != nil {+ return Snapshot{}, fmt.Errorf("failed to replicate Rustic snapshot: %w", err)+ }+ return snapshot, nil+}++func (e *Engine) executorForInternal(repositoryID string) (*actors.Executor, error) {+ if executor, ok := e.executors.Get(repositoryID); ok {+ return executor, nil+ }+ return actors.ApplyStateMap(e.lifecycleCtx, e.executors, "backup repository executor", func(values map[string]*actors.Executor) (*actors.Executor, bool, error) {+ if executor, ok := values[repositoryID]; ok {+ return executor, false, nil+ }+ executor, err := actors.NewExecutor(e.lifecycleCtx, e.runtime, "backup-repository", repositoryID, 3)+ if err != nil {+ return nil, false, err+ }+ values[repositoryID] = executor+ return executor, true, nil+ })+}++func (e *Engine) runInternal(ctx context.Context, dockerClient *client.Client, repository Repository, password string, command []string, extraMounts ...mount.Mount) (string, error) {+ if e == nil {+ return "", errors.New("backup engine is unavailable")+ }+ if strings.TrimSpace(repository.ID) == "" {+ return "", errors.New("backup repository ID is required")+ }+ executor, err := e.executorForInternal(repository.ID)+ if err != nil {+ return "", err+ }+ return actors.Execute(ctx, executor, "rustic "+command[0], func(workCtx context.Context) (string, error) {+ return e.runContainerInternal(workCtx, dockerClient, repository, password, command, extraMounts...)+ }, nil)+}++func (e *Engine) ensureImageInternal(ctx context.Context, dockerClient *client.Client) error {+ if _, err := dockerClient.ImageInspect(ctx, rusticruntime.DefaultImage); err == nil {+ return nil+ }+ if e.imageService == nil {+ return errors.New("image service is unavailable")+ }+ if err := e.imageService.PullImage(ctx, rusticruntime.DefaultImage, io.Discard, common.SystemUser, nil); err != nil {+ return fmt.Errorf("failed to pull official Rustic image: %w", err)+ }+ return nil+}+func arcaneNetworkModeInternal(ctx context.Context, dockerClient *client.Client) container.NetworkMode {+ arcane, err := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient)+ if err != nil || arcane == nil || arcane.ID == "" {+ slog.DebugContext(ctx, "backup engine: running Rustic on the default network", "error", err)+ return ""+ }+ return container.NetworkMode("container:" + arcane.ID)+}++func (e *Engine) runContainerInternal(ctx context.Context, dockerClient *client.Client, repository Repository, password string, command []string, extraMounts ...mount.Mount) (string, error) {+ if err := e.ensureImageInternal(ctx, dockerClient); err != nil {+ return "", err+ }+ mounts := append([]mount.Mount{}, repository.Mounts...)+ mounts = append(mounts, extraMounts...)+ return rusticruntime.Run(ctx, dockerClient, password, command, repository.Environment, mounts, arcaneNetworkModeInternal(ctx, dockerClient))+}diff --git a/backend/internal/backup/policies.go b/backend/internal/backup/policies.gonew file mode 100644index 000000000..7fb668538--- /dev/null+++ b/backend/internal/backup/policies.go@@ -0,0 +1,137 @@+package backup++import (+ "context"+ "fmt"+ "strings"++ "emperror.dev/errors"++ "github.com/getarcaneapp/arcane/backend/v2/internal/database"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/schedule"+ backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"+ "gorm.io/gorm"+)++// PolicyReconciliation reconciles a full set of policy updates against the+// existing rows of one backup domain: shared validation, create/update/delete+// in a single transaction, and the scheduled-job swap. Volume and system+// backups both run through it so the flow cannot drift between them.+type PolicyReconciliation[P any] struct {+ // Domain names the backup flavor in validation errors ("volume", "system").+ Domain string+ DB *database.DB+ // Existing is every policy currently persisted for the reconciled scope.+ Existing []P+ ID func(*P) string+ // New returns the blank policy for created entries, pre-scoped by the+ // caller (e.g. with the volume name set).+ New func() P+ Apply func(*P, backuptypes.UpdateBackupPolicy)+ // ValidateUpdate adds per-domain checks on top of the shared ones; nil skips.+ ValidateUpdate func(backuptypes.UpdateBackupPolicy) error+ // S3Configured verifies the destination exists and is usable; its error is+ // returned to the caller verbatim.+ S3Configured func(ctx context.Context, destinationID string) error+ Unregister func(ctx context.Context, policyID string)+ Reschedule func(ctx context.Context, policy *P)+}++func (r PolicyReconciliation[P]) Run(ctx context.Context, updates []backuptypes.UpdateBackupPolicy) error {+ if err := r.validateInternal(ctx, updates); err != nil {+ return err+ }+ policies, kept, err := r.buildInternal(updates)+ if err != nil {+ return err+ }+ err = r.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ for i := range policies {+ if saveErr := tx.Save(&policies[i]).Error; saveErr != nil {+ return saveErr+ }+ }+ for i := range r.Existing {+ if _, ok := kept[r.ID(&r.Existing[i])]; !ok {+ if deleteErr := tx.Delete(&r.Existing[i]).Error; deleteErr != nil {+ return deleteErr+ }+ }+ }+ return nil+ })+ if err != nil {+ return fmt.Errorf("failed to save %s backup policies: %w", r.Domain, err)+ }+ for i := range r.Existing {+ if _, ok := kept[r.ID(&r.Existing[i])]; !ok {+ r.Unregister(ctx, r.ID(&r.Existing[i]))+ }+ }+ for i := range policies {+ r.Reschedule(ctx, &policies[i])+ }+ return nil+}++// validateInternal checks the shared policy rules and normalizes the updates+// in place (canonical schedule, cleared destination when S3 is off).+func (r PolicyReconciliation[P]) validateInternal(ctx context.Context, updates []backuptypes.UpdateBackupPolicy) error {+ for i := range updates {+ normalized, err := schedule.NormalizeSixField(updates[i].Schedule, r.Domain+" backup")+ if err != nil {+ return err+ }+ updates[i].Schedule = normalized+ if updates[i].RetentionCount < 0 || updates[i].RetentionCount > 3650 {+ return errors.New("retentionCount must be between 0 and 3650")+ }+ if !updates[i].LocalEnabled && !updates[i].S3Enabled {+ return fmt.Errorf("select at least one %s backup destination", r.Domain)+ }+ if updates[i].S3Enabled {+ if strings.TrimSpace(updates[i].S3DestinationID) == "" {+ return fmt.Errorf("select an S3 destination for %s backups", r.Domain)+ }+ if err := r.S3Configured(ctx, updates[i].S3DestinationID); err != nil {+ return err+ }+ } else {+ updates[i].S3DestinationID = ""+ }+ if r.ValidateUpdate != nil {+ if err := r.ValidateUpdate(updates[i]); err != nil {+ return err+ }+ }+ }+ return nil+}++// buildInternal maps the updates onto existing or new policy rows and reports+// which existing IDs survive the reconciliation.+func (r PolicyReconciliation[P]) buildInternal(updates []backuptypes.UpdateBackupPolicy) ([]P, map[string]struct{}, error) {+ byID := make(map[string]P, len(r.Existing))+ for i := range r.Existing {+ byID[r.ID(&r.Existing[i])] = r.Existing[i]+ }+ policies := make([]P, 0, len(updates))+ kept := make(map[string]struct{}, len(updates))+ for _, update := range updates {+ policy := r.New()+ if update.ID != "" {+ var ok bool+ policy, ok = byID[update.ID]+ if !ok {+ return nil, nil, fmt.Errorf("%s backup policy not found", r.Domain)+ }+ if _, duplicate := kept[update.ID]; duplicate {+ return nil, nil, fmt.Errorf("duplicate %s backup policy", r.Domain)+ }+ kept[update.ID] = struct{}{}+ }+ r.Apply(&policy, update)+ policies = append(policies, policy)+ }+ return policies, kept, nil+}diff --git a/backend/internal/backup/retention.go b/backend/internal/backup/retention.gonew file mode 100644index 000000000..f7822147a--- /dev/null+++ b/backend/internal/backup/retention.go@@ -0,0 +1,25 @@+package backup++import (+ "context"++ "github.com/getarcaneapp/arcane/backend/v2/internal/database"+)++// ExpiredRunIDs returns the IDs of succeeded runs with snapshots that fall+// outside the newest keep entries for one policy, oldest last. Callers delete+// each run through their own delete path so snapshots are forgotten too.+func ExpiredRunIDs(ctx context.Context, db *database.DB, table, policyID string, keep int) ([]string, error) {+ var ids []string+ err := db.WithContext(ctx).+ Table(table).+ Where(+ "policy_id = ? AND status = ? AND (COALESCE(local_snapshot_id, '') <> '' OR COALESCE(remote_snapshot_id, '') <> '')",+ policyID,+ "succeeded",+ ).+ Order("created_at DESC").+ Offset(keep).+ Pluck("id", &ids).Error+ return ids, err+}diff --git a/backend/internal/bootstrap/jobs_bootstrap.go b/backend/internal/bootstrap/jobs_bootstrap.goindex 71923e062..6a579a1f6 100644--- a/backend/internal/bootstrap/jobs_bootstrap.go+++ b/backend/internal/bootstrap/jobs_bootstrap.go@@ -15,6 +15,8 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/job""github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/backend/v2/internal/system"+ "github.com/getarcaneapp/arcane/backend/v2/internal/systembackup"+ "github.com/getarcaneapp/arcane/backend/v2/internal/volume""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler"schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"@@ -70,12 +72,14 @@ type registerJobsParams struct {Scheduler schedulertypes.JobSchedulerActorRuntime *actors.Runtime- Activity *activity.ActivityService- GitOpsSync *gitops.GitOpsSyncService- Environment *environment.EnvironmentService- JobSchedule *job.JobService- Settings *settings.SettingsService- Admission *actors.Gate[actors.AdmissionKey]+ Activity *activity.ActivityService+ GitOpsSync *gitops.GitOpsSyncService+ Environment *environment.EnvironmentService+ JobSchedule *job.JobService+ Settings *settings.SettingsService+ Volume *volume.VolumeService+ SystemBackup *systembackup.SystemBackupService+ Admission *actors.Gate[actors.AdmissionKey]AutoUpdate *scheduler.AutoUpdateJobImageUpdateWatcher *scheduler.ImageUpdateWatcher@@ -144,13 +148,15 @@ func registerJobs(params registerJobsParams) error {// GitOps sync and environment health are no longer single global jobs; each// entity registers its own dynamic job.if err := registerDynamicJobs(dynamicJobsParams{- AppCtx: params.AppCtx,- Config: params.Config,- Scheduler: params.Scheduler,- GitOpsSync: params.GitOpsSync,- Environment: params.Environment,- JobSchedule: params.JobSchedule,- Admission: params.Admission,+ AppCtx: params.AppCtx,+ Config: params.Config,+ Scheduler: params.Scheduler,+ GitOpsSync: params.GitOpsSync,+ Environment: params.Environment,+ JobSchedule: params.JobSchedule,+ Volume: params.Volume,+ SystemBackup: params.SystemBackup,+ Admission: params.Admission,}); err != nil {return err}@@ -176,19 +182,36 @@ func registerJobs(params registerJobsParams) error {}type dynamicJobsParams struct {- AppCtx context.Context- Config *config.Config- Scheduler schedulertypes.JobScheduler- GitOpsSync *gitops.GitOpsSyncService- Environment *environment.EnvironmentService- JobSchedule *job.JobService- Admission *actors.Gate[actors.AdmissionKey]+ AppCtx context.Context+ Config *config.Config+ Scheduler schedulertypes.JobScheduler+ GitOpsSync *gitops.GitOpsSyncService+ Environment *environment.EnvironmentService+ JobSchedule *job.JobService+ Volume *volume.VolumeService+ SystemBackup *systembackup.SystemBackupService+ Admission *actors.Gate[actors.AdmissionKey]}// registerDynamicJobs injects the scheduler into the services that own per-entity// jobs and registers the jobs for already-existing entities at startup. AddJob is// an idempotent upsert, so these run safely before the scheduler is started.func registerDynamicJobs(params dynamicJobsParams) error {+ // Volume backups run on the environment that owns the Docker volume. This is+ // registered on managers and agents; environment proxying persists each policy+ // in the correct Arcane database.+ if params.Volume != nil {+ if err := params.Volume.SetScheduler(params.AppCtx, params.Scheduler, params.Admission); err != nil {+ return err+ }+ params.Volume.RegisterBackupJobsOnStartup(params.AppCtx)+ }+ if !params.Config.AgentMode && params.SystemBackup != nil {+ if err := params.SystemBackup.SetScheduler(params.AppCtx, params.Scheduler, params.Admission); err != nil {+ return err+ }+ params.SystemBackup.RegisterBackupJobOnStartup(params.AppCtx)+ }// GitOps: one job per auto-sync-enabled sync (runs on manager and agents).if params.GitOpsSync != nil {if err := params.GitOpsSync.SetScheduler(params.AppCtx, params.Scheduler, params.Admission); err != nil {diff --git a/backend/internal/dashboard/service_test.go b/backend/internal/dashboard/service_test.goindex 7e287a48b..bdb8ec077 100644--- a/backend/internal/dashboard/service_test.go+++ b/backend/internal/dashboard/service_test.go@@ -191,7 +191,7 @@ func TestDashboardService_GetSnapshot_ReturnsDashboardSnapshot(t *testing.T) {Status: project.ProjectStatusStopped,}).Error)projectSvc := project.NewProjectService(db, settingsSvc, nil, image.NewImageService(db, nil, nil, nil, nil, nil), nil, nil, nil, nil, config.Load())- svc := NewDashboardService(db, dockerSvc, nil, projectSvc, nil, settingsSvc, nil, nil, nil, volume.NewVolumeService(db, nil, nil, nil, nil, nil))+ svc := NewDashboardService(db, dockerSvc, nil, projectSvc, nil, settingsSvc, nil, nil, nil, volume.NewVolumeService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil))snapshot, err := svc.GetSnapshot(context.Background(), DashboardActionItemsOptions{}, true)require.NoError(t, err)diff --git a/backend/internal/database/database.go b/backend/internal/database/database.goindex 5ddad12e9..5ad07492b 100644--- a/backend/internal/database/database.go+++ b/backend/internal/database/database.go@@ -120,7 +120,7 @@ func connectDatabaseInternal(ctx context.Context, databaseURL string) (*DB, erroswitch {case strings.HasPrefix(databaseURL, "file:"):- connString, err := parseSqliteConnectionStringInternal(databaseURL)+ connString, err := ParseSQLiteConnectionString(databaseURL)if err != nil {return nil, errors.WrapIf(err, "failed to parse SQLite connection string")}@@ -608,7 +608,7 @@ func missingEmbeddedDowngradeMigrationsInternal(ctx context.Context, db *sql.DB,return missing, nil}-func parseSqliteConnectionStringInternal(connString string) (string, error) {+func ParseSQLiteConnectionString(connString string) (string, error) {if !strings.HasPrefix(connString, "file:") {connString = "file:" + connString}diff --git a/backend/internal/database/database_test.go b/backend/internal/database/database_test.goindex 5503173cb..811adfbe7 100644--- a/backend/internal/database/database_test.go+++ b/backend/internal/database/database_test.go@@ -202,6 +202,41 @@ func TestMigration070_PasskeysAndMFA_UpAndDown(t *testing.T) {assert.Zero(t, columnCount)}+// TestMigration073_BackupSupport_PreservesExistingBackups proves that+// pre-existing volume backup rows survive the backup-support migration as+// format=archive, and that downgrading is refused while Rustic rows exist.+func TestMigration073_BackupSupport_PreservesExistingBackups(t *testing.T) {+ ctx := context.Background()+ rawDB, _ := newSQLiteSQLDBInternal(t, t.TempDir(), "arcane-backup-support.db")++ require.NoError(t, migrateDatabaseToVersionInternal(ctx, rawDB, dbProviderSQLite, MigrationOptions{}, 72))+ _, err := rawDB.ExecContext(ctx, `INSERT INTO volume_backups (id, volume_name, size, created_at) VALUES ('legacy-1', 'app-data', 42, CURRENT_TIMESTAMP)`)+ require.NoError(t, err)++ require.NoError(t, migrateDatabaseToVersionInternal(ctx, rawDB, dbProviderSQLite, MigrationOptions{}, 73))+ var format, status, destination string+ var size int64+ require.NoError(t, rawDB.QueryRow(`SELECT format, status, destination, size FROM volume_backups WHERE id = 'legacy-1'`).Scan(&format, &status, &destination, &size))+ assert.Equal(t, "archive", format)+ assert.Equal(t, "succeeded", status)+ assert.Equal(t, "local", destination)+ assert.EqualValues(t, 42, size)++ // Downgrade is refused while a Rustic-format row exists.+ _, err = rawDB.ExecContext(ctx, `INSERT INTO volume_backups (id, volume_name, size, created_at, format, local_snapshot_id) VALUES ('rustic-1', 'app-data', 7, CURRENT_TIMESTAMP, 'rustic', 'snap-1')`)+ require.NoError(t, err)+ err = migrateDatabaseToVersionInternal(ctx, rawDB, dbProviderSQLite, MigrationOptions{AllowDowngrade: true}, 72)+ require.Error(t, err)++ // With only archive rows left, the downgrade succeeds and keeps the row.+ _, err = rawDB.ExecContext(ctx, `DELETE FROM volume_backups WHERE id = 'rustic-1'`)+ require.NoError(t, err)+ require.NoError(t, migrateDatabaseToVersionInternal(ctx, rawDB, dbProviderSQLite, MigrationOptions{AllowDowngrade: true}, 72))+ var count int+ require.NoError(t, rawDB.QueryRow(`SELECT COUNT(*) FROM volume_backups WHERE id = 'legacy-1'`).Scan(&count))+ assert.Equal(t, 1, count)+}+func TestMigration072_ProjectTags_UpDownAndCascade(t *testing.T) {ctx := context.Background()rawDB, _ := newSQLiteSQLDBInternal(t, t.TempDir(), "arcane-project-tags.db")diff --git a/backend/internal/di/di.go b/backend/internal/di/di.goindex bd7301ac8..941cb175d 100644--- a/backend/internal/di/di.go+++ b/backend/internal/di/di.go@@ -21,6 +21,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/session""github.com/getarcaneapp/arcane/backend/v2/internal/swarm""github.com/getarcaneapp/arcane/backend/v2/internal/system"+ "github.com/getarcaneapp/arcane/backend/v2/internal/systembackup""github.com/getarcaneapp/arcane/backend/v2/internal/upload""github.com/getarcaneapp/arcane/backend/v2/internal/variable""github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability"@@ -92,6 +93,9 @@ var ServiceOptions = fx.Options(provideWebhookModuleInternal,variable.NewVariableService,variable.New,+ provideS3ModuleInternal,+ provideS3ServiceInternal,+ provideBackupEngineInternal,upload.NewUploadService,upload.New,@@ -100,6 +104,9 @@ var ServiceOptions = fx.Options(provideGitRepositoryModuleInternal,provideGitRepositoryServiceInternal,provideVolumeModuleInternal,+ provideVolumeServiceInternal,+ provideSystemBackupServiceInternal,+ systembackup.New,provideAuthServiceInternal,provideAuthModuleInternal,provideContainerRegistryModuleInternal,diff --git a/backend/internal/di/di_test.go b/backend/internal/di/di_test.goindex 7452eadb3..00e583336 100644--- a/backend/internal/di/di_test.go+++ b/backend/internal/di/di_test.go@@ -8,6 +8,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/activity""github.com/getarcaneapp/arcane/backend/v2/internal/apikey""github.com/getarcaneapp/arcane/backend/v2/internal/auth"+ "github.com/getarcaneapp/arcane/backend/v2/internal/backup""github.com/getarcaneapp/arcane/backend/v2/internal/docker""github.com/getarcaneapp/arcane/backend/v2/internal/environment""github.com/getarcaneapp/arcane/backend/v2/internal/event"@@ -22,6 +23,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/project""github.com/getarcaneapp/arcane/backend/v2/internal/registry""github.com/getarcaneapp/arcane/backend/v2/internal/role"+ s3domain "github.com/getarcaneapp/arcane/backend/v2/internal/s3""github.com/getarcaneapp/arcane/backend/v2/internal/session""github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/backend/v2/internal/swarm"@@ -42,6 +44,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/port""github.com/getarcaneapp/arcane/backend/v2/internal/search""github.com/getarcaneapp/arcane/backend/v2/internal/system"+ "github.com/getarcaneapp/arcane/backend/v2/internal/systembackup""github.com/getarcaneapp/arcane/backend/v2/internal/updater""github.com/getarcaneapp/arcane/backend/v2/internal/version""github.com/getarcaneapp/arcane/backend/v2/internal/volume"@@ -68,6 +71,9 @@ type graphParams struct {BuildWorkspace *build.BuildWorkspaceServiceLifecycle *project.LifecycleServiceVolume *volume.Module+ BackupEngine *backup.Engine+ S3Destination *s3domain.Module+ SystemBackup *systembackup.ModuleNetwork *network.NetworkServicePort *port.PortServiceSwarm *swarm.SwarmServicediff --git a/backend/internal/di/providers.go b/backend/internal/di/providers.goindex 9a0cfebac..f846db47d 100644--- a/backend/internal/di/providers.go+++ b/backend/internal/di/providers.go@@ -9,6 +9,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/activity""github.com/getarcaneapp/arcane/backend/v2/internal/apikey""github.com/getarcaneapp/arcane/backend/v2/internal/auth"+ "github.com/getarcaneapp/arcane/backend/v2/internal/backup""github.com/getarcaneapp/arcane/backend/v2/internal/build""github.com/getarcaneapp/arcane/backend/v2/internal/docker""github.com/getarcaneapp/arcane/backend/v2/internal/environment"@@ -26,9 +27,11 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/project""github.com/getarcaneapp/arcane/backend/v2/internal/registry""github.com/getarcaneapp/arcane/backend/v2/internal/role"+ s3domain "github.com/getarcaneapp/arcane/backend/v2/internal/s3""github.com/getarcaneapp/arcane/backend/v2/internal/session""github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/backend/v2/internal/swarm"+ "github.com/getarcaneapp/arcane/backend/v2/internal/systembackup""github.com/getarcaneapp/arcane/backend/v2/internal/template""github.com/getarcaneapp/arcane/backend/v2/internal/user""github.com/getarcaneapp/arcane/backend/v2/internal/version"@@ -153,6 +156,12 @@ func provideSettingsModuleInternal(service *settings.SettingsService, environmen})}+func provideBackupEngineInternal(ctx context.Context, lc fx.Lifecycle, runtime *actors.Runtime, admission *actors.Gate[actors.AdmissionKey], imageService *image.ImageService) *backup.Engine {+ engine := backup.NewEngine(ctx, runtime, admission, imageService)+ lc.Append(fx.Hook{OnStop: engine.Stop})+ return engine+}+func provideAdmissionGateInternal(ctx context.Context, lc fx.Lifecycle, runtime *actors.Runtime) (*actors.Gate[actors.AdmissionKey], error) {gate, err := actors.NewGate[actors.AdmissionKey](ctx, runtime, "admission", "application")if err != nil {@@ -210,16 +219,32 @@ func provideGitRepositoryServiceInternal(module *gitrepo.Module) *gitrepo.GitRepreturn module.Service()}-func provideVolumeModuleInternal(lc fx.Lifecycle, db *database.DB, docker *docker.DockerClientService, event *event.EventService, settings *settings.SettingsService, image *image.ImageService, activity *activity.ActivityService, cfg *config.Config, uploadService *upload.UploadService) *volume.Module {+func provideS3ModuleInternal(db *database.DB, environmentService *environment.EnvironmentService) *s3domain.Module {+ return s3domain.New(s3domain.Dependencies{+ DB: db,+ SyncRemoteDestinations: environmentService.SyncS3DestinationsToRemoteEnvironments,+ CheckRemoteReferences: environmentService.CheckS3DestinationReferences,+ })+}++func provideS3ServiceInternal(module *s3domain.Module) *s3domain.S3DestinationService {+ return module.Service()+}++func provideVolumeModuleInternal(lc fx.Lifecycle, db *database.DB, dockerService *docker.DockerClientService, eventService *event.EventService, settingsService *settings.SettingsService, imageService *image.ImageService, activityService *activity.ActivityService, containerModule *container.Module, engine *backup.Engine, s3Service *s3domain.S3DestinationService, environmentService *environment.EnvironmentService, cfg *config.Config, uploadService *upload.UploadService) *volume.Module {module := volume.New(volume.Dependencies{- DB: db,- Docker: docker,- Event: event,- Settings: settings,- Image: image,- Activity: activity,- Config: cfg,- Upload: uploadService,+ DB: db,+ Docker: dockerService,+ Event: eventService,+ Settings: settingsService,+ Image: imageService,+ Activity: activityService,+ Environment: environmentService,+ Container: containerModule.Service(),+ Engine: engine,+ S3: s3Service,+ Config: cfg,+ Upload: uploadService,})lc.Append(fx.Hook{OnStop: func(ctx context.Context) error {@@ -230,6 +255,14 @@ func provideVolumeModuleInternal(lc fx.Lifecycle, db *database.DB, docker *dockereturn module}+func provideVolumeServiceInternal(module *volume.Module) *volume.VolumeService {+ return module.Service()+}++func provideSystemBackupServiceInternal(db *database.DB, dockerService *docker.DockerClientService, volumeModule *volume.Module, engine *backup.Engine, s3Service *s3domain.S3DestinationService, activityService *activity.ActivityService, cfg *config.Config) *systembackup.SystemBackupService {+ return systembackup.NewSystemBackupService(db, dockerService, volumeModule.Service(), engine, s3Service, activityService, cfg)+}+func provideContainerModuleInternal(ctx context.Context, event *event.EventService, docker *docker.DockerClientService, image *image.ImageService, settings *settings.SettingsService, project *project.ProjectService, activity *activity.ActivityService) *container.Module {return container.New(ctx, container.Dependencies{Event: event,diff --git a/backend/internal/environment/environment_health.go b/backend/internal/environment/environment_health.goindex 8428e9cb9..5923475e1 100644--- a/backend/internal/environment/environment_health.go+++ b/backend/internal/environment/environment_health.go@@ -140,6 +140,9 @@ func (s *EnvironmentService) runHealthCheckInternal(ctx context.Context, envID sif err := s.SyncRegistriesToEnvironment(syncCtx, envID); err != nil {slog.WarnContext(syncCtx, "failed to sync registries during health check", "environment_id", envID, "error", err)}+ if err := s.SyncS3DestinationsToEnvironment(syncCtx, envID); err != nil {+ slog.WarnContext(syncCtx, "failed to sync S3 destinations during health check", "environment_id", envID, "error", err)+ }if err := s.SyncRepositoriesToEnvironment(syncCtx, envID); err != nil {slog.WarnContext(syncCtx, "failed to sync git repositories during health check", "environment_id", envID, "error", err)}diff --git a/backend/internal/environment/environment_proxy.go b/backend/internal/environment/environment_proxy.goindex 29f184c3b..5de69179f 100644--- a/backend/internal/environment/environment_proxy.go+++ b/backend/internal/environment/environment_proxy.go@@ -2,12 +2,14 @@ package environmentimport ("github.com/getarcaneapp/arcane/backend/v2/internal/gitrepo"+ s3domain "github.com/getarcaneapp/arcane/backend/v2/internal/s3""context""encoding/json/v2""log/slog""maps""net/http"+ "net/url""strings""time"@@ -20,6 +22,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/remenv""github.com/getarcaneapp/arcane/backend/v2/pkg/utils"httputils "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"+ backuptypes "github.com/getarcaneapp/arcane/types/v2/backup""github.com/getarcaneapp/arcane/types/v2/containerregistry""github.com/getarcaneapp/arcane/types/v2/gitops""go.getarcane.app/sys/crypto"@@ -62,6 +65,59 @@ func (s *EnvironmentService) SyncRegistriesToRemoteEnvironments(ctx context.Contreturn nil}+// SyncS3DestinationsToRemoteEnvironments refreshes the destination cache on every enabled remote environment.+func (s *EnvironmentService) SyncS3DestinationsToRemoteEnvironments(ctx context.Context) error {+ envs, err := s.ListRemoteEnvironments(ctx)+ if err != nil {+ return errors.WrapIf(err, "failed to list remote environments for S3 destination sync")+ }++ var failedCount int+ for _, env := range envs {+ if env.AccessToken == nil || strings.TrimSpace(*env.AccessToken) == "" {+ slog.DebugContext(ctx, "Skipping S3 destination sync for environment without access token", "environmentID", env.ID, "environmentName", env.Name)+ continue+ }+ if err := s.SyncS3DestinationsToEnvironment(ctx, env.ID); err != nil {+ failedCount+++ slog.WarnContext(ctx, "Failed to sync S3 destinations to remote environment", "environmentID", env.ID, "environmentName", env.Name, "error", err)+ }+ }++ if failedCount > 0 {+ return errors.Errorf("failed to sync S3 destinations to %d remote environment(s)", failedCount)+ }+ return nil+}++// CheckS3DestinationReferences returns an error while any managed environment+// still references the destination, or when a synced environment cannot be+// checked conclusively. Deleting credentials that a remote environment still+// needs would strand its policies and retained backups.+func (s *EnvironmentService) CheckS3DestinationReferences(ctx context.Context, destinationID string) error {+ envs, err := s.ListRemoteEnvironments(ctx)+ if err != nil {+ return errors.WrapIf(err, "failed to list remote environments for S3 destination reference check")+ }+ for _, env := range envs {+ if env.AccessToken == nil || strings.TrimSpace(*env.AccessToken) == "" {+ // Environments without an access token never receive destination+ // syncs, so they cannot hold a reference.+ continue+ }+ var result struct {+ InUse bool `json:"inUse"`+ }+ if err := s.ProxyJSONRequestForEnvironment(ctx, env, http.MethodGet, "/api/backups/s3/"+url.PathEscape(destinationID)+"/in-use", nil, &result); err != nil {+ return errors.WrapIff(err, "cannot verify S3 destination references on environment %s; restore connectivity before deleting", env.Name)+ }+ if result.InUse {+ return errors.Errorf("still referenced by environment %s", env.Name)+ }+ }+ return nil+}+type remoteEnvironmentTargetInternal struct {ID stringName string@@ -322,6 +378,22 @@ func (s *EnvironmentService) SyncRegistriesToEnvironment(ctx context.Context, en)}+// SyncS3DestinationsToEnvironment sends manager-owned destinations to one remote environment.+func (s *EnvironmentService) SyncS3DestinationsToEnvironment(ctx context.Context, environmentID string) error {+ return fanOutSyncToEnvironmentInternal(ctx, s, environmentID, "S3 destinations", "/api/backups/s3/sync",+ func(_ context.Context, destination s3domain.S3Destination) (backuptypes.S3DestinationSync, bool, error) {+ secret, err := crypto.Decrypt(destination.SecretAccessKey)+ if err != nil {+ return backuptypes.S3DestinationSync{}, false, errors.WrapIff(err, "failed to decrypt S3 destination %s for sync", destination.ID)+ }+ return destination.ToSync(secret), true, nil+ },+ func(items []backuptypes.S3DestinationSync) backuptypes.S3DestinationSyncRequest {+ return backuptypes.S3DestinationSyncRequest{Destinations: items}+ },+ )+}+// SyncRepositoriesToEnvironment syncs all git repositories from this manager to a remote environmentfunc (s *EnvironmentService) SyncRepositoriesToEnvironment(ctx context.Context, environmentID string) error {return fanOutSyncToEnvironmentInternal(ctx, s, environmentID, "git repositories", "/api/git-repositories/sync",diff --git a/backend/internal/environment/handler.go b/backend/internal/environment/handler.goindex 9feb0d177..1412689fd 100644--- a/backend/internal/environment/handler.go+++ b/backend/internal/environment/handler.go@@ -908,13 +908,17 @@ func (h *EnvironmentHandler) PairAgent(ctx context.Context, input *PairAgentInpu}, nil}-// SyncEnvironment syncs container registries and git repositories to an environment.+// SyncEnvironment syncs manager-owned resources to an environment.func (h *EnvironmentHandler) SyncEnvironment(ctx context.Context, input *SyncEnvironmentInput) (*SyncEnvironmentOutput, error) {// Sync registriesif err := h.environmentService.SyncRegistriesToEnvironment(ctx, input.ID); err != nil {slog.WarnContext(ctx, "Failed to sync registries", "environmentID", input.ID, "error", err.Error())}+ if err := h.environmentService.SyncS3DestinationsToEnvironment(ctx, input.ID); err != nil {+ slog.WarnContext(ctx, "Failed to sync S3 destinations", "environmentID", input.ID, "error", err.Error())+ }+// Sync git repositoriesif err := h.environmentService.SyncRepositoriesToEnvironment(ctx, input.ID); err != nil {slog.WarnContext(ctx, "Failed to sync git repositories", "environmentID", input.ID, "error", err.Error())@@ -997,6 +1001,18 @@ func (h *EnvironmentHandler) triggerEnvironmentResourceSyncInternal(ctx context.}}(detachedCtx, environmentID, environmentName, reason)+ go func(syncCtx context.Context, envID string, envName string, syncReason string) {+ syncCtx, cancel := context.WithTimeout(syncCtx, edge.DefaultProxyTimeout)+ defer cancel()+ if err := h.environmentService.SyncS3DestinationsToEnvironment(syncCtx, envID); err != nil {+ slog.WarnContext(syncCtx, "Failed to sync S3 destinations to environment",+ "environmentID", envID,+ "environmentName", envName,+ "reason", syncReason,+ "error", err.Error())+ }+ }(detachedCtx, environmentID, environmentName, reason)+go func(syncCtx context.Context, envID string, envName string, syncReason string) {syncCtx, cancel := context.WithTimeout(syncCtx, edge.DefaultProxyTimeout)defer cancel()diff --git a/backend/internal/environment/service_test.go b/backend/internal/environment/service_test.goindex 6a522ec66..f5bc301c1 100644--- a/backend/internal/environment/service_test.go+++ b/backend/internal/environment/service_test.go@@ -28,10 +28,12 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/registry"+ s3domain "github.com/getarcaneapp/arcane/backend/v2/internal/s3""github.com/getarcaneapp/arcane/backend/v2/internal/user""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge""github.com/getarcaneapp/arcane/backend/v2/pkg/pagination""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/entityjobs"+ backuptypes "github.com/getarcaneapp/arcane/types/v2/backup""github.com/getarcaneapp/arcane/types/v2/containerregistry""github.com/getarcaneapp/arcane/types/v2/environment""github.com/getarcaneapp/arcane/types/v2/gitops"@@ -132,6 +134,7 @@ func setupEnvironmentServiceTestDB(t *testing.T) *database.DB {require.NoError(t, db.AutoMigrate(&Environment{},®istry.ContainerRegistry{},+ &s3domain.S3Destination{},&settings.SettingVariable{},&common.User{},&apikey.ApiKey{},@@ -450,6 +453,49 @@ func TestEnvironmentService_SyncRegistriesToEnvironment_IncludesECRFields(t *tesrequire.NoError(t, err)}+func TestEnvironmentService_SyncS3DestinationsToEnvironment(t *testing.T) {+ ctx := context.Background()+ db := setupEnvironmentServiceTestDB(t)+ svc := NewEnvironmentService(db, nil, nil, nil, nil, nil)++ encryptedSecret, err := crypto.Encrypt("s3-secret")+ require.NoError(t, err)+ require.NoError(t, db.WithContext(ctx).Create(&s3domain.S3Destination{+ BaseModel: database.BaseModel{ID: "s3-1"},+ Name: "Offsite",+ Endpoint: "https://s3.example.com",+ Bucket: "volume-backups",+ Region: "eu-central-1",+ AccessKeyID: "s3-access-key",+ SecretAccessKey: encryptedSecret,+ Prefix: "agents",+ UseSSL: true,+ ForcePathStyle: true,+ }).Error)++ accessToken := "agent-token"+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ require.Equal(t, http.MethodPost, r.Method)+ require.Equal(t, "/api/backups/s3/sync", r.URL.Path)+ require.Equal(t, accessToken, r.Header.Get("X-API-Key"))+ require.Equal(t, accessToken, r.Header.Get("X-Arcane-Agent-Token"))++ var syncReq backuptypes.S3DestinationSyncRequest+ require.NoError(t, json.NewDecoder(r.Body).Decode(&syncReq))+ require.Len(t, syncReq.Destinations, 1)+ require.Equal(t, "s3-1", syncReq.Destinations[0].ID)+ require.Equal(t, "s3-secret", syncReq.Destinations[0].SecretAccessKey)+ require.Equal(t, "volume-backups", syncReq.Destinations[0].Bucket)++ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"success":true,"data":{"message":"ok"}}`))+ }))+ defer server.Close()++ createTestEnvironment(t, db, "env-s3", server.URL, &accessToken)+ require.NoError(t, svc.SyncS3DestinationsToEnvironment(ctx, "env-s3"))+}+func TestEnvironmentService_SyncRepositoriesToEnvironment_UsesAgentHeaders(t *testing.T) {ctx := context.Background()db := setupEnvironmentServiceTestDB(t)diff --git a/backend/internal/middleware/environment_middleware.go b/backend/internal/middleware/environment_middleware.goindex a7bf271b1..bb2b14810 100644--- a/backend/internal/middleware/environment_middleware.go+++ b/backend/internal/middleware/environment_middleware.go@@ -430,6 +430,9 @@ func isManagementPathInternal(method, suffix string) bool {if isCentralSwarmManagementPathInternal(method, suffix) {return true}+ if isCentralVolumeBackupPolicyPathInternal(method, suffix) {+ return true+ }if suffix == "/activities" || strings.HasPrefix(suffix, "/activities/") {return true}@@ -452,6 +455,14 @@ func isManagementPathInternal(method, suffix string) bool {return isManagement}+func isCentralVolumeBackupPolicyPathInternal(method, suffix string) bool {+ if method != http.MethodGet && method != http.MethodPut {+ return false+ }+ parts := strings.Split(strings.Trim(suffix, "/"), "/")+ return len(parts) == 3 && parts[0] == "volumes" && parts[1] != "" && parts[2] == "backup-policy"+}+func isCentralSwarmManagementPathInternal(method, suffix string) bool {if method == http.MethodGet && (suffix == "/swarm/join-candidates" || suffix == "/swarm/nodes") {return truediff --git a/backend/internal/middleware/environment_middleware_test.go b/backend/internal/middleware/environment_middleware_test.goindex 11da650d1..ee7e76a5e 100644--- a/backend/internal/middleware/environment_middleware_test.go+++ b/backend/internal/middleware/environment_middleware_test.go@@ -396,3 +396,10 @@ func TestIsCentralSwarmManagementPathInternal_IsMethodAware(t *testing.T) {assert.True(t, isCentralSwarmManagementPathInternal(http.MethodPut, "/swarm/nodes/node-1/agent/binding"))assert.True(t, isCentralSwarmManagementPathInternal(http.MethodDelete, "/swarm/nodes/node-1/agent/binding"))}++func TestIsCentralVolumeBackupPolicyPathInternal_IsMethodAware(t *testing.T) {+ assert.True(t, isCentralVolumeBackupPolicyPathInternal(http.MethodGet, "/volumes/app-data/backup-policy"))+ assert.True(t, isCentralVolumeBackupPolicyPathInternal(http.MethodPut, "/volumes/app-data/backup-policy"))+ assert.False(t, isCentralVolumeBackupPolicyPathInternal(http.MethodPost, "/volumes/app-data/backup-policy"))+ assert.False(t, isCentralVolumeBackupPolicyPathInternal(http.MethodGet, "/volumes/app-data/backups"))+}diff --git a/backend/internal/recovery/request.go b/backend/internal/recovery/request.gonew file mode 100644index 000000000..0b3905940--- /dev/null+++ b/backend/internal/recovery/request.go@@ -0,0 +1,44 @@+package recovery++import (+ "time"++ "github.com/moby/moby/api/types/mount"+)++type RestoreRequest struct {+ BackupID string `json:"backupId"`+ ContainerID string `json:"containerId"`+ ContainerImage string `json:"containerImage"`+ SnapshotID string `json:"snapshotId"`+ SnapshotPath string `json:"snapshotPath"`+ LocalSnapshotID string `json:"localSnapshotId,omitempty"`+ RemoteSnapshotID string `json:"remoteSnapshotId,omitempty"`+ S3DestinationID string `json:"s3DestinationId,omitempty"`+ Size int64 `json:"size,omitempty"`+ SafetyBackup *SafetyBackup `json:"safetyBackup,omitempty"`+ RecoveryKey string `json:"recoveryKey"`+ RepositoryEnvironment []string `json:"repositoryEnvironment"`+ RepositoryMounts []mount.Mount `json:"repositoryMounts"`+ AppDataMount mount.Mount `json:"appDataMount"`+ // NetworkMode attaches the detached Rustic restore container to one of+ // Arcane's networks so network-local S3 endpoints stay reachable while+ // Arcane itself is stopped.+ NetworkMode string `json:"networkMode,omitempty"`+}++type SafetyBackup struct {+ ID string `json:"id"`+ LocalSnapshotID string `json:"localSnapshotId"`+ Size int64 `json:"size"`+ CreatedAt time.Time `json:"createdAt"`+}++type Manifest struct {+ FormatVersion int `json:"formatVersion"`+ ArcaneVersion string `json:"arcaneVersion"`+ BackupID string `json:"backupId,omitempty"`+ ActivityID string `json:"activityId,omitempty"`+ CreatedAt time.Time `json:"createdAt"`+ Environment map[string]string `json:"environment"`+}diff --git a/backend/internal/registry/handler.go b/backend/internal/registry/handler.goindex 3c8aec6d5..17ffb4a7b 100644--- a/backend/internal/registry/handler.go+++ b/backend/internal/registry/handler.go@@ -18,6 +18,8 @@ import ("go.getarcane.app/sys/crypto")+const registryRemoteSyncFailureMessageInternal = "Failed to fan out registry sync to remote environments"+// ContainerRegistryHandler handles container registry management endpoints.type ContainerRegistryHandler struct {registryService *ContainerRegistryService@@ -392,7 +394,7 @@ func (h *ContainerRegistryHandler) triggerRemoteRegistrySync(ctx context.Contextgo func(syncCtx context.Context, syncReason string) {if err := h.syncRemoteRegistries(syncCtx); err != nil {- slog.WarnContext(syncCtx, "Failed to fan out registry sync to remote environments", "reason", syncReason, "error", err.Error())+ slog.WarnContext(syncCtx, registryRemoteSyncFailureMessageInternal, "reason", syncReason, "error", err.Error())}}(detachedCtx, reason)}diff --git a/backend/internal/s3/handler.go b/backend/internal/s3/handler.gonew file mode 100644index 000000000..38ebcc006--- /dev/null+++ b/backend/internal/s3/handler.go@@ -0,0 +1,247 @@+package s3++import (+ "context"+ "log/slog"+ "net/http"++ "emperror.dev/errors"++ "github.com/danielgtaylor/huma/v2"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/authz"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"+ backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"+ "github.com/getarcaneapp/arcane/types/v2/base"+)++const (+ s3DestinationPathInternal = "/backups/s3"+ s3DestinationTagInternal = "S3 Destinations"+ s3ConnectionTestSuccessInternal = "S3 connection test succeeded"+ s3RemoteSyncFailureMessageInternal = "Failed to fan out S3 destination sync to remote environments"+)++type s3DestinationHandlerInternal struct {+ service *S3DestinationService+ syncRemoteDestinations func(context.Context) error+}++type listS3DestinationsInputInternal struct {+ Search string `query:"search" doc:"Search query"`+ Sort string `query:"sort" doc:"Column to sort by"`+ Order string `query:"order" default:"asc" doc:"Sort direction"`+ Start int `query:"start" default:"0" doc:"Start index"`+ Limit int `query:"limit" default:"20" doc:"Limit"`+}++type listS3DestinationsOutputInternal struct {+ Body struct {+ Data []backuptypes.S3Destination `json:"data"`+ Pagination base.PaginationResponse `json:"pagination"`+ }+}++type listAllS3DestinationsOutputInternal struct {+ Body []backuptypes.S3Destination+}++type s3DestinationIDInputInternal struct {+ ID string `path:"id" doc:"S3 destination ID"`+}++type createS3DestinationInputInternal struct {+ Body backuptypes.CreateS3Destination+}++type updateS3DestinationInputInternal struct {+ ID string `path:"id" doc:"S3 destination ID"`+ Body backuptypes.UpdateS3Destination+}++type s3DestinationOutputInternal struct {+ Body backuptypes.S3Destination+}++type s3DestinationMessageOutputInternal struct {+ Body base.ApiResponse[base.MessageResponse]+}++type testS3DestinationInputInternal struct {+ ID string `path:"id" doc:"S3 destination ID"`+ Body *backuptypes.UpdateS3Destination `json:"body,omitempty"`+}++type testS3DestinationConfigurationInputInternal struct {+ Body backuptypes.CreateS3Destination+}++type syncS3DestinationsInputInternal struct {+ Body backuptypes.S3DestinationSyncRequest+}++func RegisterS3Destinations(api huma.API, service *S3DestinationService, syncRemoteDestinations func(context.Context) error) {+ handler := &s3DestinationHandlerInternal{service: service, syncRemoteDestinations: syncRemoteDestinations}++ handlerutil.RegisterSecured(api,+ handlerutil.Operation("list-s3-destinations", http.MethodGet, s3DestinationPathInternal, "List S3 destinations", "List saved S3-compatible backup destinations with search, sorting, and pagination", s3DestinationTagInternal),+ authz.PermS3DestinationsList, handler.listInternal)+ handlerutil.RegisterSecured(api,+ handlerutil.Operation("list-all-s3-destinations", http.MethodGet, s3DestinationPathInternal+"/options", "List all S3 destination options", "List saved S3-compatible destinations for backup configuration selectors", s3DestinationTagInternal),+ authz.PermS3DestinationsList, handler.listAllInternal)+ handlerutil.RegisterSecured(api,+ handlerutil.Operation("sync-s3-destinations", http.MethodPost, s3DestinationPathInternal+"/sync", "Sync S3 destinations", "Synchronize manager-owned S3 destinations to an agent", s3DestinationTagInternal),+ authz.PermS3DestinationsSync, handler.syncInternal)+ handlerutil.RegisterSecured(api,+ handlerutil.Operation("get-s3-destination", http.MethodGet, s3DestinationPathInternal+"/{id}", "Get S3 destination", "", s3DestinationTagInternal),+ authz.PermS3DestinationsRead, handler.getInternal)+ handlerutil.RegisterSecured(api,+ handlerutil.Operation("create-s3-destination", http.MethodPost, s3DestinationPathInternal, "Create S3 destination", "", s3DestinationTagInternal),+ authz.PermS3DestinationsCreate, handler.createInternal)+ handlerutil.RegisterSecured(api,+ handlerutil.Operation("update-s3-destination", http.MethodPut, s3DestinationPathInternal+"/{id}", "Update S3 destination", "", s3DestinationTagInternal),+ authz.PermS3DestinationsUpdate, handler.updateInternal)+ handlerutil.RegisterSecured(api,+ handlerutil.Operation("test-s3-destination-configuration", http.MethodPost, s3DestinationPathInternal+"/test", "Test unsaved S3 destination configuration", "Verify upload, download, and delete access before saving an S3 destination", s3DestinationTagInternal),+ authz.PermS3DestinationsTest, handler.testConfigurationInternal)+ handlerutil.RegisterSecured(api,+ handlerutil.Operation("test-s3-destination", http.MethodPost, s3DestinationPathInternal+"/{id}/test", "Test S3 destination", "Verify upload, download, and delete access using the saved or supplied S3 destination configuration", s3DestinationTagInternal),+ authz.PermS3DestinationsTest, handler.testInternal)+ handlerutil.RegisterSecured(api,+ handlerutil.Operation("get-s3-destination-usage", http.MethodGet, s3DestinationPathInternal+"/{id}/in-use", "Check S3 destination references", "Report whether backup records, policies, or settings on this environment still reference the destination", s3DestinationTagInternal),+ authz.PermS3DestinationsRead, handler.inUseInternal)+ handlerutil.RegisterSecured(api,+ handlerutil.Operation("delete-s3-destination", http.MethodDelete, s3DestinationPathInternal+"/{id}", "Delete S3 destination", "", s3DestinationTagInternal),+ authz.PermS3DestinationsDelete, handler.deleteInternal)+}++type s3DestinationUsageOutputInternal struct {+ Body struct {+ InUse bool `json:"inUse"`+ }+}++func (h *s3DestinationHandlerInternal) inUseInternal(ctx context.Context, input *s3DestinationIDInputInternal) (*s3DestinationUsageOutputInternal, error) {+ inUse, err := h.service.DestinationInUse(ctx, input.ID)+ if err != nil {+ return nil, huma.Error500InternalServerError(err.Error())+ }+ output := &s3DestinationUsageOutputInternal{}+ output.Body.InUse = inUse+ return output, nil+}++func (h *s3DestinationHandlerInternal) listInternal(ctx context.Context, input *listS3DestinationsInputInternal) (*listS3DestinationsOutputInternal, error) {+ params := handlerutil.PaginationParams(input.Start, input.Limit, input.Sort, input.Order, input.Search)+ destinations, paginationResponse, err := h.service.ListS3Destinations(ctx, params)+ if err != nil {+ return nil, huma.Error500InternalServerError(err.Error())+ }+ output := &listS3DestinationsOutputInternal{}+ output.Body.Data = destinations+ output.Body.Pagination = handlerutil.PaginationResponse(paginationResponse)+ return output, nil+}++func (h *s3DestinationHandlerInternal) listAllInternal(ctx context.Context, _ *struct{}) (*listAllS3DestinationsOutputInternal, error) {+ destinations, err := h.service.ListAllS3Destinations(ctx)+ if err != nil {+ return nil, huma.Error500InternalServerError(err.Error())+ }+ return &listAllS3DestinationsOutputInternal{Body: destinations}, nil+}++func (h *s3DestinationHandlerInternal) getInternal(ctx context.Context, input *s3DestinationIDInputInternal) (*s3DestinationOutputInternal, error) {+ destination, err := h.service.GetS3Destination(ctx, input.ID)+ if errors.Is(err, ErrS3DestinationNotFound) {+ return nil, huma.Error404NotFound(err.Error())+ }+ if err != nil {+ return nil, huma.Error500InternalServerError(err.Error())+ }+ return &s3DestinationOutputInternal{Body: *destination}, nil+}++func (h *s3DestinationHandlerInternal) createInternal(ctx context.Context, input *createS3DestinationInputInternal) (*s3DestinationOutputInternal, error) {+ if err := h.service.TestS3DestinationConfiguration(ctx, input.Body); err != nil {+ return nil, huma.Error400BadRequest(err.Error())+ }+ destination, err := h.service.CreateS3Destination(ctx, input.Body)+ if err != nil {+ return nil, huma.Error400BadRequest(err.Error())+ }+ h.triggerRemoteSyncInternal(ctx, "S3 destination creation")+ return &s3DestinationOutputInternal{Body: *destination}, nil+}++func (h *s3DestinationHandlerInternal) updateInternal(ctx context.Context, input *updateS3DestinationInputInternal) (*s3DestinationOutputInternal, error) {+ if err := h.service.TestS3Destination(ctx, input.ID, &input.Body); err != nil {+ if errors.Is(err, ErrS3DestinationNotFound) {+ return nil, huma.Error404NotFound(err.Error())+ }+ return nil, huma.Error400BadRequest(err.Error())+ }+ destination, err := h.service.UpdateS3Destination(ctx, input.ID, input.Body)+ if errors.Is(err, ErrS3DestinationNotFound) {+ return nil, huma.Error404NotFound(err.Error())+ }+ if err != nil {+ return nil, huma.Error400BadRequest(err.Error())+ }+ h.triggerRemoteSyncInternal(ctx, "S3 destination update")+ return &s3DestinationOutputInternal{Body: *destination}, nil+}++func (h *s3DestinationHandlerInternal) deleteInternal(ctx context.Context, input *s3DestinationIDInputInternal) (*s3DestinationMessageOutputInternal, error) {+ err := h.service.DeleteS3Destination(ctx, input.ID)+ if errors.Is(err, ErrS3DestinationNotFound) {+ return nil, huma.Error404NotFound(err.Error())+ }+ if errors.Is(err, ErrS3DestinationInUse) {+ return nil, huma.Error409Conflict(err.Error())+ }+ if err != nil {+ return nil, huma.Error500InternalServerError(err.Error())+ }+ h.triggerRemoteSyncInternal(ctx, "S3 destination deletion")+ return &s3DestinationMessageOutputInternal{Body: successMessageResponseInternal("S3 destination deleted successfully")}, nil+}++func (h *s3DestinationHandlerInternal) testInternal(ctx context.Context, input *testS3DestinationInputInternal) (*s3DestinationMessageOutputInternal, error) {+ if err := h.service.TestS3Destination(ctx, input.ID, input.Body); err != nil {+ if errors.Is(err, ErrS3DestinationNotFound) {+ return nil, huma.Error404NotFound(err.Error())+ }+ return nil, huma.Error400BadRequest(err.Error())+ }+ return &s3DestinationMessageOutputInternal{Body: successMessageResponseInternal(s3ConnectionTestSuccessInternal)}, nil+}++func successMessageResponseInternal(message string) base.ApiResponse[base.MessageResponse] {+ return base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: message}}+}++func (h *s3DestinationHandlerInternal) triggerRemoteSyncInternal(ctx context.Context, reason string) {+ if h.syncRemoteDestinations == nil {+ return+ }+ detachedCtx := context.WithoutCancel(ctx)+ go func(syncCtx context.Context, syncReason string) {+ if err := h.syncRemoteDestinations(syncCtx); err != nil {+ slog.WarnContext(syncCtx, s3RemoteSyncFailureMessageInternal, "reason", syncReason, "error", err.Error())+ }+ }(detachedCtx, reason)+}++func (h *s3DestinationHandlerInternal) testConfigurationInternal(ctx context.Context, input *testS3DestinationConfigurationInputInternal) (*s3DestinationMessageOutputInternal, error) {+ if err := h.service.TestS3DestinationConfiguration(ctx, input.Body); err != nil {+ return nil, huma.Error400BadRequest(err.Error())+ }+ return &s3DestinationMessageOutputInternal{Body: successMessageResponseInternal(s3ConnectionTestSuccessInternal)}, nil+}++func (h *s3DestinationHandlerInternal) syncInternal(ctx context.Context, input *syncS3DestinationsInputInternal) (*s3DestinationMessageOutputInternal, error) {+ if err := h.service.SyncS3Destinations(ctx, input.Body.Destinations); err != nil {+ return nil, huma.Error400BadRequest(err.Error())+ }+ return &s3DestinationMessageOutputInternal{Body: successMessageResponseInternal("S3 destinations synced successfully")}, nil+}diff --git a/backend/internal/s3/model.go b/backend/internal/s3/model.gonew file mode 100644index 000000000..74fbf68a2--- /dev/null+++ b/backend/internal/s3/model.go@@ -0,0 +1,60 @@+package s3++import (+ "strings"++ "github.com/getarcaneapp/arcane/backend/v2/internal/database"+ backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"+)++type S3Destination struct {+ database.BaseModel++ Name string `json:"name" gorm:"column:name;type:text;not null;uniqueIndex" sortable:"true"`+ Endpoint string `json:"endpoint,omitempty" gorm:"column:endpoint;type:text" sortable:"true"`+ Bucket string `json:"bucket" gorm:"column:bucket;type:text;not null" sortable:"true"`+ Region string `json:"region" gorm:"column:region;type:text;not null" sortable:"true"`+ AccessKeyID string `json:"accessKeyId" gorm:"column:access_key_id;type:text;not null" sortable:"true"`+ SecretAccessKey string `json:"-" gorm:"column:secret_access_key;type:text;not null"`+ Prefix string `json:"prefix,omitempty" gorm:"column:prefix;type:text" sortable:"true"`+ UseSSL bool `json:"useSsl" gorm:"column:use_ssl;not null" sortable:"true"`+ ForcePathStyle bool `json:"forcePathStyle" gorm:"column:force_path_style;not null" sortable:"true"`+}++func (S3Destination) TableName() string {+ return "s3_destinations"+}++func (d S3Destination) ToDTO() backuptypes.S3Destination {+ return backuptypes.S3Destination{+ ID: d.ID,+ Name: d.Name,+ Endpoint: d.Endpoint,+ Bucket: d.Bucket,+ Region: d.Region,+ AccessKeyID: d.AccessKeyID,+ Prefix: d.Prefix,+ UseSSL: d.UseSSL,+ ForcePathStyle: d.ForcePathStyle,+ SecretConfigured: strings.TrimSpace(d.SecretAccessKey) != "",+ CreatedAt: d.CreatedAt,+ UpdatedAt: d.UpdatedAt,+ }+}++func (d S3Destination) ToSync(secretAccessKey string) backuptypes.S3DestinationSync {+ return backuptypes.S3DestinationSync{+ ID: d.ID,+ Name: d.Name,+ Endpoint: d.Endpoint,+ Bucket: d.Bucket,+ Region: d.Region,+ AccessKeyID: d.AccessKeyID,+ SecretAccessKey: secretAccessKey,+ Prefix: d.Prefix,+ UseSSL: d.UseSSL,+ ForcePathStyle: d.ForcePathStyle,+ CreatedAt: d.CreatedAt,+ UpdatedAt: d.UpdatedAt,+ }+}diff --git a/backend/internal/s3/module.go b/backend/internal/s3/module.gonew file mode 100644index 000000000..f98a3738b--- /dev/null+++ b/backend/internal/s3/module.go@@ -0,0 +1,41 @@+// Package s3 owns reusable S3-compatible backup destinations and their HTTP surface.+package s3++import (+ "context"++ "github.com/danielgtaylor/huma/v2"+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"+)++type Dependencies struct {+ DB *database.DB+ SyncRemoteDestinations func(context.Context) error+ CheckRemoteReferences func(ctx context.Context, destinationID string) error+}++type Module struct {+ service *S3DestinationService+ deps Dependencies+}++func New(deps Dependencies) *Module {+ service := NewS3DestinationService(deps.DB)+ service.checkRemoteReferences = deps.CheckRemoteReferences+ return &Module{service: service, deps: deps}+}++func (m *Module) Service() *S3DestinationService {+ if m == nil {+ return nil+ }+ return m.service+}++func (m *Module) RegisterRoutes(api huma.API) {+ if m == nil {+ RegisterS3Destinations(api, nil, nil)+ return+ }+ RegisterS3Destinations(api, m.service, m.deps.SyncRemoteDestinations)+}diff --git a/backend/internal/s3/service.go b/backend/internal/s3/service.gonew file mode 100644index 000000000..6cbf7ae11--- /dev/null+++ b/backend/internal/s3/service.go@@ -0,0 +1,331 @@+package s3++import (+ "context"+ "fmt"+ "strings"++ "emperror.dev/errors"++ "github.com/getarcaneapp/arcane/backend/v2/internal/database"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"+ s3config "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/s3"+ backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"+ "go.getarcane.app/sys/crypto"+ "gorm.io/gorm"+)++var (+ ErrS3DestinationNotFound = errors.New("S3 destination not found")+ ErrS3DestinationInUse = errors.New("S3 destination is in use")+ // errS3SecretRequiredInternal enforces the fresh-secret rule: a stored+ // secret is never combined with changed connection or trust settings.+ errS3SecretRequiredInternal = errors.New("re-enter the secret access key when changing the endpoint, bucket, region, access key ID, SSL mode, or path style")+)++type S3DestinationService struct {+ db *database.DB+ // checkRemoteReferences rejects a deletion while any managed environment+ // still references the destination (or cannot be checked conclusively).+ // Nil outside the manager's module wiring.+ checkRemoteReferences func(ctx context.Context, destinationID string) error+}++func NewS3DestinationService(db *database.DB) *S3DestinationService {+ return &S3DestinationService{db: db}+}++func destinationConfigurationInternal(id string, input backuptypes.CreateS3Destination) s3config.Configuration {+ return s3config.Configuration{+ ID: id,+ Name: input.Name,+ Endpoint: input.Endpoint,+ Bucket: input.Bucket,+ Region: input.Region,+ AccessKeyID: input.AccessKeyID,+ SecretAccessKey: input.SecretAccessKey,+ Prefix: input.Prefix,+ UseSSL: input.UseSSL,+ ForcePathStyle: input.ForcePathStyle,+ }.Normalized()+}++func s3DestinationsToDTOsInternal(destinations []S3Destination) []backuptypes.S3Destination {+ result := make([]backuptypes.S3Destination, len(destinations))+ for i := range destinations {+ result[i] = destinations[i].ToDTO()+ }+ return result+}++func (s *S3DestinationService) ListS3Destinations(ctx context.Context, params pagination.QueryParams) ([]backuptypes.S3Destination, pagination.Response, error) {+ var destinations []S3Destination+ query := s.db.WithContext(ctx).Model(&S3Destination{})+ if term := strings.TrimSpace(params.Search); term != "" {+ pattern := "%" + term + "%"+ query = query.Where("name LIKE ? OR endpoint LIKE ? OR bucket LIKE ? OR region LIKE ? OR prefix LIKE ?", pattern, pattern, pattern, pattern, pattern)+ }+ response, err := pagination.PaginateAndSortDB(params, query, &destinations)+ if err != nil {+ return nil, pagination.Response{}, fmt.Errorf("failed to list S3 destinations: %w", err)+ }+ return s3DestinationsToDTOsInternal(destinations), response, nil+}++func (s *S3DestinationService) ListAllS3Destinations(ctx context.Context) ([]backuptypes.S3Destination, error) {+ var destinations []S3Destination+ if err := s.db.WithContext(ctx).Order("name ASC").Find(&destinations).Error; err != nil {+ return nil, fmt.Errorf("failed to list S3 destinations: %w", err)+ }+ return s3DestinationsToDTOsInternal(destinations), nil+}++func (s *S3DestinationService) getS3DestinationModelInternal(ctx context.Context, id string) (*S3Destination, error) {+ var destination S3Destination+ if err := s.db.WithContext(ctx).Where("id = ?", strings.TrimSpace(id)).First(&destination).Error; err != nil {+ if errors.Is(err, gorm.ErrRecordNotFound) {+ return nil, ErrS3DestinationNotFound+ }+ return nil, fmt.Errorf("failed to load S3 destination: %w", err)+ }+ return &destination, nil+}++func (s *S3DestinationService) GetS3Destination(ctx context.Context, id string) (*backuptypes.S3Destination, error) {+ destination, err := s.getS3DestinationModelInternal(ctx, id)+ if err != nil {+ return nil, err+ }+ dto := destination.ToDTO()+ return &dto, nil+}++func applyS3ConfigurationInternal(destination *S3Destination, configuration s3config.Configuration, encryptedSecret string) {+ destination.Name = configuration.Name+ destination.Endpoint = configuration.Endpoint+ destination.Bucket = configuration.Bucket+ destination.Region = configuration.Region+ destination.AccessKeyID = configuration.AccessKeyID+ destination.SecretAccessKey = encryptedSecret+ destination.Prefix = configuration.Prefix+ destination.UseSSL = configuration.UseSSL+ destination.ForcePathStyle = configuration.ForcePathStyle+}++func (s *S3DestinationService) CreateS3Destination(ctx context.Context, input backuptypes.CreateS3Destination) (*backuptypes.S3Destination, error) {+ configuration := destinationConfigurationInternal("", input)+ if err := configuration.Validate(true); err != nil {+ return nil, err+ }+ encryptedSecret, err := crypto.Encrypt(configuration.SecretAccessKey)+ if err != nil {+ return nil, fmt.Errorf("failed to encrypt S3 secret access key: %w", err)+ }+ destination := &S3Destination{}+ applyS3ConfigurationInternal(destination, configuration, encryptedSecret)+ if err := s.db.WithContext(ctx).Create(destination).Error; err != nil {+ return nil, fmt.Errorf("failed to create S3 destination: %w", err)+ }+ dto := destination.ToDTO()+ return &dto, nil+}++// storedConfigurationInternal returns the persisted configuration without the+// decrypted secret, for comparing connection fields against an update.+func storedConfigurationInternal(destination *S3Destination) s3config.Configuration {+ return s3config.Configuration{+ ID: destination.ID,+ Name: destination.Name,+ Endpoint: destination.Endpoint,+ Bucket: destination.Bucket,+ Region: destination.Region,+ AccessKeyID: destination.AccessKeyID,+ Prefix: destination.Prefix,+ UseSSL: destination.UseSSL,+ ForcePathStyle: destination.ForcePathStyle,+ }.Normalized()+}++func (s *S3DestinationService) UpdateS3Destination(ctx context.Context, id string, input backuptypes.UpdateS3Destination) (*backuptypes.S3Destination, error) {+ configuration := destinationConfigurationInternal("", input)+ if err := configuration.Validate(false); err != nil {+ return nil, err+ }+ destination, err := s.getS3DestinationModelInternal(ctx, id)+ if err != nil {+ return nil, err+ }+ if configuration.SecretAccessKey == "" && !s3config.ConnectionFieldsEqual(storedConfigurationInternal(destination), configuration) {+ return nil, errS3SecretRequiredInternal+ }+ encryptedSecret := destination.SecretAccessKey+ if configuration.SecretAccessKey != "" {+ encryptedSecret, err = crypto.Encrypt(configuration.SecretAccessKey)+ if err != nil {+ return nil, fmt.Errorf("failed to encrypt S3 secret access key: %w", err)+ }+ }+ applyS3ConfigurationInternal(destination, configuration, encryptedSecret)+ if err := s.db.WithContext(ctx).Save(destination).Error; err != nil {+ return nil, fmt.Errorf("failed to update S3 destination: %w", err)+ }+ dto := destination.ToDTO()+ return &dto, nil+}++func (s *S3DestinationService) DeleteS3Destination(ctx context.Context, id string) error {+ destination, err := s.getS3DestinationModelInternal(ctx, id)+ if err != nil {+ return err+ }+ inUse, err := s.DestinationInUse(ctx, id)+ if err != nil {+ return err+ }+ if inUse {+ return fmt.Errorf("%w: remove it from backup configurations and retained backups first", ErrS3DestinationInUse)+ }+ if s.checkRemoteReferences != nil {+ if err := s.checkRemoteReferences(ctx, id); err != nil {+ return fmt.Errorf("%w: %s", ErrS3DestinationInUse, err.Error())+ }+ }+ if err := s.db.WithContext(ctx).Delete(destination).Error; err != nil {+ return fmt.Errorf("failed to delete S3 destination: %w", err)+ }+ return nil+}++// DestinationInUse reports whether any local backup record or policy+// references the destination.+func (s *S3DestinationService) DestinationInUse(ctx context.Context, id string) (bool, error) {+ db := s.db.WithContext(ctx)+ counts := []*gorm.DB{+ db.Table("system_backup_runs").Where("s3_destination_id = ? AND remote_snapshot_id <> ''", id),+ db.Table("system_backup_policies").Where("s3_destination_id = ? AND s3_enabled = ?", id, true),+ db.Table("volume_backups").Where("s3_destination_id = ? AND remote_snapshot_id <> ''", id),+ db.Table("volume_backup_policies").Where("s3_destination_id = ? AND s3_enabled = ?", id, true),+ }+ for _, query := range counts {+ var count int64+ if err := query.Count(&count).Error; err != nil {+ return false, fmt.Errorf("failed to check S3 destination references: %w", err)+ }+ if count > 0 {+ return true, nil+ }+ }+ return false, nil+}++func (s *S3DestinationService) S3DestinationExists(ctx context.Context, id string) bool {+ if strings.TrimSpace(id) == "" {+ return false+ }+ var count int64+ return s.db.WithContext(ctx).Model(&S3Destination{}).Where("id = ?", id).Count(&count).Error == nil && count == 1+}++// SyncS3Destinations replaces an agent's destination cache with the manager-owned destinations.+func (s *S3DestinationService) SyncS3Destinations(ctx context.Context, destinations []backuptypes.S3DestinationSync) error {+ var existing []S3Destination+ if err := s.db.WithContext(ctx).Find(&existing).Error; err != nil {+ return fmt.Errorf("failed to load existing S3 destinations: %w", err)+ }+ existingByID := make(map[string]*S3Destination, len(existing))+ for i := range existing {+ existingByID[existing[i].ID] = &existing[i]+ }++ syncedIDs := make(map[string]struct{}, len(destinations))+ for _, item := range destinations {+ configuration := destinationConfigurationInternal(item.ID, item.CreateS3Destination)+ if configuration.ID == "" {+ return errors.New("S3 destination ID is required")+ }+ if err := configuration.Validate(true); err != nil {+ return fmt.Errorf("invalid S3 destination %s: %w", configuration.ID, err)+ }+ encryptedSecret, err := crypto.Encrypt(configuration.SecretAccessKey)+ if err != nil {+ return fmt.Errorf("failed to encrypt S3 secret access key for %s: %w", configuration.ID, err)+ }+ destination, exists := existingByID[configuration.ID]+ if !exists {+ destination = &S3Destination{ID: configuration.ID}+ }+ applyS3ConfigurationInternal(destination, configuration, encryptedSecret)+ if !item.CreatedAt.IsZero() {+ destination.CreatedAt = item.CreatedAt+ }+ if item.UpdatedAt != nil {+ destination.UpdatedAt = item.UpdatedAt+ }+ if err := s.db.WithContext(ctx).Save(destination).Error; err != nil {+ return fmt.Errorf("failed to sync S3 destination %s: %w", configuration.ID, err)+ }+ syncedIDs[configuration.ID] = struct{}{}+ }++ // One undeletable destination (still referenced locally) must not abort+ // the rest of the reconciliation; report every failure after the sweep.+ var removeErr error+ for i := range existing {+ if _, ok := syncedIDs[existing[i].ID]; ok {+ continue+ }+ if err := s.DeleteS3Destination(ctx, existing[i].ID); err != nil {+ removeErr = errors.Combine(removeErr, fmt.Errorf("failed to remove unsynced S3 destination %s: %w", existing[i].ID, err))+ }+ }+ return removeErr+}++// Configuration returns a destination's decrypted runtime configuration for backup domains.+func (s *S3DestinationService) Configuration(ctx context.Context, id string) (s3config.Configuration, error) {+ destination, err := s.getS3DestinationModelInternal(ctx, id)+ if err != nil {+ return s3config.Configuration{}, err+ }+ secret, err := crypto.Decrypt(destination.SecretAccessKey)+ if err != nil {+ return s3config.Configuration{}, fmt.Errorf("failed to decrypt S3 secret access key: %w", err)+ }+ return s3config.Configuration{+ ID: destination.ID,+ Name: destination.Name,+ Endpoint: destination.Endpoint,+ Bucket: destination.Bucket,+ Region: destination.Region,+ AccessKeyID: destination.AccessKeyID,+ SecretAccessKey: secret,+ Prefix: destination.Prefix,+ UseSSL: destination.UseSSL,+ ForcePathStyle: destination.ForcePathStyle,+ }.Normalized(), nil+}++func (s *S3DestinationService) TestS3Destination(ctx context.Context, id string, input *backuptypes.UpdateS3Destination) error {+ configuration, err := s.Configuration(ctx, id)+ if err != nil {+ return err+ }+ if input != nil {+ updated := destinationConfigurationInternal("", *input)+ updated.ID = configuration.ID+ if updated.SecretAccessKey == "" {+ // The stored secret may only sign requests against the stored+ // connection settings; any change requires a fresh secret.+ if !s3config.ConnectionFieldsEqual(configuration, updated) {+ return errS3SecretRequiredInternal+ }+ updated.SecretAccessKey = configuration.SecretAccessKey+ }+ configuration = updated+ }+ return s3config.TestConnection(ctx, configuration)+}++func (s *S3DestinationService) TestS3DestinationConfiguration(ctx context.Context, input backuptypes.CreateS3Destination) error {+ return s3config.TestConnection(ctx, destinationConfigurationInternal("", input))+}diff --git a/backend/internal/s3/service_test.go b/backend/internal/s3/service_test.gonew file mode 100644index 000000000..727940f75--- /dev/null+++ b/backend/internal/s3/service_test.go@@ -0,0 +1,274 @@+package s3++import (+ "context"+ "io"+ "net/http"+ "net/http/httptest"+ "strconv"+ "strings"+ "testing"+ "time"++ "github.com/getarcaneapp/arcane/backend/v2/internal/database"+ "github.com/getarcaneapp/arcane/backend/v2/internal/settings"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"+ backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"+ sqlite "github.com/libtnb/sqlite"+ "github.com/stretchr/testify/require"+ "go.getarcane.app/sys/crypto"+ "gorm.io/gorm"+)++func setupS3DestinationServiceTestInternal(t *testing.T) (*S3DestinationService, *gorm.DB) {+ t.Helper()++ gormDB, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{})+ require.NoError(t, err)+ require.NoError(t, gormDB.AutoMigrate(+ &S3Destination{},+ &settings.SettingVariable{},+ ))+ require.NoError(t, gormDB.Exec("CREATE TABLE system_backup_runs (s3_destination_id text, remote_snapshot_id text)").Error)+ require.NoError(t, gormDB.Exec("CREATE TABLE system_backup_policies (s3_destination_id text, s3_enabled numeric)").Error)+ require.NoError(t, gormDB.Exec("CREATE TABLE volume_backups (s3_destination_id text, remote_snapshot_id text)").Error)+ require.NoError(t, gormDB.Exec("CREATE TABLE volume_backup_policies (s3_destination_id text, s3_enabled numeric)").Error)+ crypto.InitEncryption(&crypto.Config{+ EncryptionKey: "test-encryption-key-for-s3-destination-32bytes",+ Environment: "test",+ })+ return NewS3DestinationService(&database.DB{DB: gormDB}), gormDB+}++func TestS3DestinationService_CRUD(t *testing.T) {+ service, gormDB := setupS3DestinationServiceTestInternal(t)+ ctx := context.Background()++ created, err := service.CreateS3Destination(ctx, backuptypes.CreateS3Destination{+ Name: "Offsite",+ Endpoint: "https://s3.example.com",+ Bucket: "arcane-backups",+ Region: "eu-central-1",+ AccessKeyID: "access-key",+ SecretAccessKey: "secret-key",+ Prefix: "/production/",+ UseSSL: true,+ ForcePathStyle: true,+ })+ require.NoError(t, err)+ require.NotEmpty(t, created.ID)+ require.True(t, created.SecretConfigured)+ require.Equal(t, "production", created.Prefix)++ var stored S3Destination+ require.NoError(t, gormDB.First(&stored, "id = ?", created.ID).Error)+ require.NotEqual(t, "secret-key", stored.SecretAccessKey)++ listed, page, err := service.ListS3Destinations(ctx, pagination.QueryParams{+ SearchQuery: pagination.SearchQuery{Search: "arcane-backups"},+ Params: pagination.Params{Limit: 20},+ })+ require.NoError(t, err)+ require.EqualValues(t, 1, page.TotalItems)+ require.Len(t, listed, 1)++ updated, err := service.UpdateS3Destination(ctx, created.ID, backuptypes.UpdateS3Destination{+ Name: "Primary offsite",+ Endpoint: created.Endpoint,+ Bucket: created.Bucket,+ Region: created.Region,+ AccessKeyID: created.AccessKeyID,+ Prefix: created.Prefix,+ UseSSL: created.UseSSL,+ ForcePathStyle: created.ForcePathStyle,+ })+ require.NoError(t, err)+ require.Equal(t, "Primary offsite", updated.Name)++ var updatedStored S3Destination+ require.NoError(t, gormDB.First(&updatedStored, "id = ?", created.ID).Error)+ require.Equal(t, stored.SecretAccessKey, updatedStored.SecretAccessKey)++ require.NoError(t, service.DeleteS3Destination(ctx, created.ID))+ _, err = service.GetS3Destination(ctx, created.ID)+ require.ErrorIs(t, err, ErrS3DestinationNotFound)+}++func TestS3DestinationService_DeleteRejectsConfiguredDestination(t *testing.T) {+ service, gormDB := setupS3DestinationServiceTestInternal(t)+ ctx := context.Background()+ destination, err := service.CreateS3Destination(ctx, backuptypes.CreateS3Destination{+ Name: "In use",+ Bucket: "arcane-backups",+ Region: "us-east-1",+ AccessKeyID: "access-key",+ SecretAccessKey: "secret-key",+ UseSSL: true,+ })+ require.NoError(t, err)+ require.NoError(t, gormDB.Exec("INSERT INTO volume_backup_policies (s3_destination_id, s3_enabled) VALUES (?, 1)", destination.ID).Error)++ err = service.DeleteS3Destination(ctx, destination.ID)+ require.ErrorIs(t, err, ErrS3DestinationInUse)+}++func TestS3DestinationService_SyncS3Destinations(t *testing.T) {+ service, gormDB := setupS3DestinationServiceTestInternal(t)+ ctx := context.Background()++ legacy, err := service.CreateS3Destination(ctx, backuptypes.CreateS3Destination{+ Name: "Legacy",+ Bucket: "legacy-bucket",+ Region: "us-east-1",+ AccessKeyID: "legacy-access-key",+ SecretAccessKey: "legacy-secret",+ UseSSL: true,+ })+ require.NoError(t, err)++ createdAt := time.Now().Add(-time.Hour).UTC().Truncate(time.Second)+ require.NoError(t, service.SyncS3Destinations(ctx, []backuptypes.S3DestinationSync{+ {+ ID: "destination-1",+ Name: "Remote primary",+ Endpoint: "https://s3.example.com",+ Bucket: "volume-backups",+ Region: "eu-central-1",+ AccessKeyID: "remote-access-key",+ SecretAccessKey: "remote-secret",+ Prefix: "/agents/",+ UseSSL: true,+ ForcePathStyle: true,+ CreatedAt: createdAt,+ },+ }))++ var synced S3Destination+ require.NoError(t, gormDB.First(&synced, "id = ?", "destination-1").Error)+ require.Equal(t, "Remote primary", synced.Name)+ require.Equal(t, "agents", synced.Prefix)+ require.NotEqual(t, "remote-secret", synced.SecretAccessKey)+ secret, err := crypto.Decrypt(synced.SecretAccessKey)+ require.NoError(t, err)+ require.Equal(t, "remote-secret", secret)++ _, err = service.GetS3Destination(ctx, legacy.ID)+ require.ErrorIs(t, err, ErrS3DestinationNotFound)++ require.NoError(t, service.SyncS3Destinations(ctx, []backuptypes.S3DestinationSync{+ {+ ID: "destination-1",+ Name: "Remote primary updated",+ Bucket: "updated-volume-backups",+ Region: "eu-west-1",+ AccessKeyID: "updated-access-key",+ SecretAccessKey: "updated-secret",+ UseSSL: true,+ },+ }))++ require.NoError(t, gormDB.First(&synced, "id = ?", "destination-1").Error)+ require.Equal(t, "Remote primary updated", synced.Name)+ require.Equal(t, "updated-volume-backups", synced.Bucket)+ secret, err = crypto.Decrypt(synced.SecretAccessKey)+ require.NoError(t, err)+ require.Equal(t, "updated-secret", secret)+}++func TestS3DestinationService_TestS3DestinationRoundTrip(t *testing.T) {+ var object []byte+ var requestMethods []string+ expectedPathPrefix := "/arcane-backups/production/.arcane-connection-test-"+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ require.NotEmpty(t, r.Header.Get("Authorization"))+ require.True(t, strings.HasPrefix(r.URL.Path, expectedPathPrefix))+ requestMethods = append(requestMethods, r.Method)+ switch r.Method {+ case http.MethodPut:+ var err error+ object, err = io.ReadAll(r.Body)+ require.NoError(t, err)+ w.Header().Set("ETag", `"test-etag"`)+ w.WriteHeader(http.StatusOK)+ case http.MethodGet:+ w.Header().Set("Content-Length", strconv.Itoa(len(object)))+ w.WriteHeader(http.StatusOK)+ _, _ = w.Write(object)+ case http.MethodDelete:+ object = nil+ w.WriteHeader(http.StatusNoContent)+ default:+ http.Error(w, "unsupported method", http.StatusMethodNotAllowed)+ }+ }))+ defer server.Close()++ service, _ := setupS3DestinationServiceTestInternal(t)+ require.NoError(t, service.TestS3DestinationConfiguration(context.Background(), backuptypes.CreateS3Destination{+ Name: "Unsaved destination",+ Endpoint: server.URL,+ Bucket: "arcane-backups",+ Region: "",+ AccessKeyID: "access-key",+ SecretAccessKey: "secret-key",+ Prefix: "production",+ UseSSL: false,+ ForcePathStyle: true,+ }))+ destination, err := service.CreateS3Destination(context.Background(), backuptypes.CreateS3Destination{+ Name: "Test destination",+ Endpoint: server.URL,+ Bucket: "arcane-backups",+ Region: "",+ AccessKeyID: "access-key",+ SecretAccessKey: "secret-key",+ Prefix: "production",+ UseSSL: false,+ ForcePathStyle: true,+ })+ require.NoError(t, err)++ require.NoError(t, service.TestS3Destination(context.Background(), destination.ID, nil))++ // Changing a connection field without re-supplying the secret is rejected+ // before any outbound request: the stored secret must never sign requests+ // against caller-modified connection settings.+ requestsBeforeRejection := len(requestMethods)+ err = service.TestS3Destination(context.Background(), destination.ID, &backuptypes.UpdateS3Destination{+ Name: destination.Name,+ Endpoint: server.URL,+ Bucket: "edited-bucket",+ Region: "",+ AccessKeyID: destination.AccessKeyID,+ Prefix: "edited",+ UseSSL: false,+ ForcePathStyle: true,+ })+ require.ErrorContains(t, err, "re-enter the secret access key")+ require.Len(t, requestMethods, requestsBeforeRejection)++ expectedPathPrefix = "/edited-bucket/edited/.arcane-connection-test-"+ require.NoError(t, service.TestS3Destination(context.Background(), destination.ID, &backuptypes.UpdateS3Destination{+ Name: destination.Name,+ Endpoint: server.URL,+ Bucket: "edited-bucket",+ Region: "",+ AccessKeyID: destination.AccessKeyID,+ SecretAccessKey: "secret-key",+ Prefix: "edited",+ UseSSL: false,+ ForcePathStyle: true,+ }))++ require.Equal(t, []string{+ http.MethodPut, http.MethodGet, http.MethodDelete,+ http.MethodPut, http.MethodGet, http.MethodDelete,+ http.MethodPut, http.MethodGet, http.MethodDelete,+ }, requestMethods)+ require.Empty(t, object)+ persisted, err := service.GetS3Destination(context.Background(), destination.ID)+ require.NoError(t, err)+ require.Equal(t, "arcane-backups", persisted.Bucket)+ require.Equal(t, "production", persisted.Prefix)+ require.Empty(t, persisted.Region)+}diff --git a/backend/internal/settings/model.go b/backend/internal/settings/model.goindex ce107a503..12767d225 100644--- a/backend/internal/settings/model.go+++ b/backend/internal/settings/model.go@@ -174,6 +174,10 @@ type Settings struct {// API Keys category (admin management page - no actual settings)ApiKeysCategoryPlaceholder SettingVariable `key:"apiKeysCategory,internal" meta:"label=API Keys;type=internal;keywords=api,keys,tokens,authentication,access,programmatic,integration;category=apikeys;description=Manage API keys for programmatic access" catmeta:"id=apikeys;title=API Keys;icon=apikey;url=/settings/api-keys;description=Create and manage API keys for programmatic access to Arcane"`+ S3DestinationsCategoryPlaceholder SettingVariable `key:"s3DestinationsCategory,internal" meta:"label=S3 Destinations;type=internal;keywords=s3,storage,backup,aws,minio,backblaze,hetzner;category=s3destinations;description=Manage reusable S3-compatible storage destinations" catmeta:"id=s3destinations;title=S3 Destinations;icon=remoteenvironment;url=/settings/backups/s3;description=Create and manage reusable S3-compatible backup destinations"`++ SystemBackupsCategoryPlaceholder SettingVariable `key:"systemBackupsCategory,internal" meta:"label=Backups;type=internal;keywords=backup,recovery,restore,configuration,secrets;category=systembackups;description=Back up and restore this Arcane instance" catmeta:"id=systembackups;title=Backups;icon=backup;url=/settings/backups;description=Create encrypted recovery backups of Arcane data and configuration"`+FederatedCredentialsCategoryPlaceholder SettingVariable `key:"federatedCredentialsCategory,internal" meta:"label=Federated Credentials;type=internal;keywords=federated,credentials,workload,identity,oidc,token exchange,ci,github,gitlab;category=authentication;description=Manage workload identity federation credentials"`// Webhooks category (management page - no actual settings)diff --git a/backend/internal/system/upgrade_service.go b/backend/internal/system/upgrade_service.goindex 2faabd008..f30aa32ed 100644--- a/backend/internal/system/upgrade_service.go+++ b/backend/internal/system/upgrade_service.go@@ -50,12 +50,6 @@ type SystemUpgradeService struct {settingsService *settings.SettingsService}-type upgraderRuntimeOptionsInternal struct {- ContainerEnv []string- Mounts []mount.Mount- NetworkMode container.NetworkMode-}-func NewSystemUpgradeService(db *database.DB,dockerService *docker.DockerClientService,@@ -211,7 +205,7 @@ func (s *SystemUpgradeService) TriggerUpgradeViaCLI(ctx context.Context, user co}// Create the upgrader container config- runtimeOptions, err := resolveSystemUpgraderRuntimeOptionsInternal(+ containerEnv, runtimeMounts, networkMode, err := ResolveUpgraderRuntimeOptions(ctx,s.dockerService.DockerHost(),¤tContainer,@@ -242,14 +236,14 @@ func (s *SystemUpgradeService) TriggerUpgradeViaCLI(ctx context.Context, user co// is short-lived and never goes through the runtime-identity drop, so// don't rely on the image's default user.User: "0:0",- Env: runtimeOptions.ContainerEnv,+ Env: containerEnv,Labels: map[string]string{"com.getarcaneapp.arcane.upgrader": "true","com.getarcaneapp.arcane": "true",},}- mounts := append([]mount.Mount{}, runtimeOptions.Mounts...)+ mounts := append([]mount.Mount{}, runtimeMounts...)if appDataMount != nil {mounts = append(mounts, *appDataMount)}@@ -262,7 +256,7 @@ func (s *SystemUpgradeService) TriggerUpgradeViaCLI(ctx context.Context, user cohostConfig := &container.HostConfig{AutoRemove: !keepUpgraderContainer, // default: clean up after completionMounts: mounts,- NetworkMode: runtimeOptions.NetworkMode,+ NetworkMode: networkMode,}// Inherit the security context that lets the running Arcane container reach// the Docker socket (e.g. SELinux label=disable, privileged); the upgrader@@ -328,32 +322,30 @@ func determineUpgradeBinaryPathInternal(containerLabels map[string]string) strinreturn "/app/arcane"}-func resolveSystemUpgraderRuntimeOptionsInternal(+// ResolveUpgraderRuntimeOptions determines how a helper container reaches the Docker daemon.+func ResolveUpgraderRuntimeOptions(ctx context.Context,dockerHost string,currentContainer *container.InspectResponse,discoverHostPath func(context.Context, string) (string, error),isRunningInDocker func() bool,selectReachableNetwork func(context.Context, *container.InspectResponse, string) string,-) (upgraderRuntimeOptionsInternal, error) {- options := upgraderRuntimeOptionsInternal{- ContainerEnv: vuln.BuildDockerHostEnv(dockerHost),- }+) ([]string, []mount.Mount, container.NetworkMode, error) {+ containerEnv := vuln.BuildDockerHostEnv(dockerHost)scheme, socketPath, err := vuln.ParseDockerHost(dockerHost)if err != nil {- return upgraderRuntimeOptionsInternal{}, errors.WrapIff(err, "resolve docker host %q", dockerHost)+ return nil, nil, "", errors.WrapIff(err, "resolve docker host %q", dockerHost)}if scheme != "unix" {// The upgrader must reach the tcp DOCKER_HOST, so prefer a network the// daemon proxy is actually on over the plain auto heuristic (#3533).+ networkMode := dockerutils.SelectAutoNetworkMode(currentContainer)if selectReachableNetwork != nil {- options.NetworkMode = container.NetworkMode(selectReachableNetwork(ctx, currentContainer, dockerHost))- } else {- options.NetworkMode = container.NetworkMode(dockerutils.SelectAutoNetworkMode(currentContainer))+ networkMode = selectReachableNetwork(ctx, currentContainer, dockerHost)}- return options, nil+ return containerEnv, nil, container.NetworkMode(networkMode), nil}socketSource, err := vuln.ResolveUnixSocketSource(@@ -363,16 +355,16 @@ func resolveSystemUpgraderRuntimeOptionsInternal(isRunningInDocker,)if err != nil {- return upgraderRuntimeOptionsInternal{}, errors.WrapIf(err, "resolve unix socket source")+ return nil, nil, "", errors.WrapIf(err, "resolve unix socket source")}- options.Mounts = append(options.Mounts, mount.Mount{+ mounts := []mount.Mount{{Type: mount.TypeBind,Source: socketSource,Target: socketPath,- })+ }}- return options, nil+ return containerEnv, mounts, "", nil}// getCurrentContainerID detects if we're running in Docker and returns container IDdiff --git a/backend/internal/system/upgrade_service_test.go b/backend/internal/system/upgrade_service_test.goindex cd3bd81b4..494a59f65 100644--- a/backend/internal/system/upgrade_service_test.go+++ b/backend/internal/system/upgrade_service_test.go@@ -205,7 +205,7 @@ func TestResolveSystemUpgraderRuntimeOptionsInternal_TCPDockerHost(t *testing.T)},}- options, err := resolveSystemUpgraderRuntimeOptionsInternal(+ containerEnv, mounts, networkMode, err := ResolveUpgraderRuntimeOptions(context.Background(),"tcp://docker-socket-proxy:2375",currentContainer,@@ -214,13 +214,13 @@ func TestResolveSystemUpgraderRuntimeOptionsInternal_TCPDockerHost(t *testing.T)nil,)require.NoError(t, err)- require.Equal(t, []string{"DOCKER_HOST=tcp://docker-socket-proxy:2375"}, options.ContainerEnv)- require.Empty(t, options.Mounts)- require.Equal(t, containertypes.NetworkMode("arcane-test"), options.NetworkMode)+ require.Equal(t, []string{"DOCKER_HOST=tcp://docker-socket-proxy:2375"}, containerEnv)+ require.Empty(t, mounts)+ require.Equal(t, containertypes.NetworkMode("arcane-test"), networkMode)}func TestResolveSystemUpgraderRuntimeOptionsInternal_UnixDockerHost(t *testing.T) {- options, err := resolveSystemUpgraderRuntimeOptionsInternal(+ containerEnv, mounts, networkMode, err := ResolveUpgraderRuntimeOptions(context.Background(),"unix:///var/run/docker.sock",nil,@@ -231,19 +231,19 @@ func TestResolveSystemUpgraderRuntimeOptionsInternal_UnixDockerHost(t *testing.Tnil,)require.NoError(t, err)- require.Equal(t, []string{"DOCKER_HOST=unix:///var/run/docker.sock"}, options.ContainerEnv)- require.Equal(t, containertypes.NetworkMode(""), options.NetworkMode)+ require.Equal(t, []string{"DOCKER_HOST=unix:///var/run/docker.sock"}, containerEnv)+ require.Equal(t, containertypes.NetworkMode(""), networkMode)require.Equal(t, []mounttypes.Mount{{Type: mounttypes.TypeBind,Source: "/host/run/docker.sock",Target: "/var/run/docker.sock",},- }, options.Mounts)+ }, mounts)}func TestResolveSystemUpgraderRuntimeOptionsInternal_DefaultDockerHost(t *testing.T) {- options, err := resolveSystemUpgraderRuntimeOptionsInternal(+ containerEnv, mounts, _, err := ResolveUpgraderRuntimeOptions(context.Background(),"",nil,@@ -254,18 +254,18 @@ func TestResolveSystemUpgraderRuntimeOptionsInternal_DefaultDockerHost(t *testinnil,)require.NoError(t, err)- require.Nil(t, options.ContainerEnv)+ require.Nil(t, containerEnv)require.Equal(t, []mounttypes.Mount{{Type: mounttypes.TypeBind,Source: "/var/run/docker.sock",Target: "/var/run/docker.sock",},- }, options.Mounts)+ }, mounts)}func TestResolveSystemUpgraderRuntimeOptionsInternal_UnixDockerHostResolutionError(t *testing.T) {- _, err := resolveSystemUpgraderRuntimeOptionsInternal(+ _, _, _, err := ResolveUpgraderRuntimeOptions(context.Background(),"unix:///var/run/docker.sock",nil,diff --git a/backend/internal/systembackup/handler.go b/backend/internal/systembackup/handler.gonew file mode 100644index 000000000..7e009e431--- /dev/null+++ b/backend/internal/systembackup/handler.go@@ -0,0 +1,274 @@+package systembackup++import (+ "context"+ "net/http"+ "strings"++ "emperror.dev/errors"++ "github.com/danielgtaylor/huma/v2"+ "github.com/getarcaneapp/arcane/backend/v2/internal/activity"+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"+ "github.com/getarcaneapp/arcane/backend/v2/internal/middleware"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/authz"+ activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"+ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"+ backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"+ "github.com/getarcaneapp/arcane/types/v2/base"+ "github.com/samber/mo"+)++type SystemBackupHandler struct {+ service *SystemBackupService+ activity *activity.ActivityService+ appCtx context.Context+}++type ListSystemBackupsInput struct {+ Search string `query:"search"`+ Sort string `query:"sort" default:"createdAt"`+ Order string `query:"order" default:"desc"`+ Start int `query:"start" default:"0"`+ Limit int `query:"limit" default:"20"`+}++type ListSystemBackupsOutput struct {+ Body struct {+ Data []backuptypes.SystemBackupRun `json:"data"`+ Pagination base.PaginationResponse `json:"pagination"`+ }+}++type SystemBackupPoliciesOutput struct {+ Body backuptypes.SystemBackupPolicyCollection+}+type UpdateSystemBackupPoliciesInput struct {+ Body backuptypes.UpdateSystemBackupPolicies+}+type SetSystemBackupRecoveryKeyInput struct {+ Body backuptypes.SystemBackupRecoveryKey+}+type SystemBackupRecoveryKeyOutput struct {+ Body backuptypes.SystemBackupRecoveryKeyStatus+}+type GenerateSystemBackupRecoveryKeyOutput struct {+ Body backuptypes.SystemBackupRecoveryKey+}+type CreateSystemBackupInput struct {+ Body backuptypes.CreateSystemBackupRequest+}+type SystemBackupOutput struct{ Body backuptypes.SystemBackupRun }+type RestoreSystemBackupInput struct {+ ID string `path:"id"`+ Body backuptypes.RestoreSystemBackupRequest+}+type UploadSystemBackupInput struct {+ ID string `path:"id"`+ Body backuptypes.UploadSystemBackupRequest+}+type DeleteSystemBackupInput struct {+ ID string `path:"id"`+ Body backuptypes.DeleteSystemBackupRequest+}+type DiscoverSystemBackupsInput struct {+ Body backuptypes.DiscoverSystemBackupsRequest+}+type DiscoverSystemBackupsOutput struct{ Body base.ApiResponse[int] }+type SystemBackupMessageOutput struct {+ Body base.ApiResponse[base.MessageResponse]+}++func RegisterSystemBackups(api huma.API, service *SystemBackupService, activityService *activity.ActivityService, appCtx handlerutil.ActivityAppContext) {+ h := &SystemBackupHandler{service: service, activity: activityService, appCtx: appCtx.Context()}+ adminOnly := middleware.RequireGlobalAdmin(api)+ middleware.RegisterWithPermission(api, huma.Operation{OperationID: "list-system-backups", Method: http.MethodGet, Path: "/backups", Summary: "List Arcane system backups", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsRead, h.List)+ middleware.RegisterWithPermission(api, huma.Operation{OperationID: "get-system-backup-policies", Method: http.MethodGet, Path: "/backups/policies", Summary: "Get Arcane system backup policies", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsRead, h.GetPolicies)+ middleware.RegisterWithPermission(api, huma.Operation{OperationID: "update-system-backup-policies", Method: http.MethodPut, Path: "/backups/policies", Summary: "Update Arcane system backup policies", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsManage, h.UpdatePolicies)+ middleware.RegisterWithPermission(api, huma.Operation{OperationID: "generate-system-backup-recovery-key", Method: http.MethodPost, Path: "/backups/recovery-key/generate", Summary: "Generate an Arcane system backup recovery key", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsRecoveryKey, h.GenerateRecoveryKey)+ middleware.RegisterWithPermission(api, huma.Operation{OperationID: "set-system-backup-recovery-key", Method: http.MethodPut, Path: "/backups/recovery-key", Summary: "Configure Arcane system backup recovery key", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsRecoveryKey, h.SetRecoveryKey)+ middleware.RegisterWithPermission(api, huma.Operation{OperationID: "create-system-backup", Method: http.MethodPost, Path: "/backups", Summary: "Create Arcane system backup", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsManage, h.Create)+ middleware.RegisterWithPermission(api, huma.Operation{OperationID: "discover-system-backups", Method: http.MethodPost, Path: "/backups/discover", Summary: "Discover Arcane system backups in S3", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsManage, h.Discover)+ middleware.RegisterWithPermission(api, huma.Operation{OperationID: "restore-system-backup", Method: http.MethodPost, Path: "/backups/{id}/restore", Summary: "Restore Arcane system backup", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsRestore, h.Restore)+ middleware.RegisterWithPermission(api, huma.Operation{OperationID: "upload-system-backup", Method: http.MethodPost, Path: "/backups/{id}/upload", Summary: "Upload Arcane system backup", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsManage, h.Upload)+ middleware.RegisterWithPermission(api, huma.Operation{OperationID: "delete-system-backup", Method: http.MethodDelete, Path: "/backups/{id}", Summary: "Delete Arcane system backup", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsManage, h.Delete)+}++func (h *SystemBackupHandler) List(ctx context.Context, input *ListSystemBackupsInput) (*ListSystemBackupsOutput, error) {+ params := handlerutil.PaginationParams(input.Start, input.Limit, input.Sort, input.Order, input.Search)+ runs, page, err := h.service.ListBackups(ctx, params)+ if err != nil {+ return nil, huma.Error500InternalServerError(err.Error())+ }+ out := &ListSystemBackupsOutput{}+ out.Body.Data, out.Body.Pagination = runs, handlerutil.PaginationResponse(page)+ return out, nil+}++func (h *SystemBackupHandler) GetPolicies(ctx context.Context, _ *struct{}) (*SystemBackupPoliciesOutput, error) {+ policies, err := h.service.GetPolicies(ctx)+ if err != nil {+ return nil, huma.Error500InternalServerError(err.Error())+ }+ return &SystemBackupPoliciesOutput{Body: *policies}, nil+}++func (h *SystemBackupHandler) UpdatePolicies(ctx context.Context, input *UpdateSystemBackupPoliciesInput) (*SystemBackupPoliciesOutput, error) {+ user, err := handlerutil.RequireUser(ctx)+ if err != nil {+ return nil, err+ }+ var policies *backuptypes.SystemBackupPolicyCollection+ _, err = activitylib.RunHandlerActivity(utils.ActivityRuntimeContext(ctx, h.appCtx), h.activity, activitylib.HandlerOptions{+ EnvironmentID: "0", Type: activitytypes.TypeResourceAction, ResourceType: "system_backup", ResourceID: "policies", ResourceName: "Arcane", User: user,+ Step: "Saving system backup schedules", Message: "Saving Arcane system backup schedules", SuccessMessage: "Arcane system backup schedules saved",+ Metadata: database.JSON{"action": "update_system_backup_policies", "policyCount": len(input.Body.Policies)},+ }, func(activityCtx context.Context) error {+ var updateErr error+ policies, updateErr = h.service.UpdatePolicies(activityCtx, input.Body.Policies)+ return updateErr+ })+ if err != nil {+ return nil, huma.Error400BadRequest(err.Error())+ }+ return &SystemBackupPoliciesOutput{Body: *policies}, nil+}++func (h *SystemBackupHandler) GenerateRecoveryKey(_ context.Context, _ *struct{}) (*GenerateSystemBackupRecoveryKeyOutput, error) {+ recoveryKey, err := h.service.GenerateRecoveryKey()+ if err != nil {+ return nil, huma.Error500InternalServerError(err.Error())+ }+ return &GenerateSystemBackupRecoveryKeyOutput{Body: *recoveryKey}, nil+}++func (h *SystemBackupHandler) SetRecoveryKey(ctx context.Context, input *SetSystemBackupRecoveryKeyInput) (*SystemBackupRecoveryKeyOutput, error) {+ user, err := handlerutil.RequireUser(ctx)+ if err != nil {+ return nil, err+ }+ var status *backuptypes.SystemBackupRecoveryKeyStatus+ _, err = activitylib.RunHandlerActivity(utils.ActivityRuntimeContext(ctx, h.appCtx), h.activity, activitylib.HandlerOptions{+ EnvironmentID: "0", Type: activitytypes.TypeResourceAction, ResourceType: "system_backup", ResourceID: "recovery-key", ResourceName: "Arcane", User: user,+ Step: "Configuring recovery key", Message: "Configuring Arcane system backup recovery key", SuccessMessage: "Arcane system backup recovery key configured",+ Metadata: database.JSON{"action": "set_system_backup_recovery_key"},+ }, func(activityCtx context.Context) error {+ var setErr error+ status, setErr = h.service.SetRecoveryKey(activityCtx, input.Body.RecoveryKey)+ return setErr+ })+ if err != nil {+ return nil, huma.Error400BadRequest(err.Error())+ }+ return &SystemBackupRecoveryKeyOutput{Body: *status}, nil+}++func (h *SystemBackupHandler) Discover(ctx context.Context, input *DiscoverSystemBackupsInput) (*DiscoverSystemBackupsOutput, error) {+ user, err := handlerutil.RequireUser(ctx)+ if err != nil {+ return nil, err+ }+ count := 0+ _, err = activitylib.RunHandlerActivity(utils.ActivityRuntimeContext(ctx, h.appCtx), h.activity, activitylib.HandlerOptions{+ EnvironmentID: "0", Type: activitytypes.TypeResourceAction, ResourceType: "system_backup", ResourceID: "s3", ResourceName: "Arcane", User: user,+ Step: "Discovering system backups", Message: "Discovering Arcane system backups in S3", SuccessMessage: "Arcane system backup discovery completed",+ Metadata: database.JSON{"action": "discover_system_backups", "s3DestinationId": input.Body.S3DestinationID},+ }, func(activityCtx context.Context) error {+ var discoverErr error+ count, discoverErr = h.service.DiscoverRemoteBackups(activityCtx, input.Body)+ return discoverErr+ })+ if err != nil {+ return nil, huma.Error400BadRequest(err.Error())+ }+ return &DiscoverSystemBackupsOutput{Body: base.ApiResponse[int]{Success: true, Data: count}}, nil+}++func (h *SystemBackupHandler) Create(ctx context.Context, input *CreateSystemBackupInput) (*SystemBackupOutput, error) {+ user, err := handlerutil.RequireUser(ctx)+ if err != nil {+ return nil, err+ }+ var run *SystemBackupRun+ _, err = activitylib.RunHandlerActivity(utils.ActivityRuntimeContext(ctx, h.appCtx), h.activity, activitylib.HandlerOptions{+ EnvironmentID: "0", Type: activitytypes.TypeResourceAction, ResourceType: "system_backup", ResourceID: "arcane", ResourceName: "Arcane", User: user,+ Step: "Creating system backup", Message: "Creating Arcane system backup", SuccessMessage: "Arcane system backup created successfully",+ Metadata: database.JSON{"action": "create_system_backup", "destination": input.Body.Destination, "s3DestinationId": input.Body.S3DestinationID, "policyId": input.Body.PolicyID},+ }, func(activityCtx context.Context) error {+ var e error+ run, e = h.service.CreateBackup(activityCtx, *user, SystemBackupTriggerManual, input.Body)+ return e+ })+ if errors.Is(err, ErrSystemBackupAlreadyRunning) {+ return nil, huma.Error409Conflict(err.Error())+ }+ if err != nil {+ return nil, huma.Error500InternalServerError(err.Error())+ }+ dto := run.ToDTO()+ return &SystemBackupOutput{Body: dto}, nil+}++func (h *SystemBackupHandler) Restore(ctx context.Context, input *RestoreSystemBackupInput) (*SystemBackupMessageOutput, error) {+ user, err := handlerutil.RequireUser(ctx)+ if err != nil {+ return nil, err+ }+ activityID, err := activitylib.RunHandlerActivity(utils.ActivityRuntimeContext(ctx, h.appCtx), h.activity, activitylib.HandlerOptions{+ EnvironmentID: "0", Type: activitytypes.TypeResourceAction, ResourceType: "system_backup", ResourceID: input.ID, ResourceName: "Arcane", User: user,+ Step: "Preparing system restore", Message: "Preparing Arcane system restore", SuccessMessage: "Arcane system restore started",+ Metadata: database.JSON{"action": "restore_system_backup", "backupId": input.ID},+ }, func(activityCtx context.Context) error {+ return h.service.RestoreBackup(activityCtx, input.ID, input.Body.RecoveryKey, *user)+ })+ if err != nil {+ return nil, huma.Error500InternalServerError(err.Error())+ }+ return messageOutputInternal("Arcane system restore started", activityID), nil+}++func (h *SystemBackupHandler) Upload(ctx context.Context, input *UploadSystemBackupInput) (*SystemBackupOutput, error) {+ user, err := handlerutil.RequireUser(ctx)+ if err != nil {+ return nil, err+ }+ var run *SystemBackupRun+ _, err = activitylib.RunHandlerActivity(utils.ActivityRuntimeContext(ctx, h.appCtx), h.activity, activitylib.HandlerOptions{+ EnvironmentID: "0", Type: activitytypes.TypeResourceAction, ResourceType: "system_backup", ResourceID: input.ID, ResourceName: "Arcane", User: user,+ Step: "Uploading system backup", Message: "Uploading Arcane system backup", SuccessMessage: "Arcane system backup uploaded successfully",+ Metadata: database.JSON{"action": "upload_system_backup", "backupId": input.ID, "s3DestinationId": input.Body.S3DestinationID},+ }, func(activityCtx context.Context) error {+ var e error+ run, e = h.service.UploadBackup(activityCtx, input.ID, input.Body)+ return e+ })+ if err != nil {+ return nil, huma.Error500InternalServerError(err.Error())+ }+ dto := run.ToDTO()+ return &SystemBackupOutput{Body: dto}, nil+}++func (h *SystemBackupHandler) Delete(ctx context.Context, input *DeleteSystemBackupInput) (*SystemBackupMessageOutput, error) {+ user, err := handlerutil.RequireUser(ctx)+ if err != nil {+ return nil, err+ }+ activityID, err := activitylib.RunHandlerActivity(utils.ActivityRuntimeContext(ctx, h.appCtx), h.activity, activitylib.HandlerOptions{+ EnvironmentID: "0", Type: activitytypes.TypeResourceAction, ResourceType: "system_backup", ResourceID: input.ID, ResourceName: "Arcane", User: user,+ Step: "Deleting system backup", Message: "Deleting Arcane system backup", SuccessMessage: "Arcane system backup deleted successfully",+ Metadata: database.JSON{"action": "delete_system_backup", "backupId": input.ID},+ }, func(activityCtx context.Context) error {+ return h.service.DeleteBackup(activityCtx, input.ID, input.Body.RecoveryKey)+ })+ if err != nil {+ return nil, huma.Error500InternalServerError(err.Error())+ }+ return messageOutputInternal("Arcane system backup deleted successfully", activityID), nil+}++func messageOutputInternal(message, activityID string) *SystemBackupMessageOutput {+ return &SystemBackupMessageOutput{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: message, ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()}}}+}diff --git a/backend/internal/systembackup/model.go b/backend/internal/systembackup/model.gonew file mode 100644index 000000000..810b98997--- /dev/null+++ b/backend/internal/systembackup/model.go@@ -0,0 +1,83 @@+package systembackup++import (+ "time"++ "github.com/getarcaneapp/arcane/backend/v2/internal/database"+ backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"+)++type SystemBackupStatus string++const (+ SystemBackupStatusRunning SystemBackupStatus = "running"+ SystemBackupStatusSucceeded SystemBackupStatus = "succeeded"+ SystemBackupStatusFailed SystemBackupStatus = "failed"+)++type SystemBackupTrigger string++const (+ SystemBackupTriggerManual SystemBackupTrigger = "manual"+ SystemBackupTriggerScheduled SystemBackupTrigger = "scheduled"+ SystemBackupTriggerSafety SystemBackupTrigger = "safety"+)++type SystemBackupRun struct {+ database.BaseModel++ Size int64 `json:"size" gorm:"column:size" sortable:"true"`+ CreatedAt time.Time `json:"createdAt" gorm:"column:created_at" sortable:"true"`+ Status SystemBackupStatus `json:"status" gorm:"column:status;type:text;not null" sortable:"true"`+ Trigger SystemBackupTrigger `json:"trigger" gorm:"column:trigger;type:text;not null" sortable:"true"`+ Destination backuptypes.SystemBackupDestination `json:"destination" gorm:"column:destination;type:text;not null" sortable:"true"`+ LocalSnapshotID string `json:"localSnapshotId,omitempty" gorm:"column:local_snapshot_id;type:text"`+ RemoteSnapshotID string `json:"remoteSnapshotId,omitempty" gorm:"column:remote_snapshot_id;type:text"`+ S3DestinationID string `json:"s3DestinationId,omitempty" gorm:"column:s3_destination_id;type:text;index"`+ S3DestinationName string `json:"s3DestinationName,omitempty" gorm:"-"`+ PolicyID string `json:"policyId,omitempty" gorm:"column:policy_id;type:text;index"`+ Error string `json:"error,omitempty" gorm:"column:error;type:text"`+}++func (SystemBackupRun) TableName() string { return "system_backup_runs" }++func (b SystemBackupRun) ToDTO() backuptypes.SystemBackupRun {+ return backuptypes.SystemBackupRun{+ ID: b.ID, Size: b.Size, CreatedAt: b.CreatedAt, Status: string(b.Status), Trigger: string(b.Trigger),+ Destination: b.Destination, LocalSnapshotID: b.LocalSnapshotID, RemoteSnapshotID: b.RemoteSnapshotID,+ S3DestinationID: b.S3DestinationID, S3DestinationName: b.S3DestinationName, PolicyID: b.PolicyID, Error: b.Error,+ }+}++type SystemBackupPolicy struct {+ database.BaseModel++ Enabled bool `gorm:"column:enabled;not null;default:false"`+ Schedule string `gorm:"column:schedule;type:text;not null"`+ RetentionCount int `gorm:"column:retention_count;not null;default:7"`+ LocalEnabled bool `gorm:"column:local_enabled;not null;default:true"`+ S3Enabled bool `gorm:"column:s3_enabled;not null;default:false"`+ S3DestinationID string `gorm:"column:s3_destination_id;type:text;index"`+}++func (SystemBackupPolicy) TableName() string { return "system_backup_policies" }++func (p SystemBackupPolicy) ToDTO(lastRun *SystemBackupRun) backuptypes.SystemBackupPolicy {+ dto := backuptypes.SystemBackupPolicy{+ ID: p.ID, Enabled: p.Enabled, Schedule: p.Schedule, RetentionCount: p.RetentionCount,+ LocalEnabled: p.LocalEnabled, S3Enabled: p.S3Enabled, S3DestinationID: p.S3DestinationID,+ }+ if lastRun != nil {+ run := lastRun.ToDTO()+ dto.LastRun = &run+ }+ return dto+}++type SystemBackupRecoveryConfig struct {+ database.BaseModel++ EncryptedRecoveryKey string `gorm:"column:encrypted_recovery_key;type:text;not null"`+}++func (SystemBackupRecoveryConfig) TableName() string { return "system_backup_recovery_config" }diff --git a/backend/internal/systembackup/module.go b/backend/internal/systembackup/module.gonew file mode 100644index 000000000..f30343640--- /dev/null+++ b/backend/internal/systembackup/module.go@@ -0,0 +1,32 @@+// Package systembackup owns Arcane system-recovery backup orchestration and its HTTP surface.+package systembackup++import (+ "github.com/danielgtaylor/huma/v2"+ "github.com/getarcaneapp/arcane/backend/v2/internal/activity"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"+)++type Module struct {+ service *SystemBackupService+ activity *activity.ActivityService+}++func New(service *SystemBackupService, activityService *activity.ActivityService) *Module {+ return &Module{service: service, activity: activityService}+}++func (m *Module) Service() *SystemBackupService {+ if m == nil {+ return nil+ }+ return m.service+}++func (m *Module) RegisterRoutes(api huma.API, appCtx handlerutil.ActivityAppContext) {+ if m == nil {+ RegisterSystemBackups(api, nil, nil, appCtx)+ return+ }+ RegisterSystemBackups(api, m.service, m.activity, appCtx)+}diff --git a/backend/internal/systembackup/service.go b/backend/internal/systembackup/service.gonew file mode 100644index 000000000..b525285c6--- /dev/null+++ b/backend/internal/systembackup/service.go@@ -0,0 +1,1042 @@+package systembackup++import (+ "context"+ "crypto/rand"+ "encoding/base32"+ "encoding/json"+ "fmt"+ "log/slog"+ "maps"+ "net/url"+ "os"+ "path/filepath"+ "reflect"+ "regexp"+ "slices"+ "strings"+ "time"++ "emperror.dev/errors"++ "github.com/getarcaneapp/arcane/backend/v2/internal/activity"+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"+ "github.com/getarcaneapp/arcane/backend/v2/internal/backup"+ "github.com/getarcaneapp/arcane/backend/v2/internal/common"+ "github.com/getarcaneapp/arcane/backend/v2/internal/config"+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"+ "github.com/getarcaneapp/arcane/backend/v2/internal/docker"+ recoverytypes "github.com/getarcaneapp/arcane/backend/v2/internal/recovery"+ s3domain "github.com/getarcaneapp/arcane/backend/v2/internal/s3"+ "github.com/getarcaneapp/arcane/backend/v2/internal/system"+ "github.com/getarcaneapp/arcane/backend/v2/internal/volume"+ dockerutil "github.com/getarcaneapp/arcane/backend/v2/pkg/dockerutil"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"+ activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/projects"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/entityjobs"+ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"+ backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"+ "github.com/google/uuid"+ containertypes "github.com/moby/moby/api/types/container"+ "github.com/moby/moby/api/types/mount"+ "github.com/moby/moby/client"+ "go.getarcane.app/sys/cgroup"+ "go.getarcane.app/sys/crypto"+ "gorm.io/gorm"+)++const (+ defaultSystemBackupSchedule = "0 0 3 * * *"+ systemRecoveryManifestName = ".arcane-recovery.json"+ systemRecoveryHelperPath = "/app/arcane-recovery-helper"+ systemAdmissionID = "system"+)++var ErrSystemBackupAlreadyRunning = errors.New("an Arcane system backup is already running")++type SystemBackupService struct {+ db *database.DB+ dockerService *docker.DockerClientService+ volumeService *volume.VolumeService+ engine *backup.Engine+ s3Destinations *s3domain.S3DestinationService+ activityService *activity.ActivityService+ config *config.Config+ jobs *entityjobs.Registry+}++func NewSystemBackupService(db *database.DB, dockerService *docker.DockerClientService, volumeService *volume.VolumeService, engine *backup.Engine, s3Destinations *s3domain.S3DestinationService, activityService *activity.ActivityService, cfg *config.Config) *SystemBackupService {+ return &SystemBackupService{+ db: db, dockerService: dockerService, volumeService: volumeService, engine: engine,+ s3Destinations: s3Destinations, activityService: activityService, config: cfg,+ jobs: entityjobs.New("system-backup:", backup.SystemAdmissionScope),+ }+}++// SupportedDatabaseProvider reports whether system recovery works on the+// configured database. Recovery is currently SQLite-only.+func (s *SystemBackupService) SupportedDatabaseProvider() bool {+ return strings.HasPrefix(s.config.DatabaseURL, "file:")+}++// recoveryKeyFormatInternal is the only accepted key shape, generated by+// GenerateRecoveryKey: 48 base32 characters (~240 bits from crypto/rand) in 8+// hyphenated groups for hand transcription. The key is the Rustic repository+// password and is used exactly as displayed, hyphens included.+var recoveryKeyFormatInternal = regexp.MustCompile(`^[A-Z2-7]{6}(-[A-Z2-7]{6}){7}$`)++func validateRecoveryKeyInternal(key string) error {+ if !recoveryKeyFormatInternal.MatchString(strings.TrimSpace(key)) {+ return errors.New("enter the generated recovery key (8 groups of 6 characters)")+ }+ return nil+}++func recoveryHelperExecutableInternal(mounts []containertypes.MountPoint, executablePath string) (string, *mount.Mount, error) {+ executablePath = strings.TrimSpace(executablePath)+ if executablePath == "" {+ return "", nil, errors.New("arcane executable path is empty")+ }+ executableMount := dockerutil.MountForSubpath(mounts, executablePath, systemRecoveryHelperPath)+ if executableMount == nil {+ return executablePath, nil, nil+ }+ if executableMount.Type != mount.TypeBind {+ return "", nil, errors.New("the Arcane executable must come from the image or a bind mount")+ }+ executableMount.ReadOnly = true+ return systemRecoveryHelperPath, executableMount, nil+}++func (s *SystemBackupService) recoveryEnvironmentInternal() map[string]string {+ result := make(map[string]string)+ var visit func(reflect.Value)+ visit = func(value reflect.Value) {+ if value.Kind() == reflect.Pointer {+ if value.IsNil() {+ return+ }+ value = value.Elem()+ }+ if value.Kind() != reflect.Struct {+ return+ }+ typeOf := value.Type()+ for i := range value.NumField() {+ fieldType := typeOf.Field(i)+ field := value.Field(i)+ if fieldType.Anonymous {+ visit(field)+ continue+ }+ name := fieldType.Tag.Get("env")+ if name == "" || !field.CanInterface() {+ continue+ }+ if field.Type() == reflect.TypeFor[os.FileMode]() {+ mode, ok := reflect.TypeAssert[os.FileMode](field)+ if ok {+ result[name] = fmt.Sprintf("%#o", uint32(mode))+ }+ } else {+ result[name] = fmt.Sprint(field.Interface())+ }+ }+ }+ visit(reflect.ValueOf(s.config))+ return result+}++// dataDirectoryInternal resolves the directory holding the SQLite database:+// /app/data in the container, the local data directory in host development.+// The recovery manifest must live inside it so every snapshot includes it.+func (s *SystemBackupService) dataDirectoryInternal() (string, error) {+ parsed, err := url.Parse(s.config.DatabaseURL)+ if err != nil {+ return "", fmt.Errorf("failed to parse database URL: %w", err)+ }+ databasePath := parsed.Opaque+ if databasePath == "" {+ databasePath = parsed.Path+ }+ if strings.TrimSpace(databasePath) == "" {+ return "", errors.New("cannot resolve the Arcane data directory from the database URL")+ }+ absolute, err := filepath.Abs(databasePath)+ if err != nil {+ return "", fmt.Errorf("failed to resolve the Arcane data directory: %w", err)+ }+ return filepath.Dir(absolute), nil+}++func (s *SystemBackupService) manifestPathInternal() (string, error) {+ directory, err := s.dataDirectoryInternal()+ if err != nil {+ return "", err+ }+ return filepath.Join(directory, systemRecoveryManifestName), nil+}++func (s *SystemBackupService) writeManifestInternal(ctx context.Context, backupID string) error {+ manifest := recoverytypes.Manifest{FormatVersion: 1, ArcaneVersion: config.Version, BackupID: backupID, ActivityID: activitylib.IDFromContext(ctx), CreatedAt: time.Now().UTC(), Environment: s.recoveryEnvironmentInternal()}+ data, err := json.MarshalIndent(manifest, "", " ")+ if err != nil {+ return fmt.Errorf("failed to encode recovery manifest: %w", err)+ }+ manifestPath, err := s.manifestPathInternal()+ if err != nil {+ return err+ }+ if err := os.WriteFile(manifestPath, data, 0o600); err != nil {+ return fmt.Errorf("failed to write recovery manifest: %w", err)+ }+ return nil+}++func (s *SystemBackupService) appDataMountInternal(ctx context.Context, dockerClient *client.Client, target string, readOnly bool) (mount.Mount, error) {+ if _, containerErr := cgroup.CurrentContainerID(); containerErr != nil {+ // Host development: bind-mount the local data directory into the+ // Rustic helper directly instead of re-sharing the container's mount.+ directory, dirErr := s.dataDirectoryInternal()+ if dirErr != nil {+ return mount.Mount{}, dirErr+ }+ return mount.Mount{Type: mount.TypeBind, Source: directory, Target: target, ReadOnly: readOnly}, nil+ }+ dataMount, err := dockerutil.MountForCurrentContainerSubpath(ctx, dockerClient, "/app/data", target)+ if err != nil {+ return mount.Mount{}, fmt.Errorf("failed to inspect Arcane data mount: %w", err)+ }+ if dataMount == nil {+ return mount.Mount{}, errors.New("arcane system backups require /app/data to be mounted into the Arcane container")+ }+ dataMount.ReadOnly = readOnly+ return *dataMount, nil+}++func (s *SystemBackupService) localRepositoryInternal(ctx context.Context, dockerClient *client.Client, readOnly bool) (backup.Repository, error) {+ storage, err := s.volumeService.BackupStorageMount(ctx, dockerClient, "/repository", readOnly)+ if err != nil {+ return backup.Repository{}, err+ }+ return backup.Repository{+ ID: "system:local",+ Environment: []string{"RUSTIC_REPOSITORY=/repository/system-recovery"},+ Mounts: []mount.Mount{storage},+ }, nil+}++func (s *SystemBackupService) remoteRepositoryInternal(ctx context.Context, destinationID string) (backup.Repository, error) {+ configuration, err := s.s3Destinations.Configuration(ctx, destinationID)+ if err != nil {+ return backup.Repository{}, errors.New("the selected S3 backup destination is not configured")+ }+ return backup.Repository{+ ID: "system:s3:" + destinationID,+ Environment: configuration.RusticEnvironment("arcane-system-recovery"),+ }, nil+}++func (s *SystemBackupService) createSnapshotInternal(ctx context.Context, dockerClient *client.Client, repository backup.Repository, recoveryKey string) (backup.Snapshot, error) {+ dataMount, err := s.appDataMountInternal(ctx, dockerClient, "/arcane-data", true)+ if err != nil {+ return backup.Snapshot{}, err+ }+ return s.engine.CreateSnapshot(ctx, dockerClient, repository, recoveryKey, "arcane-system-recovery", dataMount)+}++func systemSnapshotPathFromFilesInternal(files []string) (string, error) {+ for _, file := range files {+ if strings.TrimPrefix(file, "/") == ".arcane-recovery.json" {+ return "/", nil+ }+ }+ for _, file := range files {+ if strings.TrimPrefix(file, "/") == "app/data/.arcane-recovery.json" {+ return "/app/data", nil+ }+ }+ return "", errors.New("system recovery snapshot does not contain an Arcane recovery manifest")+}++func (s *SystemBackupService) snapshotPathInternal(ctx context.Context, dockerClient *client.Client, repository backup.Repository, recoveryKey, snapshotID string) (string, error) {+ files, err := s.engine.ListSnapshotFiles(ctx, dockerClient, repository, recoveryKey, snapshotID)+ if err != nil {+ return "", fmt.Errorf("failed to inspect system recovery snapshot: %w", err)+ }+ return systemSnapshotPathFromFilesInternal(files)+}++func (s *SystemBackupService) recoveryKeyInternal(ctx context.Context, supplied string) (string, error) {+ if strings.TrimSpace(supplied) != "" {+ if err := validateRecoveryKeyInternal(supplied); err != nil {+ return "", err+ }+ return supplied, nil+ }+ var recoveryConfig SystemBackupRecoveryConfig+ err := s.db.WithContext(ctx).Where("id = ?", systemRecoveryConfigID).First(&recoveryConfig).Error+ if errors.Is(err, gorm.ErrRecordNotFound) {+ return "", errors.New("enter the recovery key or configure one in system backups")+ }+ if err != nil {+ return "", fmt.Errorf("failed to load stored recovery key: %w", err)+ }+ key, err := crypto.Decrypt(recoveryConfig.EncryptedRecoveryKey)+ if err != nil {+ return "", fmt.Errorf("failed to decrypt stored recovery key: %w", err)+ }+ return key, validateRecoveryKeyInternal(key)+}++// resolveBackupPlanInternal resolves which destinations a run targets, from+// either the policy or the request's destination enum.+func (s *SystemBackupService) resolveBackupPlanInternal(ctx context.Context, request backuptypes.CreateSystemBackupRequest) (localEnabled, s3Enabled bool, destinationID string, destination backuptypes.SystemBackupDestination, err error) {+ destinationID = strings.TrimSpace(request.S3DestinationID)+ if request.PolicyID != "" {+ policy, policyErr := s.loadPolicyInternal(ctx, request.PolicyID)+ if policyErr != nil {+ return false, false, "", "", policyErr+ }+ if policy == nil {+ return false, false, "", "", errors.New("system backup policy not found")+ }+ localEnabled, s3Enabled, destinationID = policy.LocalEnabled, policy.S3Enabled, policy.S3DestinationID+ } else {+ switch request.Destination {+ case backuptypes.SystemBackupDestinationLocal, "":+ localEnabled = true+ case backuptypes.SystemBackupDestinationS3:+ s3Enabled = true+ case backuptypes.SystemBackupDestinationLocalS3:+ localEnabled, s3Enabled = true, true+ default:+ return false, false, "", "", errors.New("unknown system backup destination")+ }+ }+ if !localEnabled && !s3Enabled {+ return false, false, "", "", errors.New("select at least one system backup destination")+ }+ if s3Enabled && destinationID == "" {+ return false, false, "", "", errors.New("select an S3 destination for the system backup")+ }+ destination = backuptypes.SystemBackupDestinationLocal+ if localEnabled && s3Enabled {+ destination = backuptypes.SystemBackupDestinationLocalS3+ } else if s3Enabled {+ destination = backuptypes.SystemBackupDestinationS3+ }+ return localEnabled, s3Enabled, destinationID, destination, nil+}++func (s *SystemBackupService) CreateBackup(ctx context.Context, user common.User, trigger SystemBackupTrigger, request backuptypes.CreateSystemBackupRequest) (_ *SystemBackupRun, err error) {+ if !s.SupportedDatabaseProvider() {+ return nil, errors.New("arcane system recovery currently requires the SQLite database provider")+ }+ recoveryKey, err := s.recoveryKeyInternal(ctx, request.RecoveryKey)+ if err != nil {+ return nil, err+ }+ localEnabled, s3Enabled, destinationID, destination, err := s.resolveBackupPlanInternal(ctx, request)+ if err != nil {+ return nil, err+ }+ lease, admitted, leaseErr := s.engine.TryAcquireRun(ctx, backup.SystemAdmissionScope, systemAdmissionID)+ if leaseErr != nil {+ return nil, leaseErr+ }+ if !admitted {+ return nil, ErrSystemBackupAlreadyRunning+ }+ defer lease.Release()+ run := &SystemBackupRun{CreatedAt: time.Now().UTC(), Status: SystemBackupStatusRunning, Trigger: trigger, Destination: destination, S3DestinationID: destinationID, PolicyID: request.PolicyID}+ run.ID = "system-" + uuid.NewString()+ if err := s.db.WithContext(ctx).Create(run).Error; err != nil {+ return nil, err+ }+ defer func() {+ if err != nil {+ run.Status, run.Error = SystemBackupStatusFailed, err.Error()+ } else {+ run.Status, run.Error = SystemBackupStatusSucceeded, ""+ }+ if saveErr := s.db.WithContext(context.WithoutCancel(ctx)).Save(run).Error; saveErr != nil {+ err = errors.Combine(err, fmt.Errorf("failed to save system backup result: %w", saveErr))+ }+ }()+ dockerClient, err := s.dockerService.GetClient(ctx)+ if err != nil {+ return run, err+ }+ localRepository, err := s.localRepositoryInternal(ctx, dockerClient, false)+ if err != nil {+ return run, err+ }+ // The database is locked only while the local staging snapshot is taken.+ // An S3 copy replicates that immutable snapshot afterwards, so the+ // exclusive lock never spans a network upload and the source is read once.+ stagedSnapshot, err := s.snapshotWithDatabaseLockInternal(ctx, dockerClient, localRepository, recoveryKey, run.ID)+ if err != nil {+ return run, err+ }+ // The staged snapshot is kept only for local-enabled runs. Forgetting it in+ // a defer covers the failure paths too: a failed replication otherwise+ // orphans one unreferenced snapshot in the staging repository per run.+ defer func() {+ if localEnabled {+ return+ }+ cleanupCtx := context.WithoutCancel(ctx)+ if forgetErr := s.engine.ForgetSnapshot(cleanupCtx, dockerClient, localRepository, recoveryKey, stagedSnapshot.ID); forgetErr != nil {+ slog.WarnContext(cleanupCtx, "failed to remove staged system recovery snapshot", "snapshot_id", stagedSnapshot.ID, "error", forgetErr)+ }+ }()+ if localEnabled {+ run.LocalSnapshotID, run.Size = stagedSnapshot.ID, stagedSnapshot.Size+ }+ if s3Enabled {+ remoteRepository, repoErr := s.remoteRepositoryInternal(ctx, destinationID)+ if repoErr != nil {+ return run, repoErr+ }+ remoteSnapshot, replicateErr := s.engine.Replicate(ctx, dockerClient, localRepository, stagedSnapshot.ID, remoteRepository, recoveryKey, "arcane-system-recovery")+ if replicateErr != nil {+ return run, fmt.Errorf("failed to create S3 system recovery snapshot: %w", replicateErr)+ }+ run.RemoteSnapshotID = remoteSnapshot.ID+ if run.Size == 0 {+ run.Size = remoteSnapshot.Size+ }+ }+ return run, nil+}++// snapshotWithDatabaseLockInternal takes the local staging snapshot while the+// SQLite database is checkpointed and exclusively locked, and releases the+// lock before returning.+func (s *SystemBackupService) snapshotWithDatabaseLockInternal(ctx context.Context, dockerClient *client.Client, repository backup.Repository, recoveryKey, backupID string) (backup.Snapshot, error) {+ if err := s.writeManifestInternal(ctx, backupID); err != nil {+ return backup.Snapshot{}, err+ }+ defer func() {+ if manifestPath, pathErr := s.manifestPathInternal(); pathErr == nil {+ _ = os.Remove(manifestPath)+ }+ }()+ sqlDB, err := s.db.SQLDB()+ if err != nil {+ return backup.Snapshot{}, err+ }+ if _, err := sqlDB.ExecContext(ctx, "PRAGMA wal_checkpoint(FULL)"); err != nil {+ return backup.Snapshot{}, fmt.Errorf("failed to checkpoint Arcane database: %w", err)+ }+ connection, err := sqlDB.Conn(ctx)+ if err != nil {+ return backup.Snapshot{}, fmt.Errorf("failed to reserve Arcane database connection: %w", err)+ }+ defer func() { _ = connection.Close() }()+ if _, err := connection.ExecContext(ctx, "BEGIN EXCLUSIVE"); err != nil {+ return backup.Snapshot{}, fmt.Errorf("failed to lock Arcane database for backup: %w", err)+ }+ defer func() { _, _ = connection.ExecContext(context.WithoutCancel(ctx), "ROLLBACK") }()+ snapshot, err := s.createSnapshotInternal(ctx, dockerClient, repository, recoveryKey)+ if err != nil {+ return backup.Snapshot{}, fmt.Errorf("failed to create local system recovery snapshot: %w", err)+ }+ return snapshot, nil+}++func (s *SystemBackupService) ListBackups(ctx context.Context, params pagination.QueryParams) ([]backuptypes.SystemBackupRun, pagination.Response, error) {+ var runs []SystemBackupRun+ query := s.db.WithContext(ctx).Model(&SystemBackupRun{})+ if term := strings.TrimSpace(params.Search); term != "" {+ pattern := "%" + term + "%"+ query = query.Where("status LIKE ? OR trigger LIKE ? OR destination LIKE ? OR error LIKE ?", pattern, pattern, pattern, pattern)+ }+ response, err := pagination.PaginateAndSortDB(params, query, &runs)+ if err != nil {+ return nil, pagination.Response{}, fmt.Errorf("failed to list system backups: %w", err)+ }+ names := make(map[string]string)+ if s.s3Destinations != nil {+ if destinations, listErr := s.s3Destinations.ListAllS3Destinations(ctx); listErr == nil {+ for _, destination := range destinations {+ names[destination.ID] = destination.Name+ }+ }+ }+ result := make([]backuptypes.SystemBackupRun, len(runs))+ for i := range runs {+ runs[i].S3DestinationName = names[runs[i].S3DestinationID]+ result[i] = runs[i].ToDTO()+ }+ return result, response, nil+}++func (s *SystemBackupService) backupInternal(ctx context.Context, id string) (*SystemBackupRun, error) {+ var run SystemBackupRun+ if err := s.db.WithContext(ctx).Where("id = ?", id).First(&run).Error; err != nil {+ return nil, err+ }+ return &run, nil+}++func (s *SystemBackupService) DeleteBackup(ctx context.Context, id, recoveryKey string) error {+ // Deletes contend with create/restore/upload on the system run lease so a+ // slow operation cannot resurrect or double-forget snapshots.+ lease, admitted, err := s.engine.TryAcquireRun(ctx, backup.SystemAdmissionScope, systemAdmissionID)+ if err != nil {+ return err+ }+ if !admitted {+ return ErrSystemBackupAlreadyRunning+ }+ defer lease.Release()+ run, err := s.backupInternal(ctx, id)+ if err != nil {+ return err+ }+ // A run without snapshots (a failed attempt) is just a row; deleting it+ // needs neither the key nor Rustic.+ key := ""+ if run.LocalSnapshotID != "" || run.RemoteSnapshotID != "" {+ if key, err = s.recoveryKeyInternal(ctx, recoveryKey); err != nil {+ return err+ }+ }+ dockerClient, err := s.dockerService.GetClient(ctx)+ if err != nil {+ return err+ }+ var deleteErr error+ if run.LocalSnapshotID != "" {+ repository, repoErr := s.localRepositoryInternal(ctx, dockerClient, false)+ if repoErr == nil {+ repoErr = s.engine.ForgetSnapshot(ctx, dockerClient, repository, key, run.LocalSnapshotID)+ }+ if repoErr != nil {+ deleteErr = errors.Combine(deleteErr, fmt.Errorf("failed to delete local snapshot: %w", repoErr))+ } else {+ run.LocalSnapshotID = ""+ }+ }+ if run.RemoteSnapshotID != "" {+ repository, repoErr := s.remoteRepositoryInternal(ctx, run.S3DestinationID)+ if repoErr == nil {+ repoErr = s.engine.ForgetSnapshot(ctx, dockerClient, repository, key, run.RemoteSnapshotID)+ }+ if repoErr != nil {+ deleteErr = errors.Combine(deleteErr, fmt.Errorf("failed to delete S3 snapshot: %w", repoErr))+ } else {+ run.RemoteSnapshotID, run.S3DestinationID = "", ""+ }+ }+ if deleteErr != nil {+ run.Error = deleteErr.Error()+ _ = s.db.WithContext(ctx).Save(run).Error+ return deleteErr+ }+ return s.db.WithContext(ctx).Delete(run).Error+}++func (s *SystemBackupService) DiscoverRemoteBackups(ctx context.Context, request backuptypes.DiscoverSystemBackupsRequest) (int, error) {+ recoveryKey, err := s.recoveryKeyInternal(ctx, request.RecoveryKey)+ if err != nil {+ return 0, err+ }+ repository, err := s.remoteRepositoryInternal(ctx, request.S3DestinationID)+ if err != nil {+ return 0, err+ }+ dockerClient, err := s.dockerService.GetClient(ctx)+ if err != nil {+ return 0, err+ }+ snapshots, err := s.engine.ListSnapshots(ctx, dockerClient, repository, recoveryKey)+ if err != nil {+ return 0, fmt.Errorf("failed to open system recovery repository: %w", err)+ }+ var knownIDs []string+ if err := s.db.WithContext(ctx).Model(&SystemBackupRun{}).+ Where("s3_destination_id = ? AND remote_snapshot_id <> ''", request.S3DestinationID).+ Pluck("remote_snapshot_id", &knownIDs).Error; err != nil {+ return 0, err+ }+ known := make(map[string]struct{}, len(knownIDs))+ for _, id := range knownIDs {+ known[id] = struct{}{}+ }+ created := 0+ for _, snapshot := range snapshots {+ if strings.TrimSpace(snapshot.ID) == "" {+ continue+ }+ if _, exists := known[snapshot.ID]; exists {+ continue+ }+ createdAt := snapshot.Time+ if createdAt.IsZero() {+ createdAt = time.Now().UTC()+ }+ run := &SystemBackupRun{+ Size: snapshot.Summary.TotalBytesProcessed, CreatedAt: createdAt, Status: SystemBackupStatusSucceeded,+ Trigger: SystemBackupTriggerManual, Destination: backuptypes.SystemBackupDestinationS3,+ RemoteSnapshotID: snapshot.ID, S3DestinationID: request.S3DestinationID,+ }+ run.ID = fmt.Sprintf("remote-%s-%s", request.S3DestinationID, snapshot.ID)+ if err := s.db.WithContext(ctx).Create(run).Error; err != nil {+ return created, fmt.Errorf("failed to save discovered system backup: %w", err)+ }+ created+++ }+ return created, nil+}++func (s *SystemBackupService) UploadBackup(ctx context.Context, id string, request backuptypes.UploadSystemBackupRequest) (*SystemBackupRun, error) {+ lease, admitted, err := s.engine.TryAcquireRun(ctx, backup.SystemAdmissionScope, systemAdmissionID)+ if err != nil {+ return nil, err+ }+ if !admitted {+ return nil, ErrSystemBackupAlreadyRunning+ }+ defer lease.Release()+ run, err := s.backupInternal(ctx, id)+ if err != nil {+ return nil, err+ }+ if run.Status != SystemBackupStatusSucceeded || run.LocalSnapshotID == "" {+ return nil, errors.New("only successful local system backups can be uploaded")+ }+ if run.RemoteSnapshotID != "" {+ return nil, errors.New("system backup has already been uploaded")+ }+ key, err := s.recoveryKeyInternal(ctx, request.RecoveryKey)+ if err != nil {+ return nil, err+ }+ if strings.TrimSpace(request.S3DestinationID) == "" {+ return nil, errors.New("select an S3 destination for the upload")+ }+ dockerClient, err := s.dockerService.GetClient(ctx)+ if err != nil {+ return nil, err+ }+ localRepository, err := s.localRepositoryInternal(ctx, dockerClient, true)+ if err != nil {+ return nil, err+ }+ remoteRepository, err := s.remoteRepositoryInternal(ctx, request.S3DestinationID)+ if err != nil {+ return nil, err+ }+ snapshot, err := s.engine.Replicate(ctx, dockerClient, localRepository, run.LocalSnapshotID, remoteRepository, key, "arcane-system-recovery")+ if err != nil {+ return nil, fmt.Errorf("failed to upload system recovery snapshot: %w", err)+ }+ run.RemoteSnapshotID, run.S3DestinationID = snapshot.ID, request.S3DestinationID+ run.Destination = backuptypes.SystemBackupDestinationLocalS3+ if err := s.db.WithContext(ctx).Save(run).Error; err != nil {+ return nil, fmt.Errorf("failed to save uploaded system backup: %w", err)+ }+ return run, nil+}++func (s *SystemBackupService) RestoreBackup(ctx context.Context, id, recoveryKey string, user common.User) error {+ run, err := s.backupInternal(ctx, id)+ if err != nil {+ return err+ }+ if run.Status != SystemBackupStatusSucceeded {+ return errors.New("only successful system backups can be restored")+ }+ key, err := s.recoveryKeyInternal(ctx, recoveryKey)+ if err != nil {+ return err+ }+ // A local safety snapshot is created before the detached helper is allowed to+ // stop Arcane and replace its data.+ safetyBackup, err := s.CreateBackup(ctx, user, SystemBackupTriggerSafety, backuptypes.CreateSystemBackupRequest{+ Destination: backuptypes.SystemBackupDestinationLocal,+ RecoveryKey: key,+ })+ if err != nil {+ return fmt.Errorf("failed to create pre-restore system backup: %w", err)+ }+ dockerClient, err := s.dockerService.GetClient(ctx)+ if err != nil {+ return err+ }+ containerID, err := cgroup.CurrentContainerID()+ if err != nil {+ return errors.New("arcane system restore requires Arcane to run in Docker")+ }+ inspectResult, err := libarcane.ContainerInspectWithCompatibility(ctx, dockerClient, containerID, client.ContainerInspectOptions{})+ if err != nil {+ return fmt.Errorf("inspect Arcane container: %w", err)+ }+ current := inspectResult.Container+ appDataMount := dockerutil.MountForDestination(current.Mounts, "/app/data", "/app/data")+ if appDataMount == nil {+ return errors.New("arcane system restore requires /app/data to be mounted")+ }+ var repository backup.Repository+ var snapshotID string+ switch {+ case run.LocalSnapshotID != "":+ repository, err = s.localRepositoryInternal(ctx, dockerClient, true)+ snapshotID = run.LocalSnapshotID+ case run.RemoteSnapshotID != "":+ repository, err = s.remoteRepositoryInternal(ctx, run.S3DestinationID)+ snapshotID = run.RemoteSnapshotID+ default:+ return errors.New("system backup has no Rustic snapshot")+ }+ if err != nil {+ return err+ }+ snapshotPath, err := s.snapshotPathInternal(ctx, dockerClient, repository, key, snapshotID)+ if err != nil {+ return err+ }+ request := recoverytypes.RestoreRequest{+ BackupID: run.ID, ContainerID: current.ID, ContainerImage: current.Config.Image, SnapshotID: snapshotID, SnapshotPath: snapshotPath, RecoveryKey: key,+ LocalSnapshotID: run.LocalSnapshotID, RemoteSnapshotID: run.RemoteSnapshotID, S3DestinationID: run.S3DestinationID,+ Size: run.Size, NetworkMode: rusticRestoreNetworkModeInternal(¤t),+ SafetyBackup: &recoverytypes.SafetyBackup{+ ID: safetyBackup.ID, LocalSnapshotID: safetyBackup.LocalSnapshotID,+ Size: safetyBackup.Size, CreatedAt: safetyBackup.CreatedAt,+ },+ RepositoryEnvironment: repository.Environment, RepositoryMounts: repository.Mounts, AppDataMount: *appDataMount,+ }+ requestData, err := json.Marshal(request)+ if err != nil {+ return fmt.Errorf("encode recovery request: %w", err)+ }+ requestFile := "/app/data/.arcane-recovery-request.json"+ if err := os.WriteFile(requestFile, requestData, 0o600); err != nil {+ return fmt.Errorf("write recovery request: %w", err)+ }+ cleanupRequest := true+ defer func() {+ if cleanupRequest {+ _ = os.Remove(requestFile)+ }+ }()+ containerEnv, runtimeMounts, networkMode, err := system.ResolveUpgraderRuntimeOptions(ctx, s.dockerService.DockerHost(), ¤t,+ func(ctx context.Context, containerPath string) (string, error) {+ return projects.GetHostPathForContainerPath(ctx, dockerClient, containerPath)+ },+ func() bool { _, err := cgroup.CurrentContainerID(); return err == nil },+ func(ctx context.Context, inspect *containertypes.InspectResponse, dockerHost string) string {+ return dockerutil.SelectDockerHostReachableNetworkMode(ctx, dockerClient, inspect, dockerHost)+ },+ )+ if err != nil {+ return fmt.Errorf("resolve recovery helper runtime: %w", err)+ }+ executablePath, err := os.Executable()+ if err != nil {+ return fmt.Errorf("resolve Arcane executable: %w", err)+ }+ helperExecutable, executableMount, err := recoveryHelperExecutableInternal(current.Mounts, executablePath)+ if err != nil {+ return fmt.Errorf("resolve recovery helper executable: %w", err)+ }+ mounts := append([]mount.Mount{}, runtimeMounts...)+ mounts = append(mounts, *appDataMount)+ if executableMount != nil {+ mounts = append(mounts, *executableMount)+ }+ hostConfig := &containertypes.HostConfig{AutoRemove: true, Mounts: mounts, NetworkMode: networkMode}+ if current.HostConfig != nil {+ hostConfig.SecurityOpt = append([]string{}, current.HostConfig.SecurityOpt...)+ hostConfig.Privileged = current.HostConfig.Privileged+ }+ created, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{+ Config: &containertypes.Config{+ Image: current.Config.Image, Cmd: []string{helperExecutable, "recovery-restore", "--request", requestFile}, User: "0:0",+ Env: containerEnv,+ Labels: map[string]string{"com.getarcaneapp.arcane.recovery": "true", "com.getarcaneapp.arcane": "true"},+ },+ HostConfig: hostConfig,+ Name: fmt.Sprintf("%s-recovery-%d", strings.TrimPrefix(current.Name, "/"), time.Now().Unix()),+ })+ if err != nil {+ return fmt.Errorf("create recovery helper container: %w", err)+ }+ if _, err := dockerClient.ContainerStart(ctx, created.ID, client.ContainerStartOptions{}); err != nil {+ _, _ = dockerClient.ContainerRemove(ctx, created.ID, client.ContainerRemoveOptions{Force: true})+ return fmt.Errorf("start recovery helper container: %w", err)+ }+ cleanupRequest = false+ return nil+}++const systemRecoveryConfigID = "system-recovery"++func (s *SystemBackupService) loadPoliciesInternal(ctx context.Context) ([]SystemBackupPolicy, error) {+ var policies []SystemBackupPolicy+ if err := s.db.WithContext(ctx).Order("created_at ASC").Find(&policies).Error; err != nil {+ return nil, fmt.Errorf("failed to load system backup policies: %w", err)+ }+ return policies, nil+}++func (s *SystemBackupService) loadPolicyInternal(ctx context.Context, policyID string) (*SystemBackupPolicy, error) {+ if strings.TrimSpace(policyID) == "" {+ return nil, nil+ }+ var policy SystemBackupPolicy+ err := s.db.WithContext(ctx).Where("id = ?", policyID).First(&policy).Error+ if errors.Is(err, gorm.ErrRecordNotFound) {+ return nil, nil+ }+ if err != nil {+ return nil, fmt.Errorf("failed to load system backup policy: %w", err)+ }+ return &policy, nil+}++// rusticRestoreNetworkModeInternal picks a named network Arcane is attached to+// so the detached restore's Rustic container can still resolve network-local+// S3 endpoints once Arcane is stopped. The default bridge carries no DNS, so+// it (and none) fall back to the default network.+func rusticRestoreNetworkModeInternal(current *containertypes.InspectResponse) string {+ if current == nil || current.NetworkSettings == nil {+ return ""+ }+ for _, name := range slices.Sorted(maps.Keys(current.NetworkSettings.Networks)) {+ if name != "bridge" && name != "none" {+ return name+ }+ }+ return ""+}++func (s *SystemBackupService) recoveryKeyConfiguredInternal(ctx context.Context) (bool, error) {+ var count int64+ if err := s.db.WithContext(ctx).Model(&SystemBackupRecoveryConfig{}).+ Where("id = ? AND encrypted_recovery_key <> ''", systemRecoveryConfigID).Count(&count).Error; err != nil {+ return false, fmt.Errorf("failed to load recovery key status: %w", err)+ }+ return count > 0, nil+}++func (s *SystemBackupService) GenerateRecoveryKey() (*backuptypes.SystemBackupRecoveryKey, error) {+ raw := make([]byte, 30)+ if _, err := rand.Read(raw); err != nil {+ return nil, fmt.Errorf("failed to generate recovery key: %w", err)+ }+ encoded := base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(raw)+ groups := make([]string, 0, len(encoded)/6)+ for i := 0; i < len(encoded); i += 6 {+ groups = append(groups, encoded[i:i+6])+ }+ return &backuptypes.SystemBackupRecoveryKey{RecoveryKey: strings.Join(groups, "-")}, nil+}++func (s *SystemBackupService) SetRecoveryKey(ctx context.Context, recoveryKey string) (*backuptypes.SystemBackupRecoveryKeyStatus, error) {+ if err := validateRecoveryKeyInternal(recoveryKey); err != nil {+ return nil, err+ }+ // The repositories are encrypted with the key that initialized them and a+ // different key cannot open them, so a rotation would silently break every+ // scheduled backup and keyless restore. Refuse it while backups exist.+ configured, err := s.recoveryKeyConfiguredInternal(ctx)+ if err != nil {+ return nil, err+ }+ if configured {+ if current, keyErr := s.recoveryKeyInternal(ctx, ""); keyErr == nil && current != recoveryKey {+ var runs int64+ if err := s.db.WithContext(ctx).Model(&SystemBackupRun{}).Count(&runs).Error; err != nil {+ return nil, err+ }+ if runs > 0 {+ return nil, errors.New("delete the existing system backups before replacing the recovery key; they can only be opened with the current key")+ }+ }+ }+ encrypted, err := crypto.Encrypt(recoveryKey)+ if err != nil {+ return nil, fmt.Errorf("failed to encrypt recovery key: %w", err)+ }+ config := SystemBackupRecoveryConfig{ID: systemRecoveryConfigID, EncryptedRecoveryKey: encrypted}+ if err := s.db.WithContext(ctx).Save(&config).Error; err != nil {+ return nil, fmt.Errorf("failed to save recovery key: %w", err)+ }+ return &backuptypes.SystemBackupRecoveryKeyStatus{Configured: true}, nil+}++func (s *SystemBackupService) GetPolicies(ctx context.Context) (*backuptypes.SystemBackupPolicyCollection, error) {+ policies, err := s.loadPoliciesInternal(ctx)+ if err != nil {+ return nil, err+ }+ configured, err := s.recoveryKeyConfiguredInternal(ctx)+ if err != nil {+ return nil, err+ }+ result := &backuptypes.SystemBackupPolicyCollection{+ Policies: make([]backuptypes.SystemBackupPolicy, 0, len(policies)), RecoveryKeyStored: configured,+ }+ destinations := make(map[string]string)+ if s.s3Destinations != nil {+ if available, listErr := s.s3Destinations.ListAllS3Destinations(ctx); listErr == nil {+ for _, destination := range available {+ destinations[destination.ID] = destination.Name+ }+ }+ }+ for i := range policies {+ var lastRun *SystemBackupRun+ var run SystemBackupRun+ if runErr := s.db.WithContext(ctx).Where("policy_id = ?", policies[i].ID).Order("created_at DESC").First(&run).Error; runErr == nil {+ run.S3DestinationName = destinations[run.S3DestinationID]+ lastRun = &run+ } else if !errors.Is(runErr, gorm.ErrRecordNotFound) {+ return nil, fmt.Errorf("failed to load latest system backup: %w", runErr)+ }+ dto := policies[i].ToDTO(lastRun)+ dto.S3DestinationName = destinations[policies[i].S3DestinationID]+ result.Policies = append(result.Policies, dto)+ }+ return result, nil+}++func (s *SystemBackupService) UpdatePolicies(ctx context.Context, updates []backuptypes.UpdateSystemBackupPolicy) (*backuptypes.SystemBackupPolicyCollection, error) {+ configured, err := s.recoveryKeyConfiguredInternal(ctx)+ if err != nil {+ return nil, err+ }+ existing, err := s.loadPoliciesInternal(ctx)+ if err != nil {+ return nil, err+ }+ reconcile := backup.PolicyReconciliation[SystemBackupPolicy]{+ Domain: "system",+ DB: s.db,+ Existing: existing,+ ID: func(policy *SystemBackupPolicy) string { return policy.ID },+ New: func() SystemBackupPolicy { return SystemBackupPolicy{} },+ Apply: func(policy *SystemBackupPolicy, update backuptypes.UpdateBackupPolicy) {+ policy.Enabled, policy.Schedule, policy.RetentionCount = update.Enabled, update.Schedule, update.RetentionCount+ policy.LocalEnabled, policy.S3Enabled, policy.S3DestinationID = update.LocalEnabled, update.S3Enabled, update.S3DestinationID+ },+ ValidateUpdate: func(update backuptypes.UpdateBackupPolicy) error {+ if update.Enabled && !configured {+ return errors.New("configure a recovery key before enabling scheduled system backups")+ }+ if update.Enabled && !s.SupportedDatabaseProvider() {+ return errors.New("scheduled system backups require the SQLite database provider")+ }+ return nil+ },+ S3Configured: func(ctx context.Context, destinationID string) error {+ if _, err := s.s3Destinations.Configuration(ctx, destinationID); err != nil {+ return errors.New("select a valid S3 destination for system backups")+ }+ return nil+ },+ Unregister: s.jobs.Unregister,+ Reschedule: s.rescheduleSystemBackupPolicyInternal,+ }+ if err := reconcile.Run(ctx, updates); err != nil {+ return nil, err+ }+ return s.GetPolicies(ctx)+}++// SetScheduler injects the dynamic scheduler and admission gate for per-policy+// system backup jobs. Agent mode leaves them unset.+func (s *SystemBackupService) SetScheduler(ctx context.Context, scheduler schedulertypes.DynamicScheduler, admissionGate *actors.Gate[actors.AdmissionKey]) error {+ return s.jobs.SetScheduler(ctx, scheduler, admissionGate)+}++func (s *SystemBackupService) runScheduledBackupInternal(ctx context.Context, policyID string) {+ policy, loadErr := s.loadPolicyInternal(ctx, policyID)+ if loadErr != nil || policy == nil || !policy.Enabled {+ return+ }+ var run *SystemBackupRun+ _, runErr := activitylib.RunHandlerActivity(ctx, s.activityService, activitylib.HandlerOptions{+ EnvironmentID: "0", Type: activitytypes.TypeResourceAction, ResourceType: "system_backup",+ ResourceID: policy.ID, ResourceName: "Arcane", User: &common.SystemUser,+ Step: "Creating scheduled system backup", Message: "Creating scheduled Arcane system backup",+ SuccessMessage: "Scheduled Arcane system backup created successfully",+ Metadata: database.JSON{"action": "scheduled_system_backup", "policyId": policy.ID, "schedule": policy.Schedule,+ "retentionCount": policy.RetentionCount, "localEnabled": policy.LocalEnabled,+ "s3Enabled": policy.S3Enabled, "s3DestinationId": policy.S3DestinationID},+ }, func(activityCtx context.Context) error {+ var backupErr error+ run, backupErr = s.CreateBackup(activityCtx, common.SystemUser, SystemBackupTriggerScheduled,+ backuptypes.CreateSystemBackupRequest{PolicyID: policy.ID})+ return backupErr+ })+ if errors.Is(runErr, ErrSystemBackupAlreadyRunning) {+ slog.InfoContext(ctx, "Scheduled Arcane system backup skipped; another backup is running", "policyId", policy.ID)+ return+ }+ if runErr != nil {+ slog.ErrorContext(ctx, "Scheduled Arcane system backup failed", "policyId", policy.ID, "error", runErr)+ return+ }+ if policy.RetentionCount > 0 {+ if retentionErr := s.applyRetentionInternal(ctx, policy.ID, policy.RetentionCount); retentionErr != nil {+ slog.ErrorContext(ctx, "System backup retention failed", "policyId", policy.ID, "error", retentionErr)+ }+ }+ slog.InfoContext(ctx, "Scheduled Arcane system backup completed", "backupId", run.ID, "policyId", policy.ID)+}++func (s *SystemBackupService) rescheduleSystemBackupPolicyInternal(ctx context.Context, policy *SystemBackupPolicy) {+ if policy == nil {+ return+ }+ if !policy.Enabled {+ s.jobs.Unregister(ctx, policy.ID)+ return+ }+ policyID := policy.ID+ s.jobs.Register(ctx, policyID,+ func(ctx context.Context) string {+ current, err := s.loadPolicyInternal(ctx, policyID)+ if err != nil || current == nil {+ return defaultSystemBackupSchedule+ }+ return current.Schedule+ },+ func(ctx context.Context) {+ s.runScheduledBackupInternal(ctx, policyID)+ },+ )+}++func (s *SystemBackupService) RegisterBackupJobOnStartup(ctx context.Context) {+ policies, err := s.loadPoliciesInternal(ctx)+ if err != nil {+ slog.ErrorContext(ctx, "Failed to load Arcane system backup policies", "error", err)+ return+ }+ for i := range policies {+ s.rescheduleSystemBackupPolicyInternal(ctx, &policies[i])+ }+ slog.InfoContext(ctx, "Registered scheduled Arcane system backup jobs", "count", len(policies))+}++func (s *SystemBackupService) applyRetentionInternal(ctx context.Context, policyID string, keep int) error {+ expired, err := backup.ExpiredRunIDs(ctx, s.db, "system_backup_runs", policyID, keep)+ if err != nil {+ return err+ }+ for _, id := range expired {+ if err := s.DeleteBackup(ctx, id, ""); err != nil {+ return err+ }+ }+ return nil+}diff --git a/backend/internal/systembackup/service_test.go b/backend/internal/systembackup/service_test.gonew file mode 100644index 000000000..124ea7963--- /dev/null+++ b/backend/internal/systembackup/service_test.go@@ -0,0 +1,157 @@+package systembackup++import (+ "context"+ "testing"+ "time"++ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"+ "github.com/getarcaneapp/arcane/backend/v2/internal/backup"+ "github.com/getarcaneapp/arcane/backend/v2/internal/config"+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/entityjobs"+ backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"+ sqlite "github.com/libtnb/sqlite"+ containertypes "github.com/moby/moby/api/types/container"+ mounttypes "github.com/moby/moby/api/types/mount"+ "github.com/stretchr/testify/require"+ "go.getarcane.app/sys/crypto"+ "go.uber.org/fx/fxtest"+ "gorm.io/gorm"+)++func newSystemBackupAdmissionGateForTestInternal(t testing.TB) *actors.Gate[actors.AdmissionKey] {+ t.Helper()+ fxLifecycle := fxtest.NewLifecycle(t)+ runtime, err := actors.NewRuntime(t.Context(), fxLifecycle)+ require.NoError(t, err)+ gate, err := actors.NewGate[actors.AdmissionKey](t.Context(), runtime, "system-backup-test-admission", t.Name())+ require.NoError(t, err)+ t.Cleanup(func() {+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ require.NoError(t, gate.Stop(stopCtx))+ require.NoError(t, fxLifecycle.Stop(stopCtx))+ })+ return gate+}++type systemBackupPolicySchedulerInternal struct {+ jobs map[string]schedulertypes.Job+}++func (s *systemBackupPolicySchedulerInternal) AddJob(_ context.Context, job schedulertypes.Job) error {+ s.jobs[job.Name()] = job+ return nil+}++func (s *systemBackupPolicySchedulerInternal) RemoveJob(_ context.Context, name string) {+ delete(s.jobs, name)+}++func (s *systemBackupPolicySchedulerInternal) HasJob(name string) bool {+ _, ok := s.jobs[name]+ return ok+}++func TestRecoveryHelperExecutableInternal(t *testing.T) {+ path, executableMount, err := recoveryHelperExecutableInternal(nil, "/app/arcane")+ require.NoError(t, err)+ require.Equal(t, "/app/arcane", path)+ require.Nil(t, executableMount)++ path, executableMount, err = recoveryHelperExecutableInternal([]containertypes.MountPoint{{+ Type: mounttypes.TypeBind, Source: "/workspace/backend", Destination: "/app/backend", RW: true,+ }}, "/app/backend/.bin/arcane")+ require.NoError(t, err)+ require.Equal(t, systemRecoveryHelperPath, path)+ require.Equal(t, mounttypes.TypeBind, executableMount.Type)+ require.Equal(t, "/workspace/backend/.bin/arcane", executableMount.Source)+ require.Equal(t, systemRecoveryHelperPath, executableMount.Target)+ require.True(t, executableMount.ReadOnly)+}++func TestSystemBackupPoliciesRegisterIndependentJobs(t *testing.T) {+ gormDB, err := gorm.Open(sqlite.Open("file:system-backup-schedules?mode=memory&cache=shared"), &gorm.Config{})+ require.NoError(t, err)+ require.NoError(t, gormDB.AutoMigrate(&SystemBackupPolicy{}, &SystemBackupRun{}, &SystemBackupRecoveryConfig{}))+ crypto.InitEncryption(&crypto.Config{EncryptionKey: "system-backup-policy-test-key-32bytes", Environment: "test"})+ service := &SystemBackupService{+ db: &database.DB{DB: gormDB},+ config: &config.Config{DatabaseURL: "file:system-backup-schedules-test.db"},+ jobs: entityjobs.New("system-backup:", backup.SystemAdmissionScope),+ }+ scheduler := &systemBackupPolicySchedulerInternal{jobs: make(map[string]schedulertypes.Job)}+ require.NoError(t, service.SetScheduler(context.Background(), scheduler, newSystemBackupAdmissionGateForTestInternal(t)))++ status, err := service.SetRecoveryKey(context.Background(), "QWERTY-ABCDEF-234567-GHIJKL-MNOPQR-STUVWX-YZ2345-ZXCVBN")+ require.NoError(t, err)+ require.True(t, status.Configured)+ storedKey, err := service.recoveryKeyInternal(context.Background(), "")+ require.NoError(t, err)+ require.Equal(t, "QWERTY-ABCDEF-234567-GHIJKL-MNOPQR-STUVWX-YZ2345-ZXCVBN", storedKey)++ collection, err := service.UpdatePolicies(context.Background(), []backuptypes.UpdateSystemBackupPolicy{+ {Enabled: true, Schedule: "0 0 2 * * *", RetentionCount: 5, LocalEnabled: true},+ {Enabled: true, Schedule: "0 0 14 * * *", RetentionCount: 30, LocalEnabled: true},+ })+ require.NoError(t, err)+ require.True(t, collection.RecoveryKeyStored)+ require.Len(t, collection.Policies, 2)+ require.Len(t, scheduler.jobs, 2)++ firstID := collection.Policies[0].ID+ collection, err = service.UpdatePolicies(context.Background(), []backuptypes.UpdateSystemBackupPolicy{{+ ID: firstID, Enabled: true, Schedule: "0 */30 * * * *", RetentionCount: 9, LocalEnabled: true,+ }})+ require.NoError(t, err)+ require.Len(t, collection.Policies, 1)+ require.Equal(t, firstID, collection.Policies[0].ID)+ require.Equal(t, 9, collection.Policies[0].RetentionCount)+ require.Len(t, scheduler.jobs, 1)+}++func TestSystemBackupPolicyRequiresConfiguredRecoveryKeyWhenEnabled(t *testing.T) {+ gormDB, err := gorm.Open(sqlite.Open("file:system-backup-key-required?mode=memory&cache=shared"), &gorm.Config{})+ require.NoError(t, err)+ require.NoError(t, gormDB.AutoMigrate(&SystemBackupPolicy{}, &SystemBackupRecoveryConfig{}))+ service := &SystemBackupService{+ db: &database.DB{DB: gormDB},+ config: &config.Config{DatabaseURL: "file:system-backup-key-required-test.db"},+ }++ _, err = service.UpdatePolicies(context.Background(), []backuptypes.UpdateSystemBackupPolicy{{+ Enabled: true, Schedule: "0 0 2 * * *", RetentionCount: 7, LocalEnabled: true,+ }})+ require.ErrorContains(t, err, "configure a recovery key")+}++func TestRecoveryEnvironmentInternalIncludesRuntimeSecrets(t *testing.T) {+ cfg := &config.Config{+ JWTSecret: "jwt-secret",+ EncryptionKey: "encryption-secret",+ AdminStaticAPIKey: "admin-key",+ OidcClientSecret: "oidc-secret",+ FilePerm: 0o640,+ }+ environment := (&SystemBackupService{config: cfg}).recoveryEnvironmentInternal()+ require.Equal(t, "jwt-secret", environment["JWT_SECRET"])+ require.Equal(t, "encryption-secret", environment["ENCRYPTION_KEY"])+ require.Equal(t, "admin-key", environment["ADMIN_STATIC_API_KEY"])+ require.Equal(t, "oidc-secret", environment["OIDC_CLIENT_SECRET"])+ require.Equal(t, "0640", environment["FILE_PERM"])+}++func TestSystemSnapshotPathFromFilesInternal(t *testing.T) {+ path, err := systemSnapshotPathFromFilesInternal([]string{"/arcane.db", "/.arcane-recovery.json"})+ require.NoError(t, err)+ require.Equal(t, "/", path)++ path, err = systemSnapshotPathFromFilesInternal([]string{"/app/data/arcane.db", "/app/data/.arcane-recovery.json"})+ require.NoError(t, err)+ require.Equal(t, "/app/data", path)++ _, err = systemSnapshotPathFromFilesInternal([]string{"/arcane.db"})+ require.ErrorContains(t, err, "does not contain an Arcane recovery manifest")+}diff --git a/backend/internal/volume/backup.go b/backend/internal/volume/backup.goindex 161e1b1c4..61479710b 100644--- a/backend/internal/volume/backup.go+++ b/backend/internal/volume/backup.go@@ -1,30 +1,34 @@package volumeimport (- "github.com/getarcaneapp/arcane/backend/v2/internal/event"-- "github.com/getarcaneapp/arcane/backend/v2/internal/common"-- "github.com/getarcaneapp/arcane/backend/v2/internal/database"-"archive/tar""compress/gzip""context"+ "crypto/sha256"+ "encoding/hex""fmt""io""log/slog""path"- "strconv""strings""time""uuid""emperror.dev/errors"+ "gorm.io/gorm"+ "github.com/getarcaneapp/arcane/backend/v2/internal/backup"+ "github.com/getarcaneapp/arcane/backend/v2/internal/common"+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"+ "github.com/getarcaneapp/arcane/backend/v2/internal/event"+ s3domain "github.com/getarcaneapp/arcane/backend/v2/internal/s3"docker "github.com/getarcaneapp/arcane/backend/v2/pkg/dockerutil""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"+ activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/volumehelper""github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"+ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"+ volumetypes "github.com/getarcaneapp/arcane/types/v2/volume""github.com/moby/moby/api/types/container""github.com/moby/moby/api/types/mount""github.com/moby/moby/client"@@ -45,6 +49,11 @@ const (const backupMountMissingWarning = "No volume is mounted at /backups in the Arcane container. Backups will only live inside Docker unless you mount a host path."+const (+ volumeBackupContainerRecoveryTimeout = 30 * time.Second+ volumeBackupContainerRecoveryInterval = 500 * time.Millisecond+)+type backupStorageMountInternal struct {mode backupStorageModemount mount.Mount@@ -140,67 +149,13 @@ func (s *VolumeService) BackupMountWarning(ctx context.Context) string {return backupMountWarningFromArcaneMountsInternal(inspect.Mounts)}-func (s *VolumeService) createBackupTempContainerWithMountInternal(ctx context.Context, dockerClient *client.Client, helperImage string, backupMount mount.Mount) (string, func(), error) {- var err error- if dockerClient == nil {- dockerClient, err = s.dockerService.GetClient(ctx)- if err != nil {- return "", nil, err- }- }-- if strings.TrimSpace(helperImage) == "" {- helperImage, err = getVolumeHelperImageInternal(ctx, s.dockerService, s.imageService, dockerClient)- if err != nil {- return "", nil, err- }- }-- config := &container.Config{- Image: helperImage,- Cmd: []string{"sleep", "infinity"},- NetworkDisabled: true,- Labels: volumehelper.Labels(),- }-- hostConfig := volumehelper.HostConfig(helperImage, nil, []mount.Mount{backupMount})-- resp, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{- Config: config,- HostConfig: hostConfig,- })- if err != nil {- return "", nil, errors.WrapIf(err, "failed to create backup temp container")- }-- if _, err := dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); err != nil {- _, _ = dockerClient.ContainerRemove(ctx, resp.ID, volumehelper.RemoveOptions())- return "", nil, errors.WrapIf(err, "failed to start backup temp container")- }-- cleanup := func() {- _, _ = dockerClient.ContainerRemove(ctx, resp.ID, volumehelper.RemoveOptions())- }-- return resp.ID, cleanup, nil-}--func (s *VolumeService) createBackupTempContainerInternal(ctx context.Context, dockerClient *client.Client, target string, readOnly bool) (string, func(), error) {- slog.DebugContext(ctx, "volume service: create backup temp container", "target", target, "read_only", readOnly)- var err error- if dockerClient == nil {- dockerClient, err = s.dockerService.GetClient(ctx)- if err != nil {- return "", nil, err- }- }-- backupStorage, err := s.resolveUsableBackupStorageMountInternal(ctx, dockerClient, target, readOnly)+// BackupStorageMount resolves the repository mount shared with system-backup operations.+func (s *VolumeService) BackupStorageMount(ctx context.Context, dockerClient *client.Client, target string, readOnly bool) (mount.Mount, error) {+ storage, err := s.resolveUsableBackupStorageMountInternal(ctx, dockerClient, target, readOnly)if err != nil {- return "", nil, err+ return mount.Mount{}, err}-- return s.createBackupTempContainerWithMountInternal(ctx, dockerClient, "", backupStorage.mount)+ return storage.mount, nil}func (s *VolumeService) ensureBackupVolumeInternal(ctx context.Context) error {@@ -222,110 +177,145 @@ func (s *VolumeService) ensureBackupVolumeInternal(ctx context.Context) error {return nil}-func (s *VolumeService) CreateBackup(ctx context.Context, volumeName string, user common.User) (*VolumeBackup, error) {- slog.DebugContext(ctx, "volume service: create backup", "volume", volumeName, "user", user.ID)- workspaceLock, _ := ctx.Value(volumeWorkspaceLockContextKeyInternal{}).(volumeWorkspaceLockContextInternal)- if workspaceLock.service != s || workspaceLock.volumeName != volumeName {- defer s.workspaceLocks.Lock(volumeName)()- }-- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return nil, err- }-- backupID := fmt.Sprintf("%s-%d-%s", volumeName, time.Now().UnixNano(), uuid.New().String()[:8])- filename, err := s.backupArchiveFilenameInternal(backupID)- if err != nil {- return nil, err- }-- helperImage, err := getVolumeHelperImageInternal(ctx, s.dockerService, s.imageService, dockerClient)- if err != nil {- return nil, err- }-- backupStorage, err := s.resolveUsableBackupStorageMountInternal(ctx, dockerClient, "/backups", false)- if err != nil {- return nil, err- }-- config := &container.Config{- Image: helperImage,- Cmd: []string{"sh", "-c", fmt.Sprintf("tar -czf /backups/%s -C /volume .", filename)},- Labels: volumehelper.Labels(),+func (s *VolumeService) stopRunningContainersForBackupInternal(ctx context.Context, dockerClient *client.Client, volumeName string, user common.User, refuseArcaneWriters bool) ([]container.Summary, error) {+ if s.containerService == nil {+ return nil, errors.New("container service is unavailable")}-- hostConfig := volumehelper.HostConfig(helperImage, []string{- volumeName + ":/volume:ro",- }, []mount.Mount{backupStorage.mount})-- resp, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{- Config: config,- HostConfig: hostConfig,- })+ containers, err := dockerClient.ContainerList(ctx, client.ContainerListOptions{})if err != nil {- return nil, errors.WrapIf(err, "failed to create backup container")- }-- if _, err := dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); err != nil {- _, _ = dockerClient.ContainerRemove(ctx, resp.ID, volumehelper.RemoveOptions())- return nil, errors.WrapIf(err, "failed to start backup container")+ return nil, fmt.Errorf("failed to list running containers before volume backup: %w", err)}- waitResult := dockerClient.ContainerWait(ctx, resp.ID, client.ContainerWaitOptions{Condition: container.WaitConditionNotRunning})- select {- case err := <-waitResult.Error:- if err != nil {- return nil, err+ eligible := make([]container.Summary, 0, len(containers.Items))+ arcaneOwned := make([]container.Summary, 0, 2)+ for _, candidate := range containers.Items {+ if strings.EqualFold(candidate.Labels["com.getarcaneapp.arcane"], "true") || strings.EqualFold(candidate.Labels["com.getarcaneapp.arcane.agent"], "true") {+ arcaneOwned = append(arcaneOwned, candidate)+ continue}- case status := <-waitResult.Result:- if status.StatusCode != 0 {- return nil, errors.Errorf("backup container exited with status %d", status.StatusCode)+ // Arcane's own helper containers mount the volume as well, but they are+ // auto-removed when stopped and recreated on demand. Stopping one here+ // would leave nothing for the restart pass to find, failing an+ // otherwise successful backup or restore.+ if strings.EqualFold(candidate.Labels[libarcane.InternalResourceLabel], "true") {+ continue}+ eligible = append(eligible, candidate)}-- sizeCheckMount := backupStorage.mount- sizeCheckMount.Target = "/volume"- sizeCheckMount.ReadOnly = true-- tempContainerID, cleanup, err := s.createBackupTempContainerWithMountInternal(ctx, dockerClient, "", sizeCheckMount)- if err != nil {- return nil, err- }- defer cleanup()-- sizeStr, _, err := s.execInContainerInternal(ctx, tempContainerID, []string{"stat", "-c", "%s", path.Join("/volume", filename)})- if err != nil {- return nil, err- }- size, err := strconv.ParseInt(strings.TrimSpace(sizeStr), 10, 64)- if err != nil {- return nil, err+ // Restores rewrite the volume in place; refusing beats corrupting the data+ // under a labeled container this pass deliberately never stops.+ if refuseArcaneWriters {+ if writers := docker.FilterContainersUsingVolume(arcaneOwned, volumeName); len(writers) > 0 {+ return nil, fmt.Errorf("volume %s is mounted by a running Arcane-managed container; restoring under it would corrupt its data", volumeName)+ }}-- backup := &VolumeBackup{- VolumeName: volumeName,- Size: size,- CreatedAt: time.Now(),+ containerIDs := docker.FilterContainersUsingVolume(eligible, volumeName)+ containersByID := make(map[string]container.Summary, len(eligible))+ for _, candidate := range eligible {+ containersByID[candidate.ID] = candidate+ }+ stopped := make([]container.Summary, 0, len(containerIDs))+ for _, containerID := range containerIDs {+ candidate := containersByID[containerID]+ if err := s.containerService.StopContainer(ctx, containerID, user); err != nil {+ stillStopped, restartErr := s.startContainersAfterBackupInternal(context.WithoutCancel(ctx), dockerClient, stopped, user)+ return stillStopped, errors.Combine(fmt.Errorf("failed to stop container %s before volume backup: %w", containerID, err), restartErr)+ }+ stopped = append(stopped, candidate)}- backup.ID = backupID+ return stopped, nil+}- if err := s.db.WithContext(ctx).Create(backup).Error; err != nil {- return nil, err- }+//nolint:gocognit // recovery retries must reconcile IDs, names, and Compose identities in one bounded loop+func (s *VolumeService) startContainersAfterBackupInternal(ctx context.Context, dockerClient *client.Client, stoppedContainers []container.Summary, user common.User) ([]container.Summary, error) {+ recoveryCtx, cancel := context.WithTimeout(ctx, volumeBackupContainerRecoveryTimeout)+ defer cancel()++ remaining := append([]container.Summary(nil), stoppedContainers...)+ lastErrors := make(map[string]error, len(stoppedContainers))+ for len(remaining) > 0 {+ currentContainers, listErr := dockerClient.ContainerList(recoveryCtx, client.ContainerListOptions{All: true})+ if listErr == nil {+ currentByID := make(map[string]container.Summary, len(currentContainers.Items))+ currentByName := make(map[string]container.Summary, len(currentContainers.Items))+ for _, current := range currentContainers.Items {+ currentByID[current.ID] = current+ if name := docker.ContainerNameFromNames(current.Names); name != "" {+ currentByName[name] = current+ }+ }++ nextRemaining := make([]container.Summary, 0, len(remaining))+ for _, stopped := range remaining {+ current, found := currentByID[stopped.ID]+ if !found {+ name := docker.ContainerNameFromNames(stopped.Names)+ current, found = currentByName[name]+ }+ if !found {+ projectName := docker.ComposeProjectLabel(stopped.Labels)+ serviceName := docker.ComposeServiceLabel(stopped.Labels)+ containerNumber := strings.TrimSpace(stopped.Labels["com.docker.compose.container-number"])+ if projectName != "" && serviceName != "" {+ for _, candidate := range currentContainers.Items {+ if docker.ComposeProjectLabel(candidate.Labels) != projectName || docker.ComposeServiceLabel(candidate.Labels) != serviceName {+ continue+ }+ if containerNumber != "" && strings.TrimSpace(candidate.Labels["com.docker.compose.container-number"]) != containerNumber {+ continue+ }+ current = candidate+ found = true+ break+ }+ }+ }+ if !found {+ nextRemaining = append(nextRemaining, stopped)+ continue+ }+ if current.State == container.StateRunning || current.State == container.StateRestarting {+ if current.ID != stopped.ID {+ slog.InfoContext(ctx, "volume service: container was replaced during backup and is already running", "previous_container", stopped.ID, "current_container", current.ID)+ }+ continue+ }+ if startErr := s.containerService.StartContainer(recoveryCtx, current.ID, user); startErr != nil {+ lastErrors[stopped.ID] = startErr+ nextRemaining = append(nextRemaining, stopped)+ continue+ }+ if current.ID != stopped.ID {+ slog.InfoContext(ctx, "volume service: restarted replacement container after backup", "previous_container", stopped.ID, "current_container", current.ID)+ }+ }+ remaining = nextRemaining+ } else {+ for _, stopped := range remaining {+ lastErrors[stopped.ID] = listErr+ }+ }- metadata := database.JSON{- "action": "backup_create",- "backup_id": backup.ID,- "filename": filename,- "size": size,- }- if logErr := s.eventService.LogVolumeEvent(ctx, event.EventTypeVolumeBackupCreate, volumeName, volumeName, user.ID, user.Username, "0", metadata); logErr != nil {- slog.WarnContext(ctx, "could not log volume backup create event", "volume", volumeName, "error", logErr.Error())+ if len(remaining) == 0 {+ return nil, nil+ }+ timer := time.NewTimer(volumeBackupContainerRecoveryInterval)+ select {+ case <-recoveryCtx.Done():+ timer.Stop()+ var restartErr error+ for _, stopped := range remaining {+ if lastErr := lastErrors[stopped.ID]; lastErr != nil {+ restartErr = errors.Combine(restartErr, fmt.Errorf("failed to restart container %s after volume backup: %w", stopped.ID, lastErr))+ } else {+ restartErr = errors.Combine(restartErr, fmt.Errorf("failed to restart container %s after volume backup: replacement did not appear within %s", stopped.ID, volumeBackupContainerRecoveryTimeout))+ }+ }+ return remaining, restartErr+ case <-timer.C:+ }}-- return backup, nil+ return nil, nil}func (s *VolumeService) ListBackupsPaginated(ctx context.Context, volumeName string, params pagination.QueryParams) ([]VolumeBackup, pagination.Response, error) {@@ -334,7 +324,8 @@ func (s *VolumeService) ListBackupsPaginated(ctx context.Context, volumeName strquery := s.db.WithContext(ctx).Model(&VolumeBackup{}).Where("volume_name = ?", volumeName)if params.Search != "" {- query = query.Where("id LIKE ?", "%"+params.Search+"%")+ pattern := "%" + params.Search + "%"+ query = query.Where("id LIKE ? OR status LIKE ? OR trigger LIKE ? OR destination LIKE ? OR COALESCE(local_snapshot_id, '') LIKE ? OR COALESCE(remote_snapshot_id, '') LIKE ? OR COALESCE(error, '') LIKE ?", pattern, pattern, pattern, pattern, pattern, pattern, pattern)}var totalItems int64@@ -352,6 +343,14 @@ func (s *VolumeService) ListBackupsPaginated(ctx context.Context, volumeName strsortCol = "id"case "size":sortCol = "size"+ case "status":+ sortCol = "status"+ case "trigger":+ sortCol = "trigger"+ case "destination":+ sortCol = "destination"+ case "remoteSnapshotId", "remote_snapshot_id":+ sortCol = "remote_snapshot_id"default:sortCol = "created_at"}@@ -371,9 +370,30 @@ func (s *VolumeService) ListBackupsPaginated(ctx context.Context, volumeName strif err := query.Find(&backups).Error; err != nil {return nil, pagination.Response{}, err}+ hasS3Destination := false+ for i := range backups {+ if strings.TrimSpace(backups[i].S3DestinationID) != "" {+ hasS3Destination = true+ break+ }+ }+ if s.s3Destinations != nil && hasS3Destination {+ // Destination names are decoration; never fail the listing over them.+ destinations, destinationErr := s.s3Destinations.ListAllS3Destinations(ctx)+ if destinationErr != nil {+ slog.WarnContext(ctx, "could not resolve volume backup S3 destination names", "volume", volumeName, "error", destinationErr)+ } else {+ destinationNames := make(map[string]string, len(destinations))+ for _, destination := range destinations {+ destinationNames[destination.ID] = destination.Name+ }+ for i := range backups {+ backups[i].S3DestinationName = destinationNames[backups[i].S3DestinationID]+ }+ }+ }- paginationResp := pagination.BuildResponse(totalItems, totalItems, params)- return backups, paginationResp, nil+ return backups, pagination.BuildResponse(totalItems, totalItems, params), nil}func (s *VolumeService) ListBackups(ctx context.Context, volumeName string) ([]VolumeBackup, error) {@@ -383,313 +403,856 @@ func (s *VolumeService) ListBackups(ctx context.Context, volumeName string) ([]Vreturn backups, err}-func (s *VolumeService) DeleteBackup(ctx context.Context, backupID string, user *common.User) error {- slog.DebugContext(ctx, "volume service: delete backup", "backup_id", backupID)- var backup VolumeBackup- if err := s.db.WithContext(ctx).Where("id = ?", backupID).First(&backup).Error; err != nil {- return err- }-- // Delete from DB first - if this fails, no changes are made.- // If file deletion fails afterward, we just have an orphan file (easier to clean up)- // rather than an orphan DB record pointing to a non-existent file.- volumeName := backup.VolumeName // Save before deletion- if err := s.db.WithContext(ctx).Delete(&backup).Error; err != nil {- return err- }-- // Now delete the actual file - best effort since DB record is already gone- containerID, cleanup, err := s.createBackupTempContainerInternal(ctx, nil, "/volume", false)- if err != nil {- slog.WarnContext(ctx, "failed to create container for backup file cleanup", "backup_id", backupID, "error", err.Error())- } else {- defer cleanup()- filename, filenameErr := s.backupArchiveFilenameInternal(backupID)- if filenameErr != nil {- slog.WarnContext(ctx, "failed to sanitize backup id for file cleanup", "backup_id", backupID, "error", filenameErr.Error())- } else if _, _, err = s.execInContainerInternal(ctx, containerID, []string{"rm", "-f", path.Join("/volume", filename)}); err != nil {- slog.WarnContext(ctx, "failed to delete backup file (orphan file may remain)", "backup_id", backupID, "error", err.Error())- }+func (s *VolumeService) sanitizeBackupPathInternal(input string) (string, error) {+ trimmed := strings.TrimSpace(input)+ if trimmed == "" {+ return "", errors.New("invalid path: empty")}-- actingUser := user- if actingUser == nil {- actingUser = &common.SystemUser+ cleaned := path.Clean(trimmed)+ if cleaned == "." || cleaned == "/" {+ return "", errors.Errorf("invalid path: %s", input)}- metadata := database.JSON{- "action": "backup_delete",- "backup_id": backupID,+ if path.IsAbs(cleaned) {+ cleaned = strings.TrimPrefix(cleaned, "/")}- if logErr := s.eventService.LogVolumeEvent(ctx, event.EventTypeVolumeBackupDelete, volumeName, volumeName, actingUser.ID, actingUser.Username, "0", metadata); logErr != nil {- slog.WarnContext(ctx, "could not log volume backup delete event", "volume", volumeName, "error", logErr.Error())+ if cleaned == "" || cleaned == "." || cleaned == "/" || strings.HasPrefix(cleaned, "..") || strings.Contains(cleaned, "/../") {+ return "", errors.Errorf("invalid path: %s", input)}-- return nil+ return cleaned, nil}-func (s *VolumeService) RestoreBackup(ctx context.Context, volumeName, backupID string, user common.User) error {- slog.DebugContext(ctx, "volume service: restore backup", "volume", volumeName, "backup_id", backupID, "user", user.ID)- var backup VolumeBackup- if err := s.db.WithContext(ctx).Where("id = ?", backupID).First(&backup).Error; err != nil {- return err+func (s *VolumeService) sanitizeBackupIDInternal(backupID string) (string, error) {+ cleaned, err := s.sanitizeBackupPathInternal(backupID)+ if err != nil {+ return "", errors.WrapIf(err, "invalid backup id")}-- // Validate backup belongs to volume- if backup.VolumeName != volumeName {- return errors.Errorf("backup does not belong to volume %s", volumeName)+ if strings.Contains(cleaned, "/") {+ return "", errors.New("invalid backup id: path separators not allowed")}- unlock := s.workspaceLocks.Lock(volumeName)- defer unlock()- ctx = context.WithValue(ctx, volumeWorkspaceLockContextKeyInternal{}, volumeWorkspaceLockContextInternal{- service: s,- volumeName: volumeName,- })+ return cleaned, nil+}- // Check if volume is in use by running containers- inUse, containerIDs, err := s.GetVolumeUsage(ctx, volumeName)+func (s *VolumeService) backupArchiveFilenameInternal(backupID string) (string, error) {+ sanitizedBackupID, err := s.sanitizeBackupIDInternal(backupID)if err != nil {- slog.WarnContext(ctx, "could not check volume usage", "volume", volumeName, "error", err.Error())- } else if inUse {- return errors.Errorf("volume is in use by %d container(s): restoring while containers are running may cause data corruption. Stop the containers first or use selective file restore", len(containerIDs))+ return "", err}+ return sanitizedBackupID + ".tar.gz", nil+}- preBackup, err := s.CreateBackup(ctx, volumeName, user)- if err != nil {- return errors.WrapIf(err, "failed to create pre-restore backup")- }+const (+ volumeRusticRepositoryPath = "/repository/volumes"+ localVolumeRepositoryID = "volumes:local"+)- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return err- }+func (s *VolumeService) rusticPasswordInternal() string {+ sum := sha256.Sum256([]byte("arcane-volume-backups:" + s.encryptionKey))+ return hex.EncodeToString(sum[:])+}- filename, err := s.backupArchiveFilenameInternal(backupID)+func (s *VolumeService) localRusticRepositoryInternal(ctx context.Context, dockerClient *client.Client, readOnly bool) (backup.Repository, error) {+ storage, err := s.resolveUsableBackupStorageMountInternal(ctx, dockerClient, "/repository", readOnly)if err != nil {- return err+ return backup.Repository{}, err}+ return backup.Repository{+ ID: localVolumeRepositoryID,+ Environment: []string{"RUSTIC_REPOSITORY=" + volumeRusticRepositoryPath},+ Mounts: []mount.Mount{storage.mount},+ }, nil+}- helperImage, err := getVolumeHelperImageInternal(ctx, s.dockerService, s.imageService, dockerClient)- if err != nil {- return err+func (s *VolumeService) remoteRusticRepositoryInternal(ctx context.Context, destinationID string) (backup.Repository, error) {+ if s.s3Destinations == nil {+ return backup.Repository{}, errors.New("S3 backup service is unavailable")}-- backupStorage, err := s.resolveUsableBackupStorageMountInternal(ctx, dockerClient, "/backups", true)+ configuration, err := s.s3Destinations.Configuration(ctx, destinationID)if err != nil {- return err+ return backup.Repository{}, errors.New("the selected S3 backup destination is not configured")+ }+ instanceID := strings.TrimSpace(s.settingsService.GetSettingsConfig().InstanceID.Value)+ if instanceID == "" {+ return backup.Repository{}, errors.New("arcane instance ID is unavailable")}+ return backup.Repository{+ ID: "volumes:s3:" + destinationID,+ Environment: configuration.RusticEnvironment("arcane-volume-backups", instanceID),+ }, nil+}- config := &container.Config{- Image: helperImage,- Cmd: []string{- "sh",- "-c",- fmt.Sprintf("set -e; tmp=$(mktemp -d /volume/.restore_tmp.XXXXXX); tar -tzf /backups/%s >/dev/null; tar -xzf /backups/%s -C \"$tmp\"; find /volume -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +; find \"$tmp\" -mindepth 1 -maxdepth 1 -exec mv -- {} /volume/ \\;; rmdir \"$tmp\"", filename, filename),- },- Labels: volumehelper.Labels(),+func (s *VolumeService) rusticRepositoryForBackupInternal(ctx context.Context, dockerClient *client.Client, entry *VolumeBackup) (backup.Repository, string, error) {+ if entry.LocalSnapshotID != "" {+ repository, err := s.localRusticRepositoryInternal(ctx, dockerClient, true)+ return repository, entry.LocalSnapshotID, err}+ if entry.RemoteSnapshotID != "" {+ repository, err := s.remoteRusticRepositoryInternal(ctx, entry.S3DestinationID)+ return repository, entry.RemoteSnapshotID, err+ }+ return backup.Repository{}, "", errors.New("volume backup has no Rustic snapshot")+}- hostConfig := volumehelper.HostConfig(helperImage, []string{- volumeName + ":/volume",- }, []mount.Mount{backupStorage.mount})+func volumeSourceMountInternal(volumeName string) mount.Mount {+ return mount.Mount{Type: mount.TypeVolume, Source: volumeName, Target: "/volume", ReadOnly: true}+}- resp, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{- Config: config,- HostConfig: hostConfig,- })- if err != nil {- return errors.WrapIf(err, "failed to create restore container")- }+var ErrVolumeBackupAlreadyRunning = errors.New("a backup is already running for this volume")- if _, err := dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); err != nil {- _, _ = dockerClient.ContainerRemove(ctx, resp.ID, volumehelper.RemoveOptions())- return errors.WrapIf(err, "failed to start restore container")- }+// backupPlanInternal is the resolved destination selection for one backup run.+type backupPlanInternal struct {+ policy *VolumeBackupPolicy+ localEnabled bool+ s3Enabled bool+ s3DestinationID string+ destination volumetypes.BackupDestination+}- waitResult := dockerClient.ContainerWait(ctx, resp.ID, client.ContainerWaitOptions{Condition: container.WaitConditionNotRunning})- var waitBody container.WaitResponse- select {- case err := <-waitResult.Error:+func (s *VolumeService) resolveBackupPlanInternal(ctx context.Context, volumeName string, trigger VolumeBackupTrigger, request volumetypes.CreateBackupRequest) (backupPlanInternal, error) {+ destination := request.Destination+ if destination != "" && destination != volumetypes.BackupDestinationLocal && destination != volumetypes.BackupDestinationS3 && destination != volumetypes.BackupDestinationLocalS3 {+ return backupPlanInternal{}, errors.New("invalid volume backup destination")+ }+ var policy *VolumeBackupPolicy+ if trigger != VolumeBackupTriggerSafety {+ var err error+ policy, err = s.loadVolumeBackupPolicyInternal(ctx, volumeName, request.PolicyID)if err != nil {- return err+ return backupPlanInternal{}, err}- case waitBody = <-waitResult.Result:+ if request.PolicyID != "" && policy == nil {+ return backupPlanInternal{}, errors.New("volume backup policy not found")+ }+ }+ localEnabled, s3Enabled, s3DestinationID := true, false, strings.TrimSpace(request.S3DestinationID)+ if policy != nil {+ localEnabled, s3Enabled, s3DestinationID = policy.LocalEnabled, policy.S3Enabled, policy.S3DestinationID+ }+ if trigger == VolumeBackupTriggerSafety {+ localEnabled, s3Enabled = true, false+ } else if destination != "" {+ localEnabled = destination != volumetypes.BackupDestinationS3+ s3Enabled = destination != volumetypes.BackupDestinationLocal+ }+ if !localEnabled && !s3Enabled {+ return backupPlanInternal{}, errors.New("select at least one volume backup destination")+ }+ if s3Enabled && strings.TrimSpace(s3DestinationID) == "" {+ return backupPlanInternal{}, errors.New("select an S3 destination for the volume backup")+ }+ if s3Enabled {+ if s.s3Destinations == nil {+ return backupPlanInternal{}, errors.New("S3 backup destinations are unavailable")+ }+ if _, err := s.s3Destinations.Configuration(ctx, s3DestinationID); err != nil {+ return backupPlanInternal{}, errors.New("select a valid S3 destination for the volume backup")+ }+ } else {+ s3DestinationID = ""+ }+ switch {+ case localEnabled && s3Enabled:+ destination = volumetypes.BackupDestinationLocalS3+ case s3Enabled:+ destination = volumetypes.BackupDestinationS3+ default:+ destination = volumetypes.BackupDestinationLocal}+ return backupPlanInternal{policy: policy, localEnabled: localEnabled, s3Enabled: s3Enabled, s3DestinationID: s3DestinationID, destination: destination}, nil+}- if waitBody.StatusCode != 0 {- return errors.Errorf("restore container exited with code %d (volume may be partially wiped)", waitBody.StatusCode)+//nolint:gocognit,gocyclo // backup orchestration keeps cleanup and persistence transitions in one transaction-like flow+func (s *VolumeService) CreateBackup(ctx context.Context, volumeName string, user common.User, trigger VolumeBackupTrigger, request volumetypes.CreateBackupRequest) (_ *VolumeBackup, err error) {+ workspaceLock, _ := ctx.Value(volumeWorkspaceLockContextKeyInternal{}).(volumeWorkspaceLockContextInternal)+ if workspaceLock.service != s || workspaceLock.volumeName != volumeName {+ defer s.workspaceLocks.Lock(volumeName)()+ }+ if trigger == "" {+ trigger = VolumeBackupTriggerManual+ }+ plan, err := s.resolveBackupPlanInternal(ctx, volumeName, trigger, request)+ if err != nil {+ return nil, err+ }+ lease, admitted, leaseErr := s.engine.TryAcquireRun(ctx, backup.VolumeAdmissionScope, volumeName)+ if leaseErr != nil {+ return nil, leaseErr}+ if !admitted {+ return nil, ErrVolumeBackupAlreadyRunning+ }+ defer lease.Release()+ entry := &VolumeBackup{+ VolumeName: volumeName, CreatedAt: time.Now(), Status: VolumeBackupStatusRunning,+ Trigger: trigger, Destination: plan.destination, Format: VolumeBackupFormatRustic,+ S3DestinationID: plan.s3DestinationID, PolicyID: request.PolicyID,+ }+ entry.ID = fmt.Sprintf("%s-%d-%s", volumeName, time.Now().UnixNano(), uuid.New().String()[:8])+ if err := s.db.WithContext(ctx).Create(entry).Error; err != nil {+ return nil, err+ }+ defer func() {+ if err != nil {+ entry.Status, entry.Error = VolumeBackupStatusFailed, err.Error()+ } else {+ entry.Status, entry.Error = VolumeBackupStatusSucceeded, ""+ }+ if saveErr := s.db.WithContext(context.WithoutCancel(ctx)).Save(entry).Error; saveErr != nil {+ err = errors.Combine(err, fmt.Errorf("failed to save volume backup result: %w", saveErr))+ }+ }()+ dockerClient, err := s.dockerService.GetClient(ctx)+ if err != nil {+ return entry, err+ }+ var stopped []container.Summary+ containersStopped := false+ if plan.policy != nil && plan.policy.StopContainers {+ stopped, err = s.stopRunningContainersForBackupInternal(ctx, dockerClient, volumeName, user, false)+ containersStopped = len(stopped) > 0+ defer func() {+ if containersStopped {+ _, restartErr := s.startContainersAfterBackupInternal(context.WithoutCancel(ctx), dockerClient, stopped, user)+ err = errors.Combine(err, restartErr)+ }+ }()+ if err != nil {+ return entry, err+ }+ }+ // The live volume is read once. A local+S3 backup replicates the local+ // snapshot to S3 instead of scanning the source a second time.+ var localSnapshot backup.Snapshot+ if plan.localEnabled {+ repository, repoErr := s.localRusticRepositoryInternal(ctx, dockerClient, false)+ if repoErr != nil {+ return entry, repoErr+ }+ localSnapshot, err = s.engine.CreateSnapshot(ctx, dockerClient, repository, s.rusticPasswordInternal(), volumeName, volumeSourceMountInternal(volumeName))+ if err != nil {+ return entry, fmt.Errorf("failed to create local Rustic snapshot: %w", err)+ }+ entry.LocalSnapshotID = localSnapshot.ID+ entry.Size = localSnapshot.Size+ }+ if plan.s3Enabled {+ remoteRepository, repoErr := s.remoteRusticRepositoryInternal(ctx, plan.s3DestinationID)+ if repoErr != nil {+ return entry, repoErr+ }+ var remoteSnapshot backup.Snapshot+ if plan.localEnabled {+ localRepository, localErr := s.localRusticRepositoryInternal(ctx, dockerClient, true)+ if localErr != nil {+ return entry, localErr+ }+ remoteSnapshot, err = s.engine.Replicate(ctx, dockerClient, localRepository, localSnapshot.ID, remoteRepository, s.rusticPasswordInternal(), volumeName)+ } else {+ remoteSnapshot, err = s.engine.CreateSnapshot(ctx, dockerClient, remoteRepository, s.rusticPasswordInternal(), volumeName, volumeSourceMountInternal(volumeName))+ }+ if err != nil {+ return entry, fmt.Errorf("failed to create S3 Rustic snapshot: %w", err)+ }+ entry.RemoteSnapshotID = remoteSnapshot.ID+ if entry.Size == 0 {+ entry.Size = remoteSnapshot.Size+ }+ }+ if containersStopped {+ stopped, err = s.startContainersAfterBackupInternal(context.WithoutCancel(ctx), dockerClient, stopped, user)+ containersStopped = len(stopped) > 0+ if err != nil {+ return entry, err+ }+ }+ // Retention failures must not fail the run: the backup itself succeeded,+ // and the deferred status-save would otherwise mark it failed and hide it+ // from ExpiredRunIDs forever.+ if trigger != VolumeBackupTriggerSafety && plan.policy != nil && plan.policy.RetentionCount > 0 {+ if retentionErr := s.applyVolumeBackupRetentionInternal(ctx, plan.policy.ID, plan.policy.RetentionCount); retentionErr != nil {+ slog.ErrorContext(ctx, "Volume backup retention failed", "volume", volumeName, "policy", plan.policy.ID, "error", retentionErr)+ }+ }+ metadata := database.JSON{"action": "backup_create", "backup_id": entry.ID, "size": entry.Size, "destination": entry.Destination}+ if logErr := s.eventService.LogVolumeEvent(ctx, event.EventTypeVolumeBackupCreate, volumeName, volumeName, user.ID, user.Username, "0", metadata); logErr != nil {+ slog.WarnContext(ctx, "could not log volume backup create event", "volume", volumeName, "error", logErr)+ }+ return entry, nil+}- metadata := database.JSON{- "action": "backup_restore",- "backup_id": backupID,- "pre_restore_backupId": preBackup.ID,+func (s *VolumeService) UploadBackup(ctx context.Context, backupID, s3DestinationID string) (*VolumeBackup, error) {+ var entry VolumeBackup+ if err := s.db.WithContext(ctx).Where("id = ?", backupID).First(&entry).Error; err != nil {+ return nil, err}- if logErr := s.eventService.LogVolumeEvent(ctx, event.EventTypeVolumeBackupRestore, volumeName, volumeName, user.ID, user.Username, "0", metadata); logErr != nil {- slog.WarnContext(ctx, "could not log volume backup restore event", "volume", volumeName, "error", logErr.Error())+ lease, admitted, err := s.engine.TryAcquireRun(ctx, backup.VolumeAdmissionScope, entry.VolumeName)+ if err != nil {+ return nil, err+ }+ if !admitted {+ return nil, ErrVolumeBackupAlreadyRunning+ }+ defer lease.Release()+ // Reload under the lease: a delete may have raced the first read.+ if err := s.db.WithContext(ctx).Where("id = ?", backupID).First(&entry).Error; err != nil {+ return nil, err+ }+ if entry.Status != VolumeBackupStatusSucceeded || entry.LocalSnapshotID == "" {+ return nil, errors.New("only successful local volume backups can be uploaded")+ }+ if entry.RemoteSnapshotID != "" {+ return nil, errors.New("volume backup has already been uploaded")+ }+ if strings.TrimSpace(s3DestinationID) == "" {+ return nil, errors.New("select an S3 destination for the upload")+ }+ if s.s3Destinations == nil {+ return nil, errors.New("S3 backup destinations are unavailable")+ }+ if _, err := s.s3Destinations.Configuration(ctx, s3DestinationID); err != nil {+ return nil, errors.New("select a valid S3 destination for the upload")+ }+ dockerClient, err := s.dockerService.GetClient(ctx)+ if err != nil {+ return nil, err+ }+ localRepository, err := s.localRusticRepositoryInternal(ctx, dockerClient, true)+ if err != nil {+ return nil, err+ }+ remoteRepository, err := s.remoteRusticRepositoryInternal(ctx, s3DestinationID)+ if err != nil {+ return nil, err+ }+ snapshot, err := s.engine.Replicate(ctx, dockerClient, localRepository, entry.LocalSnapshotID, remoteRepository, s.rusticPasswordInternal(), entry.VolumeName)+ if err != nil {+ return nil, fmt.Errorf("failed to upload Rustic snapshot to S3: %w", err)+ }+ entry.RemoteSnapshotID = snapshot.ID+ entry.S3DestinationID = s3DestinationID+ entry.Destination = volumetypes.BackupDestinationLocalS3+ if err := s.db.WithContext(ctx).Save(&entry).Error; err != nil {+ return nil, fmt.Errorf("failed to save uploaded volume backup: %w", err)+ }+ return &entry, nil+}++func (s *VolumeService) DeleteBackup(ctx context.Context, backupID string, user *common.User) error {+ var entry VolumeBackup+ if err := s.db.WithContext(ctx).Where("id = ?", backupID).First(&entry).Error; err != nil {+ return err+ }+ // Deletes contend with create/restore/upload on the volume's run lease so+ // a slow operation cannot resurrect or double-forget snapshots. Retention+ // deletes run under CreateBackup's lease and call the internal path.+ lease, admitted, err := s.engine.TryAcquireRun(ctx, backup.VolumeAdmissionScope, entry.VolumeName)+ if err != nil {+ return err+ }+ if !admitted {+ return ErrVolumeBackupAlreadyRunning}+ defer lease.Release()+ return s.deleteBackupInternal(ctx, backupID, user)+}+func (s *VolumeService) deleteBackupInternal(ctx context.Context, backupID string, user *common.User) error {+ var entry VolumeBackup+ if err := s.db.WithContext(ctx).Where("id = ?", backupID).First(&entry).Error; err != nil {+ return err+ }+ if entry.Format == VolumeBackupFormatArchive {+ return s.deleteArchiveBackupInternal(ctx, &entry, user)+ }+ dockerClient, err := s.dockerService.GetClient(ctx)+ if err != nil {+ return err+ }+ var deleteErr error+ if entry.LocalSnapshotID != "" {+ repository, repoErr := s.localRusticRepositoryInternal(ctx, dockerClient, false)+ if repoErr == nil {+ repoErr = s.engine.ForgetSnapshot(ctx, dockerClient, repository, s.rusticPasswordInternal(), entry.LocalSnapshotID)+ }+ if repoErr != nil {+ deleteErr = errors.Combine(deleteErr, fmt.Errorf("failed to delete local Rustic snapshot: %w", repoErr))+ } else {+ entry.LocalSnapshotID = ""+ }+ }+ if entry.RemoteSnapshotID != "" {+ repository, repoErr := s.remoteRusticRepositoryInternal(ctx, entry.S3DestinationID)+ if repoErr == nil {+ repoErr = s.engine.ForgetSnapshot(ctx, dockerClient, repository, s.rusticPasswordInternal(), entry.RemoteSnapshotID)+ }+ if repoErr != nil {+ deleteErr = errors.Combine(deleteErr, fmt.Errorf("failed to delete S3 Rustic snapshot: %w", repoErr))+ } else {+ entry.RemoteSnapshotID = ""+ entry.S3DestinationID = ""+ }+ }+ if deleteErr != nil {+ switch {+ case entry.LocalSnapshotID != "" && entry.RemoteSnapshotID != "":+ entry.Destination = volumetypes.BackupDestinationLocalS3+ case entry.LocalSnapshotID != "":+ entry.Destination = volumetypes.BackupDestinationLocal+ case entry.RemoteSnapshotID != "":+ entry.Destination = volumetypes.BackupDestinationS3+ }+ entry.Error = deleteErr.Error()+ if saveErr := s.db.WithContext(ctx).Save(&entry).Error; saveErr != nil {+ deleteErr = errors.Combine(deleteErr, saveErr)+ }+ return deleteErr+ }+ if err := s.db.WithContext(ctx).Delete(&entry).Error; err != nil {+ return fmt.Errorf("failed to delete volume backup record: %w", err)+ }+ s.logBackupDeleteEventInternal(ctx, entry.VolumeName, backupID, user)return nil}-func (s *VolumeService) sanitizeBackupPathInternal(input string) (string, error) {- trimmed := strings.TrimSpace(input)- if trimmed == "" {- return "", errors.New("invalid path: empty")+func (s *VolumeService) logBackupDeleteEventInternal(ctx context.Context, volumeName, backupID string, user *common.User) {+ actingUser := user+ if actingUser == nil {+ actingUser = &common.SystemUser}- cleaned := path.Clean(trimmed)- if cleaned == "." || cleaned == "/" {- return "", errors.Errorf("invalid path: %s", input)+ if logErr := s.eventService.LogVolumeEvent(ctx, event.EventTypeVolumeBackupDelete, volumeName, volumeName, actingUser.ID, actingUser.Username, "0", database.JSON{"action": "backup_delete", "backup_id": backupID}); logErr != nil {+ slog.WarnContext(ctx, "could not log volume backup delete event", "volume", volumeName, "error", logErr)}- if path.IsAbs(cleaned) {- cleaned = strings.TrimPrefix(cleaned, "/")+}++func (s *VolumeService) RestoreBackup(ctx context.Context, volumeName, backupID string, user common.User) (err error) {+ var entry VolumeBackup+ if err := s.db.WithContext(ctx).Where("id = ?", backupID).First(&entry).Error; err != nil {+ return err}- if cleaned == "" || cleaned == "." || cleaned == "/" || strings.HasPrefix(cleaned, "..") || strings.Contains(cleaned, "/../") {- return "", errors.Errorf("invalid path: %s", input)+ if entry.VolumeName != volumeName {+ return errors.New("backup does not belong to volume")}- return cleaned, nil+ unlock := s.workspaceLocks.Lock(volumeName)+ defer unlock()+ ctx = context.WithValue(ctx, volumeWorkspaceLockContextKeyInternal{}, volumeWorkspaceLockContextInternal{+ service: s,+ volumeName: volumeName,+ })+ dockerClient, err := s.dockerService.GetClient(ctx)+ if err != nil {+ return err+ }+ var repository backup.Repository+ var snapshotID string+ if entry.Format != VolumeBackupFormatArchive {+ repository, snapshotID, err = s.rusticRepositoryForBackupInternal(ctx, dockerClient, &entry)+ if err != nil {+ return err+ }+ }+ stopped, err := s.stopRunningContainersForBackupInternal(ctx, dockerClient, volumeName, user, true)+ containersStopped := len(stopped) > 0+ defer func() {+ if containersStopped {+ _, restartErr := s.startContainersAfterBackupInternal(context.WithoutCancel(ctx), dockerClient, stopped, user)+ err = errors.Combine(err, restartErr)+ }+ }()+ if err != nil {+ return err+ }+ preBackup, err := s.CreateBackup(ctx, volumeName, user, VolumeBackupTriggerSafety, volumetypes.CreateBackupRequest{Destination: volumetypes.BackupDestinationLocal})+ if err != nil {+ return fmt.Errorf("failed to create pre-restore backup: %w", err)+ }+ if entry.Format == VolumeBackupFormatArchive {+ if err := s.restoreArchiveBackupInternal(ctx, dockerClient, volumeName, backupID); err != nil {+ return err+ }+ } else if err := s.engine.RestoreSnapshot(ctx, dockerClient, repository, s.rusticPasswordInternal(), snapshotID, mount.Mount{Type: mount.TypeVolume, Source: volumeName, Target: "/volume"}, backup.RestoreOptions{DeleteExtra: true}); err != nil {+ return fmt.Errorf("failed to restore Rustic snapshot: %w", err)+ }+ if containersStopped {+ stopped, err = s.startContainersAfterBackupInternal(context.WithoutCancel(ctx), dockerClient, stopped, user)+ containersStopped = len(stopped) > 0+ if err != nil {+ return err+ }+ }+ metadata := database.JSON{"action": "backup_restore", "backup_id": backupID, "pre_restore_backupId": preBackup.ID}+ if logErr := s.eventService.LogVolumeEvent(ctx, event.EventTypeVolumeBackupRestore, volumeName, volumeName, user.ID, user.Username, "0", metadata); logErr != nil {+ slog.WarnContext(ctx, "could not log volume backup restore event", "volume", volumeName, "error", logErr)+ }+ return nil}-func (s *VolumeService) sanitizeBackupIDInternal(backupID string) (string, error) {- cleaned, err := s.sanitizeBackupPathInternal(backupID)+func (s *VolumeService) ListBackupFiles(ctx context.Context, backupID string) ([]string, error) {+ var entry VolumeBackup+ if err := s.db.WithContext(ctx).Where("id = ?", backupID).First(&entry).Error; err != nil {+ return nil, err+ }+ if entry.Format == VolumeBackupFormatArchive {+ return s.listArchiveBackupFilesInternal(ctx, backupID)+ }+ dockerClient, err := s.dockerService.GetClient(ctx)if err != nil {- return "", errors.WrapIf(err, "invalid backup id")+ return nil, err}- if strings.Contains(cleaned, "/") {- return "", errors.New("invalid backup id: path separators not allowed")+ repository, snapshotID, err := s.rusticRepositoryForBackupInternal(ctx, dockerClient, &entry)+ if err != nil {+ return nil, err}- return cleaned, nil+ return s.engine.ListSnapshotFiles(ctx, dockerClient, repository, s.rusticPasswordInternal(), snapshotID)}-func (s *VolumeService) backupArchiveFilenameInternal(backupID string) (string, error) {- sanitizedBackupID, err := s.sanitizeBackupIDInternal(backupID)+func (s *VolumeService) BackupHasPath(ctx context.Context, backupID, filePath string) (bool, error) {+ cleaned, err := s.sanitizeBackupPathInternal(filePath)if err != nil {- return "", err+ return false, err}+ files, err := s.ListBackupFiles(ctx, backupID)+ if err != nil {+ return false, err+ }+ for _, candidate := range files {+ if strings.TrimPrefix(candidate, "/") == cleaned {+ return true, nil+ }+ }+ return false, nil+}- return sanitizedBackupID + ".tar.gz", nil+func (s *VolumeService) RestoreBackupFiles(ctx context.Context, volumeName, backupID string, paths []string, user common.User) (err error) {+ if len(paths) == 0 {+ return errors.New("no paths provided")+ }+ var entry VolumeBackup+ if err := s.db.WithContext(ctx).Where("id = ?", backupID).First(&entry).Error; err != nil {+ return err+ }+ if entry.VolumeName != volumeName {+ return errors.New("backup does not belong to volume")+ }+ unlock := s.workspaceLocks.Lock(volumeName)+ defer unlock()+ ctx = context.WithValue(ctx, volumeWorkspaceLockContextKeyInternal{}, volumeWorkspaceLockContextInternal{+ service: s,+ volumeName: volumeName,+ })+ cleanedPaths := make([]string, 0, len(paths))+ for _, requestedPath := range paths {+ cleaned, cleanErr := s.sanitizeBackupPathInternal(requestedPath)+ if cleanErr != nil {+ return cleanErr+ }+ cleanedPaths = append(cleanedPaths, cleaned)+ }+ dockerClient, err := s.dockerService.GetClient(ctx)+ if err != nil {+ return err+ }+ var repository backup.Repository+ var snapshotID string+ if entry.Format != VolumeBackupFormatArchive {+ repository, snapshotID, err = s.rusticRepositoryForBackupInternal(ctx, dockerClient, &entry)+ if err != nil {+ return err+ }+ }+ stopped, err := s.stopRunningContainersForBackupInternal(ctx, dockerClient, volumeName, user, true)+ containersStopped := len(stopped) > 0+ defer func() {+ if containersStopped {+ _, restartErr := s.startContainersAfterBackupInternal(context.WithoutCancel(ctx), dockerClient, stopped, user)+ err = errors.Combine(err, restartErr)+ }+ }()+ if err != nil {+ return err+ }+ preBackup, err := s.CreateBackup(ctx, volumeName, user, VolumeBackupTriggerSafety, volumetypes.CreateBackupRequest{Destination: volumetypes.BackupDestinationLocal})+ if err != nil {+ return fmt.Errorf("failed to create pre-restore backup: %w", err)+ }+ if entry.Format == VolumeBackupFormatArchive {+ if err := s.restoreArchiveBackupFilesInternal(ctx, dockerClient, volumeName, backupID, cleanedPaths); err != nil {+ return err+ }+ } else {+ for _, cleaned := range cleanedPaths {+ if err := s.engine.RestoreSnapshot(ctx, dockerClient, repository, s.rusticPasswordInternal(), snapshotID, mount.Mount{Type: mount.TypeVolume, Source: volumeName, Target: "/volume"}, backup.RestoreOptions{SourcePath: cleaned, DestinationPath: path.Join("/volume", cleaned)}); err != nil {+ return fmt.Errorf("failed to restore %s from Rustic snapshot: %w", cleaned, err)+ }+ }+ }+ if containersStopped {+ stopped, err = s.startContainersAfterBackupInternal(context.WithoutCancel(ctx), dockerClient, stopped, user)+ containersStopped = len(stopped) > 0+ if err != nil {+ return err+ }+ }+ metadata := database.JSON{"action": "backup_restore_files", "backup_id": backupID, "pre_restore_backupId": preBackup.ID, "paths_count": len(cleanedPaths)}+ if logErr := s.eventService.LogVolumeEvent(ctx, event.EventTypeVolumeBackupRestoreFiles, volumeName, volumeName, user.ID, user.Username, "0", metadata); logErr != nil {+ slog.WarnContext(ctx, "could not log volume backup restore files event", "volume", volumeName, "error", logErr)+ }+ return nil}-func (s *VolumeService) restoreBackupFilesInContainerInternal(ctx context.Context, containerID, filename string, cleanedPaths []string) (string, error) {- args := make([]string, 0, len(cleanedPaths)+5)- args = append(args, "sh", "-c", restoreBackupFilesScriptInternal, "sh", path.Join("/backups", filename))- for _, cleaned := range cleanedPaths {- args = append(args, "./"+cleaned)+// DownloadBackup streams a backup as a tar.gz archive. Legacy archive rows+// stream their stored file; local Rustic rows are restored into a scratch+// volume and packaged on the fly.+func (s *VolumeService) DownloadBackup(ctx context.Context, backupID string, user *common.User) (io.ReadCloser, int64, error) {+ var entry VolumeBackup+ if err := s.db.WithContext(ctx).Where("id = ?", backupID).First(&entry).Error; err != nil {+ return nil, 0, err}- _, stderr, err := s.execInContainerInternal(ctx, containerID, args)- return stderr, errors.WrapIf(err, "failed to restore files")+ var reader io.ReadCloser+ var size int64+ var err error+ if entry.Format == VolumeBackupFormatArchive {+ reader, size, err = s.downloadArchiveBackupInternal(ctx, backupID)+ } else {+ reader, size, err = s.downloadRusticBackupInternal(ctx, &entry)+ }+ if err != nil {+ return nil, 0, err+ }+ actingUser := user+ if actingUser == nil {+ actingUser = &common.SystemUser+ }+ metadata := database.JSON{"action": "backup_download", "backup_id": backupID, "size": size}+ if logErr := s.eventService.LogVolumeEvent(ctx, event.EventTypeVolumeBackupDownload, entry.VolumeName, entry.VolumeName, actingUser.ID, actingUser.Username, "0", metadata); logErr != nil {+ slog.WarnContext(ctx, "could not log volume backup download event", "volume", entry.VolumeName, "error", logErr)+ }+ return reader, size, nil}-const restoreBackupFilesScriptInternal = `set -e-archive="$1"-shift-archive_mode=$(stat -c '%A' -- "$archive" 2>/dev/null) || { echo ARCANE_NOT_FOUND >&2; exit 44; }-case "$archive_mode" in -*) ;; *) echo ARCANE_NOT_FOUND >&2; exit 44 ;; esac-for member do- if ! tar -tzf "$archive" -- "$member" >/dev/null 2>&1; then echo ARCANE_NOT_FOUND >&2; exit 44; fi-done-tar -xzf "$archive" -C /volume -- "$@"`+func (s *VolumeService) downloadRusticBackupInternal(ctx context.Context, entry *VolumeBackup) (io.ReadCloser, int64, error) {+ if entry.LocalSnapshotID == "" {+ return nil, 0, errors.New("only local volume backups can be downloaded")+ }+ dockerClient, err := s.dockerService.GetClient(ctx)+ if err != nil {+ return nil, 0, err+ }+ repository, err := s.localRusticRepositoryInternal(ctx, dockerClient, true)+ if err != nil {+ return nil, 0, err+ }+ scratchVolume := "arcane-rustic-download-" + uuid.New().String()+ if _, err := dockerClient.VolumeCreate(ctx, client.VolumeCreateOptions{Name: scratchVolume, Labels: volumehelper.Labels()}); err != nil {+ return nil, 0, fmt.Errorf("failed to create download scratch volume: %w", err)+ }+ removeScratch := func() {+ _, _ = dockerClient.VolumeRemove(context.WithoutCancel(ctx), scratchVolume, client.VolumeRemoveOptions{Force: true})+ }+ if err := s.engine.RestoreSnapshot(ctx, dockerClient, repository, s.rusticPasswordInternal(), entry.LocalSnapshotID, mount.Mount{Type: mount.TypeVolume, Source: scratchVolume, Target: "/volume"}, backup.RestoreOptions{DeleteExtra: true}); err != nil {+ removeScratch()+ return nil, 0, fmt.Errorf("failed to restore Rustic snapshot for download: %w", err)+ }+ helperImage, err := getVolumeHelperImageInternal(ctx, s.dockerService, s.imageService, dockerClient)+ if err != nil {+ removeScratch()+ return nil, 0, err+ }+ archiveMount := mount.Mount{Type: mount.TypeVolume, Source: scratchVolume, Target: "/volume", ReadOnly: false}+ containerID, removeContainer, err := s.createBackupTempContainerWithMountInternal(ctx, dockerClient, helperImage, archiveMount)+ if err != nil {+ removeScratch()+ return nil, 0, err+ }+ cleanup := func() {+ removeContainer()+ removeScratch()+ }+ archivePath := "/tmp/" + entry.ID + ".tar.gz"+ if _, _, err := s.execInContainerInternal(ctx, containerID, []string{"tar", "-czf", archivePath, "-C", "/volume", "."}); err != nil {+ cleanup()+ return nil, 0, fmt.Errorf("failed to package Rustic snapshot for download: %w", err)+ }+ return volumehelper.DownloadFileFromContainer(ctx, dockerClient, containerID, archivePath, cleanup)+}-func (s *VolumeService) BackupHasPath(ctx context.Context, backupID string, filePath string) (bool, error) {- slog.DebugContext(ctx, "volume service: backup has path", "backup_id", backupID, "path", filePath)- cleaned, err := s.sanitizeBackupPathInternal(filePath)+func (s *VolumeService) createBackupTempContainerWithMountInternal(ctx context.Context, dockerClient *client.Client, helperImage string, backupMount mount.Mount) (string, func(), error) {+ var err error+ if dockerClient == nil {+ dockerClient, err = s.dockerService.GetClient(ctx)+ if err != nil {+ return "", nil, err+ }+ }++ if strings.TrimSpace(helperImage) == "" {+ helperImage, err = getVolumeHelperImageInternal(ctx, s.dockerService, s.imageService, dockerClient)+ if err != nil {+ return "", nil, err+ }+ }++ config := &container.Config{+ Image: helperImage,+ Cmd: []string{"sleep", "infinity"},+ NetworkDisabled: true,+ Labels: volumehelper.Labels(),+ }++ hostConfig := volumehelper.HostConfig(helperImage, nil, []mount.Mount{backupMount})++ resp, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{+ Config: config,+ HostConfig: hostConfig,+ })+ if err != nil {+ return "", nil, errors.WrapIf(err, "failed to create backup temp container")+ }++ if _, err := dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); err != nil {+ _, _ = dockerClient.ContainerRemove(ctx, resp.ID, volumehelper.RemoveOptions())+ return "", nil, errors.WrapIf(err, "failed to start backup temp container")+ }++ cleanup := func() {+ _, _ = dockerClient.ContainerRemove(context.WithoutCancel(ctx), resp.ID, volumehelper.RemoveOptions())+ }++ return resp.ID, cleanup, nil+}++func (s *VolumeService) createBackupTempContainerInternal(ctx context.Context, dockerClient *client.Client, target string, readOnly bool) (string, func(), error) {+ slog.DebugContext(ctx, "volume service: create backup temp container", "target", target, "read_only", readOnly)+ var err error+ if dockerClient == nil {+ dockerClient, err = s.dockerService.GetClient(ctx)+ if err != nil {+ return "", nil, err+ }+ }++ backupStorage, err := s.resolveUsableBackupStorageMountInternal(ctx, dockerClient, target, readOnly)+ if err != nil {+ return "", nil, err+ }++ return s.createBackupTempContainerWithMountInternal(ctx, dockerClient, "", backupStorage.mount)+}++func (s *VolumeService) downloadArchiveBackupInternal(ctx context.Context, backupID string) (io.ReadCloser, int64, error) {+ filename, err := s.backupArchiveFilenameInternal(backupID)+ if err != nil {+ return nil, 0, err+ }+ dockerClient, err := s.dockerService.GetClient(ctx)if err != nil {- return false, err+ return nil, 0, err}- filename, err := s.backupArchiveFilenameInternal(backupID)+ containerID, cleanup, err := s.createBackupTempContainerInternal(ctx, dockerClient, "/volume", true)if err != nil {- return false, err+ return nil, 0, err}+ return volumehelper.DownloadFileFromContainer(ctx, dockerClient, containerID, path.Join("/volume", filename), cleanup)+}- var backup VolumeBackup- if err := s.db.WithContext(ctx).Where("id = ?", backupID).First(&backup).Error; err != nil {- return false, err+func (s *VolumeService) deleteArchiveBackupInternal(ctx context.Context, entry *VolumeBackup, user *common.User) error {+ // Delete from DB first - if this fails, no changes are made. If file+ // deletion fails afterward we just have an orphan file, easier to clean up+ // than an orphan DB record pointing at a missing file.+ volumeName := entry.VolumeName+ backupID := entry.ID+ if err := s.db.WithContext(ctx).Delete(entry).Error; err != nil {+ return err}-- dockerClient, err := s.dockerService.GetClient(ctx)+ containerID, cleanup, err := s.createBackupTempContainerInternal(ctx, nil, "/volume", false)if err != nil {- return false, err+ slog.WarnContext(ctx, "failed to create container for backup file cleanup", "backup_id", backupID, "error", err.Error())+ } else {+ defer cleanup()+ filename, filenameErr := s.backupArchiveFilenameInternal(backupID)+ if filenameErr != nil {+ slog.WarnContext(ctx, "failed to sanitize backup id for file cleanup", "backup_id", backupID, "error", filenameErr.Error())+ } else if _, _, err = s.execInContainerInternal(ctx, containerID, []string{"rm", "-f", path.Join("/volume", filename)}); err != nil {+ slog.WarnContext(ctx, "failed to delete backup file (orphan file may remain)", "backup_id", backupID, "error", err.Error())+ }}+ s.logBackupDeleteEventInternal(ctx, volumeName, backupID, user)+ return nil+}- helperImage, err := getVolumeHelperImageInternal(ctx, s.dockerService, s.imageService, dockerClient)+func (s *VolumeService) restoreArchiveBackupInternal(ctx context.Context, dockerClient *client.Client, volumeName, backupID string) error {+ filename, err := s.backupArchiveFilenameInternal(backupID)if err != nil {- return false, err+ return err}-- backupStorage, err := s.resolveUsableBackupStorageMountInternal(ctx, dockerClient, "/volume", true)+ helperImage, err := getVolumeHelperImageInternal(ctx, s.dockerService, s.imageService, dockerClient)if err != nil {- return false, err+ return err}-- containerID, cleanup, err := s.createBackupTempContainerWithMountInternal(ctx, dockerClient, helperImage, backupStorage.mount)+ backupStorage, err := s.resolveUsableBackupStorageMountInternal(ctx, dockerClient, "/backups", true)if err != nil {- return false, err+ return err}- defer cleanup()-- archivePath := path.Join("/volume", filename)- cmd := []string{"tar", "-tzf", archivePath}- stdout, stderr, err := s.execInContainerInternal(ctx, containerID, cmd)+ config := &container.Config{+ Image: helperImage,+ Cmd: []string{+ "sh",+ "-c",+ fmt.Sprintf("set -e; tmp=$(mktemp -d /volume/.restore_tmp.XXXXXX); tar -tzf /backups/%s >/dev/null; tar -xzf /backups/%s -C \"$tmp\"; find /volume -mindepth 1 -maxdepth 1 -not -path \"$tmp\" -exec rm -rf -- {} +; find \"$tmp\" -mindepth 1 -maxdepth 1 -exec mv -- {} /volume/ \\;; rmdir \"$tmp\"", filename, filename),+ },+ Labels: volumehelper.Labels(),+ }+ hostConfig := volumehelper.HostConfig(helperImage, []string{+ volumeName + ":/volume",+ }, []mount.Mount{backupStorage.mount})+ resp, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{+ Config: config,+ HostConfig: hostConfig,+ })if err != nil {- return false, err+ return errors.WrapIf(err, "failed to create restore container")}- if strings.TrimSpace(stderr) != "" {- return false, errors.Errorf("failed to list backup contents: %s", strings.TrimSpace(stderr))+ defer func() {+ _, _ = dockerClient.ContainerRemove(context.WithoutCancel(ctx), resp.ID, volumehelper.RemoveOptions())+ }()+ if _, err := dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); err != nil {+ return errors.WrapIf(err, "failed to start restore container")}-- for line := range strings.SplitSeq(stdout, "\n") {- entry := strings.TrimSpace(line)- if entry == "" {- continue- }- entry = strings.TrimPrefix(entry, "./")- if entry == cleaned || strings.TrimSuffix(entry, "/") == cleaned {- return true, nil+ waitResult := dockerClient.ContainerWait(ctx, resp.ID, client.ContainerWaitOptions{Condition: container.WaitConditionNotRunning})+ var waitBody container.WaitResponse+ select {+ case err := <-waitResult.Error:+ if err != nil {+ return err}+ case waitBody = <-waitResult.Result:}-- return false, nil+ if waitBody.StatusCode != 0 {+ return errors.Errorf("restore container exited with code %d (volume may be partially wiped)", waitBody.StatusCode)+ }+ return nil}-func (s *VolumeService) ListBackupFiles(ctx context.Context, backupID string) ([]string, error) {- slog.DebugContext(ctx, "volume service: list backup files", "backup_id", backupID)+func (s *VolumeService) listArchiveBackupFilesInternal(ctx context.Context, backupID string) ([]string, error) {filename, err := s.backupArchiveFilenameInternal(backupID)if err != nil {return nil, err}-- var backup VolumeBackup- if err := s.db.WithContext(ctx).Where("id = ?", backupID).First(&backup).Error; err != nil {- return nil, err- }-dockerClient, err := s.dockerService.GetClient(ctx)if err != nil {return nil, err}-- helperImage, err := getVolumeHelperImageInternal(ctx, s.dockerService, s.imageService, dockerClient)- if err != nil {- return nil, err- }-backupStorage, err := s.resolveUsableBackupStorageMountInternal(ctx, dockerClient, "/volume", true)if err != nil {return nil, err}-- containerID, cleanup, err := s.createBackupTempContainerWithMountInternal(ctx, dockerClient, helperImage, backupStorage.mount)+ containerID, cleanup, err := s.createBackupTempContainerWithMountInternal(ctx, dockerClient, "", backupStorage.mount)if err != nil {return nil, err}defer cleanup()-- archivePath := path.Join("/volume", filename)- cmd := []string{"tar", "-tzf", archivePath}- stdout, _, err := s.execInContainerInternal(ctx, containerID, cmd)+ stdout, _, err := s.execInContainerInternal(ctx, containerID, []string{"tar", "-tzf", path.Join("/volume", filename)})if err != nil {return nil, err}-lines := strings.Split(strings.TrimSpace(stdout), "\n")files := make([]string, 0, len(lines))seen := make(map[string]struct{})@@ -708,79 +1271,51 @@ func (s *VolumeService) ListBackupFiles(ctx context.Context, backupID string) ([seen[clean] = struct{}{}files = append(files, clean)}-return files, nil}-func (s *VolumeService) RestoreBackupFiles(ctx context.Context, volumeName, backupID string, paths []string, user common.User) error {- slog.DebugContext(ctx, "volume service: restore backup files", "volume", volumeName, "backup_id", backupID, "paths_count", len(paths), "user", user.ID)- if len(paths) == 0 {- return errors.New("no paths provided")- }- filename, err := s.backupArchiveFilenameInternal(backupID)- if err != nil {- return err- }-- var backup VolumeBackup- if err := s.db.WithContext(ctx).Where("id = ?", backupID).First(&backup).Error; err != nil {- return err- }- if backup.VolumeName != volumeName {- return errors.New("backup does not belong to volume")- }- unlock := s.workspaceLocks.Lock(volumeName)- defer unlock()- ctx = context.WithValue(ctx, volumeWorkspaceLockContextKeyInternal{}, volumeWorkspaceLockContextInternal{- service: s,- volumeName: volumeName,- })-- // Create pre-restore backup for safety (consistent with RestoreBackup behavior)- preBackup, err := s.CreateBackup(ctx, volumeName, user)- if err != nil {- return errors.WrapIf(err, "failed to create pre-restore backup")+func (s *VolumeService) restoreBackupFilesInContainerInternal(ctx context.Context, containerID, filename string, cleanedPaths []string) (string, error) {+ args := make([]string, 0, len(cleanedPaths)+5)+ args = append(args, "sh", "-c", restoreBackupFilesScriptInternal, "sh", path.Join("/backups", filename))+ for _, cleaned := range cleanedPaths {+ args = append(args, "./"+cleaned)}- slog.DebugContext(ctx, "created pre-restore backup", "volume", volumeName, "pre_backup_id", preBackup.ID)+ _, stderr, err := s.execInContainerInternal(ctx, containerID, args)+ return stderr, errors.WrapIf(err, "failed to restore files")+}- cleanedPaths := make([]string, 0, len(paths))- for _, p := range paths {- cleaned, err := s.sanitizeBackupPathInternal(p)- if err != nil {- return err- }- cleanedPaths = append(cleanedPaths, cleaned)- }- if len(cleanedPaths) == 0 {- return errors.New("no valid paths provided")- }+const restoreBackupFilesScriptInternal = `set -e+archive="$1"+shift+archive_mode=$(stat -c '%A' -- "$archive" 2>/dev/null) || { echo ARCANE_NOT_FOUND >&2; exit 44; }+case "$archive_mode" in -*) ;; *) echo ARCANE_NOT_FOUND >&2; exit 44 ;; esac+for member do+ if ! tar -tzf "$archive" -- "$member" >/dev/null 2>&1; then echo ARCANE_NOT_FOUND >&2; exit 44; fi+done+tar -xzf "$archive" -C /volume -- "$@"`- dockerClient, err := s.dockerService.GetClient(ctx)+func (s *VolumeService) restoreArchiveBackupFilesInternal(ctx context.Context, dockerClient *client.Client, volumeName, backupID string, cleanedPaths []string) error {+ filename, err := s.backupArchiveFilenameInternal(backupID)if err != nil {return err}-helperImage, err := getVolumeHelperImageInternal(ctx, s.dockerService, s.imageService, dockerClient)if err != nil {return err}-backupStorage, err := s.resolveUsableBackupStorageMountInternal(ctx, dockerClient, "/backups", true)if err != nil {return err}-config := &container.Config{Image: helperImage,Cmd: []string{"sleep", "infinity"},NetworkDisabled: true,Labels: volumehelper.Labels(),}-hostConfig := volumehelper.HostConfig(helperImage, []string{volumeName + ":/volume",}, []mount.Mount{backupStorage.mount})-resp, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{Config: config,HostConfig: hostConfig,@@ -788,17 +1323,12 @@ func (s *VolumeService) RestoreBackupFiles(ctx context.Context, volumeName, backif err != nil {return errors.WrapIf(err, "failed to create restore container")}-+ defer func() {+ _, _ = dockerClient.ContainerRemove(context.WithoutCancel(ctx), resp.ID, volumehelper.RemoveOptions())+ }()if _, err := dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); err != nil {- _, _ = dockerClient.ContainerRemove(ctx, resp.ID, volumehelper.RemoveOptions())return errors.WrapIf(err, "failed to start restore container")}-- cleanup := func() {- _, _ = dockerClient.ContainerRemove(ctx, resp.ID, volumehelper.RemoveOptions())- }- defer cleanup()-stderr, err := s.restoreBackupFilesInContainerInternal(ctx, resp.ID, filename, cleanedPaths)if err != nil {return errors.WrapIf(err, "failed to restore files")@@ -806,68 +1336,9 @@ func (s *VolumeService) RestoreBackupFiles(ctx context.Context, volumeName, backif strings.TrimSpace(stderr) != "" {slog.DebugContext(ctx, "volume service: restore files stderr", "backup_id", backupID, "stderr", strings.TrimSpace(stderr))}-- metadata := database.JSON{- "action": "backup_restore_files",- "backup_id": backupID,- "pre_restore_backupId": preBackup.ID,- "paths_count": len(cleanedPaths),- }- if len(cleanedPaths) > 0 {- limit := min(len(cleanedPaths), 5)- metadata["paths_sample"] = cleanedPaths[:limit]- }- if logErr := s.eventService.LogVolumeEvent(ctx, event.EventTypeVolumeBackupRestoreFiles, volumeName, volumeName, user.ID, user.Username, "0", metadata); logErr != nil {- slog.WarnContext(ctx, "could not log volume backup restore files event", "volume", volumeName, "error", logErr.Error())- }-return nil}-func (s *VolumeService) DownloadBackup(ctx context.Context, backupID string, user *common.User) (io.ReadCloser, int64, error) {- slog.DebugContext(ctx, "volume service: download backup", "backup_id", backupID)- filename, err := s.backupArchiveFilenameInternal(backupID)- if err != nil {- return nil, 0, err- }- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return nil, 0, err- }-- containerID, cleanup, err := s.createBackupTempContainerInternal(ctx, dockerClient, "/volume", true)- if err != nil {- return nil, 0, err- }-- reader, size, err := volumehelper.DownloadFileFromContainer(ctx, dockerClient, containerID, path.Join("/volume", filename), cleanup)- if err != nil {- return nil, 0, err- }-- actingUser := user- if actingUser == nil {- actingUser = &common.SystemUser- }- volumeName := ""- var backup VolumeBackup- if err := s.db.WithContext(ctx).Where("id = ?", backupID).First(&backup).Error; err == nil {- volumeName = backup.VolumeName- }- if volumeName != "" {- metadata := database.JSON{- "action": "backup_download",- "backup_id": backupID,- "size": size,- }- if logErr := s.eventService.LogVolumeEvent(ctx, event.EventTypeVolumeBackupDownload, volumeName, volumeName, actingUser.ID, actingUser.Username, "0", metadata); logErr != nil {- slog.WarnContext(ctx, "could not log volume backup download event", "volume", volumeName, "error", logErr.Error())- }- }-- return reader, size, nil-}-func (s *VolumeService) UploadAndRestore(ctx context.Context, volumeName string, archive io.ReadSeeker, filename string, user common.User) error {slog.DebugContext(ctx, "volume service: upload and restore", "volume", volumeName, "filename", filename, "user", user.ID)@@ -880,14 +1351,14 @@ func (s *VolumeService) UploadAndRestore(ctx context.Context, volumeName string,return errors.WrapIf(err, "invalid archive")}_ = gzr.Close()+unlock := s.workspaceLocks.Lock(volumeName)defer unlock()ctx = context.WithValue(ctx, volumeWorkspaceLockContextKeyInternal{}, volumeWorkspaceLockContextInternal{service: s,volumeName: volumeName,})-- preBackup, err := s.CreateBackup(ctx, volumeName, user)+ preBackup, err := s.CreateBackup(ctx, volumeName, user, VolumeBackupTriggerSafety, volumetypes.CreateBackupRequest{Destination: volumetypes.BackupDestinationLocal})if err != nil {return errors.WrapIf(err, "failed to create pre-restore backup")}@@ -959,3 +1430,207 @@ func (s *VolumeService) UploadAndRestore(ctx context.Context, volumeName string,return nil}++const defaultVolumeBackupSchedule = "0 0 2 * * *"++func (s *VolumeService) loadVolumeBackupPoliciesInternal(ctx context.Context, volumeName string) ([]VolumeBackupPolicy, error) {+ var policies []VolumeBackupPolicy+ if err := s.db.WithContext(ctx).Where("volume_name = ?", volumeName).Order("created_at ASC").Find(&policies).Error; err != nil {+ return nil, fmt.Errorf("failed to load volume backup policies: %w", err)+ }+ return policies, nil+}++func (s *VolumeService) loadVolumeBackupPolicyInternal(ctx context.Context, volumeName, policyID string) (*VolumeBackupPolicy, error) {+ if strings.TrimSpace(policyID) == "" {+ return nil, nil+ }+ var policy VolumeBackupPolicy+ err := s.db.WithContext(ctx).Where("id = ? AND volume_name = ?", policyID, volumeName).First(&policy).Error+ if errors.Is(err, gorm.ErrRecordNotFound) {+ return nil, nil+ }+ if err != nil {+ return nil, fmt.Errorf("failed to load volume backup policy: %w", err)+ }+ return &policy, nil+}++func (s *VolumeService) GetBackupPolicies(ctx context.Context, volumeName string) (*volumetypes.BackupPolicyCollection, error) {+ policies, err := s.loadVolumeBackupPoliciesInternal(ctx, volumeName)+ if err != nil {+ return nil, err+ }+ result := &volumetypes.BackupPolicyCollection{Policies: make([]volumetypes.BackupPolicy, 0, len(policies))}+ destinations := make(map[string]s3domain.S3Destination)+ if s.s3Destinations != nil {+ available, listErr := s.s3Destinations.ListAllS3Destinations(ctx)+ if listErr == nil {+ result.S3Available = len(available) > 0+ for _, destination := range available {+ destinations[destination.ID] = s3domain.S3Destination{ID: destination.ID, Name: destination.Name, Bucket: destination.Bucket}+ }+ }+ }+ for i := range policies {+ var lastRun *VolumeBackup+ var entry VolumeBackup+ if runErr := s.db.WithContext(ctx).Where("policy_id = ?", policies[i].ID).Order("created_at DESC").First(&entry).Error; runErr == nil {+ if destination, ok := destinations[entry.S3DestinationID]; ok {+ entry.S3DestinationName = destination.Name+ }+ lastRun = &entry+ } else if !errors.Is(runErr, gorm.ErrRecordNotFound) {+ return nil, fmt.Errorf("failed to load latest volume backup: %w", runErr)+ }+ dto := policies[i].ToDTO(lastRun)+ dto.S3Available = result.S3Available+ if destination, ok := destinations[policies[i].S3DestinationID]; ok {+ dto.S3Bucket = destination.Bucket+ dto.S3DestinationName = destination.Name+ }+ result.Policies = append(result.Policies, dto)+ }+ return result, nil+}++func (s *VolumeService) UpdateBackupPolicies(ctx context.Context, volumeName string, updates []volumetypes.UpdateBackupPolicy) (*volumetypes.BackupPolicyCollection, error) {+ existing, err := s.loadVolumeBackupPoliciesInternal(ctx, volumeName)+ if err != nil {+ return nil, err+ }+ reconcile := backup.PolicyReconciliation[VolumeBackupPolicy]{+ Domain: "volume",+ DB: s.db,+ Existing: existing,+ ID: func(policy *VolumeBackupPolicy) string { return policy.ID },+ New: func() VolumeBackupPolicy { return VolumeBackupPolicy{VolumeName: volumeName} },+ Apply: func(policy *VolumeBackupPolicy, update volumetypes.UpdateBackupPolicy) {+ policy.Enabled, policy.Schedule, policy.RetentionCount = update.Enabled, update.Schedule, update.RetentionCount+ policy.StopContainers, policy.LocalEnabled, policy.S3Enabled = update.StopContainers, update.LocalEnabled, update.S3Enabled+ policy.S3DestinationID = update.S3DestinationID+ },+ S3Configured: func(ctx context.Context, destinationID string) error {+ if s.s3Destinations == nil {+ return errors.New("S3 backup destinations are unavailable")+ }+ if _, err := s.s3Destinations.Configuration(ctx, destinationID); err != nil {+ return errors.New("select a valid S3 destination for volume backups")+ }+ return nil+ },+ Unregister: s.jobs.Unregister,+ Reschedule: s.rescheduleVolumeBackupPolicyInternal,+ }+ if err := reconcile.Run(ctx, updates); err != nil {+ return nil, err+ }+ return s.GetBackupPolicies(ctx, volumeName)+}++func (s *VolumeService) applyVolumeBackupRetentionInternal(ctx context.Context, policyID string, retentionCount int) error {+ expired, err := backup.ExpiredRunIDs(ctx, s.db, "volume_backups", policyID, retentionCount)+ if err != nil {+ return err+ }+ for _, id := range expired {+ if err := s.deleteBackupInternal(ctx, id, nil); err != nil {+ return err+ }+ }+ return nil+}++func (s *VolumeService) runScheduledBackupInternal(ctx context.Context, policyID string) {+ var policy VolumeBackupPolicy+ if err := s.db.WithContext(ctx).Where("id = ? AND enabled = ?", policyID, true).First(&policy).Error; err != nil {+ return+ }+ var entry *VolumeBackup+ _, err := activitylib.RunHandlerActivity(ctx, s.activityService, activitylib.HandlerOptions{+ EnvironmentID: "0",+ Type: activitytypes.TypeResourceAction,+ ResourceType: "volume_backup",+ ResourceID: policy.VolumeName,+ ResourceName: policy.VolumeName,+ User: &common.SystemUser,+ Step: "Creating scheduled backup",+ Message: "Creating scheduled volume backup",+ SuccessMessage: "Scheduled volume backup created successfully",+ Metadata: database.JSON{+ "action": "scheduled_volume_backup",+ "policyId": policy.ID,+ "schedule": policy.Schedule,+ "volumeName": policy.VolumeName,+ "retentionCount": policy.RetentionCount,+ "stopContainers": policy.StopContainers,+ "localEnabled": policy.LocalEnabled,+ "s3Enabled": policy.S3Enabled,+ "s3DestinationId": policy.S3DestinationID,+ },+ }, func(activityCtx context.Context) error {+ var backupErr error+ entry, backupErr = s.CreateBackup(activityCtx, policy.VolumeName, common.SystemUser, VolumeBackupTriggerScheduled, volumetypes.CreateBackupRequest{PolicyID: policy.ID})+ return backupErr+ })+ if errors.Is(err, ErrVolumeBackupAlreadyRunning) {+ slog.InfoContext(ctx, "Scheduled volume backup skipped; another backup is running", "volume", policy.VolumeName)+ return+ }+ if err != nil {+ slog.ErrorContext(ctx, "Scheduled volume backup failed", "volume", policy.VolumeName, "error", err)+ return+ }+ slog.InfoContext(ctx, "Scheduled volume backup completed", "volume", policy.VolumeName, "backup_id", entry.ID, "remote_snapshot_id", entry.RemoteSnapshotID)+}++func (s *VolumeService) rescheduleVolumeBackupPolicyInternal(ctx context.Context, policy *VolumeBackupPolicy) {+ if policy == nil {+ return+ }+ if !policy.Enabled {+ s.jobs.Unregister(ctx, policy.ID)+ return+ }+ policyID := policy.ID+ s.jobs.Register(ctx, policyID,+ func(ctx context.Context) string {+ var current VolumeBackupPolicy+ if err := s.db.WithContext(ctx).Where("id = ?", policyID).First(¤t).Error; err != nil {+ return defaultVolumeBackupSchedule+ }+ return current.Schedule+ },+ func(ctx context.Context) {+ s.runScheduledBackupInternal(ctx, policyID)+ },+ )+}++func (s *VolumeService) RegisterBackupJobsOnStartup(ctx context.Context) {+ if !s.jobs.Enabled() {+ return+ }+ var policies []VolumeBackupPolicy+ if err := s.db.WithContext(ctx).Where("enabled = ?", true).Find(&policies).Error; err != nil {+ slog.ErrorContext(ctx, "Failed to load scheduled volume backups", "error", err)+ return+ }+ for i := range policies {+ s.rescheduleVolumeBackupPolicyInternal(ctx, &policies[i])+ }+ slog.InfoContext(ctx, "Registered scheduled volume backup jobs", "count", len(policies))+}++func (s *VolumeService) removeVolumeBackupPolicyInternal(ctx context.Context, volumeName string) {+ policies, err := s.loadVolumeBackupPoliciesInternal(ctx, volumeName)+ if err != nil {+ return+ }+ for i := range policies {+ s.jobs.Unregister(ctx, policies[i].ID)+ }+ if err := s.db.WithContext(ctx).Where("volume_name = ?", volumeName).Delete(&VolumeBackupPolicy{}).Error; err != nil {+ slog.WarnContext(ctx, "Failed to delete volume backup policy", "volume", volumeName, "error", err)+ }+}diff --git a/backend/internal/volume/handler.go b/backend/internal/volume/handler.goindex 5099f5306..0dbb539f3 100644--- a/backend/internal/volume/handler.go+++ b/backend/internal/volume/handler.go@@ -1,19 +1,19 @@package volumeimport (- "github.com/getarcaneapp/arcane/backend/v2/internal/common"-- "github.com/getarcaneapp/arcane/backend/v2/internal/database"-"context"+ "fmt""io"+ "log/slog""net/http"+ "net/url""strconv""strings"- activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"-+ "github.com/getarcaneapp/arcane/backend/v2/internal/common"+ "github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/docker"+ "github.com/getarcaneapp/arcane/backend/v2/internal/environment""emperror.dev/errors""github.com/danielgtaylor/huma/v2"@@ -25,6 +25,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/pagination""github.com/getarcaneapp/arcane/backend/v2/pkg/utils""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"+ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity""github.com/getarcaneapp/arcane/types/v2/base"uploadtypes "github.com/getarcaneapp/arcane/types/v2/upload"volumetypes "github.com/getarcaneapp/arcane/types/v2/volume"@@ -34,11 +35,12 @@ import (// VolumeHandler provides Huma-based volume management endpoints.type VolumeHandler struct {- volumeService *VolumeService- dockerService *docker.DockerClientService- activityService *activity.ActivityService- uploadService *upload.UploadService- appCtx context.Context+ volumeService *VolumeService+ dockerService *docker.DockerClientService+ activityService *activity.ActivityService+ environmentService *environment.EnvironmentService+ uploadService *upload.UploadService+ appCtx context.Context}// --- Huma Input/Output Wrappers ---@@ -173,14 +175,34 @@ type ListBackupsOutput struct {}type CreateBackupInput struct {- EnvironmentID string `path:"id" doc:"Environment ID"`- VolumeName string `path:"volumeName" doc:"Volume name"`+ EnvironmentID string `path:"id" doc:"Environment ID"`+ VolumeName string `path:"volumeName" doc:"Volume name"`+ Body *volumetypes.CreateBackupRequest `json:"body,omitempty"`}type CreateBackupOutput struct {Body base.ApiResponse[*VolumeBackup]}+type GetVolumeBackupPolicyInput struct {+ EnvironmentID string `path:"id" doc:"Environment ID"`+ VolumeName string `path:"volumeName" doc:"Volume name"`+}++type GetVolumeBackupPolicyOutput struct {+ Body base.ApiResponse[volumetypes.BackupPolicyCollection]+}++type UpdateVolumeBackupPolicyInput struct {+ EnvironmentID string `path:"id" doc:"Environment ID"`+ VolumeName string `path:"volumeName" doc:"Volume name"`+ Body volumetypes.UpdateBackupPolicies `doc:"Scheduled volume backup policies"`+}++type UpdateVolumeBackupPolicyOutput struct {+ Body base.ApiResponse[volumetypes.BackupPolicyCollection]+}+type RestoreBackupInput struct {EnvironmentID string `path:"id" doc:"Environment ID"`VolumeName string `path:"volumeName" doc:"Volume name"`@@ -232,15 +254,15 @@ type DeleteBackupInput struct {BackupID string `path:"backupId" doc:"Backup ID"`}-type DeleteBackupOutput struct {- Body base.ApiResponse[base.MessageResponse]-}-type DownloadBackupInput struct {EnvironmentID string `path:"id" doc:"Environment ID"`BackupID string `path:"backupId" doc:"Backup ID"`}+type DeleteBackupOutput struct {+ Body base.ApiResponse[base.MessageResponse]+}+type UploadAndRestoreInput struct {EnvironmentID string `path:"id" doc:"Environment ID"`VolumeName string `path:"volumeName" doc:"Volume name"`@@ -251,14 +273,25 @@ type UploadAndRestoreOutput struct {Body base.ApiResponse[base.MessageResponse]}+type UploadBackupInput struct {+ EnvironmentID string `path:"id" doc:"Environment ID"`+ BackupID string `path:"backupId" doc:"Backup ID"`+ Body volumetypes.UploadBackupRequest+}++type UploadBackupOutput struct {+ Body base.ApiResponse[*VolumeBackup]+}+// RegisterVolumes registers volume management routes using Huma.-func RegisterVolumes(api huma.API, dockerService *docker.DockerClientService, volumeService *VolumeService, activityService *activity.ActivityService, uploadService *upload.UploadService, appCtx handlerutil.ActivityAppContext) {+func RegisterVolumes(api huma.API, dockerService *docker.DockerClientService, volumeService *VolumeService, activityService *activity.ActivityService, environmentService *environment.EnvironmentService, uploadService *upload.UploadService, appCtx handlerutil.ActivityAppContext) {h := &VolumeHandler{- volumeService: volumeService,- dockerService: dockerService,- activityService: activityService,- uploadService: uploadService,- appCtx: appCtx.Context(),+ volumeService: volumeService,+ dockerService: dockerService,+ activityService: activityService,+ environmentService: environmentService,+ uploadService: uploadService,+ appCtx: appCtx.Context(),}middleware.RegisterWithPermission(api, huma.Operation{@@ -347,6 +380,24 @@ func RegisterVolumes(api huma.API, dockerService *docker.DockerClientService, vo// --- Volume Backup Endpoints ---+ middleware.RegisterWithPermission(api, huma.Operation{+ OperationID: "get-volume-backup-policy",+ Method: http.MethodGet,+ Path: "/environments/{id}/volumes/{volumeName}/backup-policy",+ Summary: "Get volume backup policies",+ Tags: []string{"Volume Backup"},+ Security: handlerutil.DefaultOperationSecurity(),+ }, authz.PermVolumesRead, h.GetBackupPolicy)++ middleware.RegisterWithPermission(api, huma.Operation{+ OperationID: "update-volume-backup-policy",+ Method: http.MethodPut,+ Path: "/environments/{id}/volumes/{volumeName}/backup-policy",+ Summary: "Update volume backup policies",+ Tags: []string{"Volume Backup"},+ Security: handlerutil.DefaultOperationSecurity(),+ }, authz.PermVolumesBackup, h.UpdateBackupPolicy)+middleware.RegisterWithPermission(api, huma.Operation{OperationID: "list-volume-backups",Method: http.MethodGet,@@ -392,6 +443,16 @@ func RegisterVolumes(api huma.API, dockerService *docker.DockerClientService, voSecurity: handlerutil.DefaultOperationSecurity(),}, authz.PermVolumesBackup, h.DeleteBackup)+ middleware.RegisterWithPermission(api, huma.Operation{+ OperationID: "upload-retained-volume-backup-to-s3",+ Method: http.MethodPost,+ Path: "/environments/{id}/volumes/backups/{backupId}/upload",+ Summary: "Upload volume backup",+ Description: "Upload an existing local volume backup to the selected S3 destination",+ Tags: []string{"Volume Backup"},+ Security: handlerutil.DefaultOperationSecurity(),+ }, authz.PermVolumesBackup, h.UploadBackup)+middleware.RegisterWithPermission(api, huma.Operation{OperationID: "download-volume-backup",Method: http.MethodGet,@@ -431,6 +492,117 @@ func RegisterVolumes(api huma.API, dockerService *docker.DockerClientService, vo}, authz.PermVolumesUpload, h.UploadAndRestore)}+func (h *VolumeHandler) DownloadBackup(ctx context.Context, input *DownloadBackupInput) (*huma.StreamResponse, error) {+ user, _ := common.CurrentUserFromContext(ctx)+ reader, size, err := h.volumeService.DownloadBackup(ctx, input.BackupID, user)+ if err != nil {+ return nil, huma.Error500InternalServerError(err.Error())+ }++ return &huma.StreamResponse{+ Body: func(humaCtx huma.Context) {+ defer func() { _ = reader.Close() }()++ humaCtx.SetHeader("Content-Type", "application/x-gzip")+ humaCtx.SetHeader("Content-Disposition", "attachment; filename="+input.BackupID+".tar.gz")+ humaCtx.SetHeader("Content-Length", strconv.FormatInt(size, 10))++ writer := humaCtx.BodyWriter()+ _, _ = io.Copy(writer, reader)+ },+ }, nil+}++func (h *VolumeHandler) GetBackupPolicy(ctx context.Context, input *GetVolumeBackupPolicyInput) (*GetVolumeBackupPolicyOutput, error) {+ if input.EnvironmentID != "0" {+ if h.environmentService == nil {+ return nil, huma.Error500InternalServerError("environment service not available")+ }+ remotePath := fmt.Sprintf("/api/environments/0/volumes/%s/backup-policy", url.PathEscape(input.VolumeName))+ response, err := handlerutil.ProxyRemoteJSON[base.ApiResponse[volumetypes.BackupPolicyCollection]](ctx, h.environmentService.ProxyJSONRequest, input.EnvironmentID, http.MethodGet, remotePath, nil)+ if err != nil {+ return nil, err+ }+ return &GetVolumeBackupPolicyOutput{Body: *response}, nil+ }++ policies, err := h.volumeService.GetBackupPolicies(ctx, input.VolumeName)+ if err != nil {+ return nil, huma.Error500InternalServerError(err.Error())+ }+ return &GetVolumeBackupPolicyOutput{Body: base.ApiResponse[volumetypes.BackupPolicyCollection]{Success: true, Data: *policies}}, nil+}++func (h *VolumeHandler) UpdateBackupPolicy(ctx context.Context, input *UpdateVolumeBackupPolicyInput) (*UpdateVolumeBackupPolicyOutput, error) {+ user, _ := common.CurrentUserFromContext(ctx)+ var policies *volumetypes.BackupPolicyCollection+ var remoteResponse *base.ApiResponse[volumetypes.BackupPolicyCollection]+ errorStatus := http.StatusBadRequest+ runtimeCtx := utils.ActivityRuntimeContext(ctx, h.appCtx)+ _, err := activitylib.RunHandlerActivity(runtimeCtx, h.activityService, activitylib.HandlerOptions{+ EnvironmentID: input.EnvironmentID,+ Type: activitytypes.TypeResourceAction,+ ResourceType: "volume_backup_policy",+ ResourceID: input.VolumeName,+ ResourceName: input.VolumeName,+ User: user,+ Step: "Saving backup policies",+ Message: "Saving volume backup policies",+ SuccessMessage: "Volume backup policies saved successfully",+ Metadata: database.JSON{+ "action": "update_volume_backup_policy",+ "policyCount": len(input.Body.Policies),+ },+ }, func(activityCtx context.Context) error {+ if input.EnvironmentID != "0" {+ if h.environmentService == nil {+ errorStatus = http.StatusInternalServerError+ return errors.New("environment service not available")+ }+ hasS3 := false+ for _, policy := range input.Body.Policies {+ if policy.S3Enabled {+ hasS3 = true+ break+ }+ }+ // Destinations are reconciled before every policy write so the agent+ // can validate S3 references. Only batches that actually need S3+ // fail on a sync error; local-only edits proceed.+ if syncErr := h.environmentService.SyncS3DestinationsToEnvironment(activityCtx, input.EnvironmentID); syncErr != nil {+ if hasS3 {+ errorStatus = http.StatusBadGateway+ return fmt.Errorf("failed to synchronize S3 destinations to environment: %w", syncErr)+ }+ slog.WarnContext(activityCtx, "S3 destination sync failed before local-only volume backup policy write", "environment_id", input.EnvironmentID, "error", syncErr)+ }+ remotePath := fmt.Sprintf("/api/environments/0/volumes/%s/backup-policy", url.PathEscape(input.VolumeName))+ var proxyErr error+ remoteResponse, proxyErr = handlerutil.ProxyRemoteJSON[base.ApiResponse[volumetypes.BackupPolicyCollection]](activityCtx, h.environmentService.ProxyJSONRequest, input.EnvironmentID, http.MethodPut, remotePath, input.Body)+ if proxyErr != nil {+ errorStatus = http.StatusBadGateway+ }+ return proxyErr+ }+ var updateErr error+ policies, updateErr = h.volumeService.UpdateBackupPolicies(activityCtx, input.VolumeName, input.Body.Policies)+ return updateErr+ })+ if err != nil {+ if errorStatus == http.StatusInternalServerError {+ return nil, huma.Error500InternalServerError(err.Error())+ }+ if errorStatus == http.StatusBadGateway {+ return nil, huma.NewError(http.StatusBadGateway, err.Error())+ }+ return nil, huma.Error400BadRequest(err.Error())+ }+ if remoteResponse != nil {+ return &UpdateVolumeBackupPolicyOutput{Body: *remoteResponse}, nil+ }+ return &UpdateVolumeBackupPolicyOutput{Body: base.ApiResponse[volumetypes.BackupPolicyCollection]{Success: true, Data: *policies}}, nil+}+// ListVolumes returns a paginated list of volumes.func (h *VolumeHandler) ListVolumes(ctx context.Context, input *ListVolumesInput) (*ListVolumesOutput, error) {params := handlerutil.PaginationParams(input.Start, input.Limit, input.Sort, input.Order, input.Search)@@ -707,6 +879,14 @@ func (h *VolumeHandler) CreateBackup(ctx context.Context, input *CreateBackupInp}var backup *VolumeBackup+ destination := volumetypes.BackupDestination("")+ policyID := ""+ s3DestinationID := ""+ if input.Body != nil {+ destination = input.Body.Destination+ policyID = input.Body.PolicyID+ s3DestinationID = input.Body.S3DestinationID+ }runtimeCtx := utils.ActivityRuntimeContext(ctx, h.appCtx)activityID, err := activitylib.RunHandlerActivity(runtimeCtx, h.activityService, activitylib.HandlerOptions{EnvironmentID: input.EnvironmentID,@@ -718,12 +898,17 @@ func (h *VolumeHandler) CreateBackup(ctx context.Context, input *CreateBackupInpStep: "Creating backup",Message: "Creating volume backup",SuccessMessage: "Volume backup created successfully",- Metadata: database.JSON{"action": "create_volume_backup"},+ Metadata: database.JSON{"action": "create_volume_backup", "destination": destination, "policyId": policyID, "s3DestinationId": s3DestinationID},}, func(runtimeCtx context.Context) error {var backupErr error- backup, backupErr = h.volumeService.CreateBackup(runtimeCtx, input.VolumeName, *user)+ backup, backupErr = h.volumeService.CreateBackup(runtimeCtx, input.VolumeName, *user, VolumeBackupTriggerManual, volumetypes.CreateBackupRequest{+ Destination: destination, PolicyID: policyID, S3DestinationID: s3DestinationID,+ })return backupErr})+ if errors.Is(err, ErrVolumeBackupAlreadyRunning) {+ return nil, huma.Error409Conflict(err.Error())+ }if err != nil {return nil, huma.Error500InternalServerError(err.Error())}@@ -875,25 +1060,38 @@ func (h *VolumeHandler) DeleteBackup(ctx context.Context, input *DeleteBackupInp}, nil}-func (h *VolumeHandler) DownloadBackup(ctx context.Context, input *DownloadBackupInput) (*huma.StreamResponse, error) {+func (h *VolumeHandler) UploadBackup(ctx context.Context, input *UploadBackupInput) (*UploadBackupOutput, error) {+ if h.volumeService == nil {+ return nil, huma.Error500InternalServerError("service not available")+ }user, _ := common.CurrentUserFromContext(ctx)- reader, size, err := h.volumeService.DownloadBackup(ctx, input.BackupID, user)+ var backup *VolumeBackup+ runtimeCtx := utils.ActivityRuntimeContext(ctx, h.appCtx)+ activityID, err := activitylib.RunHandlerActivity(runtimeCtx, h.activityService, activitylib.HandlerOptions{+ EnvironmentID: input.EnvironmentID,+ Type: activitytypes.TypeResourceAction,+ ResourceType: "volume_backup",+ ResourceID: input.BackupID,+ ResourceName: input.BackupID,+ User: user,+ Step: "Uploading backup",+ Message: "Uploading volume backup to S3",+ SuccessMessage: "Volume backup uploaded successfully",+ Metadata: database.JSON{+ "action": "upload_volume_backup",+ "backupId": input.BackupID,+ "s3DestinationId": input.Body.S3DestinationID,+ },+ }, func(runtimeCtx context.Context) error {+ var uploadErr error+ backup, uploadErr = h.volumeService.UploadBackup(runtimeCtx, input.BackupID, input.Body.S3DestinationID)+ return uploadErr+ })if err != nil {return nil, huma.Error500InternalServerError(err.Error())}-- return &huma.StreamResponse{- Body: func(humaCtx huma.Context) {- defer func() { _ = reader.Close() }()-- humaCtx.SetHeader("Content-Type", "application/x-gzip")- humaCtx.SetHeader("Content-Disposition", "attachment; filename="+input.BackupID+".tar.gz")- humaCtx.SetHeader("Content-Length", strconv.FormatInt(size, 10))-- writer := humaCtx.BodyWriter()- _, _ = io.Copy(writer, reader)- },- }, nil+ backup.ActivityID = mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()+ return &UploadBackupOutput{Body: base.ApiResponse[*VolumeBackup]{Success: true, Data: backup}}, nil}func (h *VolumeHandler) UploadAndRestore(ctx context.Context, input *UploadAndRestoreInput) (*UploadAndRestoreOutput, error) {diff --git a/backend/internal/volume/model.go b/backend/internal/volume/model.goindex 91ecb356c..e8ab26ed8 100644--- a/backend/internal/volume/model.go+++ b/backend/internal/volume/model.go@@ -8,13 +8,46 @@ import ("github.com/getarcaneapp/arcane/types/v2/volume")+type VolumeBackupStatus string++const (+ VolumeBackupStatusRunning VolumeBackupStatus = "running"+ VolumeBackupStatusSucceeded VolumeBackupStatus = "succeeded"+ VolumeBackupStatusFailed VolumeBackupStatus = "failed"+)++type VolumeBackupTrigger string++const (+ VolumeBackupTriggerManual VolumeBackupTrigger = "manual"+ VolumeBackupTriggerScheduled VolumeBackupTrigger = "scheduled"+ VolumeBackupTriggerSafety VolumeBackupTrigger = "safety"+)++type VolumeBackupFormat string++const (+ VolumeBackupFormatArchive VolumeBackupFormat = "archive"+ VolumeBackupFormatRustic VolumeBackupFormat = "rustic"+)+type VolumeBackup struct {database.BaseModel- VolumeName string `json:"volumeName" gorm:"column:volume_name;index"`- Size int64 `json:"size" gorm:"column:size"`- CreatedAt time.Time `json:"createdAt" gorm:"column:created_at"`- ActivityID *string `json:"activityId,omitempty" gorm:"-"`+ VolumeName string `json:"volumeName" gorm:"column:volume_name;index"`+ Size int64 `json:"size" gorm:"column:size"`+ CreatedAt time.Time `json:"createdAt" gorm:"column:created_at"`+ Status VolumeBackupStatus `json:"status" gorm:"column:status;type:text;not null;default:succeeded"`+ Trigger VolumeBackupTrigger `json:"trigger" gorm:"column:trigger;type:text;not null;default:manual"`+ Destination volume.BackupDestination `json:"destination" gorm:"column:destination;type:text;not null;default:local"`+ Format VolumeBackupFormat `json:"format" gorm:"column:format;type:text;not null;default:archive"`+ LocalSnapshotID string `json:"localSnapshotId,omitempty" gorm:"column:local_snapshot_id;type:text"`+ RemoteSnapshotID string `json:"remoteSnapshotId,omitempty" gorm:"column:remote_snapshot_id;type:text"`+ S3DestinationID string `json:"s3DestinationId,omitempty" gorm:"column:s3_destination_id;type:text;index"`+ S3DestinationName string `json:"s3DestinationName,omitempty" gorm:"-"`+ PolicyID string `json:"policyId,omitempty" gorm:"column:policy_id;type:text;index"`+ Error string `json:"error,omitempty" gorm:"column:error;type:text"`+ ActivityID *string `json:"activityId,omitempty" gorm:"-"`}func (*VolumeBackup) TableName() string {@@ -23,9 +56,55 @@ func (*VolumeBackup) TableName() string {func (b *VolumeBackup) ToDTO() volume.BackupEntry {return volume.BackupEntry{- ID: b.ID,- VolumeName: b.VolumeName,- Size: b.Size,- CreatedAt: b.CreatedAt.Format(time.RFC3339),+ ID: b.ID,+ VolumeName: b.VolumeName,+ Size: b.Size,+ CreatedAt: b.CreatedAt.Format(time.RFC3339),+ Status: string(b.Status),+ Trigger: string(b.Trigger),+ Destination: b.Destination,+ Format: volume.BackupFormat(b.Format),+ LocalSnapshotID: b.LocalSnapshotID,+ RemoteSnapshotID: b.RemoteSnapshotID,+ S3DestinationID: b.S3DestinationID,+ S3DestinationName: b.S3DestinationName,+ PolicyID: b.PolicyID,+ Error: b.Error,+ }+}++type VolumeBackupPolicy struct {+ database.BaseModel++ VolumeName string `json:"volumeName" gorm:"column:volume_name;not null;index"`+ Enabled bool `json:"enabled" gorm:"column:enabled;not null;default:false"`+ Schedule string `json:"schedule" gorm:"column:schedule;type:text;not null"`+ RetentionCount int `json:"retentionCount" gorm:"column:retention_count;not null;default:7"`+ StopContainers bool `json:"stopContainers" gorm:"column:stop_containers;not null;default:false"`+ LocalEnabled bool `json:"localEnabled" gorm:"column:local_enabled;not null"`+ S3Enabled bool `json:"s3Enabled" gorm:"column:s3_enabled;not null;default:false"`+ S3DestinationID string `json:"s3DestinationId,omitempty" gorm:"column:s3_destination_id;type:text;index"`+}++func (VolumeBackupPolicy) TableName() string {+ return "volume_backup_policies"+}++func (p VolumeBackupPolicy) ToDTO(lastRun *VolumeBackup) volume.BackupPolicy {+ dto := volume.BackupPolicy{+ ID: p.ID,+ VolumeName: p.VolumeName,+ Enabled: p.Enabled,+ Schedule: p.Schedule,+ RetentionCount: p.RetentionCount,+ StopContainers: p.StopContainers,+ LocalEnabled: p.LocalEnabled,+ S3Enabled: p.S3Enabled,+ S3DestinationID: p.S3DestinationID,+ }+ if lastRun != nil {+ lastRunDTO := lastRun.ToDTO()+ dto.LastRun = &lastRunDTO}+ return dto}diff --git a/backend/internal/volume/module.go b/backend/internal/volume/module.goindex 8cbec5492..2f0955301 100644--- a/backend/internal/volume/module.go+++ b/backend/internal/volume/module.go@@ -4,8 +4,12 @@ package volumeimport ("github.com/danielgtaylor/huma/v2"+ "github.com/getarcaneapp/arcane/backend/v2/internal/backup"+ "github.com/getarcaneapp/arcane/backend/v2/internal/container""github.com/getarcaneapp/arcane/backend/v2/internal/docker"+ "github.com/getarcaneapp/arcane/backend/v2/internal/environment""github.com/getarcaneapp/arcane/backend/v2/internal/image"+ s3domain "github.com/getarcaneapp/arcane/backend/v2/internal/s3""github.com/getarcaneapp/arcane/backend/v2/internal/activity""github.com/getarcaneapp/arcane/backend/v2/internal/config"@@ -18,14 +22,18 @@ import (// Dependencies are the collaborators the volume domain needs.type Dependencies struct {- DB *database.DB- Docker *docker.DockerClientService- Event *event.EventService- Settings *settings.SettingsService- Image *image.ImageService- Activity *activity.ActivityService- Config *config.Config- Upload *upload.UploadService+ DB *database.DB+ Docker *docker.DockerClientService+ Event *event.EventService+ Settings *settings.SettingsService+ Image *image.ImageService+ Activity *activity.ActivityService+ Environment *environment.EnvironmentService+ Container *container.ContainerService+ Engine *backup.Engine+ S3 *s3domain.S3DestinationService+ Config *config.Config+ Upload *upload.UploadService}// Module wires the volume domain and mounts its routes.@@ -37,7 +45,7 @@ type Module struct {// New builds the volume domain from its dependencies.func New(deps Dependencies) *Module {return &Module{- service: NewVolumeService(deps.DB, deps.Docker, deps.Event, deps.Settings, deps.Image, deps.Config),+ service: NewVolumeService(deps.DB, deps.Docker, deps.Event, deps.Activity, deps.Settings, deps.Container, deps.Image, deps.Engine, deps.S3, deps.Config),deps: deps,}}@@ -55,8 +63,8 @@ func (m *Module) Service() *VolumeService {// OpenAPI spec generation can discover the routes without a service graph.func (m *Module) RegisterRoutes(api huma.API, appCtx handlerutil.ActivityAppContext) {if m == nil {- RegisterVolumes(api, nil, nil, nil, nil, appCtx)+ RegisterVolumes(api, nil, nil, nil, nil, nil, appCtx)return}- RegisterVolumes(api, m.deps.Docker, m.service, m.deps.Activity, m.deps.Upload, appCtx)+ RegisterVolumes(api, m.deps.Docker, m.service, m.deps.Activity, m.deps.Environment, m.deps.Upload, appCtx)}diff --git a/backend/internal/volume/service.go b/backend/internal/volume/service.goindex 1ed238ed1..883767d19 100644--- a/backend/internal/volume/service.go+++ b/backend/internal/volume/service.go@@ -8,20 +8,27 @@ import ("strings""sync"+ "github.com/getarcaneapp/arcane/backend/v2/internal/activity"+ "github.com/getarcaneapp/arcane/backend/v2/internal/backup""github.com/getarcaneapp/arcane/backend/v2/internal/config"+ "github.com/getarcaneapp/arcane/backend/v2/internal/container""github.com/getarcaneapp/arcane/backend/v2/internal/docker""github.com/getarcaneapp/arcane/backend/v2/internal/image"+ s3domain "github.com/getarcaneapp/arcane/backend/v2/internal/s3""emperror.dev/errors"+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors""github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/event""github.com/getarcaneapp/arcane/backend/v2/internal/settings"dockerutil "github.com/getarcaneapp/arcane/backend/v2/pkg/dockerutil""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/entityjobs""github.com/getarcaneapp/arcane/backend/v2/pkg/utils"workspacepkg "github.com/getarcaneapp/arcane/backend/v2/pkg/workspace"+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"volumetypes "github.com/getarcaneapp/arcane/types/v2/volume""github.com/moby/moby/client""golang.org/x/sync/singleflight"@@ -31,9 +38,14 @@ type VolumeService struct {db *database.DBdockerService *docker.DockerClientServiceeventService *event.EventService+ activityService *activity.ActivityServicesettingsService *settings.SettingsService+ containerService *container.ContainerServiceimageService *image.ImageService+ engine *backup.Engine+ s3Destinations *s3domain.S3DestinationServicebackupVolumeName string+ encryptionKey stringworkspaceMaxDepth intworkspaceMaxEntries intworkspaceMaxFileSizeBytes int64@@ -45,6 +57,13 @@ type VolumeService struct {// second overwrites the first in helperByVolume, orphaning a `sleep infinity`// container that pins the volume until restart.helperGroup singleflight.Group+ jobs *entityjobs.Registry+}++// SetScheduler injects the dynamic scheduler and admission gate for per-policy+// backup jobs. Agent mode passes them too: agents run their own volume backups.+func (s *VolumeService) SetScheduler(ctx context.Context, scheduler schedulertypes.DynamicScheduler, admissionGate *actors.Gate[actors.AdmissionKey]) error {+ return s.jobs.SetScheduler(ctx, scheduler, admissionGate)}type volumeWorkspaceLockContextKeyInternal struct{}@@ -56,14 +75,16 @@ type volumeWorkspaceLockContextInternal struct {const trivyCacheVolumePruneFilterValue = libarcane.InternalResourceLabel + "=true"-func NewVolumeService(db *database.DB, dockerService *docker.DockerClientService, eventService *event.EventService, settingsService *settings.SettingsService, imageService *image.ImageService, cfg *config.Config) *VolumeService {+func NewVolumeService(db *database.DB, dockerService *docker.DockerClientService, eventService *event.EventService, activityService *activity.ActivityService, settingsService *settings.SettingsService, containerService *container.ContainerService, imageService *image.ImageService, engine *backup.Engine, s3Destinations *s3domain.S3DestinationService, cfg *config.Config) *VolumeService {slog.Debug("volume service: new")backupVolumeName := ""+ encryptionKey := ""workspaceMaxDepth := 50workspaceMaxEntries := 10000workspaceMaxFileSizeMB := workspacepkg.DefaultMaxFileSizeMBif cfg != nil {backupVolumeName = cfg.BackupVolumeName+ encryptionKey = cfg.EncryptionKeyworkspaceMaxDepth = cfg.VolumeWorkspaceMaxDepthworkspaceMaxEntries = cfg.VolumeWorkspaceMaxEntriesworkspaceMaxFileSizeMB = cfg.VolumeWorkspaceMaxFileSizeMB@@ -75,13 +96,19 @@ func NewVolumeService(db *database.DB, dockerService *docker.DockerClientServicedb: db,dockerService: dockerService,eventService: eventService,+ activityService: activityService,settingsService: settingsService,+ containerService: containerService,imageService: imageService,+ engine: engine,+ s3Destinations: s3Destinations,backupVolumeName: backupVolumeName,+ encryptionKey: encryptionKey,workspaceMaxDepth: workspaceMaxDepth,workspaceMaxEntries: workspaceMaxEntries,workspaceMaxFileSizeBytes: workspacepkg.MaxFileSizeBytes(workspaceMaxFileSizeMB),helperByVolume: make(map[string]*volumeHelper),+ jobs: entityjobs.New("volume-backup:", backup.VolumeAdmissionScope),}}@@ -190,6 +217,7 @@ func (s *VolumeService) DeleteVolume(ctx context.Context, name string, force boo}s.removeHelperEntry(name)+ s.removeVolumeBackupPolicyInternal(ctx, name)dockerutil.InvalidateVolumeUsageCache(dockerClient)return nil}@@ -230,6 +258,7 @@ func (s *VolumeService) PruneVolumesWithOptions(ctx context.Context, all bool) (for _, volumeName := range volumePruneResult.Report.VolumesDeleted {s.removeHelperEntry(volumeName)+ s.removeVolumeBackupPolicyInternal(ctx, volumeName)}dockerutil.InvalidateVolumeUsageCache(dockerClient)diff --git a/backend/internal/volume/service_test.go b/backend/internal/volume/service_test.goindex 251d1cfc7..dc45d211d 100644--- a/backend/internal/volume/service_test.go+++ b/backend/internal/volume/service_test.go@@ -6,20 +6,56 @@ import ("net/http""net/http/httptest""strings"+ "sync""testing""time"+ "github.com/getarcaneapp/arcane/backend/v2/internal/activity"+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"+ "github.com/getarcaneapp/arcane/backend/v2/internal/backup"+ "github.com/getarcaneapp/arcane/backend/v2/internal/common"+ "github.com/getarcaneapp/arcane/backend/v2/internal/config"+ containerdomain "github.com/getarcaneapp/arcane/backend/v2/internal/container"+ "github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/docker"+ "github.com/getarcaneapp/arcane/backend/v2/internal/event"+ s3domain "github.com/getarcaneapp/arcane/backend/v2/internal/s3""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/volumehelper"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/entityjobs"+ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"volumetypes "github.com/getarcaneapp/arcane/types/v2/volume"+ "github.com/libtnb/sqlite""github.com/moby/moby/api/types/container""github.com/moby/moby/api/types/mount""github.com/moby/moby/api/types/volume""github.com/moby/moby/client""github.com/stretchr/testify/require"+ "go.getarcane.app/sys/crypto"+ "go.uber.org/fx/fxtest"+ "gorm.io/gorm")+func newVolumeBackupEngineForTestInternal(t testing.TB) *backup.Engine {+ t.Helper()+ fxLifecycle := fxtest.NewLifecycle(t)+ runtime, err := actors.NewRuntime(t.Context(), fxLifecycle)+ require.NoError(t, err)+ gate, err := actors.NewGate[actors.AdmissionKey](t.Context(), runtime, "volume-backup-test-admission", t.Name())+ require.NoError(t, err)+ engine := backup.NewEngine(t.Context(), runtime, gate, nil)+ t.Cleanup(func() {+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ require.NoError(t, engine.Stop(stopCtx))+ require.NoError(t, gate.Stop(stopCtx))+ require.NoError(t, fxLifecycle.Stop(stopCtx))+ })+ return engine+}+func newVolumeServiceTestDockerClientInternal(t *testing.T, server *httptest.Server) *client.Client {t.Helper()dockerClient, err := client.New(@@ -477,51 +513,6 @@ func TestBackupMountWarningFromArcaneMountsInternal(t *testing.T) {}}-func TestBackupArchiveFilenameInternal(t *testing.T) {- svc := &VolumeService{}-- tests := []struct {- name string- backupID string- want string- wantErr bool- }{- {- name: "valid backup id",- backupID: "volume-123-abc",- want: "volume-123-abc.tar.gz",- },- {- name: "trims surrounding whitespace",- backupID: " volume-123-abc ",- want: "volume-123-abc.tar.gz",- },- {- name: "rejects traversal attempts",- backupID: "../../bin/busybox",- wantErr: true,- },- {- name: "rejects path separators",- backupID: "nested/path",- wantErr: true,- },- }-- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := svc.backupArchiveFilenameInternal(tt.backupID)- if tt.wantErr {- require.Error(t, err)- return- }-- require.NoError(t, err)- require.Equal(t, tt.want, got)- })- }-}-func TestCollectStaleHelperIDsInternal(t *testing.T) {now := time.Now()s := &VolumeService{@@ -605,3 +596,403 @@ func TestTouchHelperInternal(t *testing.T) {s.touchHelperInternal("missing")require.NotContains(t, s.helperByVolume, "missing")}++type volumeBackupPolicySchedulerInternal struct {+ jobs map[string]schedulertypes.Job+}++func (s *volumeBackupPolicySchedulerInternal) AddJob(_ context.Context, job schedulertypes.Job) error {+ s.jobs[job.Name()] = job+ return nil+}++func (s *volumeBackupPolicySchedulerInternal) RemoveJob(_ context.Context, name string) {+ delete(s.jobs, name)+}++func (s *volumeBackupPolicySchedulerInternal) HasJob(name string) bool {+ _, ok := s.jobs[name]+ return ok+}++func TestVolumeBackupPolicy_UpdateRegistersIndependentJobsAndSettings(t *testing.T) {+ gormDB, err := gorm.Open(sqlite.Open("file:volume-backup-schedule?mode=memory&cache=shared"), &gorm.Config{})+ require.NoError(t, err)+ require.NoError(t, gormDB.AutoMigrate(&VolumeBackupPolicy{}, &VolumeBackup{}))+ db := &database.DB{DB: gormDB}+ scheduler := &volumeBackupPolicySchedulerInternal{jobs: make(map[string]schedulertypes.Job)}+ service := &VolumeService{db: db, jobs: entityjobs.New("volume-backup:", backup.VolumeAdmissionScope)}+ fxLifecycle := fxtest.NewLifecycle(t)+ runtime, err := actors.NewRuntime(t.Context(), fxLifecycle)+ require.NoError(t, err)+ gate, err := actors.NewGate[actors.AdmissionKey](t.Context(), runtime, "volume-policy-test-admission", t.Name())+ require.NoError(t, err)+ t.Cleanup(func() {+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ require.NoError(t, gate.Stop(stopCtx))+ require.NoError(t, fxLifecycle.Stop(stopCtx))+ })+ require.NoError(t, service.SetScheduler(context.Background(), scheduler, gate))++ collection, err := service.UpdateBackupPolicies(context.Background(), "app-data", []volumetypes.UpdateBackupPolicy{+ {Enabled: true, Schedule: "0 */15 * * * *", RetentionCount: 5, StopContainers: true, LocalEnabled: true},+ {Enabled: true, Schedule: "0 0 2 * * *", RetentionCount: 30, LocalEnabled: true},+ })+ require.NoError(t, err)+ require.Len(t, collection.Policies, 2)+ require.Equal(t, 5, collection.Policies[0].RetentionCount)+ require.True(t, collection.Policies[0].StopContainers)+ require.Equal(t, 30, collection.Policies[1].RetentionCount)+ require.Len(t, scheduler.jobs, 2)++ firstID := collection.Policies[0].ID+ collection, err = service.UpdateBackupPolicies(context.Background(), "app-data", []volumetypes.UpdateBackupPolicy{{+ ID: firstID, Schedule: "0 */30 * * * *", RetentionCount: 9, LocalEnabled: true,+ }})+ require.NoError(t, err)+ require.Len(t, collection.Policies, 1)+ require.Equal(t, firstID, collection.Policies[0].ID)+ require.Equal(t, 9, collection.Policies[0].RetentionCount)+ require.Empty(t, scheduler.jobs)+}++func TestVolumeBackupPolicy_GetReturnsLastRunForEachPolicy(t *testing.T) {+ gormDB, err := gorm.Open(sqlite.Open("file:volume-backup-last-runs?mode=memory&cache=shared"), &gorm.Config{})+ require.NoError(t, err)+ require.NoError(t, gormDB.AutoMigrate(&VolumeBackupPolicy{}, &VolumeBackup{}))+ first := &VolumeBackupPolicy{VolumeName: "app-data", Schedule: "0 0 2 * * *", LocalEnabled: true}+ second := &VolumeBackupPolicy{VolumeName: "app-data", Schedule: "0 0 14 * * *", LocalEnabled: true}+ require.NoError(t, gormDB.Create(first).Error)+ require.NoError(t, gormDB.Create(second).Error)+ require.NoError(t, gormDB.Create(&VolumeBackup{VolumeName: "app-data", PolicyID: first.ID, Status: VolumeBackupStatusSucceeded}).Error)+ require.NoError(t, gormDB.Create(&VolumeBackup{VolumeName: "app-data", PolicyID: second.ID, Status: VolumeBackupStatusFailed}).Error)++ service := &VolumeService{db: &database.DB{DB: gormDB}}+ collection, err := service.GetBackupPolicies(context.Background(), "app-data")+ require.NoError(t, err)+ require.Len(t, collection.Policies, 2)+ require.Equal(t, "succeeded", collection.Policies[0].LastRun.Status)+ require.Equal(t, "failed", collection.Policies[1].LastRun.Status)+}++func TestVolumeBackupPolicy_RetentionIgnoresFailedRuns(t *testing.T) {+ gormDB, err := gorm.Open(sqlite.Open("file:volume-backup-retention-failed?mode=memory&cache=shared"), &gorm.Config{})+ require.NoError(t, err)+ require.NoError(t, gormDB.AutoMigrate(&VolumeBackup{}))++ policyID := "policy-1"+ require.NoError(t, gormDB.Create(&VolumeBackup{+ VolumeName: "app-data", PolicyID: policyID, Status: VolumeBackupStatusSucceeded,+ LocalSnapshotID: "snapshot-1", CreatedAt: time.Now().Add(-time.Hour),+ }).Error)+ require.NoError(t, gormDB.Create(&VolumeBackup{+ VolumeName: "app-data", PolicyID: policyID, Status: VolumeBackupStatusFailed,+ CreatedAt: time.Now(),+ }).Error)++ service := &VolumeService{db: &database.DB{DB: gormDB}}+ require.NoError(t, service.applyVolumeBackupRetentionInternal(context.Background(), policyID, 1))++ var backups []VolumeBackup+ require.NoError(t, gormDB.Order("created_at ASC").Find(&backups).Error)+ require.Len(t, backups, 2)+ require.Equal(t, "snapshot-1", backups[0].LocalSnapshotID)+ require.Equal(t, VolumeBackupStatusFailed, backups[1].Status)+}++func TestVolumeBackup_ValidationErrors(t *testing.T) {+ gormDB, err := gorm.Open(sqlite.Open("file:volume-backup-validation?mode=memory&cache=shared"), &gorm.Config{})+ require.NoError(t, err)+ require.NoError(t, gormDB.AutoMigrate(&VolumeBackupPolicy{}))+ service := &VolumeService{db: &database.DB{DB: gormDB}}+ ctx := context.Background()++ _, err = service.UpdateBackupPolicies(ctx, "app-data", []volumetypes.UpdateBackupPolicy{{Schedule: "not a cron", LocalEnabled: true}})+ require.ErrorContains(t, err, "invalid volume backup schedule")++ _, err = service.UpdateBackupPolicies(ctx, "app-data", []volumetypes.UpdateBackupPolicy{{Schedule: "0 0 2 * * *", RetentionCount: 7}})+ require.ErrorContains(t, err, "select at least one volume backup destination")++ _, err = service.CreateBackup(ctx, "app-data", common.User{}, VolumeBackupTriggerManual, volumetypes.CreateBackupRequest{Destination: volumetypes.BackupDestination("invalid")})+ require.EqualError(t, err, "invalid volume backup destination")++ _, err = service.CreateBackup(ctx, "app-data", common.User{}, VolumeBackupTriggerManual, volumetypes.CreateBackupRequest{Destination: volumetypes.BackupDestinationS3})+ require.EqualError(t, err, "select an S3 destination for the volume backup")+}++func TestVolumeBackupPolicy_ScheduledRunCreatesActivity(t *testing.T) {+ gormDB, err := gorm.Open(sqlite.Open("file:volume-backup-scheduled-activity?mode=memory&cache=shared"), &gorm.Config{})+ require.NoError(t, err)+ require.NoError(t, gormDB.AutoMigrate(&VolumeBackupPolicy{}, &VolumeBackup{}, &activity.Activity{}))+ db := &database.DB{DB: gormDB}+ policy := &VolumeBackupPolicy{+ VolumeName: "app-data",+ Enabled: true,+ Schedule: "0 0 2 * * *",+ RetentionCount: 7,+ LocalEnabled: true,+ }+ require.NoError(t, gormDB.Create(policy).Error)+ engine := newVolumeBackupEngineForTestInternal(t)+ service := &VolumeService{+ db: db,+ activityService: activity.NewActivityService(db, nil),+ engine: engine,+ jobs: entityjobs.New("volume-backup:", backup.VolumeAdmissionScope),+ }+ // Holding the volume's admission lease makes the scheduled run fail with+ // "already running", which still must record a failed activity.+ lease, admitted, err := engine.TryAcquireRun(context.Background(), backup.VolumeAdmissionScope, policy.VolumeName)+ require.NoError(t, err)+ require.True(t, admitted)+ defer lease.Release()++ service.runScheduledBackupInternal(context.Background(), policy.ID)++ var activity activity.Activity+ require.NoError(t, gormDB.Where("resource_type = ?", "volume_backup").First(&activity).Error)+ require.Equal(t, activitytypes.StatusFailed, activity.Status)+ require.Equal(t, "scheduled_volume_backup", activity.Metadata["action"])+ require.Equal(t, policy.Schedule, activity.Metadata["schedule"])+}++func TestVolumeBackupPolicy_UpdateUsesSelectedS3Destination(t *testing.T) {+ gormDB, err := gorm.Open(sqlite.Open("file:volume-backup-s3-secret?mode=memory&cache=shared"), &gorm.Config{})+ require.NoError(t, err)+ require.NoError(t, gormDB.AutoMigrate(&VolumeBackupPolicy{}, &VolumeBackup{}, &s3domain.S3Destination{}))+ db := &database.DB{DB: gormDB}+ crypto.InitEncryption(&crypto.Config{+ EncryptionKey: "test-encryption-key-for-volume-backups-32bytes",+ Environment: "test",+ })+ encryptedSecret, err := crypto.Encrypt("destination-s3-secret")+ require.NoError(t, err)+ destination := &s3domain.S3Destination{+ Name: "Offsite",+ Bucket: "volume-backups",+ Region: "us-east-1",+ AccessKeyID: "destination-access-key",+ SecretAccessKey: encryptedSecret,+ UseSSL: true,+ }+ require.NoError(t, gormDB.Create(destination).Error)+ service := &VolumeService{+ db: db,+ s3Destinations: s3domain.NewS3DestinationService(db),+ jobs: entityjobs.New("volume-backup:", backup.VolumeAdmissionScope),+ }+ collection, err := service.UpdateBackupPolicies(context.Background(), "app-data", []volumetypes.UpdateBackupPolicy{{+ Schedule: "0 0 2 * * *",+ RetentionCount: 7,+ S3Enabled: true,+ S3DestinationID: destination.ID,+ }})+ require.NoError(t, err)+ require.Len(t, collection.Policies, 1)+ policy := collection.Policies[0]+ require.False(t, policy.LocalEnabled)+ require.True(t, policy.S3Enabled)+ require.True(t, policy.S3Available)+ require.Equal(t, destination.ID, policy.S3DestinationID)+ require.Equal(t, "Offsite", policy.S3DestinationName)+ require.Equal(t, "volume-backups", policy.S3Bucket)++ var stored VolumeBackupPolicy+ require.NoError(t, gormDB.Where("volume_name = ?", "app-data").First(&stored).Error)+ require.False(t, stored.LocalEnabled)+ require.True(t, stored.S3Enabled)+ require.Equal(t, destination.ID, stored.S3DestinationID)+}++func TestVolumeBackup_ListResolvesDestinationName(t *testing.T) {+ gormDB, err := gorm.Open(sqlite.Open("file:volume-backup-destination-name?mode=memory&cache=shared"), &gorm.Config{})+ require.NoError(t, err)+ require.NoError(t, gormDB.AutoMigrate(&VolumeBackup{}, &s3domain.S3Destination{}))+ db := &database.DB{DB: gormDB}+ destination := &s3domain.S3Destination{+ Name: "Offsite",+ Bucket: "volume-backups",+ Region: "us-east-1",+ AccessKeyID: "access-key",+ SecretAccessKey: "encrypted-secret",+ }+ require.NoError(t, gormDB.Create(destination).Error)+ require.NoError(t, gormDB.Create(&VolumeBackup{+ VolumeName: "app-data",+ Status: VolumeBackupStatusSucceeded,+ Trigger: VolumeBackupTriggerManual,+ Destination: volumetypes.BackupDestinationLocalS3,+ S3DestinationID: destination.ID,+ }).Error)++ service := &VolumeService{db: db, s3Destinations: s3domain.NewS3DestinationService(db)}+ backups, _, err := service.ListBackupsPaginated(context.Background(), "app-data", pagination.QueryParams{})+ require.NoError(t, err)+ require.Len(t, backups, 1)+ require.Equal(t, volumetypes.BackupDestinationLocalS3, backups[0].Destination)+ require.Equal(t, "Offsite", backups[0].S3DestinationName)+}++func setupVolumeBackupLifecycleTestInternal(t *testing.T, handler http.Handler) (*VolumeService, *client.Client) {+ t.Helper()++ server := httptest.NewServer(handler)+ t.Cleanup(server.Close)+ dockerClient, err := client.New(client.WithHost(server.URL), client.WithAPIVersion("1.41"))+ require.NoError(t, err)+ t.Cleanup(func() { _ = dockerClient.Close() })++ gormDB, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{})+ require.NoError(t, err)+ require.NoError(t, gormDB.AutoMigrate(&event.Event{}))+ db := &database.DB{DB: gormDB}+ dockerService := docker.NewDockerClientService(t.Context(), db, &config.Config{}, nil).WithClient(dockerClient)+ eventService := event.NewEventService(db, &config.Config{}, nil)+ containerService := containerdomain.NewContainerService(t.Context(), eventService, dockerService, nil, nil, nil)+ return &VolumeService{containerService: containerService}, dockerClient+}++func TestVolumeBackupContainerLifecycleStopsAndRestartsOnlyRunningContainersUsingVolume(t *testing.T) {+ var mu sync.Mutex+ var operations []string+ serverHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ switch {+ case strings.HasSuffix(r.URL.Path, "/containers/json"):+ require.NoError(t, json.NewEncoder(w).Encode([]container.Summary{+ {ID: "uses-volume", Mounts: []container.MountPoint{{Type: mount.TypeVolume, Name: "app-data"}}},+ {ID: "other-volume", Mounts: []container.MountPoint{{Type: mount.TypeVolume, Name: "other-data"}}},+ {ID: "arcane", Labels: map[string]string{"com.getarcaneapp.arcane": "true"}, Mounts: []container.MountPoint{{Type: mount.TypeVolume, Name: "app-data"}}},+ }))+ case strings.HasSuffix(r.URL.Path, "/containers/uses-volume/stop"):+ mu.Lock()+ operations = append(operations, "stop:uses-volume")+ mu.Unlock()+ w.WriteHeader(http.StatusNoContent)+ case strings.HasSuffix(r.URL.Path, "/containers/uses-volume/start"):+ mu.Lock()+ operations = append(operations, "start:uses-volume")+ mu.Unlock()+ w.WriteHeader(http.StatusNoContent)+ default:+ http.NotFound(w, r)+ }+ })++ service, dockerClient := setupVolumeBackupLifecycleTestInternal(t, serverHandler)+ actor := common.User{BaseModel: database.BaseModel{ID: "user-1"}, Username: "tester"}+ stopped, err := service.stopRunningContainersForBackupInternal(context.Background(), dockerClient, "app-data", actor, false)+ require.NoError(t, err)+ require.Len(t, stopped, 1)+ require.Equal(t, "uses-volume", stopped[0].ID)++ remaining, err := service.startContainersAfterBackupInternal(context.Background(), dockerClient, stopped, actor)+ require.NoError(t, err)+ require.Empty(t, remaining)+ require.Equal(t, []string{"stop:uses-volume", "start:uses-volume"}, operations)+}++func TestVolumeBackupContainerLifecycleRollsBackStoppedContainersOnStopFailure(t *testing.T) {+ var mu sync.Mutex+ var operations []string+ serverHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ switch {+ case strings.HasSuffix(r.URL.Path, "/containers/json"):+ require.NoError(t, json.NewEncoder(w).Encode([]container.Summary{+ {ID: "first", Mounts: []container.MountPoint{{Type: mount.TypeVolume, Name: "app-data"}}},+ {ID: "second", Mounts: []container.MountPoint{{Type: mount.TypeVolume, Name: "app-data"}}},+ }))+ case strings.HasSuffix(r.URL.Path, "/containers/first/stop"):+ mu.Lock()+ operations = append(operations, "stop:first")+ mu.Unlock()+ w.WriteHeader(http.StatusNoContent)+ case strings.HasSuffix(r.URL.Path, "/containers/second/stop"):+ mu.Lock()+ operations = append(operations, "stop:second")+ mu.Unlock()+ http.Error(w, "stop failed", http.StatusInternalServerError)+ case strings.HasSuffix(r.URL.Path, "/containers/first/start"):+ mu.Lock()+ operations = append(operations, "start:first")+ mu.Unlock()+ w.WriteHeader(http.StatusNoContent)+ default:+ http.NotFound(w, r)+ }+ })++ service, dockerClient := setupVolumeBackupLifecycleTestInternal(t, serverHandler)+ actor := common.User{BaseModel: database.BaseModel{ID: "user-1"}, Username: "tester"}+ stillStopped, err := service.stopRunningContainersForBackupInternal(context.Background(), dockerClient, "app-data", actor, false)+ require.ErrorContains(t, err, "failed to stop container second")+ require.Empty(t, stillStopped)+ require.Equal(t, []string{"stop:first", "stop:second", "start:first"}, operations)+}++func TestVolumeBackupContainerLifecycleWaitsForRunningComposeReplacement(t *testing.T) {+ var mu sync.Mutex+ var operations []string+ listCalls := 0+ serverHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ switch {+ case strings.HasSuffix(r.URL.Path, "/containers/json"):+ mu.Lock()+ listCalls+++ call := listCalls+ mu.Unlock()+ if call == 1 {+ require.NoError(t, json.NewEncoder(w).Encode([]container.Summary{+ {+ ID: "old-id",+ Names: []string{"/old-name"},+ State: container.StateRunning,+ Labels: map[string]string{"com.docker.compose.project": "vault", "com.docker.compose.service": "vaultwarden", "com.docker.compose.container-number": "1"},+ Mounts: []container.MountPoint{{Type: mount.TypeVolume, Name: "app-data"}},+ },+ }))+ return+ }+ if call == 2 {+ require.NoError(t, json.NewEncoder(w).Encode([]container.Summary{}))+ return+ }+ require.NoError(t, json.NewEncoder(w).Encode([]container.Summary{+ {+ ID: "new-id",+ Names: []string{"/new-name"},+ State: container.StateRunning,+ Labels: map[string]string{"com.docker.compose.project": "vault", "com.docker.compose.service": "vaultwarden", "com.docker.compose.container-number": "1"},+ Mounts: []container.MountPoint{{Type: mount.TypeVolume, Name: "app-data"}},+ },+ }))+ case strings.HasSuffix(r.URL.Path, "/containers/old-id/stop"):+ mu.Lock()+ operations = append(operations, "stop:old-id")+ mu.Unlock()+ w.WriteHeader(http.StatusNoContent)+ case strings.Contains(r.URL.Path, "/start"):+ mu.Lock()+ operations = append(operations, "unexpected:"+r.URL.Path)+ mu.Unlock()+ w.WriteHeader(http.StatusNoContent)+ default:+ http.NotFound(w, r)+ }+ })++ service, dockerClient := setupVolumeBackupLifecycleTestInternal(t, serverHandler)+ actor := common.User{BaseModel: database.BaseModel{ID: "user-1"}, Username: "tester"}+ stopped, err := service.stopRunningContainersForBackupInternal(context.Background(), dockerClient, "app-data", actor, false)+ require.NoError(t, err)+ require.Len(t, stopped, 1)+ require.Equal(t, "old-id", stopped[0].ID)++ remaining, err := service.startContainersAfterBackupInternal(context.Background(), dockerClient, stopped, actor)+ require.NoError(t, err)+ require.Empty(t, remaining)+ require.Equal(t, []string{"stop:old-id"}, operations)+}diff --git a/backend/pkg/authz/access_policy.go b/backend/pkg/authz/access_policy.goindex 480d50280..c17951792 100644--- a/backend/pkg/authz/access_policy.go+++ b/backend/pkg/authz/access_policy.go@@ -48,6 +48,8 @@ var accessSurfacesInternal = []AccessSurface{"settings.category.jobschedule","settings.category.notifications","settings.category.roles",+ "settings.category.s3destinations",+ "settings.category.systembackups","settings.category.timeouts","settings.category.users","settings.category.webhooks",@@ -99,6 +101,8 @@ var accessSurfacesInternal = []AccessSurface{settingsCategorySurfaceInternal("jobschedule", "/settings/jobs", "Automations", AccessScopeModeSelectedEnvPlusGlobal, []string{PermJobsManage}),settingsCategorySurfaceInternal("notifications", "/settings/notifications", "Notifications", AccessScopeModeGlobalOnly, []string{PermNotificationsManage}),settingsCategorySurfaceInternal("roles", "/settings/roles", "Roles", AccessScopeModeGlobalOnly, []string{PermRolesList, PermRolesRead}),+ settingsCategorySurfaceInternal("s3destinations", "/settings/backups/s3", "S3 Destinations", AccessScopeModeGlobalOnly, []string{PermS3DestinationsList, PermS3DestinationsRead}),+ globalAdminSettingsCategorySurfaceInternal("systembackups", "/settings/backups", "Backups"),routeSurfaceInternal("route.settings.roles.new", "/settings/roles/new", "Create role", AccessScopeModeGlobalOnly, []string{PermRolesList, PermRolesRead}, 0),routeSurfaceInternal("route.settings.roles.detail", "/settings/roles/{id}", "Role", AccessScopeModeGlobalOnly, []string{PermRolesList, PermRolesRead}, 0),settingsCategorySurfaceInternal("timeouts", "/settings/timeouts", "Timeouts", AccessScopeModeGlobalOnly, []string{PermSettingsRead}),@@ -243,6 +247,14 @@ func globalAdminRouteSurfaceInternal(id, url, label string) AccessSurface {return surface}+// globalAdminSettingsCategorySurfaceInternal approximates the global-admin+// gate its routes enforce: only admins hold every permission.+func globalAdminSettingsCategorySurfaceInternal(categoryID, url, label string) AccessSurface {+ surface := settingsCategorySurfaceInternal(categoryID, url, label, AccessScopeModeGlobalOnly, AllPermissions())+ surface.MatchMode = AccessMatchModeAllOf+ return surface+}+func settingsCategorySurfaceInternal(categoryID, url, label, scopeMode string, permissions []string) AccessSurface {return AccessSurface{ID: "settings.category." + categoryID,diff --git a/backend/pkg/authz/catalog.go b/backend/pkg/authz/catalog.goindex cd3b69f34..761daaded 100644--- a/backend/pkg/authz/catalog.go+++ b/backend/pkg/authz/catalog.go@@ -93,6 +93,21 @@ var permissionCatalog = []PermissionCatalogResource{{"test", PermGitReposTest, "Test", ""},{"sync", PermGitReposSync, "Sync", ""},}},+ {"s3-destinations", "S3 Destinations", PermissionScopeGlobal, []PermissionCatalogAction{+ {"list", PermS3DestinationsList, "List", ""},+ {"read", PermS3DestinationsRead, "Read", ""},+ {"create", PermS3DestinationsCreate, "Create", ""},+ {"update", PermS3DestinationsUpdate, "Update", ""},+ {"delete", PermS3DestinationsDelete, "Delete", ""},+ {"test", PermS3DestinationsTest, "Test", ""},+ {"sync", PermS3DestinationsSync, "Sync", ""},+ }},+ {"system-backups", "System Backups", PermissionScopeGlobal, []PermissionCatalogAction{+ {"read", PermSystemBackupsRead, "Read", ""},+ {"manage", PermSystemBackupsManage, "Manage", ""},+ {"restore", PermSystemBackupsRestore, "Restore", ""},+ {"recovery-key", PermSystemBackupsRecoveryKey, "Recovery Key", ""},+ }},{"events", "Events", PermissionScopeGlobal, []PermissionCatalogAction{{"read", PermEventsRead, "Read", ""},{"delete", PermEventsDelete, "Delete", ""},diff --git a/backend/pkg/authz/permission_set_test.go b/backend/pkg/authz/permission_set_test.goindex 19adff6b6..a0b879a93 100644--- a/backend/pkg/authz/permission_set_test.go+++ b/backend/pkg/authz/permission_set_test.go@@ -283,6 +283,85 @@ func TestVariablePermissionsAreSeparateGlobalGrantsWithBuiltInAccess(t *testing.}}+func TestSystemBackupPermissionsAreAdminOnlyGlobalGrants(t *testing.T) {+ builtInRoles := map[string][]string{+ "Editor": BuiltInEditorPermissions(),+ "No-Shell Editor": BuiltInNoShellEditorPermissions(),+ "Viewer": BuiltInViewerPermissions(),+ "Monitor": BuiltInMonitorPermissions(),+ "Deployer": BuiltInDeployerPermissions(),+ }+ for _, permission := range []string{PermSystemBackupsRead, PermSystemBackupsManage, PermSystemBackupsRestore, PermSystemBackupsRecoveryKey} {+ require.True(t, IsKnownPermission(permission), "system backup permission %q must be known", permission)+ require.True(t, IsOrgLevel(permission), "system backup permission %q must be global", permission)+ require.Contains(t, AllPermissions(), permission, "Admin must receive %q", permission)+ for name, permissions := range builtInRoles {+ require.NotContains(t, permissions, permission, "%s must not receive %q", name, permission)+ }+ }+}++func TestS3DestinationPermissionsAreSeparateGlobalGrantsWithBuiltInAccess(t *testing.T) {+ s3Permissions := []string{+ PermS3DestinationsList,+ PermS3DestinationsRead,+ PermS3DestinationsCreate,+ PermS3DestinationsUpdate,+ PermS3DestinationsDelete,+ PermS3DestinationsTest,+ PermS3DestinationsSync,+ }++ for _, permission := range s3Permissions {++ require.True(t, IsKnownPermission(permission),+ "S3 destination permission %q must be known", permission)++ require.True(t, IsOrgLevel(permission),+ "S3 destination permission %q must be global", permission)++ require.Contains(t, AllPermissions(), permission,+ "Admin must receive %q", permission)++ require.NotContains(t, BuiltInMonitorPermissions(), permission,+ "Monitor must not receive %q", permission)++ require.NotContains(t, BuiltInDeployerPermissions(), permission,+ "Deployer must not receive %q", permission)++ }++ // Roles that can run backups pick a destination, so they read but never+ // manage the stored credentials.+ for name, permissions := range map[string][]string{+ "Editor": BuiltInEditorPermissions(),+ "No-Shell Editor": BuiltInNoShellEditorPermissions(),+ "Viewer": BuiltInViewerPermissions(),+ } {+ require.Contains(t, permissions, PermS3DestinationsList,+ "%s must receive %q", name, PermS3DestinationsList)++ require.Contains(t, permissions, PermS3DestinationsRead,+ "%s must receive %q", name, PermS3DestinationsRead)++ for _, permission := range s3Permissions[2:] {++ require.NotContains(t, permissions, permission,+ "%s must not receive %q", name, permission)++ }+ }++ settingsOnly := NewPermissionSet()+ settingsOnly.AddGlobal(PermSettingsRead, PermSettingsWrite)+ for _, permission := range s3Permissions {++ require.False(t, settingsOnly.Allows(permission, ""),+ "settings grants must not satisfy %q", permission)++ }+}+func TestPermissionCatalogDerivesKnownPermissionsAndScopes(t *testing.T) {catalog := PermissionCatalog()diff --git a/backend/pkg/authz/permissions.go b/backend/pkg/authz/permissions.goindex 30060afff..07bd18d35 100644--- a/backend/pkg/authz/permissions.go+++ b/backend/pkg/authz/permissions.go@@ -83,6 +83,29 @@ const (PermGitReposTest = "git-repositories:test"PermGitReposSync = "git-repositories:sync"+ // PermS3DestinationsList and its siblings gate S3 backup destinations,+ // which hold credentials and are manager-owned, so they are granted+ // separately from general settings, like container registries.+ PermS3DestinationsList = "s3-destinations:list"+ PermS3DestinationsRead = "s3-destinations:read"+ PermS3DestinationsCreate = "s3-destinations:create"+ PermS3DestinationsUpdate = "s3-destinations:update"+ PermS3DestinationsDelete = "s3-destinations:delete"+ PermS3DestinationsTest = "s3-destinations:test"+ PermS3DestinationsSync = "s3-destinations:sync"++ // PermSystemBackupsRead and siblings gate Arcane's own system backups.+ // Every route additionally requires a global admin; the dedicated grants+ // exist so admin-owned scoped API keys can be limited to this surface+ // instead of carrying settings:read/write. Restore is separate because it+ // replaces the whole instance.+ PermSystemBackupsRead = "system-backups:read"+ PermSystemBackupsManage = "system-backups:manage"+ PermSystemBackupsRestore = "system-backups:restore"+ // PermSystemBackupsRecoveryKey is separate from manage: regenerating the+ // key invalidates access to the existing repositories.+ PermSystemBackupsRecoveryKey = "system-backups:recovery-key"+PermEventsRead = "events:read"PermEventsDelete = "events:delete"PermCustomizeManage = "customize:manage"@@ -293,6 +316,8 @@ func BuiltInEditorPermissions() []string {PermTemplatesList, PermTemplatesRead, PermTemplatesCreate, PermTemplatesUpdate, PermTemplatesDelete,PermVariablesRead, PermVariablesCreate, PermVariablesUpdate, PermVariablesDelete, PermVariablesSync,PermGitReposList, PermGitReposRead,+ // Editor can run volume backups, so it needs to pick a destination.+ PermS3DestinationsList, PermS3DestinationsRead,PermEventsRead,// Full env-scoped Docker managementPermContainersList, PermContainersRead, PermContainersLogs, PermContainersCreate, PermContainersStart, PermContainersStop, PermContainersRestart, PermContainersRedeploy, PermContainersKill, PermContainersPause, PermContainersDelete, PermContainersExec, PermContainersAutoUpdate,@@ -351,6 +376,7 @@ func BuiltInViewerPermissions() []string {PermTemplatesList, PermTemplatesRead,PermVariablesRead,PermGitReposList, PermGitReposRead,+ PermS3DestinationsList, PermS3DestinationsRead,PermEventsRead,PermContainersList, PermContainersRead, PermContainersLogs,PermProjectsList, PermProjectsRead, PermProjectsLogs,diff --git a/backend/pkg/libarcane/activity/handler.go b/backend/pkg/libarcane/activity/handler.goindex 2f7abd111..e2827f5a5 100644--- a/backend/pkg/libarcane/activity/handler.go+++ b/backend/pkg/libarcane/activity/handler.go@@ -26,6 +26,16 @@ import (// cancelled (rather than failed) terminal status.const ErrCanceled = errors.Sentinel("activity cancelled by user")+type handlerActivityIDContextKey struct{}++func IDFromContext(ctx context.Context) string {+ if ctx == nil {+ return ""+ }+ activityID, _ := ctx.Value(handlerActivityIDContextKey{}).(string)+ return strings.TrimSpace(activityID)+}+// cancelledMessage is the latest-message recorded when work is cancelled.const cancelledMessage = "Cancelled by user"@@ -125,6 +135,7 @@ func StartHandlerActivity(if tracker, ok := activityService.(Tracker); ok {workCtx = tracker.Track(ctx, activity.ID)}+ workCtx = context.WithValue(workCtx, handlerActivityIDContextKey{}, activity.ID)return activity.ID, workCtx}diff --git a/backend/pkg/libarcane/edge/commands.go b/backend/pkg/libarcane/edge/commands.goindex a7afc0c9d..04c3c9b84 100644--- a/backend/pkg/libarcane/edge/commands.go+++ b/backend/pkg/libarcane/edge/commands.go@@ -14,6 +14,8 @@ var commandRoutes = []commandRoute{{Method: http.MethodGet, PathPattern: "/api/swarm/node-identity", CommandName: "swarm.node_identity"},{Method: http.MethodPost, PathPattern: "/api/container-registries/sync", CommandName: "container_registry.sync"},{Method: http.MethodPost, PathPattern: "/api/git-repositories/sync", CommandName: "git_repository.sync"},+ {Method: http.MethodPost, PathPattern: "/api/backups/s3/sync", CommandName: "s3_destination.sync"},+ {Method: http.MethodGet, PathPattern: "/api/backups/s3/{id}/in-use", CommandName: "s3_destination.in_use"},{Method: http.MethodGet, PathPattern: "/api/environments/{id}/containers", CommandName: "container.list"},{Method: http.MethodGet, PathPattern: "/api/environments/{id}/containers/counts", CommandName: "container.counts"},@@ -122,9 +124,11 @@ var commandRoutes = []commandRoute{{Method: http.MethodPost, PathPattern: "/api/environments/{id}/volumes/{volumeName}/backups/{backupId}/restore-files", CommandName: "volume.backup.restore_files"},{Method: http.MethodDelete, PathPattern: "/api/environments/{id}/volumes/backups/{backupId}", CommandName: "volume.backup.delete"},{Method: http.MethodGet, PathPattern: "/api/environments/{id}/volumes/backups/{backupId}/download", CommandName: "volume.backup.download"},+ {Method: http.MethodGet, PathPattern: "/api/environments/{id}/volumes/{volumeName}/backup-policy", CommandName: "volume.backup_policy.get"},+ {Method: http.MethodPut, PathPattern: "/api/environments/{id}/volumes/{volumeName}/backup-policy", CommandName: "volume.backup_policy.update"},{Method: http.MethodGet, PathPattern: "/api/environments/{id}/volumes/backups/{backupId}/has-path", CommandName: "volume.backup.has_path"},{Method: http.MethodGet, PathPattern: "/api/environments/{id}/volumes/backups/{backupId}/files", CommandName: "volume.backup.files"},- {Method: http.MethodPost, PathPattern: "/api/environments/{id}/volumes/{volumeName}/backups/upload", CommandName: "volume.backup.upload"},+ {Method: http.MethodPost, PathPattern: "/api/environments/{id}/volumes/backups/{backupId}/upload", CommandName: "volume.backup.upload"},{Method: http.MethodGet, PathPattern: "/api/environments/{id}/builds/browse", CommandName: "build_workspace.browse.list"},{Method: http.MethodGet, PathPattern: "/api/environments/{id}/builds/browse/content", CommandName: "build_workspace.browse.read_file"},diff --git a/backend/pkg/libarcane/rustic/image.go b/backend/pkg/libarcane/rustic/image.gonew file mode 100644index 000000000..c52487117--- /dev/null+++ b/backend/pkg/libarcane/rustic/image.go@@ -0,0 +1,5 @@+// Package rustic provides shared Rustic runtime configuration.+package rustic++// DefaultImage is the official Rustic container image used by backup and recovery flows.+const DefaultImage = "ghcr.io/rustic-rs/rustic:v0.11.2"diff --git a/backend/pkg/libarcane/rustic/run.go b/backend/pkg/libarcane/rustic/run.gonew file mode 100644index 000000000..a42917e20--- /dev/null+++ b/backend/pkg/libarcane/rustic/run.go@@ -0,0 +1,77 @@+package rustic++import (+ "bytes"+ "context"+ "fmt"+ "strings"++ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/volumehelper"+ "github.com/moby/moby/api/pkg/stdcopy"+ "github.com/moby/moby/api/types/container"+ "github.com/moby/moby/api/types/mount"+ "github.com/moby/moby/client"+)++// Run executes one Rustic command in an ephemeral container built from+// DefaultImage and returns its trimmed stdout. It is the single Rustic+// invocation contract, shared by the backup engine and the detached recovery+// helper; the image must already be present. The standard RUSTIC_* variables+// are appended to environment, and networkMode is applied verbatim ("" keeps+// the default network).+func Run(ctx context.Context, dockerClient *client.Client, password string, command, environment []string, mounts []mount.Mount, networkMode container.NetworkMode) (string, error) {+ env := append([]string{}, environment...)+ env = append(env,+ "RUSTIC_PASSWORD="+password,+ "RUSTIC_NO_PROGRESS=true",+ "RUSTIC_LOG_LEVEL=error",+ )+ hostConfig := volumehelper.HostConfig(DefaultImage, nil, mounts)+ hostConfig.AutoRemove = false+ hostConfig.NetworkMode = networkMode+ created, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{+ Config: &container.Config{+ Image: DefaultImage,+ Cmd: command,+ Env: env,+ Labels: volumehelper.Labels(),+ },+ HostConfig: hostConfig,+ })+ if err != nil {+ return "", fmt.Errorf("failed to create Rustic container: %w", err)+ }+ defer func() {+ _, _ = dockerClient.ContainerRemove(context.WithoutCancel(ctx), created.ID, volumehelper.RemoveOptions())+ }()+ if _, err := dockerClient.ContainerStart(ctx, created.ID, client.ContainerStartOptions{}); err != nil {+ return "", fmt.Errorf("failed to start Rustic container: %w", err)+ }+ wait := dockerClient.ContainerWait(ctx, created.ID, client.ContainerWaitOptions{Condition: container.WaitConditionNotRunning})+ var status container.WaitResponse+ select {+ case err := <-wait.Error:+ if err != nil {+ return "", fmt.Errorf("failed to wait for Rustic container: %w", err)+ }+ case status = <-wait.Result:+ }+ logs, err := dockerClient.ContainerLogs(ctx, created.ID, client.ContainerLogsOptions{ShowStdout: true, ShowStderr: true})+ if err != nil {+ return "", fmt.Errorf("failed to read Rustic output: %w", err)+ }+ defer func() { _ = logs.Close() }()+ var stdout bytes.Buffer+ var stderr bytes.Buffer+ if _, err := stdcopy.StdCopy(&stdout, &stderr, logs); err != nil {+ return "", fmt.Errorf("failed to decode Rustic output: %w", err)+ }+ if status.StatusCode != 0 {+ message := strings.TrimSpace(stderr.String())+ if message == "" {+ message = strings.TrimSpace(stdout.String())+ }+ return "", fmt.Errorf("rustic exited with code %d: %s", status.StatusCode, message)+ }+ return strings.TrimSpace(stdout.String()), nil+}diff --git a/backend/pkg/utils/s3/s3.go b/backend/pkg/utils/s3/s3.gonew file mode 100644index 000000000..3983212ae--- /dev/null+++ b/backend/pkg/utils/s3/s3.go@@ -0,0 +1,254 @@+// Package s3 provides reusable S3 configuration and connectivity helpers. It+// deals in infrastructure configuration only; mapping from API DTOs belongs to+// the backup destination domain.+package s3++import (+ "bytes"+ "context"+ "fmt"+ "io"+ "net/url"+ "path"+ "strings"++ "emperror.dev/errors"++ "github.com/aws/aws-sdk-go-v2/aws"+ awsconfig "github.com/aws/aws-sdk-go-v2/config"+ "github.com/aws/aws-sdk-go-v2/credentials"+ awss3 "github.com/aws/aws-sdk-go-v2/service/s3"+ "github.com/google/uuid"+)++const connectionTestPayload = "arcane-s3-connection-test"++// Configuration contains the credentials and addressing options for an S3 destination.+type Configuration struct {+ ID string+ Name string+ Endpoint string+ Bucket string+ Region string+ AccessKeyID string+ SecretAccessKey string+ Prefix string+ UseSSL bool+ ForcePathStyle bool+}++// Normalized returns a copy with surrounding whitespace and prefix separators removed.+func (c Configuration) Normalized() Configuration {+ c.ID = strings.TrimSpace(c.ID)+ c.Name = strings.TrimSpace(c.Name)+ c.Endpoint = strings.TrimSpace(c.Endpoint)+ c.Bucket = strings.TrimSpace(c.Bucket)+ c.Region = strings.TrimSpace(c.Region)+ c.AccessKeyID = strings.TrimSpace(c.AccessKeyID)+ c.SecretAccessKey = strings.TrimSpace(c.SecretAccessKey)+ c.Prefix = strings.Trim(strings.TrimSpace(c.Prefix), "/")+ return c+}++// ValidateEndpoint checks a custom endpoint: only HTTP or HTTPS with a host,+// no userinfo, query, or fragment. Private and self-hosted hosts are allowed.+func ValidateEndpoint(endpoint string) error {+ endpoint = strings.TrimSpace(endpoint)+ if endpoint == "" {+ return nil+ }+ candidate := endpoint+ if !strings.Contains(candidate, "://") {+ candidate = "https://" + candidate+ }+ parsed, err := url.Parse(candidate)+ if err != nil {+ return fmt.Errorf("invalid S3 endpoint: %w", err)+ }+ if parsed.Scheme != "http" && parsed.Scheme != "https" {+ return errors.New("invalid S3 endpoint: only http and https are supported")+ }+ if parsed.Host == "" {+ return errors.New("invalid S3 endpoint: host is required")+ }+ if parsed.User != nil {+ return errors.New("invalid S3 endpoint: credentials in the URL are not allowed")+ }+ if parsed.RawQuery != "" || parsed.Fragment != "" || parsed.RawFragment != "" {+ return errors.New("invalid S3 endpoint: query strings and fragments are not allowed")+ }+ return nil+}++// Validate verifies the fields required to connect to the configured destination.+func (c Configuration) Validate(requireSecret bool) error {+ c = c.Normalized()+ if c.Name == "" {+ return errors.New("name is required")+ }+ if c.Bucket == "" {+ return errors.New("bucket is required")+ }+ if c.Endpoint == "" && c.Region == "" {+ return errors.New("region is required for AWS S3")+ }+ if err := ValidateEndpoint(c.Endpoint); err != nil {+ return err+ }+ if c.AccessKeyID == "" {+ return errors.New("access key ID is required")+ }+ if requireSecret && c.SecretAccessKey == "" {+ return errors.New("secret access key is required")+ }+ return nil+}++// ConnectionFieldsEqual reports whether two configurations address the same+// service the same way. A change in any of these fields means a stored secret+// may no longer be combined with the configuration.+func ConnectionFieldsEqual(a, b Configuration) bool {+ a, b = a.Normalized(), b.Normalized()+ return a.Endpoint == b.Endpoint &&+ a.Bucket == b.Bucket &&+ a.Region == b.Region &&+ a.AccessKeyID == b.AccessKeyID &&+ a.UseSSL == b.UseSSL &&+ a.ForcePathStyle == b.ForcePathStyle+}++// EndpointURL returns the normalized endpoint URL, applying UseSSL to custom endpoints.+func (c Configuration) EndpointURL() string {+ endpoint := strings.TrimSpace(c.Endpoint)+ if endpoint == "" {+ return ""+ }+ endpoint = strings.TrimPrefix(strings.TrimPrefix(endpoint, "https://"), "http://")+ scheme := "http://"+ if c.UseSSL {+ scheme = "https://"+ }+ return scheme + strings.TrimRight(endpoint, "/")+}++// RusticEnvironment builds the environment used by Rustic's OpenDAL S3 backend.+func (c Configuration) RusticEnvironment(rootParts ...string) []string {+ c = c.Normalized()+ repositoryParts := append([]string{"/", c.Prefix}, rootParts...)+ region := c.Region+ if region == "" {+ region = "auto"+ }+ environment := []string{+ "RUSTIC_REPOSITORY=opendal:s3",+ "RUSTIC_REPO_OPT_BUCKET=" + c.Bucket,+ "RUSTIC_REPO_OPT_ROOT=" + path.Join(repositoryParts...),+ "AWS_ACCESS_KEY_ID=" + c.AccessKeyID,+ "AWS_SECRET_ACCESS_KEY=" + c.SecretAccessKey,+ "AWS_EC2_METADATA_DISABLED=true",+ }+ if endpoint := c.EndpointURL(); endpoint != "" {+ environment = append(environment, "RUSTIC_REPO_OPT_ENDPOINT="+endpoint)+ }+ environment = append(environment,+ "RUSTIC_REPO_OPT_REGION="+region,+ "AWS_REGION="+region,+ )+ if !c.ForcePathStyle {+ environment = append(environment, "RUSTIC_REPO_OPT_ENABLE_VIRTUAL_HOST_STYLE=true")+ }+ return environment+}++// TestConnection verifies upload, download, and delete access for a destination.+func TestConnection(ctx context.Context, configuration Configuration) (err error) {+ configuration = configuration.Normalized()+ if err := configuration.Validate(true); err != nil {+ return err+ }+ client, err := newClientInternal(ctx, configuration)+ if err != nil {+ return fmt.Errorf("failed to upload S3 connection test object: %w", err)+ }+ payload := []byte(connectionTestPayload)+ remoteKey := path.Join(configuration.Prefix, ".arcane-connection-test-"+uuid.NewString())+ if err := putObjectInternal(ctx, client, configuration.Bucket, remoteKey, payload); err != nil {+ return fmt.Errorf("failed to upload S3 connection test object: %w", err)+ }++ deleted := false+ defer func() {+ if deleted {+ return+ }+ if cleanupErr := deleteObjectInternal(context.WithoutCancel(ctx), client, configuration.Bucket, remoteKey); cleanupErr != nil {+ err = errors.Combine(err, fmt.Errorf("failed to clean up S3 connection test object: %w", cleanupErr))+ }+ }()++ downloaded, err := getObjectInternal(ctx, client, configuration.Bucket, remoteKey)+ if err != nil {+ return fmt.Errorf("failed to download S3 connection test object: %w", err)+ }+ if !bytes.Equal(downloaded, payload) {+ return errors.New("S3 connection test object contents did not match")+ }+ if err := deleteObjectInternal(ctx, client, configuration.Bucket, remoteKey); err != nil {+ return fmt.Errorf("failed to delete S3 connection test object: %w", err)+ }+ deleted = true+ return nil+}++func newClientInternal(ctx context.Context, configuration Configuration) (*awss3.Client, error) {+ region := configuration.Region+ if region == "" {+ region = "us-east-1"+ }+ awsConfiguration, err := awsconfig.LoadDefaultConfig(ctx,+ awsconfig.WithRegion(region),+ awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(configuration.AccessKeyID, configuration.SecretAccessKey, "")),+ )+ if err != nil {+ return nil, err+ }+ endpoint := configuration.EndpointURL()+ return awss3.NewFromConfig(awsConfiguration, func(options *awss3.Options) {+ options.UsePathStyle = configuration.ForcePathStyle+ if endpoint != "" {+ options.BaseEndpoint = aws.String(endpoint)+ }+ }), nil+}++func putObjectInternal(ctx context.Context, client *awss3.Client, bucket, key string, payload []byte) error {+ _, err := client.PutObject(ctx, &awss3.PutObjectInput{+ Bucket: aws.String(bucket),+ Key: aws.String(key),+ Body: bytes.NewReader(payload),+ ContentLength: new(int64(len(payload))),+ ContentType: aws.String("application/gzip"),+ })+ return err+}++func getObjectInternal(ctx context.Context, client *awss3.Client, bucket, key string) ([]byte, error) {+ output, err := client.GetObject(ctx, &awss3.GetObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)})+ if err != nil {+ return nil, err+ }+ payload, readErr := io.ReadAll(output.Body)+ closeErr := output.Body.Close()+ if readErr != nil {+ return nil, errors.Combine(readErr, closeErr)+ }+ if closeErr != nil {+ return nil, closeErr+ }+ return payload, nil+}++func deleteObjectInternal(ctx context.Context, client *awss3.Client, bucket, key string) error {+ _, err := client.DeleteObject(ctx, &awss3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)})+ return err+}diff --git a/backend/pkg/utils/s3/s3_test.go b/backend/pkg/utils/s3/s3_test.gonew file mode 100644index 000000000..4b459c440--- /dev/null+++ b/backend/pkg/utils/s3/s3_test.go@@ -0,0 +1,53 @@+package s3++import (+ "testing"++ "github.com/stretchr/testify/require"+)++func TestConfigurationValidateRequiresRegionOnlyForAWS(t *testing.T) {+ configuration := Configuration{Name: "Offsite", Bucket: "backups", AccessKeyID: "access-key", SecretAccessKey: "secret-key"}+ require.EqualError(t, configuration.Validate(true), "region is required for AWS S3")++ configuration.Endpoint = "s3.example.com"+ require.NoError(t, configuration.Validate(true))+}++func TestValidateEndpoint(t *testing.T) {+ require.NoError(t, ValidateEndpoint(""))+ require.NoError(t, ValidateEndpoint("minio.internal:9000"))+ require.NoError(t, ValidateEndpoint("http://10.0.0.5:9000"))+ require.NoError(t, ValidateEndpoint("https://s3.example.com"))++ require.ErrorContains(t, ValidateEndpoint("ftp://s3.example.com"), "only http and https")+ require.ErrorContains(t, ValidateEndpoint("https://user:[email protected]"), "credentials in the URL")+ require.ErrorContains(t, ValidateEndpoint("https://s3.example.com?x=1"), "query strings and fragments")+ require.ErrorContains(t, ValidateEndpoint("https://s3.example.com#frag"), "query strings and fragments")+ require.ErrorContains(t, ValidateEndpoint("https://"), "host is required")+}++func TestConfigurationRusticEnvironment(t *testing.T) {+ configuration := Configuration{+ Endpoint: "http://s3.example.com/",+ Bucket: "backups",+ AccessKeyID: "access-key",+ SecretAccessKey: "secret-key",+ Prefix: "/production/",+ UseSSL: true,+ ForcePathStyle: false,+ }++ require.Equal(t, []string{+ "RUSTIC_REPOSITORY=opendal:s3",+ "RUSTIC_REPO_OPT_BUCKET=backups",+ "RUSTIC_REPO_OPT_ROOT=/production/arcane-volume-backups/instance-1",+ "AWS_ACCESS_KEY_ID=access-key",+ "AWS_SECRET_ACCESS_KEY=secret-key",+ "AWS_EC2_METADATA_DISABLED=true",+ "RUSTIC_REPO_OPT_ENDPOINT=https://s3.example.com",+ "RUSTIC_REPO_OPT_REGION=auto",+ "AWS_REGION=auto",+ "RUSTIC_REPO_OPT_ENABLE_VIRTUAL_HOST_STYLE=true",+ }, configuration.RusticEnvironment("arcane-volume-backups", "instance-1"))+}diff --git a/backend/pkg/utils/schedule/cron.go b/backend/pkg/utils/schedule/cron.gonew file mode 100644index 000000000..30ac6f37e--- /dev/null+++ b/backend/pkg/utils/schedule/cron.go@@ -0,0 +1,26 @@+// Package schedule provides reusable schedule validation.+package schedule++import (+ "fmt"+ "strings"++ "github.com/robfig/cron/v3"+)++// NormalizeSixField validates and normalizes a six-field cron schedule.+func NormalizeSixField(value, subject string) (string, error) {+ fields := strings.Fields(value)+ if len(fields) == 0 {+ return "", fmt.Errorf("%s schedule is required", subject)+ }+ if len(fields) != 6 {+ return "", fmt.Errorf("invalid %s schedule %q: expected six fields", subject, strings.TrimSpace(value))+ }+ normalized := strings.Join(fields, " ")+ parser := cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor)+ if _, err := parser.Parse(normalized); err != nil {+ return "", fmt.Errorf("invalid %s schedule %q: %w", subject, normalized, err)+ }+ return normalized, nil+}diff --git a/backend/resources/migrations/postgres/073_add_backup_support.sql b/backend/resources/migrations/postgres/073_add_backup_support.sqlnew file mode 100644index 000000000..d19148cea--- /dev/null+++ b/backend/resources/migrations/postgres/073_add_backup_support.sql@@ -0,0 +1,112 @@+-- +goose Up+CREATE TABLE s3_destinations (+ id TEXT PRIMARY KEY,+ name TEXT NOT NULL UNIQUE,+ endpoint TEXT,+ bucket TEXT NOT NULL,+ region TEXT NOT NULL,+ access_key_id TEXT NOT NULL,+ secret_access_key TEXT NOT NULL,+ prefix TEXT,+ use_ssl BOOLEAN NOT NULL DEFAULT TRUE,+ force_path_style BOOLEAN NOT NULL DEFAULT TRUE,+ created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,+ updated_at TIMESTAMPTZ+);++ALTER TABLE volume_backups ADD COLUMN status TEXT NOT NULL DEFAULT 'succeeded';+ALTER TABLE volume_backups ADD COLUMN trigger TEXT NOT NULL DEFAULT 'manual';+ALTER TABLE volume_backups ADD COLUMN s3_destination_id TEXT;+ALTER TABLE volume_backups ADD COLUMN error TEXT;+ALTER TABLE volume_backups ADD COLUMN destination TEXT NOT NULL DEFAULT 'local';+ALTER TABLE volume_backups ADD COLUMN local_snapshot_id TEXT;+ALTER TABLE volume_backups ADD COLUMN remote_snapshot_id TEXT;+ALTER TABLE volume_backups ADD COLUMN policy_id TEXT;+-- Pre-existing rows are tar.gz archive backups; new Rustic-snapshot rows set 'rustic'.+ALTER TABLE volume_backups ADD COLUMN format TEXT NOT NULL DEFAULT 'archive';+CREATE INDEX idx_volume_backups_s3_destination_id ON volume_backups(s3_destination_id);+CREATE INDEX idx_volume_backups_policy_id ON volume_backups(policy_id);++CREATE TABLE volume_backup_policies (+ id TEXT PRIMARY KEY,+ volume_name TEXT NOT NULL,+ enabled BOOLEAN NOT NULL DEFAULT FALSE,+ schedule TEXT NOT NULL,+ retention_count INTEGER NOT NULL DEFAULT 7,+ stop_containers BOOLEAN NOT NULL DEFAULT FALSE,+ local_enabled BOOLEAN NOT NULL DEFAULT TRUE,+ s3_enabled BOOLEAN NOT NULL DEFAULT FALSE,+ s3_destination_id TEXT,+ created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,+ updated_at TIMESTAMPTZ+);+CREATE INDEX idx_volume_backup_policies_volume_name ON volume_backup_policies(volume_name);+CREATE INDEX idx_volume_backup_policies_s3_destination_id ON volume_backup_policies(s3_destination_id);++CREATE TABLE system_backup_runs (+ id TEXT PRIMARY KEY,+ size BIGINT NOT NULL DEFAULT 0,+ created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,+ updated_at TIMESTAMPTZ,+ status TEXT NOT NULL,+ trigger TEXT NOT NULL,+ destination TEXT NOT NULL,+ local_snapshot_id TEXT,+ remote_snapshot_id TEXT,+ s3_destination_id TEXT,+ policy_id TEXT,+ error TEXT+);+CREATE INDEX idx_system_backup_runs_s3_destination_id ON system_backup_runs(s3_destination_id);+CREATE INDEX idx_system_backup_runs_policy_id ON system_backup_runs(policy_id);++CREATE TABLE system_backup_policies (+ id TEXT PRIMARY KEY,+ enabled BOOLEAN NOT NULL DEFAULT FALSE,+ schedule TEXT NOT NULL,+ retention_count INTEGER NOT NULL DEFAULT 7,+ local_enabled BOOLEAN NOT NULL DEFAULT TRUE,+ s3_enabled BOOLEAN NOT NULL DEFAULT FALSE,+ s3_destination_id TEXT,+ created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,+ updated_at TIMESTAMPTZ+);+CREATE INDEX idx_system_backup_policies_s3_destination_id ON system_backup_policies(s3_destination_id);++CREATE TABLE system_backup_recovery_config (+ id TEXT PRIMARY KEY,+ encrypted_recovery_key TEXT NOT NULL,+ created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,+ updated_at TIMESTAMPTZ+);++-- +goose Down+-- Refuse to downgrade while Rustic-format backups exist: the pre-073 schema cannot+-- represent them and their snapshots would become unreachable.+-- +goose StatementBegin+DO $$+BEGIN+ IF EXISTS (SELECT 1 FROM volume_backups WHERE format <> 'archive') THEN+ RAISE EXCEPTION 'downgrade blocked: Rustic-format volume backups exist; delete them first or stay on this version';+ END IF;+END+$$;+-- +goose StatementEnd++DROP TABLE system_backup_recovery_config;+DROP TABLE system_backup_policies;+DROP TABLE system_backup_runs;+DROP TABLE volume_backup_policies;++DROP INDEX idx_volume_backups_policy_id;+DROP INDEX idx_volume_backups_s3_destination_id;+ALTER TABLE volume_backups DROP COLUMN format;+ALTER TABLE volume_backups DROP COLUMN policy_id;+ALTER TABLE volume_backups DROP COLUMN remote_snapshot_id;+ALTER TABLE volume_backups DROP COLUMN local_snapshot_id;+ALTER TABLE volume_backups DROP COLUMN destination;+ALTER TABLE volume_backups DROP COLUMN error;+ALTER TABLE volume_backups DROP COLUMN s3_destination_id;+ALTER TABLE volume_backups DROP COLUMN trigger;+ALTER TABLE volume_backups DROP COLUMN status;+DROP TABLE s3_destinations;diff --git a/backend/resources/migrations/sqlite/073_add_backup_support.sql b/backend/resources/migrations/sqlite/073_add_backup_support.sqlnew file mode 100644index 000000000..cde9d6cb2--- /dev/null+++ b/backend/resources/migrations/sqlite/073_add_backup_support.sql@@ -0,0 +1,108 @@+-- +goose Up+CREATE TABLE s3_destinations (+ id TEXT PRIMARY KEY,+ name TEXT NOT NULL UNIQUE,+ endpoint TEXT,+ bucket TEXT NOT NULL,+ region TEXT NOT NULL,+ access_key_id TEXT NOT NULL,+ secret_access_key TEXT NOT NULL,+ prefix TEXT,+ use_ssl INTEGER NOT NULL DEFAULT 1,+ force_path_style INTEGER NOT NULL DEFAULT 1,+ created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,+ updated_at DATETIME+);++ALTER TABLE volume_backups ADD COLUMN status TEXT NOT NULL DEFAULT 'succeeded';+ALTER TABLE volume_backups ADD COLUMN trigger TEXT NOT NULL DEFAULT 'manual';+ALTER TABLE volume_backups ADD COLUMN s3_destination_id TEXT;+ALTER TABLE volume_backups ADD COLUMN error TEXT;+ALTER TABLE volume_backups ADD COLUMN destination TEXT NOT NULL DEFAULT 'local';+ALTER TABLE volume_backups ADD COLUMN local_snapshot_id TEXT;+ALTER TABLE volume_backups ADD COLUMN remote_snapshot_id TEXT;+ALTER TABLE volume_backups ADD COLUMN policy_id TEXT;+-- Pre-existing rows are tar.gz archive backups; new Rustic-snapshot rows set 'rustic'.+ALTER TABLE volume_backups ADD COLUMN format TEXT NOT NULL DEFAULT 'archive';+CREATE INDEX idx_volume_backups_s3_destination_id ON volume_backups(s3_destination_id);+CREATE INDEX idx_volume_backups_policy_id ON volume_backups(policy_id);++CREATE TABLE volume_backup_policies (+ id TEXT PRIMARY KEY,+ volume_name TEXT NOT NULL,+ enabled INTEGER NOT NULL DEFAULT 0,+ schedule TEXT NOT NULL,+ retention_count INTEGER NOT NULL DEFAULT 7,+ stop_containers INTEGER NOT NULL DEFAULT 0,+ local_enabled INTEGER NOT NULL DEFAULT 1,+ s3_enabled INTEGER NOT NULL DEFAULT 0,+ s3_destination_id TEXT,+ created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,+ updated_at DATETIME+);+CREATE INDEX idx_volume_backup_policies_volume_name ON volume_backup_policies(volume_name);+CREATE INDEX idx_volume_backup_policies_s3_destination_id ON volume_backup_policies(s3_destination_id);++CREATE TABLE system_backup_runs (+ id TEXT PRIMARY KEY,+ size INTEGER NOT NULL DEFAULT 0,+ created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,+ updated_at DATETIME,+ status TEXT NOT NULL,+ trigger TEXT NOT NULL,+ destination TEXT NOT NULL,+ local_snapshot_id TEXT,+ remote_snapshot_id TEXT,+ s3_destination_id TEXT,+ policy_id TEXT,+ error TEXT+);+CREATE INDEX idx_system_backup_runs_s3_destination_id ON system_backup_runs(s3_destination_id);+CREATE INDEX idx_system_backup_runs_policy_id ON system_backup_runs(policy_id);++CREATE TABLE system_backup_policies (+ id TEXT PRIMARY KEY,+ enabled INTEGER NOT NULL DEFAULT 0,+ schedule TEXT NOT NULL,+ retention_count INTEGER NOT NULL DEFAULT 7,+ local_enabled INTEGER NOT NULL DEFAULT 1,+ s3_enabled INTEGER NOT NULL DEFAULT 0,+ s3_destination_id TEXT,+ created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,+ updated_at DATETIME+);+CREATE INDEX idx_system_backup_policies_s3_destination_id ON system_backup_policies(s3_destination_id);++CREATE TABLE system_backup_recovery_config (+ id TEXT PRIMARY KEY,+ encrypted_recovery_key TEXT NOT NULL,+ created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,+ updated_at DATETIME+);++-- +goose Down+-- Refuse to downgrade while Rustic-format backups exist: the pre-073 schema cannot+-- represent them and their snapshots would become unreachable. The INSERT below+-- violates NOT NULL only when such a row exists, aborting the downgrade.+CREATE TABLE _downgrade_blocked_rustic_backups_exist (reason TEXT NOT NULL);+INSERT INTO _downgrade_blocked_rustic_backups_exist (reason)+SELECT NULL FROM volume_backups WHERE format <> 'archive' LIMIT 1;+DROP TABLE _downgrade_blocked_rustic_backups_exist;++DROP TABLE system_backup_recovery_config;+DROP TABLE system_backup_policies;+DROP TABLE system_backup_runs;+DROP TABLE volume_backup_policies;++DROP INDEX idx_volume_backups_policy_id;+DROP INDEX idx_volume_backups_s3_destination_id;+ALTER TABLE volume_backups DROP COLUMN format;+ALTER TABLE volume_backups DROP COLUMN policy_id;+ALTER TABLE volume_backups DROP COLUMN remote_snapshot_id;+ALTER TABLE volume_backups DROP COLUMN local_snapshot_id;+ALTER TABLE volume_backups DROP COLUMN destination;+ALTER TABLE volume_backups DROP COLUMN error;+ALTER TABLE volume_backups DROP COLUMN s3_destination_id;+ALTER TABLE volume_backups DROP COLUMN trigger;+ALTER TABLE volume_backups DROP COLUMN status;+DROP TABLE s3_destinations;diff --git a/frontend/messages/en.json b/frontend/messages/en.jsonindex 1bb45a6c6..5aa551eb4 100644--- a/frontend/messages/en.json+++ b/frontend/messages/en.json@@ -56,6 +56,8 @@"common_unused": "Unused","common_total": "Total","common_running": "Running",+ "common_configured": "Configured",+ "common_not_configured": "Not configured","common_stopped": "Stopped","common_size": "Size","common_overview": "Overview",@@ -922,6 +924,8 @@"volumes_workspace_restore_staged": "The selected backup restore will be applied when you save.","volumes_workspace_save_success": "Volume workspace updated successfully.","volumes_backups_title": "Volume Backups",+ "volume_backups_remove_selected_title": "Remove {count} Volume Backup(s)",+ "volume_backups_remove_selected_message": "Are you sure you want to remove the selected volume backup(s)? Arcane will also attempt to remove their local and S3 copies. This action cannot be undone.","volumes_backups_restore": "Restore","volumes_remove_in_use_warning": "Warning: This volume is currently in use by containers. Forcing removal may cause data loss or container issues.","volumes_containers_using_title": "Containers Using This Volume",@@ -942,10 +946,11 @@"volumes_backup_delete_confirm": "Are you sure you want to delete this backup?","volumes_backup_restore_title": "Restore Volume","volumes_backup_restore_message": "This will replace ALL data in the volume \"{volumeName}\" with the contents of this backup.\n\nA safety backup will be created automatically before restoring, so you can undo this operation if needed.",- "volumes_backup_restore_in_use_warning": "\n\n⚠️ Warning: This volume is currently in use by {count} container(s). Restoring while containers are running may cause data corruption. Stop the containers first or use \"Restore files\" to restore specific files.",+ "volumes_backup_restore_in_use_warning": "\n\nThis volume is currently in use by {count} container(s). Arcane will stop them before creating the safety backup and restart them after the restore.","volumes_backup_restore_success": "Volume restored successfully. A safety backup was created before restoring.","volumes_backup_restore_files_success": "{count} file(s) restored successfully. A safety backup was created before restoring.","volumes_backup_safety_info": "A safety backup will be created automatically before restoring, so you can undo this operation if needed.",+ "volume_backup_restore_files_lifecycle_info": "Arcane will stop containers using this volume, create a safety backup, restore the selected files, and then restart the containers.","volumes_backup_overwrite_warning": "Selected files will overwrite existing files at those paths.","volumes_backup_file_restore_success": "File restored successfully. A safety backup was created before restoring.","volumes_symlink_external_tooltip": "This symlink points to a location outside the volume",@@ -1895,8 +1900,6 @@"git_sync_pre_deploy_acknowledgement": "Arcane runs this script on every deploy. Anyone who can push to the repo can change what it does.","git_sync_pre_deploy_managed_hint": "Managed by administrators","git_sync_pre_deploy_swarm_unsupported": "Pre-deploy scripts are only available for project syncs.",- "git_sync_pre_deploy_status_configured": "Configured",- "git_sync_pre_deploy_status_none": "Not configured","lifecycle_indicator_tooltip": "Runs a pre-deploy script: {path}","lifecycle_inline_runner_summary": "(image: {image}, network: {network})","common_syncing": "Syncing...",@@ -3122,6 +3125,7 @@"environments_connect_error": "Failed to Connect to Environment","common_access_denied": "Access denied","volumes_backup_restore_desc": "Select files from this backup to restore.",+ "volumes_backup_load_failed": "Failed to load backups","volumes_workspace_backup_restore_desc": "Select a backup to restore the chosen file.","editor_commands_desc": "Run editor commands","environments_swarm_stack_source_label": "Swarm Stack Sources Directory",@@ -3135,5 +3139,132 @@"containers_status_removing": "Removing","paused": "Paused","containers_status_exited": "Exited",- "containers_status_dead": "Dead"+ "containers_status_dead": "Dead",+ "backups_schedule_description": "Six-field cron expression interpreted in Arcane's configured timezone.",+ "backups_retention_label": "Backups to keep",+ "backups_retention_description": "Applied separately to local and remote backups. Set to 0 to keep all backups.",+ "backups_s3_endpoint_label": "Endpoint",+ "backups_s3_endpoint_description": "Leave empty for AWS S3, or enter a custom provider endpoint.",+ "backups_s3_bucket_label": "Bucket",+ "backups_s3_region_label": "Region",+ "backups_s3_region_description": "Required for AWS S3. Optional for custom S3-compatible endpoints; Arcane uses us-east-1 for request signing when left empty.",+ "backups_s3_access_key_label": "Access key ID",+ "backups_s3_secret_key_label": "Secret access key",+ "backups_s3_prefix_label": "Path prefix",+ "backups_s3_prefix_description": "Optional folder-like prefix inside the bucket.",+ "backups_s3_ssl_label": "Use SSL",+ "backups_s3_ssl_description": "Use HTTPS when connecting to the custom endpoint.",+ "backups_s3_path_style_label": "Use path-style addressing",+ "backups_s3_path_style_description": "Recommended for MinIO and many S3-compatible providers.",+ "backups_s3_bucket_required": "Bucket is required when S3 upload is enabled.",+ "backups_s3_region_required": "Region is required when using AWS S3.",+ "backups_s3_access_key_required": "Access key ID is required when S3 upload is enabled.",+ "backups_s3_secret_key_required": "Secret access key is required when S3 upload is enabled.",+ "backups_trigger_manual": "Manual",+ "backups_trigger_scheduled": "Scheduled",+ "backups_trigger_safety": "Pre-restore safety",+ "backups_destination_s3": "S3",+ "backups_destination_local_s3": "Local + S3",+ "s3_destinations_title": "S3 Destinations",+ "s3_destinations_description": "Create and manage reusable S3-compatible storage destinations for Arcane and volume backups.",+ "s3_destinations_load_failed": "Failed to load S3 destinations",+ "s3_destination_add": "Add destination",+ "s3_destination_add_title": "Add S3 destination",+ "s3_destination_edit_title": "Edit S3 destination",+ "s3_destination_dialog_description": "Configure an AWS S3, Backblaze B2, MinIO, Hetzner, or compatible storage destination.",+ "s3_destination_secret_keep_description": "Leave empty to preserve the saved encrypted secret.",+ "s3_destination_aws_default": "AWS S3 default",+ "s3_destination_created": "Created {name}",+ "s3_destination_updated": "Updated {name}",+ "s3_destination_save_failed": "Failed to save S3 destination",+ "s3_destination_test_success": "Connection to {name} succeeded",+ "s3_destination_test_failed": "Failed to connect to {name}",+ "s3_destination_test_required": "Test this connection successfully before saving.",+ "s3_destination_test_verified": "Connection verified. You can now save this destination.",+ "s3_destination_delete_message": "This destination can only be deleted when no backup configuration or retained remote backup uses it.",+ "s3_destination_deleted": "Deleted {name}",+ "s3_destination_delete_failed": "Failed to delete S3 destination",+ "volume_backup_policy_title": "Automatic volume backups",+ "volume_backup_policy_description": "Add independent schedules with their own destination, retention, and container shutdown settings.",+ "volume_backup_encryption_note": "Volume backups are encrypted automatically with Arcane's internal encryption key; no separate recovery key is required. Keep the original encryption key to restore these backups on another Arcane instance.",+ "volume_backup_policy_configure": "Configure schedules",+ "volume_backup_policy_saved": "Volume backup schedules saved",+ "volume_backup_policy_save_failed": "Failed to save volume backup policy",+ "backups_enabled": "Enable scheduled backups",+ "volume_backup_policy_enabled_description": "Create backups of this volume according to this cron schedule.",+ "volume_backup_add_schedule": "Add schedule",+ "backups_remove_schedule": "Remove schedule",+ "volume_backup_schedule_removed": "Volume backup schedule removed",+ "volume_backup_retention_summary": "Keep {count}",+ "volume_backup_retention_all": "Keep all",+ "volume_backup_containers_stopped": "Stops containers",+ "volume_backup_containers_running": "Containers stay running",+ "volume_backup_schedule_number": "Cron expression {number}",+ "volume_backup_retention_invalid": "Backups to keep must be a whole number between 0 and 3650.",+ "volume_backup_stop_containers": "Stop containers during backup",+ "volume_backup_stop_containers_description": "Stop running containers that use this volume while Rustic creates the snapshot, then start them again.",+ "backups_destination_label": "Backup destination",+ "volume_backup_destination_description": "Keep backups locally, remotely in a saved S3 destination, or in both places.",+ "volume_backup_destination_local": "Local only",+ "volume_backup_destination_local_description": "Keep Rustic snapshots in Arcane's local backup storage.",+ "volume_backup_destination_s3": "S3 only",+ "volume_backup_destination_s3_description": "Store Rustic snapshots directly in the selected S3 destination. Pre-restore safety backups remain local.",+ "volume_backup_destination_local_s3_description": "Keep Rustic snapshots both locally and in the selected S3 destination.",+ "volume_backup_s3_destination_label": "S3 destination",+ "volume_backup_s3_destination_description": "Select the saved destination used for scheduled and manual volume backup uploads.",+ "volume_backup_choose_s3_destination": "Choose S3 destination",+ "volume_backup_choose_s3_destination_description": "Select where this on-demand backup should be stored remotely.",+ "volume_backup_no_s3_destinations": "No S3 destinations are configured. Add one in Settings before creating an S3 backup.",+ "volume_backup_s3_destination_required": "Select an S3 destination.",+ "volume_backup_destination_unavailable": "The selected S3 destination is no longer available.",+ "volume_backup_status_succeeded": "Succeeded",+ "volume_backup_trigger": "Trigger",+ "volume_backup_remote_snapshot": "S3 snapshot",+ "volume_backup_not_uploaded": "Not uploaded",+ "backups_upload_s3": "Upload to S3",+ "backups_upload_s3_success": "Backup uploaded to S3",+ "backups_upload_s3_failed": "Failed to upload backup to S3",+ "system_backups_title": "Backups",+ "system_backups_description": "Create encrypted recovery snapshots of Arcane's database, files, settings, and runtime secrets.",+ "system_backups_id": "Backup ID",+ "system_backups_configure": "Configure schedule",+ "system_backups_schedules": "Backup schedules",+ "system_backups_add_schedule": "Create schedule",+ "system_backups_schedule_removed": "System backup schedule removed.",+ "system_backups_no_schedules": "No system backup schedules configured yet.",+ "system_backups_schedule_description": "Each schedule has its own destination and retention policy.",+ "system_backups_dialog_description": "The separate recovery key encrypts this Rustic repository and is required to restore it on a fresh Arcane instance.",+ "system_backups_recovery_key": "Recovery key",+ "system_backups_recovery_key_saved_description": "Leave empty to use the configured recovery key.",+ "system_backups_recovery_key_required": "Enter the generated recovery key (8 groups of 6 characters).",+ "system_backups_recovery_key_needed": "Set up a recovery key before creating system backups.",+ "system_backups_reset_recovery_key": "Reset recovery key",+ "system_backups_recovery_key_alert": "Save this key outside Arcane now. It will not be shown again, and without it your system backups cannot be restored.",+ "system_backups_recovery_key_enter_description": "Enter the recovery key that protects this backup.",+ "system_backups_recovery_key_schedule_required": "Configure a recovery key before enabling this schedule.",+ "system_backups_setup_recovery_key": "Set up recovery key",+ "system_backups_recovery_key_saved": "Recovery key configured.",+ "system_backups_recovery_key_save_failed": "Failed to configure the recovery key.",+ "system_backups_enabled_description": "Run an encrypted Arcane recovery backup on the configured cron schedule.",+ "system_backups_created": "Arcane system backup created successfully.",+ "system_backups_create_failed": "Failed to create Arcane system backup.",+ "system_backups_policy_saved": "System backup schedule saved.",+ "system_backups_policy_failed": "Failed to save the system backup schedule.",+ "system_backups_restore_title": "Restore Arcane",+ "system_backups_restore_description": "Arcane will create a safety backup, stop itself, restore all data and runtime secrets, recreate its container, and start the recovered instance.",+ "system_backups_restore_started": "Arcane restore started. This page will disconnect while Arcane restarts.",+ "system_backups_restore_failed": "Failed to start the Arcane restore.",+ "system_backups_upload_title": "Upload system backup",+ "system_backups_delete_title": "Delete system backup?",+ "system_backups_delete_description": "The Rustic snapshot will be removed from every reachable storage destination.",+ "system_backups_deleted": "System backup deleted.",+ "system_backups_delete_failed": "Failed to delete system backup.",+ "system_backups_backup_configuration": "Backup configuration",+ "system_backups_custom_configuration": "Custom destination",+ "system_backups_custom_configuration_description": "Choose a destination for this backup only.",+ "system_backups_discover": "Find S3 backups",+ "system_backups_discover_title": "Find existing S3 backups",+ "system_backups_discover_description": "Open an Arcane system-recovery repository with its separate recovery key and add its snapshots to this table.",+ "system_backups_discovered": "Found {count} new system backup(s).",+ "system_backups_discover_failed": "Failed to discover system backups. Check the destination and recovery key."}diff --git a/frontend/src/lib/components/arcane-table/cells/backup-destination-cell.svelte b/frontend/src/lib/components/arcane-table/cells/backup-destination-cell.sveltenew file mode 100644index 000000000..f6978986f--- /dev/null+++ b/frontend/src/lib/components/arcane-table/cells/backup-destination-cell.svelte@@ -0,0 +1,14 @@+<script lang="ts">+ import { Badge } from '#lib/components/ui/badge';+ import type { BackupRun } from '#lib/types/backup';+ import { backupDestinationLabel, backupDestinationName } from '#lib/utils/backups';++ let { item }: { item: Pick<BackupRun, 'destination' | 's3DestinationId' | 's3DestinationName'> } = $props();+</script>++<div class="flex items-center gap-2">+ <Badge variant={item.destination === 'local' ? 'gray' : 'blue'}>{backupDestinationLabel(item.destination)}</Badge>+ {#if item.destination !== 'local' && backupDestinationName(item)}+ <span class="max-w-48 truncate text-xs text-muted-foreground">{backupDestinationName(item)}</span>+ {/if}+</div>diff --git a/frontend/src/lib/components/arcane-table/cells/backup-size-cell.svelte b/frontend/src/lib/components/arcane-table/cells/backup-size-cell.sveltenew file mode 100644index 000000000..c11941b3b--- /dev/null+++ b/frontend/src/lib/components/arcane-table/cells/backup-size-cell.svelte@@ -0,0 +1,7 @@+<script lang="ts">+ import { bytes } from '#lib/utils/formatting';++ let { size }: { size: number } = $props();+</script>++{bytes.format(size, { unitSeparator: ' ' }) ?? '-'}diff --git a/frontend/src/lib/components/arcane-table/cells/backup-status-cell.svelte b/frontend/src/lib/components/arcane-table/cells/backup-status-cell.sveltenew file mode 100644index 000000000..423b337b8--- /dev/null+++ b/frontend/src/lib/components/arcane-table/cells/backup-status-cell.svelte@@ -0,0 +1,9 @@+<script lang="ts">+ import { Badge } from '#lib/components/ui/badge';+ import type { BackupStatus } from '#lib/types/backup';+ import { backupStatusLabel, backupStatusVariant } from '#lib/utils/backups';++ let { status }: { status: BackupStatus } = $props();+</script>++<Badge variant={backupStatusVariant(status)}>{backupStatusLabel(status)}</Badge>diff --git a/frontend/src/lib/components/arcane-table/cells/backup-trigger-cell.svelte b/frontend/src/lib/components/arcane-table/cells/backup-trigger-cell.sveltenew file mode 100644index 000000000..d5b7de6da--- /dev/null+++ b/frontend/src/lib/components/arcane-table/cells/backup-trigger-cell.svelte@@ -0,0 +1,8 @@+<script lang="ts">+ import type { BackupTrigger } from '#lib/types/backup';+ import { backupTriggerLabel } from '#lib/utils/backups';++ let { trigger }: { trigger: BackupTrigger } = $props();+</script>++{backupTriggerLabel(trigger)}diff --git a/frontend/src/lib/components/backup-policy-card.svelte b/frontend/src/lib/components/backup-policy-card.sveltenew file mode 100644index 000000000..88605ee58--- /dev/null+++ b/frontend/src/lib/components/backup-policy-card.svelte@@ -0,0 +1,67 @@+<script lang="ts">+ import { ArcaneButton } from '#lib/components/arcane-button';+ import { Badge } from '#lib/components/ui/badge';+ import { EditIcon } from '#lib/icons';+ import type { BackupPolicy, BackupStatus } from '#lib/types/backup';+ import { backupDestinationFromFlags, backupDestinationLabel, backupStatusLabel } from '#lib/utils/backups';+ import { formatDateTimeShort } from '#lib/utils/formatting';+ import * as m from '#lib/paraglide/messages.js';++ type CardPolicy = BackupPolicy & {+ s3DestinationName?: string;+ s3Bucket?: string;+ lastRun?: { status: BackupStatus; createdAt: string };+ };++ let {+ policy,+ showStopContainers = false,+ onEdit,+ editDisabled = false+ }: {+ policy: CardPolicy;+ showStopContainers?: boolean;+ onEdit?: () => void;+ editDisabled?: boolean;+ } = $props();++ const s3Name = $derived(policy.s3DestinationName || policy.s3Bucket || policy.s3DestinationId || '');+</script>++<div class="grid min-w-0 grid-cols-[auto_minmax(0,1fr)_auto] items-center gap-x-1.5 gap-y-0.5 rounded-md border px-2 py-1">+ <Badge variant={policy.enabled ? 'green' : 'gray'}>{policy.enabled ? m.common_enabled() : m.common_disabled()}</Badge>+ <code class="truncate">{policy.schedule}</code>+ <div class="flex items-center gap-1.5 justify-self-end">+ <Badge variant="blue">{backupDestinationLabel(backupDestinationFromFlags(policy.localEnabled, policy.s3Enabled))}</Badge>+ {#if onEdit}+ <ArcaneButton+ action="edit"+ size="icon"+ icon={EditIcon}+ showLabel={false}+ customLabel={m.jobs_edit_schedule()}+ onclick={onEdit}+ class="size-7"+ disabled={editDisabled}+ />+ {/if}+ </div>+ <div class="col-span-2 flex min-w-0 items-center gap-1.5">+ <span>+ {policy.retentionCount === 0+ ? m.volume_backup_retention_all()+ : m.volume_backup_retention_summary({ count: policy.retentionCount })}+ </span>+ {#if showStopContainers}+ <span>·</span>+ <span>{policy.stopContainers ? m.volume_backup_containers_stopped() : m.volume_backup_containers_running()}</span>+ {/if}+ {#if policy.lastRun}+ <span>·</span>+ <span class="truncate">{backupStatusLabel(policy.lastRun.status)} · {formatDateTimeShort(policy.lastRun.createdAt)}</span>+ {/if}+ </div>+ {#if policy.s3Enabled && s3Name}+ <span class="max-w-28 justify-self-end truncate" title={s3Name}>{s3Name}</span>+ {/if}+</div>diff --git a/frontend/src/lib/components/backup-policy-dialog.svelte b/frontend/src/lib/components/backup-policy-dialog.sveltenew file mode 100644index 000000000..29a0cc378--- /dev/null+++ b/frontend/src/lib/components/backup-policy-dialog.svelte@@ -0,0 +1,206 @@+<script lang="ts" generics="TPolicy extends BackupPolicy">+ import { ResponsiveDialog } from '#lib/components/ui/responsive-dialog';+ import { ArcaneButton } from '#lib/components/arcane-button';+ import BackupPolicyFields from '#lib/components/backup-policy-fields.svelte';+ import type { BackupPolicy, BackupPolicyForm, BackupPolicyUpdate } from '#lib/types/backup';+ import type { S3Destination } from '#lib/types/s3-destination';+ import { s3DestinationService } from '#lib/services/s3-destination-service';+ import { backupDestinationFromFlags, backupPolicyDestinationValues } from '#lib/utils/backups';+ import { toast } from 'svelte-sonner';+ import * as m from '#lib/paraglide/messages.js';++ type PolicyForm = BackupPolicyForm & { id: string; serverError?: string };++ let {+ open = $bindable(),+ idPrefix,+ policies,+ policyId,+ addTitle,+ description,+ enabledDescription,+ enabledError = null,+ defaultSchedule,+ defaultEnabled = true,+ showStopContainers = false,+ destinations,+ updatePolicies,+ messages,+ onSaved+ }: {+ open: boolean;+ idPrefix: string;+ policies: TPolicy[];+ policyId?: string;+ addTitle: string;+ description: string;+ enabledDescription: string;+ enabledError?: string | null;+ defaultSchedule: string;+ defaultEnabled?: boolean;+ showStopContainers?: boolean;+ destinations?: S3Destination[];+ updatePolicies: (policies: BackupPolicyUpdate[]) => Promise<TPolicy[]>;+ messages: { saved: string; saveFailed: string; removed: string };+ onSaved: (policies: TPolicy[]) => void;+ } = $props();++ let saving = $state(false);+ let deleting = $state(false);+ let loadedDestinations = $state<S3Destination[]>([]);+ let destinationsLoading = $state(false);+ let form = $state<PolicyForm>(newPolicy());+ const destinationList = $derived(destinations ?? loadedDestinations);+ const scheduleError = $derived(+ !form.schedule.trim()+ ? m.jobs_cron_required()+ : form.schedule.trim().split(/\s+/).length !== 6+ ? m.jobs_cron_invalid()+ : (form.serverError ?? null)+ );+ const retentionError = $derived(+ Number.isInteger(Number(form.retentionCount)) && form.retentionCount >= 0 && form.retentionCount <= 3650+ ? null+ : m.volume_backup_retention_invalid()+ );+ const destinationError = $derived.by(() => {+ if (form.destination === 'local' || destinationsLoading) return null;+ if (!form.s3DestinationId) return m.volume_backup_s3_destination_required();+ if (!destinations && !loadedDestinations.some((item) => item.id === form.s3DestinationId))+ return m.volume_backup_destination_unavailable();+ return null;+ });+ const activeEnabledError = $derived(form.enabled ? enabledError : null);+ const formInvalid = $derived(Boolean(scheduleError || retentionError || destinationError || activeEnabledError));++ function newPolicy(): PolicyForm {+ return {+ id: '',+ enabled: defaultEnabled,+ schedule: defaultSchedule,+ retentionCount: 7,+ stopContainers: false,+ destination: 'local',+ s3DestinationId: ''+ };+ }++ async function loadDestinations() {+ destinationsLoading = true;+ try {+ loadedDestinations = await s3DestinationService.listAll();+ } catch (error) {+ toast.error(error instanceof Error ? error.message : m.s3_destinations_load_failed());+ } finally {+ destinationsLoading = false;+ }+ }++ $effect(() => {+ if (!open) return;+ const policy = policies.find((item) => item.id === policyId);+ form = policy+ ? {+ id: policy.id,+ enabled: policy.enabled,+ schedule: policy.schedule,+ retentionCount: policy.retentionCount,+ stopContainers: policy.stopContainers ?? false,+ s3DestinationId: policy.s3DestinationId || '',+ destination: backupDestinationFromFlags(policy.localEnabled, policy.s3Enabled)+ }+ : newPolicy();+ if (!destinations) void loadDestinations();+ });++ function updateForm(values: Partial<BackupPolicyForm>) {+ form = { ...form, ...values, serverError: undefined };+ }++ function policyPayload(policy: BackupPolicy): BackupPolicyUpdate {+ return {+ id: policy.id,+ enabled: policy.enabled,+ schedule: policy.schedule,+ retentionCount: policy.retentionCount,+ localEnabled: policy.localEnabled,+ s3Enabled: policy.s3Enabled,+ s3DestinationId: policy.s3DestinationId ?? '',+ ...(showStopContainers ? { stopContainers: policy.stopContainers ?? false } : {})+ };+ }++ async function savePolicies() {+ if (formInvalid) return;+ saving = true;+ try {+ const current: BackupPolicyUpdate = {+ id: form.id,+ enabled: form.enabled,+ schedule: form.schedule,+ retentionCount: Number(form.retentionCount),+ ...backupPolicyDestinationValues(form.destination, form.s3DestinationId),+ ...(showStopContainers ? { stopContainers: form.stopContainers ?? false } : {})+ };+ const existing = policies.map(policyPayload);+ const next = policyId ? existing.map((policy) => (policy.id === policyId ? current : policy)) : [...existing, current];+ onSaved(await updatePolicies(next));+ open = false;+ toast.success(messages.saved);+ } catch (error) {+ const message = error instanceof Error ? error.message : messages.saveFailed;+ if (/cron|schedule/i.test(message)) form = { ...form, serverError: message };+ else toast.error(message);+ } finally {+ saving = false;+ }+ }++ async function deletePolicy() {+ if (!policyId) return;+ deleting = true;+ try {+ onSaved(await updatePolicies(policies.filter((policy) => policy.id !== policyId).map(policyPayload)));+ open = false;+ toast.success(messages.removed);+ } catch (error) {+ toast.error(error instanceof Error ? error.message : messages.saveFailed);+ } finally {+ deleting = false;+ }+ }+</script>++<ResponsiveDialog bind:open title={policyId ? m.jobs_edit_schedule() : addTitle} {description} contentClass="sm:max-w-[720px]">+ {#snippet children()}+ <div class="space-y-4 py-2">+ <BackupPolicyFields+ {idPrefix}+ {form}+ destinations={destinationList}+ {scheduleError}+ {retentionError}+ {destinationError}+ enabledError={activeEnabledError}+ {enabledDescription}+ schedulePlaceholder={defaultSchedule}+ {showStopContainers}+ {destinationsLoading}+ onChange={updateForm}+ />+ </div>+ {/snippet}+ {#snippet footer()}+ {#if policyId}+ <ArcaneButton+ action="remove"+ customLabel={m.backups_remove_schedule()}+ onclick={deletePolicy}+ loading={deleting}+ disabled={saving || deleting}+ />+ {/if}+ <ArcaneButton action="cancel" onclick={() => (open = false)} />+ <ArcaneButton action="save" onclick={savePolicies} loading={saving} disabled={saving || deleting || formInvalid} />+ {/snippet}+</ResponsiveDialog>diff --git a/frontend/src/lib/components/backup-policy-fields.svelte b/frontend/src/lib/components/backup-policy-fields.sveltenew file mode 100644index 000000000..67cbd0ce7--- /dev/null+++ b/frontend/src/lib/components/backup-policy-fields.svelte@@ -0,0 +1,99 @@+<script lang="ts">+ import LabeledSwitch from '#lib/components/form/labeled-switch.svelte';+ import SelectWithLabel from '#lib/components/form/select-with-label.svelte';+ import TextInputWithLabel from '#lib/components/form/text-input-with-label.svelte';+ import type { BackupDestination, BackupPolicyForm } from '#lib/types/backup';+ import type { S3Destination } from '#lib/types/s3-destination';+ import { backupDestinationOptions, s3DestinationOptions } from '#lib/utils/backups';+ import * as m from '#lib/paraglide/messages.js';++ let {+ idPrefix,+ form,+ destinations,+ scheduleError,+ retentionError,+ destinationError,+ enabledError,+ enabledDescription,+ schedulePlaceholder,+ showStopContainers = false,+ destinationsLoading = false,+ onChange+ }: {+ idPrefix: string;+ form: BackupPolicyForm;+ destinations: S3Destination[];+ scheduleError?: string | null;+ retentionError?: string | null;+ destinationError?: string | null;+ enabledError?: string | null;+ enabledDescription: string;+ schedulePlaceholder: string;+ showStopContainers?: boolean;+ destinationsLoading?: boolean;+ onChange: (values: Partial<BackupPolicyForm>) => void;+ } = $props();++ const destinationOptions = $derived(backupDestinationOptions(destinations.length > 0, true));+ const s3Options = $derived(s3DestinationOptions(destinations));+</script>++<div class="space-y-5">+ <LabeledSwitch+ id={`${idPrefix}-enabled`}+ checked={form.enabled}+ onCheckedChange={(enabled) => onChange({ enabled })}+ label={m.backups_enabled()}+ description={enabledDescription}+ />+ {#if enabledError}<p class="text-sm text-destructive">{enabledError}</p>{/if}+ <TextInputWithLabel+ value={form.schedule}+ onChange={(schedule) => onChange({ schedule })}+ error={scheduleError}+ label={m.jobs_schedule()}+ description={m.backups_schedule_description()}+ helpText={scheduleError ? undefined : m.jobs_cron_expression_help()}+ reserveHelpTextSpace+ placeholder={schedulePlaceholder}+ />+ <TextInputWithLabel+ value={form.retentionCount}+ onChange={(retentionCount) => onChange({ retentionCount: Number(retentionCount) })}+ error={retentionError}+ label={m.backups_retention_label()}+ description={m.backups_retention_description()}+ reserveHelpTextSpace+ type="number"+ />+ {#if showStopContainers}+ <LabeledSwitch+ id={`${idPrefix}-stop-containers`}+ checked={form.stopContainers ?? false}+ onCheckedChange={(stopContainers) => onChange({ stopContainers })}+ label={m.volume_backup_stop_containers()}+ description={m.volume_backup_stop_containers_description()}+ />+ {/if}+ <SelectWithLabel+ id={`${idPrefix}-destination`}+ value={form.destination}+ onValueChange={(destination) => onChange({ destination: destination as BackupDestination })}+ label={m.backups_destination_label()}+ description={m.volume_backup_destination_description()}+ options={destinationOptions}+ />+ {#if form.destination !== 'local'}+ <SelectWithLabel+ id={`${idPrefix}-s3-destination`}+ value={form.s3DestinationId}+ onValueChange={(s3DestinationId) => onChange({ s3DestinationId })}+ label={m.volume_backup_s3_destination_label()}+ description={m.volume_backup_s3_destination_description()}+ error={destinationError}+ disabled={destinationsLoading}+ options={s3Options}+ />+ {/if}+</div>diff --git a/frontend/src/lib/components/dialogs/gitops-sync-dialog.svelte b/frontend/src/lib/components/dialogs/gitops-sync-dialog.svelteindex 05d866ee5..6b06efcc6 100644--- a/frontend/src/lib/components/dialogs/gitops-sync-dialog.svelte+++ b/frontend/src/lib/components/dialogs/gitops-sync-dialog.svelte@@ -600,9 +600,9 @@<span class="mt-0.5 inline-flex shrink-0 items-center gap-1.5 text-xs">{#if hookConfigured}<CodeIcon class="size-3.5" />- {m.git_sync_pre_deploy_status_configured()}+ {m.common_configured()}{:else}- {m.git_sync_pre_deploy_status_none()}+ {m.common_not_configured()}{/if}</span></div>diff --git a/frontend/src/lib/config/navigation-config.ts b/frontend/src/lib/config/navigation-config.tsindex 3cc750878..f306cc4a7 100644--- a/frontend/src/lib/config/navigation-config.ts+++ b/frontend/src/lib/config/navigation-config.ts@@ -25,7 +25,8 @@ import {GlobeIcon,UpdateIcon,VariableIcon,- ActivityIcon+ ActivityIcon,+ BackupIcon} from '#lib/icons';import { m } from '#lib/paraglide/messages';import type { ShortcutKey } from '#lib/utils/navigation';@@ -43,6 +44,7 @@ export type NavigationItem = {* item visible to every authenticated user.*/accessSurfaceId?: string;+ adminOnly?: boolean;};export type NavigationSections = {@@ -221,6 +223,13 @@ export const navigationItems: NavigationSections = {icon: ActivityIcon,accessSurfaceId: 'settings.category.activity'},+ {+ title: m.system_backups_title(),+ url: '/settings/backups',+ icon: BackupIcon,+ accessSurfaceId: 'settings.category.systembackups',+ adminOnly: true+ },{title: m.builds(),url: '/settings/builds',@@ -313,6 +322,7 @@ function canSeeItem(currentEnvId: string | undefined,accessManifest: PermissionsManifest | null | undefined): boolean {+ if (item.adminOnly && !user.isGlobalAdmin) return false;if (!item.accessSurfaceId) return true;if (!accessManifest?.accessSurfaces?.length) return true;return canReachAccessSurface(accessManifest, item.accessSurfaceId, user, currentEnvId);diff --git a/frontend/src/lib/icons/index.ts b/frontend/src/lib/icons/index.tsindex f0932c472..2b58d1a85 100644--- a/frontend/src/lib/icons/index.ts+++ b/frontend/src/lib/icons/index.ts@@ -53,6 +53,7 @@ export { default as UserIcon } from 'virtual:icons/solar/user-circle-bold';export { default as ClockIcon } from 'virtual:icons/solar/clock-circle-linear';export { default as RegistryIcon } from 'virtual:icons/pajamas/infrastructure-registry';export { default as DownloadIcon } from 'virtual:icons/solar/download-linear';+export { default as BackupIcon } from 'virtual:icons/solar/archive-down-minimlistic-linear';export { default as SuccessIcon } from 'virtual:icons/qlementine-icons/success-12';export { default as ExternalLinkIcon } from 'virtual:icons/majesticons/external-link-line';export { default as GithubIcon } from 'virtual:icons/garden/github-stroke-16';@@ -118,3 +119,4 @@ export { default as ShieldXIcon } from 'virtual:icons/solar/shield-cross-linear'export { default as ScanIcon } from 'virtual:icons/solar/magnifer-linear';export { default as AlertTriangleIcon } from 'virtual:icons/solar/danger-triangle-linear';export { default as HammerIcon } from 'virtual:icons/gravity-ui/hammer';+export { default as CloudStorageIcon } from 'virtual:icons/solar/cloud-storage-linear';diff --git a/frontend/src/lib/layouts/settings-page-layout.svelte b/frontend/src/lib/layouts/settings-page-layout.svelteindex 407958e2e..3f0f4c7cc 100644--- a/frontend/src/lib/layouts/settings-page-layout.svelte+++ b/frontend/src/lib/layouts/settings-page-layout.svelte@@ -6,7 +6,7 @@import * as DropdownMenu from '#lib/components/ui/dropdown-menu/index.js';import { getContext } from 'svelte';import { m } from '#lib/paraglide/messages';- import { EllipsisIcon, ResetIcon, type IconType } from '#lib/icons';+ import { EllipsisIcon, ResetIcon, type IconType, ArrowDownIcon } from '#lib/icons';import { cn } from '#lib/utils';import type { SettingsActionButton, SettingsPageType, SettingsStatCard } from './types.js';@@ -49,15 +49,51 @@{#snippet ActionButtonList(buttons: SettingsActionButton[])}{#each buttons as button}- <ArcaneButton- action={button.action}- customLabel={button.label}- loadingLabel={button.loadingLabel}- loading={button.loading}- disabled={button.disabled}- onclick={button.onclick}- size="sm"- />+ {#if button.options?.length}+ <DropdownMenu.Root>+ <DropdownMenu.Trigger>+ {#snippet child({ props })}+ <ArcaneButton+ {...props}+ action={button.action}+ icon={button.icon}+ customLabel={button.label}+ disabled={button.disabled}+ size={button.size ?? 'default'}+ >+ <ArrowDownIcon class="size-3.5 opacity-60" />+ </ArcaneButton>+ {/snippet}+ </DropdownMenu.Trigger>+ <DropdownMenu.Content+ align="end"+ class="z-[var(--arcane-z-surface)] min-w-[160px] rounded-xl border bg-popover/90 p-1 shadow-lg backdrop-blur-md"+ >+ <DropdownMenu.Group>+ {#each button.options as option}+ <DropdownMenu.Item onclick={option.onclick} disabled={option.disabled}>+ {#if option.icon}+ {@const OptionIcon = option.icon}+ <OptionIcon class="size-4" />+ {/if}+ {option.label}+ </DropdownMenu.Item>+ {/each}+ </DropdownMenu.Group>+ </DropdownMenu.Content>+ </DropdownMenu.Root>+ {:else}+ <ArcaneButton+ action={button.action}+ icon={button.icon}+ customLabel={button.label}+ loadingLabel={button.loadingLabel}+ loading={button.loading}+ disabled={button.disabled}+ onclick={button.onclick}+ size={button.size ?? 'default'}+ />+ {/if}{/each}{/snippet}@@ -163,9 +199,18 @@><DropdownMenu.Group>{#each mobileDropdownButtons as button}- <DropdownMenu.Item onclick={button.onclick} disabled={button.disabled || button.loading}>- {button.loading ? button.loadingLabel || button.label : button.label}- </DropdownMenu.Item>+ {#if button.options?.length}+ <DropdownMenu.Label>{button.label}</DropdownMenu.Label>+ {#each button.options as option}+ <DropdownMenu.Item onclick={option.onclick} disabled={button.disabled || option.disabled}>+ {option.label}+ </DropdownMenu.Item>+ {/each}+ {:else}+ <DropdownMenu.Item onclick={button.onclick} disabled={button.disabled || button.loading}>+ {button.loading ? button.loadingLabel || button.label : button.label}+ </DropdownMenu.Item>+ {/if}{/each}</DropdownMenu.Group></DropdownMenu.Content>diff --git a/frontend/src/lib/layouts/types.ts b/frontend/src/lib/layouts/types.tsindex 411bab82e..d86b2ce37 100644--- a/frontend/src/lib/layouts/types.ts+++ b/frontend/src/lib/layouts/types.ts@@ -1,14 +1,25 @@-import type { Action } from '#lib/components/arcane-button/index.js';+import type { Action, ArcaneButtonSize } from '#lib/components/arcane-button/index.js';import type { IconType } from '#lib/icons';+export interface SettingsActionOption {+ label: string;+ icon?: IconType;+ onclick: () => void;+ disabled?: boolean;+}+export interface SettingsActionButton {id: string;action: Action;+ icon?: IconType;label: string;loadingLabel?: string;loading?: boolean;disabled?: boolean;- onclick: () => void;+ size?: ArcaneButtonSize;+ onclick?: () => void;+ // When set, the button renders as a dropdown trigger and onclick is unused.+ options?: SettingsActionOption[];showOnMobile?: boolean;}diff --git a/frontend/src/lib/services/s3-destination-service.ts b/frontend/src/lib/services/s3-destination-service.tsnew file mode 100644index 000000000..64de09bd2--- /dev/null+++ b/frontend/src/lib/services/s3-destination-service.ts@@ -0,0 +1,38 @@+import BaseAPIService from './api-service';+import type { CreateS3Destination, S3Destination, UpdateS3Destination } from '#lib/types/s3-destination';+import type { Paginated, SearchPaginationSortRequest } from '#lib/types/shared';+import { transformPaginationParams } from '#lib/utils/tables';++class S3DestinationService extends BaseAPIService {+ async list(options?: SearchPaginationSortRequest): Promise<Paginated<S3Destination>> {+ const params = transformPaginationParams(options);+ const response = await this.api.get('/backups/s3', { params });+ return response.data;+ }++ async listAll(): Promise<S3Destination[]> {+ return this.handleResponse(this.api.get('/backups/s3/options'));+ }++ async create(destination: CreateS3Destination): Promise<S3Destination> {+ return this.handleResponse(this.api.post('/backups/s3', destination));+ }++ async update(id: string, destination: UpdateS3Destination): Promise<S3Destination> {+ return this.handleResponse(this.api.put(`/backups/s3/${id}`, destination));+ }++ async test(id: string, destination?: UpdateS3Destination): Promise<void> {+ await this.handleResponse(this.api.post(`/backups/s3/${id}/test`, destination));+ }++ async testConfiguration(destination: CreateS3Destination): Promise<void> {+ await this.handleResponse(this.api.post('/backups/s3/test', destination));+ }++ async delete(id: string): Promise<void> {+ await this.handleResponse(this.api.delete(`/backups/s3/${id}`));+ }+}++export const s3DestinationService = new S3DestinationService();diff --git a/frontend/src/lib/services/system-backup-service.ts b/frontend/src/lib/services/system-backup-service.tsnew file mode 100644index 000000000..5359b03a2--- /dev/null+++ b/frontend/src/lib/services/system-backup-service.ts@@ -0,0 +1,55 @@+import BaseAPIService from './api-service';+import type { Paginated, SearchPaginationSortRequest } from '#lib/types/shared';+import type {+ CreateSystemBackup,+ SystemBackupPolicyCollection,+ SystemBackupRecoveryKey,+ SystemBackupRun,+ UpdateSystemBackupPolicy+} from '#lib/types/system-backup';+import { transformPaginationParams } from '#lib/utils/tables';++class SystemBackupService extends BaseAPIService {+ async list(options?: SearchPaginationSortRequest): Promise<Paginated<SystemBackupRun>> {+ const response = await this.api.get('/backups', { params: transformPaginationParams(options) });+ return response.data;+ }++ async getPolicies(): Promise<SystemBackupPolicyCollection> {+ return this.handleResponse(this.api.get('/backups/policies'));+ }++ async updatePolicies(policies: UpdateSystemBackupPolicy[]): Promise<SystemBackupPolicyCollection> {+ return this.handleResponse(this.api.put('/backups/policies', { policies }));+ }++ async generateRecoveryKey(): Promise<SystemBackupRecoveryKey> {+ return this.handleResponse(this.api.post('/backups/recovery-key/generate'));+ }++ async setRecoveryKey(recoveryKey: string): Promise<{ configured: boolean }> {+ return this.handleResponse(this.api.put('/backups/recovery-key', { recoveryKey }));+ }++ async create(input: CreateSystemBackup): Promise<SystemBackupRun> {+ return this.handleResponse(this.api.post('/backups', input));+ }++ async restore(id: string, recoveryKey: string): Promise<void> {+ await this.handleResponse(this.api.post(`/backups/${id}/restore`, { recoveryKey }));+ }++ async upload(id: string, s3DestinationId: string, recoveryKey: string): Promise<SystemBackupRun> {+ return this.handleResponse(this.api.post(`/backups/${id}/upload`, { s3DestinationId, recoveryKey }));+ }++ async delete(id: string, recoveryKey: string): Promise<void> {+ await this.handleResponse(this.api.delete(`/backups/${id}`, { data: { recoveryKey } }));+ }++ async discover(s3DestinationId: string, recoveryKey: string): Promise<number> {+ return this.handleResponse(this.api.post('/backups/discover', { s3DestinationId, recoveryKey }));+ }+}++export const systemBackupService = new SystemBackupService();diff --git a/frontend/src/lib/services/volume-backup-service.ts b/frontend/src/lib/services/volume-backup-service.tsindex 9413e9c11..3607a65cd 100644--- a/frontend/src/lib/services/volume-backup-service.ts+++ b/frontend/src/lib/services/volume-backup-service.ts@@ -1,16 +1,31 @@import BaseAPIService from './api-service';import { uploadService, type UploadProgressCallback } from './upload-service';import { environmentStore } from '#lib/stores/environment.store.svelte';-import type { BackupEntry } from '#lib/types/shared';+import type {+ BackupEntry,+ CreateVolumeBackupRequest,+ UpdateVolumeBackupPolicy,+ VolumeBackupPolicyCollection+} from '#lib/types/shared';import type { SearchPaginationSortRequest, Paginated } from '#lib/types/shared';import { transformPaginationParams } from '#lib/utils/tables';export type VolumeBackupListResponse = Paginated<BackupEntry> & { warnings?: string[] };class VolumeBackupService extends BaseAPIService {- async createBackup(volumeName: string): Promise<BackupEntry> {+ async getPolicies(volumeName: string): Promise<VolumeBackupPolicyCollection> {const envId = await environmentStore.getCurrentEnvironmentId();- const res = await this.api.post(`/environments/${envId}/volumes/${volumeName}/backups`);+ return this.handleResponse(this.api.get(`/environments/${envId}/volumes/${volumeName}/backup-policy`));+ }++ async updatePolicies(volumeName: string, policies: UpdateVolumeBackupPolicy[]): Promise<VolumeBackupPolicyCollection> {+ const envId = await environmentStore.getCurrentEnvironmentId();+ return this.handleResponse(this.api.put(`/environments/${envId}/volumes/${volumeName}/backup-policy`, { policies }));+ }++ async createBackup(volumeName: string, request?: CreateVolumeBackupRequest): Promise<BackupEntry> {+ const envId = await environmentStore.getCurrentEnvironmentId();+ const res = await this.api.post(`/environments/${envId}/volumes/${volumeName}/backups`, request);return res.data.data;}@@ -49,11 +64,6 @@ class VolumeBackupService extends BaseAPIService {return res.data.data ?? [];}- async deleteBackup(backupId: string): Promise<any> {- const envId = await environmentStore.getCurrentEnvironmentId();- return this.handleResponse(this.api.delete(`/environments/${envId}/volumes/backups/${backupId}`));- }-async downloadBackup(backupId: string): Promise<void> {const envId = await environmentStore.getCurrentEnvironmentId();const res = await this.api.get(`/environments/${envId}/volumes/backups/${backupId}/download`, {@@ -69,6 +79,16 @@ class VolumeBackupService extends BaseAPIService {link.remove();}+ async deleteBackup(backupId: string): Promise<any> {+ const envId = await environmentStore.getCurrentEnvironmentId();+ return this.handleResponse(this.api.delete(`/environments/${envId}/volumes/backups/${backupId}`));+ }++ async uploadBackup(backupId: string, s3DestinationId: string): Promise<BackupEntry> {+ const envId = await environmentStore.getCurrentEnvironmentId();+ return this.handleResponse(this.api.post(`/environments/${envId}/volumes/backups/${backupId}/upload`, { s3DestinationId }));+ }+async uploadAndRestore(volumeName: string, file: File, onProgress?: UploadProgressCallback): Promise<any> {const envId = await environmentStore.getCurrentEnvironmentId();const uploadId = await uploadService.uploadFile(envId, 'volume-backup', file, onProgress);diff --git a/frontend/src/lib/types/backup.ts b/frontend/src/lib/types/backup.tsnew file mode 100644index 000000000..d99d22f02--- /dev/null+++ b/frontend/src/lib/types/backup.ts@@ -0,0 +1,50 @@+export type BackupDestination = 'local' | 's3' | 'local_s3';+export type BackupStatus = 'running' | 'succeeded' | 'failed';+export type BackupTrigger = 'manual' | 'scheduled' | 'safety';++export type BackupRun = {+ id: string;+ size: number;+ createdAt: string;+ status: BackupStatus;+ trigger: BackupTrigger;+ destination: BackupDestination;+ format?: 'archive' | 'rustic';+ localSnapshotId?: string;+ remoteSnapshotId?: string;+ s3DestinationId?: string;+ s3DestinationName?: string;+ policyId?: string;+ error?: string;+};++export type BackupPolicyForm = {+ enabled: boolean;+ schedule: string;+ retentionCount: number;+ destination: BackupDestination;+ s3DestinationId: string;+ stopContainers?: boolean;+};++export type BackupPolicy = {+ id: string;+ enabled: boolean;+ schedule: string;+ retentionCount: number;+ stopContainers?: boolean;+ localEnabled: boolean;+ s3Enabled: boolean;+ s3DestinationId?: string;+};++export type BackupPolicyUpdate = {+ id: string;+ enabled: boolean;+ schedule: string;+ retentionCount: number;+ localEnabled: boolean;+ s3Enabled: boolean;+ s3DestinationId: string;+ stopContainers?: boolean;+};diff --git a/frontend/src/lib/types/s3-destination.ts b/frontend/src/lib/types/s3-destination.tsnew file mode 100644index 000000000..4da0f7c4b--- /dev/null+++ b/frontend/src/lib/types/s3-destination.ts@@ -0,0 +1,28 @@+export type S3Destination = {+ id: string;+ name: string;+ endpoint?: string;+ bucket: string;+ region: string;+ accessKeyId: string;+ prefix?: string;+ useSsl: boolean;+ forcePathStyle: boolean;+ secretConfigured: boolean;+ createdAt: string;+ updatedAt?: string;+};++export type CreateS3Destination = {+ name: string;+ endpoint: string;+ bucket: string;+ region: string;+ accessKeyId: string;+ secretAccessKey: string;+ prefix: string;+ useSsl: boolean;+ forcePathStyle: boolean;+};++export type UpdateS3Destination = CreateS3Destination;diff --git a/frontend/src/lib/types/shared.ts b/frontend/src/lib/types/shared.tsindex a880f3045..08a773210 100644--- a/frontend/src/lib/types/shared.ts+++ b/frontend/src/lib/types/shared.ts@@ -105,11 +105,41 @@ export interface FileContentResponse {mimeType: string;}-export interface BackupEntry {+import type { BackupDestination, BackupPolicyUpdate, BackupRun } from './backup';++export type VolumeBackupDestination = BackupDestination;++export interface CreateVolumeBackupRequest {+ destination?: VolumeBackupDestination;+ policyId?: string;+ s3DestinationId?: string;+}++export interface BackupEntry extends BackupRun {+ volumeName: string;+}++export interface VolumeBackupPolicy {id: string;volumeName: string;- size: number;- createdAt: string;+ enabled: boolean;+ schedule: string;+ retentionCount: number;+ stopContainers: boolean;+ localEnabled: boolean;+ s3Enabled: boolean;+ s3DestinationId: string;+ s3DestinationName?: string;+ s3Available: boolean;+ s3Bucket: string;+ lastRun?: BackupEntry;+}++export type UpdateVolumeBackupPolicy = BackupPolicyUpdate;++export interface VolumeBackupPolicyCollection {+ policies: VolumeBackupPolicy[];+ s3Available: boolean;}// --- Customize search ---diff --git a/frontend/src/lib/types/system-backup.ts b/frontend/src/lib/types/system-backup.tsnew file mode 100644index 000000000..c105888eb--- /dev/null+++ b/frontend/src/lib/types/system-backup.ts@@ -0,0 +1,35 @@+import type { BackupDestination, BackupRun } from './backup';++export type SystemBackupDestination = BackupDestination;++export type SystemBackupRun = BackupRun;++export type SystemBackupPolicy = {+ id: string;+ enabled: boolean;+ schedule: string;+ retentionCount: number;+ localEnabled: boolean;+ s3Enabled: boolean;+ s3DestinationId?: string;+ s3DestinationName?: string;+ lastRun?: SystemBackupRun;+};++export type UpdateSystemBackupPolicy = Omit<SystemBackupPolicy, 's3DestinationName' | 'lastRun'>;++export type SystemBackupPolicyCollection = {+ policies: SystemBackupPolicy[];+ recoveryKeyStored: boolean;+};++export type SystemBackupRecoveryKey = {+ recoveryKey: string;+};++export type CreateSystemBackup = {+ destination?: SystemBackupDestination;+ s3DestinationId?: string;+ recoveryKey?: string;+ policyId?: string;+};diff --git a/frontend/src/lib/utils/backups.ts b/frontend/src/lib/utils/backups.tsnew file mode 100644index 000000000..07d2f6500--- /dev/null+++ b/frontend/src/lib/utils/backups.ts@@ -0,0 +1,87 @@+import { m } from '#lib/paraglide/messages';+import type { BackupDestination, BackupRun, BackupStatus, BackupTrigger } from '#lib/types/backup';+import type { S3Destination } from '#lib/types/s3-destination';++export function backupStatusLabel(status: BackupStatus): string {+ if (status === 'succeeded') return m.volume_backup_status_succeeded();+ if (status === 'failed') return m.common_failed();+ return m.common_running();+}++export function backupStatusVariant(status: BackupStatus): 'green' | 'red' | 'blue' {+ if (status === 'succeeded') return 'green';+ if (status === 'failed') return 'red';+ return 'blue';+}++export function backupTriggerLabel(trigger: BackupTrigger): string {+ if (trigger === 'scheduled') return m.backups_trigger_scheduled();+ if (trigger === 'safety') return m.backups_trigger_safety();+ return m.backups_trigger_manual();+}++export function backupDestinationLabel(destination: BackupDestination): string {+ if (destination === 'local_s3') return m.backups_destination_local_s3();+ if (destination === 's3') return m.backups_destination_s3();+ return m.local();+}++export function backupDestinationFromFlags(localEnabled: boolean, s3Enabled: boolean): BackupDestination {+ if (!s3Enabled) return 'local';+ return localEnabled ? 'local_s3' : 's3';+}++export function backupDestinationName(item: Pick<BackupRun, 's3DestinationId' | 's3DestinationName'>): string {+ return item.s3DestinationName || item.s3DestinationId || '';+}++export function backupDestinationDisplay(item: Pick<BackupRun, 'destination' | 's3DestinationId' | 's3DestinationName'>): string {+ const label = backupDestinationLabel(item.destination);+ const name = backupDestinationName(item);+ return name && item.destination !== 'local' ? `${label} · ${name}` : label;+}++export function backupPolicyDestinationDisplay(policy: {+ localEnabled: boolean;+ s3Enabled: boolean;+ s3DestinationName?: string;+}): string {+ const label = backupDestinationLabel(backupDestinationFromFlags(policy.localEnabled, policy.s3Enabled));+ return policy.s3DestinationName ? `${label} · ${policy.s3DestinationName}` : label;+}++export function backupDestinationOptions(hasS3Destinations: boolean, descriptions = false) {+ return [+ {+ label: m.volume_backup_destination_local(),+ value: 'local',+ description: descriptions ? m.volume_backup_destination_local_description() : undefined+ },+ ...(hasS3Destinations+ ? [+ {+ label: m.volume_backup_destination_s3(),+ value: 's3',+ description: descriptions ? m.volume_backup_destination_s3_description() : undefined+ },+ {+ label: m.backups_destination_local_s3(),+ value: 'local_s3',+ description: descriptions ? m.volume_backup_destination_local_s3_description() : undefined+ }+ ]+ : [])+ ];+}++export function s3DestinationOptions(destinations: S3Destination[]) {+ return destinations.map((item) => ({ label: item.name, value: item.id, description: item.bucket }));+}++export function backupPolicyDestinationValues(destination: BackupDestination, s3DestinationId: string) {+ return {+ localEnabled: destination !== 's3',+ s3Enabled: destination !== 'local',+ s3DestinationId: destination === 'local' ? '' : s3DestinationId+ };+}diff --git a/frontend/src/lib/utils/bulk-actions.ts b/frontend/src/lib/utils/bulk-actions.tsindex 2e4c4e974..e39e2ea43 100644--- a/frontend/src/lib/utils/bulk-actions.ts+++ b/frontend/src/lib/utils/bulk-actions.ts@@ -40,7 +40,7 @@ export interface RunBulkOperationOptions<T> {/** Clears the table selection after completion. Always called when there were ids. */clearSelection?: () => void;/** Optional per-item failure side effect, such as an item-specific toast. */- onItemFailure?: (id: string) => void;+ onItemFailure?: (id: string, error: Error) => void;/** Run operations one at a time instead of concurrently. Defaults to concurrent. */sequential?: boolean;}@@ -69,7 +69,7 @@ async function runBulkOperation<T>({const tally = (id: string, outcome: Result<T>) => {if (outcome.error) {result.failed += 1;- onItemFailure?.(id);+ onItemFailure?.(id, outcome.error);} else {result.success += 1;const activityId = extractActivityId(outcome.data);diff --git a/frontend/src/routes/(app)/settings/+page.svelte b/frontend/src/routes/(app)/settings/+page.svelteindex 3263d63d1..f18388d45 100644--- a/frontend/src/routes/(app)/settings/+page.svelte+++ b/frontend/src/routes/(app)/settings/+page.svelte@@ -12,7 +12,9 @@JobsIcon,CodeIcon,GlobeIcon,- ActivityIcon+ ActivityIcon,+ RemoteEnvironmentIcon,+ BackupIcon} from '#lib/icons';import { m } from '#lib/paraglide/messages';import { settingsSearchService } from '#lib/services/settings-search';@@ -47,7 +49,9 @@jobs: JobsIcon,code: CodeIcon,globe: GlobeIcon,- activity: ActivityIcon+ activity: ActivityIcon,+ remoteenvironment: RemoteEnvironmentIcon,+ backup: BackupIcon};const categoryMessages = {@@ -109,6 +113,7 @@}function isAccessibleCategory(category: SettingsCategory) {+ if (category.id === 'systembackups' && !user?.isGlobalAdmin) return false;if (!permissionsManifest?.accessSurfaces?.length) return true;return canReachAccessSurfaceUrl(permissionsManifest, category.url, user, environmentStore.selected?.id);}diff --git a/frontend/src/routes/(app)/settings/backups/+page.svelte b/frontend/src/routes/(app)/settings/backups/+page.sveltenew file mode 100644index 000000000..c989aa489--- /dev/null+++ b/frontend/src/routes/(app)/settings/backups/+page.svelte@@ -0,0 +1,434 @@+<script lang="ts">+ import { untrack } from 'svelte';+ import { goto } from '$app/navigation';+ import { toast } from 'svelte-sonner';+ import settingsStore from '#lib/stores/config-store';+ import { SettingsPageLayout, type SettingsActionButton } from '#lib/layouts';+ import { AlertIcon, BackupIcon, CloudStorageIcon, LockIcon, ResetIcon } from '#lib/icons';+ import * as Alert from '#lib/components/ui/alert';+ import { CopyButton } from '#lib/components/ui/copy-button';+ import { Input } from '#lib/components/ui/input';+ import { ResponsiveDialog } from '#lib/components/ui/responsive-dialog';+ import { ArcaneButton } from '#lib/components/arcane-button';+ import SelectWithLabel from '#lib/components/form/select-with-label.svelte';+ import TextInputWithLabel from '#lib/components/form/text-input-with-label.svelte';+ import { systemBackupService } from '#lib/services/system-backup-service';+ import { hasPermission } from '#lib/utils/auth';+ import { backupDestinationOptions, backupPolicyDestinationDisplay, s3DestinationOptions } from '#lib/utils/backups';+ import type { SearchPaginationSortRequest } from '#lib/types/shared';+ import type { SystemBackupDestination, SystemBackupRun } from '#lib/types/system-backup';+ import * as m from '#lib/paraglide/messages.js';+ import SystemBackupTable from './system-backup-table.svelte';+ import BackupPolicyDialog from '#lib/components/backup-policy-dialog.svelte';+ import BackupPolicyCard from '#lib/components/backup-policy-card.svelte';++ let { data } = $props();+ let backups = $state(untrack(() => data.backups));+ let policyCollection = $state(untrack(() => data.policyCollection));+ let requestOptions = $state<SearchPaginationSortRequest>(untrack(() => data.requestOptions));+ let policyOpen = $state(false);+ let editingPolicyId = $state<string | undefined>();+ let keyOpen = $state(false);+ let actionOpen = $state(false);+ let action = $state<'create' | 'restore' | 'upload' | 'delete' | 'discover'>('create');+ let selected = $state<SystemBackupRun | null>(null);+ let backupConfiguration = $state('custom');+ let destination = $state<SystemBackupDestination>('local');+ let s3DestinationId = $state('');+ let recoveryKey = $state('');+ let newRecoveryKey = $state('');+ let loading = $state(false);+ let generatingKey = $state(false);+ const isReadOnly = $derived.by(() => $settingsStore.uiConfigDisabled);+ const canManageRecoveryKey = $derived(hasPermission('system-backups:recovery-key'));+ const destinationOptions = $derived(backupDestinationOptions(data.destinations.length > 0));+ const s3Options = $derived(s3DestinationOptions(data.destinations));+ const configurationOptions = $derived([+ {+ label: m.system_backups_custom_configuration(),+ value: 'custom',+ description: m.system_backups_custom_configuration_description()+ },+ ...policyCollection.policies.map((item) => ({+ label: item.schedule,+ value: item.id,+ description: backupPolicyDestinationDisplay(item)+ }))+ ]);+ // Restore and discover may target backups keyed by another instance, so+ // they accept a typed (previously generated) key. Everything else only+ // ever uses this instance's stored key.+ const actionNeedsTypedKey = $derived(action === 'restore' || action === 'discover');+ const recoveryKeyPattern = /^[A-Z2-7]{6}(-[A-Z2-7]{6}){7}$/;+ const keyError = $derived(+ actionNeedsTypedKey && recoveryKey.length > 0 && !recoveryKeyPattern.test(recoveryKey.trim())+ ? m.system_backups_recovery_key_required()+ : ''+ );+ const actionKeyInvalid = $derived.by(() => {+ if (actionNeedsTypedKey) return !policyCollection.recoveryKeyStored && !recoveryKeyPattern.test(recoveryKey.trim());+ // Deleting is always allowed; the backend only asks for the key when+ // the run still has snapshots to forget.+ if (action === 'delete') return false;+ return !policyCollection.recoveryKeyStored;+ });+ const destinationError = $derived(+ backupConfiguration === 'custom' &&+ ((action === 'create' && destination !== 'local') || action === 'upload' || action === 'discover') &&+ !s3DestinationId+ ? m.volume_backup_s3_destination_required()+ : ''+ );+ const invalid = $derived(Boolean(actionKeyInvalid || keyError || destinationError));++ function openPolicy(id?: string) {+ editingPolicyId = id;+ policyOpen = true;+ }++ // Generating persists the key in the same step; the dialog only ever shows+ // an already-saved key, with one job: get the user to store it elsewhere.+ async function openRecoveryKey() {+ newRecoveryKey = '';+ generatingKey = true;+ try {+ const generated = await systemBackupService.generateRecoveryKey();+ await systemBackupService.setRecoveryKey(generated.recoveryKey);+ newRecoveryKey = generated.recoveryKey;+ policyCollection = { ...policyCollection, recoveryKeyStored: true };+ keyOpen = true;+ } catch (error) {+ toast.error(error instanceof Error ? error.message : m.system_backups_recovery_key_save_failed());+ } finally {+ generatingKey = false;+ }+ }++ function openAction(next: typeof action, backup: SystemBackupRun | null = null) {+ action = next;+ selected = backup;+ backupConfiguration = 'custom';+ destination = 'local';+ s3DestinationId =+ backup?.s3DestinationId ||+ policyCollection.policies.find((item) => item.s3DestinationId)?.s3DestinationId ||+ data.destinations[0]?.id ||+ '';+ recoveryKey = '';+ actionOpen = true;+ }+ function dialogTitle() {+ if (action === 'restore') return m.system_backups_restore_title();+ if (action === 'upload') return m.system_backups_upload_title();+ if (action === 'delete') return m.system_backups_delete_title();+ if (action === 'discover') return m.system_backups_discover_title();+ return m.volumes_backup_create();+ }+ function dialogDescription() {+ if (action === 'restore') return m.system_backups_restore_description();+ if (action === 'delete') return m.system_backups_delete_description();+ if (action === 'discover') return m.system_backups_discover_description();+ return m.system_backups_dialog_description();+ }+ async function refresh() {+ backups = await systemBackupService.list(requestOptions);+ }++ // With a stored key the S3 repositories are scanned automatically, so+ // remote snapshots just appear in the table; the manual Find S3 backups+ // flow only exists for fresh instances that must supply an old key.+ let autoDiscovered = false;+ $effect(() => {+ if (autoDiscovered || !policyCollection.recoveryKeyStored || data.destinations.length === 0) return;+ autoDiscovered = true;+ void (async () => {+ try {+ const counts = await Promise.all(data.destinations.map((item) => systemBackupService.discover(item.id, '')));+ if (counts.some((count) => count > 0)) await refresh();+ } catch (error) {+ console.warn('S3 backup discovery failed', error);+ }+ })();+ });++ async function submitAction() {+ if (loading || invalid) return;+ loading = true;+ try {+ if (action === 'create') {+ await systemBackupService.create(+ backupConfiguration === 'custom'+ ? { destination, s3DestinationId: destination === 'local' ? '' : s3DestinationId, recoveryKey }+ : { policyId: backupConfiguration, recoveryKey }+ );+ toast.success(m.system_backups_created());+ await refresh();+ } else if (action === 'restore' && selected) {+ await systemBackupService.restore(selected.id, recoveryKey);+ toast.success(m.system_backups_restore_started());+ } else if (action === 'upload' && selected) {+ await systemBackupService.upload(selected.id, s3DestinationId, recoveryKey);+ toast.success(m.backups_upload_s3_success());+ await refresh();+ } else if (action === 'delete' && selected) {+ await systemBackupService.delete(selected.id, recoveryKey);+ toast.success(m.system_backups_deleted());+ await refresh();+ } else if (action === 'discover') {+ const count = await systemBackupService.discover(s3DestinationId, recoveryKey);+ toast.success(m.system_backups_discovered({ count }));+ await refresh();+ }+ actionOpen = false;+ } catch (error) {+ const fallback =+ action === 'restore'+ ? m.system_backups_restore_failed()+ : action === 'delete'+ ? m.system_backups_delete_failed()+ : action === 'upload'+ ? m.backups_upload_s3_failed()+ : action === 'discover'+ ? m.system_backups_discover_failed()+ : m.system_backups_create_failed();+ toast.error(error instanceof Error ? error.message : fallback);+ } finally {+ loading = false;+ }+ }++ const actionButtons: SettingsActionButton[] = $derived.by(() => [+ ...(policyCollection.recoveryKeyStored && canManageRecoveryKey+ ? [+ {+ id: 'reset-recovery-key',+ action: 'refresh',+ icon: ResetIcon,+ label: m.system_backups_reset_recovery_key(),+ loading: generatingKey,+ onclick: openRecoveryKey,+ disabled: isReadOnly+ } satisfies SettingsActionButton+ ]+ : []),+ {+ id: 's3-destinations',+ action: 'edit',+ icon: CloudStorageIcon,+ label: m.s3_destinations_title(),+ onclick: () => goto('/settings/backups/s3')+ },+ ...(!policyCollection.recoveryKeyStored+ ? [+ {+ id: 'discover',+ action: 'inspect',+ label: m.system_backups_discover(),+ onclick: () => openAction('discover'),+ disabled: isReadOnly || data.destinations.length === 0+ } satisfies SettingsActionButton+ ]+ : []),+ {+ id: 'create',+ action: 'create',+ label: m.common_create(),+ disabled: isReadOnly,+ options: [+ { label: m.jobs_schedule(), onclick: () => openPolicy() },+ { label: m.volumes_workspace_backup(), onclick: () => openAction('create') }+ ]+ }+ ]);+</script>++{#snippet recoveryKeySummary()}+ {#if !policyCollection.recoveryKeyStored}+ <div class="flex items-center justify-between gap-3 rounded-md border border-amber-500/40 bg-amber-500/5 px-3 py-2">+ <div class="flex min-w-0 items-center gap-2">+ <LockIcon class="size-4 shrink-0 text-amber-500" />+ <p class="text-sm">{m.system_backups_recovery_key_needed()}</p>+ </div>+ <ArcaneButton+ action="edit"+ size="sm"+ customLabel={m.system_backups_setup_recovery_key()}+ loading={generatingKey}+ onclick={openRecoveryKey}+ disabled={isReadOnly || !canManageRecoveryKey}+ />+ </div>+ {/if}+{/snippet}++{#snippet recoveryKeyDialog()}+ <ResponsiveDialog+ bind:open={keyOpen}+ title={m.system_backups_recovery_key()}+ description={m.system_backups_recovery_key_saved()}+ contentClass="sm:max-w-[560px]"+ >+ {#snippet children()}+ <div class="space-y-3 py-2">+ <div class="flex items-center gap-2">+ <Input+ id="system-backup-recovery-key"+ value={newRecoveryKey}+ readonly+ spellcheck={false}+ class="font-mono tracking-wide uppercase"+ />+ <CopyButton text={newRecoveryKey} variant="outline" tabindex={0} class="shrink-0" />+ </div>+ <Alert.Root variant="destructive">+ <AlertIcon class="size-4" />+ <Alert.Description>{m.system_backups_recovery_key_alert()}</Alert.Description>+ </Alert.Root>+ </div>+ {/snippet}+ {#snippet footer()}+ <ArcaneButton action="confirm" customLabel={m.common_done()} onclick={() => (keyOpen = false)} />+ {/snippet}+ </ResponsiveDialog>+{/snippet}++{#snippet actionDialog()}+ <ResponsiveDialog+ bind:open={actionOpen}+ title={dialogTitle()}+ description={dialogDescription()}+ contentClass="sm:max-w-[560px]"+ >+ {#snippet children()}+ <div class="space-y-5 py-2">+ {#if action === 'create'}+ <SelectWithLabel+ id="manual-system-backup-configuration"+ value={backupConfiguration}+ onValueChange={(value) => (backupConfiguration = value)}+ label={m.system_backups_backup_configuration()}+ options={configurationOptions}+ />+ {#if backupConfiguration === 'custom'}+ <SelectWithLabel+ id="manual-system-backup-destination"+ value={destination}+ onValueChange={(value) => (destination = value as SystemBackupDestination)}+ label={m.backups_destination_label()}+ options={destinationOptions}+ />+ {/if}+ {/if}+ {#if action === 'upload' || action === 'discover' || (action === 'create' && backupConfiguration === 'custom' && destination !== 'local')}+ <SelectWithLabel+ id="manual-system-backup-s3"+ value={s3DestinationId}+ onValueChange={(value) => (s3DestinationId = value)}+ label={m.volume_backup_s3_destination_label()}+ error={destinationError || null}+ options={s3Options}+ />+ {/if}+ {#if actionNeedsTypedKey}+ <TextInputWithLabel+ value={recoveryKey}+ onChange={(value) => (recoveryKey = value)}+ error={keyError || null}+ label={m.system_backups_recovery_key()}+ description={policyCollection.recoveryKeyStored+ ? m.system_backups_recovery_key_saved_description()+ : m.system_backups_recovery_key_enter_description()}+ type="password"+ autocomplete="current-password"+ />+ {:else if action !== 'delete' && !policyCollection.recoveryKeyStored}+ <div class="flex items-center justify-between gap-3 rounded-md border px-3 py-2">+ <p class="text-sm text-muted-foreground">{m.system_backups_recovery_key_needed()}</p>+ <ArcaneButton+ action="edit"+ size="sm"+ customLabel={m.system_backups_setup_recovery_key()}+ loading={generatingKey}+ onclick={() => {+ actionOpen = false;+ openRecoveryKey();+ }}+ />+ </div>+ {/if}+ </div>+ {/snippet}+ {#snippet footer()}+ <ArcaneButton action="cancel" onclick={() => (actionOpen = false)} disabled={loading} />+ <ArcaneButton+ action={action === 'delete' ? 'remove' : action === 'restore' ? 'confirm' : action === 'create' ? 'create' : 'save'}+ customLabel={dialogTitle()}+ onclick={submitAction}+ {loading}+ disabled={loading || invalid}+ />+ {/snippet}+ </ResponsiveDialog>+{/snippet}++<SettingsPageLayout+ title={m.system_backups_title()}+ description={m.system_backups_description()}+ icon={BackupIcon}+ pageType="management"+ showReadOnlyTag={isReadOnly}+ {actionButtons}+>+ {#snippet mainContent()}+ <div class="space-y-4">+ {@render recoveryKeySummary()}++ <div class="space-y-2">+ <h2 class="text-lg font-semibold">{m.system_backups_schedules()}</h2>+ {#if policyCollection.policies.length}+ <div class="grid grid-cols-1 gap-1.5 text-xs text-muted-foreground sm:grid-cols-2 xl:grid-cols-3">+ {#each policyCollection.policies as policy (policy.id)}+ <BackupPolicyCard {policy} onEdit={() => openPolicy(policy.id)} editDisabled={isReadOnly} />+ {/each}+ </div>+ {:else}+ <p class="text-sm text-muted-foreground">{m.system_backups_no_schedules()}</p>+ {/if}+ </div>++ <SystemBackupTable+ bind:backups+ bind:requestOptions+ onChanged={(options) => systemBackupService.list(options)}+ onRestore={(item) => openAction('restore', item)}+ onUpload={(item) => openAction('upload', item)}+ onDelete={(item) => openAction('delete', item)}+ />+ </div>+ {/snippet}+ {#snippet additionalContent()}+ <BackupPolicyDialog+ bind:open={policyOpen}+ idPrefix="system-backup-policy"+ policies={policyCollection.policies}+ policyId={editingPolicyId}+ addTitle={m.system_backups_add_schedule()}+ description={m.system_backups_schedule_description()}+ enabledDescription={m.system_backups_enabled_description()}+ enabledError={policyCollection.recoveryKeyStored ? null : m.system_backups_recovery_key_schedule_required()}+ defaultSchedule="0 0 3 * * *"+ defaultEnabled={policyCollection.recoveryKeyStored}+ destinations={data.destinations}+ updatePolicies={async (policies) => (await systemBackupService.updatePolicies(policies)).policies}+ messages={{+ saved: m.system_backups_policy_saved(),+ saveFailed: m.system_backups_policy_failed(),+ removed: m.system_backups_schedule_removed()+ }}+ onSaved={(policies) => (policyCollection = { ...policyCollection, policies })}+ />++ {@render recoveryKeyDialog()}+ {@render actionDialog()}+ {/snippet}+</SettingsPageLayout>diff --git a/frontend/src/routes/(app)/settings/backups/+page.ts b/frontend/src/routes/(app)/settings/backups/+page.tsnew file mode 100644index 000000000..7cb2f0f63--- /dev/null+++ b/frontend/src/routes/(app)/settings/backups/+page.ts@@ -0,0 +1,18 @@+import type { PageLoad } from './$types';+import { systemBackupService } from '#lib/services/system-backup-service';+import { s3DestinationService } from '#lib/services/s3-destination-service';+import { resolveInitialTableRequest } from '#lib/utils/tables';+import type { SearchPaginationSortRequest } from '#lib/types/shared';++export const load: PageLoad = async () => {+ const requestOptions = resolveInitialTableRequest('arcane-system-backups-table', {+ pagination: { page: 1, limit: 20 },+ sort: { column: 'createdAt', direction: 'desc' }+ } satisfies SearchPaginationSortRequest);+ const [backups, policyCollection, destinations] = await Promise.all([+ systemBackupService.list(requestOptions),+ systemBackupService.getPolicies(),+ s3DestinationService.listAll()+ ]);+ return { backups, policyCollection, destinations, requestOptions };+};diff --git a/frontend/src/routes/(app)/settings/backups/s3/+page.svelte b/frontend/src/routes/(app)/settings/backups/s3/+page.sveltenew file mode 100644index 000000000..9c83591af--- /dev/null+++ b/frontend/src/routes/(app)/settings/backups/s3/+page.svelte@@ -0,0 +1,104 @@+<script lang="ts">+ import { untrack } from 'svelte';+ import { toast } from 'svelte-sonner';+ import settingsStore from '#lib/stores/config-store';+ import { SettingsPageLayout, type SettingsActionButton } from '#lib/layouts';+ import { RemoteEnvironmentIcon } from '#lib/icons';+ import { openConfirmDialog } from '#lib/components/confirm-dialog';+ import { s3DestinationService } from '#lib/services/s3-destination-service';+ import type { CreateS3Destination, S3Destination } from '#lib/types/s3-destination';+ import type { SearchPaginationSortRequest } from '#lib/types/shared';+ import * as m from '#lib/paraglide/messages.js';+ import S3DestinationDialog from './s3-destination-dialog.svelte';+ import S3DestinationTable from './s3-destination-table.svelte';++ let { data } = $props();+ let destinations = $state(untrack(() => data.destinations));+ let requestOptions = $state<SearchPaginationSortRequest>(untrack(() => data.requestOptions));+ let selected = $state<S3Destination | null>(null);+ let dialogOpen = $state(false);+ let saving = $state(false);+ const isReadOnly = $derived.by(() => $settingsStore.uiConfigDisabled);++ function openCreate() {+ selected = null;+ dialogOpen = true;+ }++ function openEdit(destination: S3Destination) {+ selected = destination;+ dialogOpen = true;+ }++ async function saveDestination(input: CreateS3Destination) {+ saving = true;+ try {+ if (selected) {+ await s3DestinationService.update(selected.id, input);+ toast.success(m.s3_destination_updated({ name: input.name }));+ } else {+ await s3DestinationService.create(input);+ toast.success(m.s3_destination_created({ name: input.name }));+ }+ destinations = await s3DestinationService.list(requestOptions);+ dialogOpen = false;+ selected = null;+ } catch (error) {+ toast.error(error instanceof Error ? error.message : m.s3_destination_save_failed());+ } finally {+ saving = false;+ }+ }++ function deleteDestination(destination: S3Destination) {+ openConfirmDialog({+ title: m.delete_name({ name: destination.name }),+ message: m.s3_destination_delete_message(),+ confirm: {+ label: m.common_delete(),+ destructive: true,+ action: async () => {+ try {+ await s3DestinationService.delete(destination.id);+ destinations = await s3DestinationService.list(requestOptions);+ toast.success(m.s3_destination_deleted({ name: destination.name }));+ } catch (error) {+ toast.error(error instanceof Error ? error.message : m.s3_destination_delete_failed());+ }+ }+ }+ });+ }++ const actionButtons: SettingsActionButton[] = $derived.by(() => [+ {+ id: 'create',+ action: 'create',+ label: m.s3_destination_add(),+ onclick: openCreate,+ disabled: isReadOnly+ }+ ]);+</script>++<SettingsPageLayout+ title={m.s3_destinations_title()}+ description={m.s3_destinations_description()}+ icon={RemoteEnvironmentIcon}+ pageType="management"+ showReadOnlyTag={isReadOnly}+ {actionButtons}+>+ {#snippet mainContent()}+ <S3DestinationTable+ bind:destinations+ bind:requestOptions+ onDestinationsChanged={(options) => s3DestinationService.list(options)}+ onEdit={openEdit}+ onDelete={deleteDestination}+ />+ {/snippet}+ {#snippet additionalContent()}+ <S3DestinationDialog bind:open={dialogOpen} destination={selected} {saving} onSubmit={saveDestination} />+ {/snippet}+</SettingsPageLayout>diff --git a/frontend/src/routes/(app)/settings/backups/s3/+page.ts b/frontend/src/routes/(app)/settings/backups/s3/+page.tsnew file mode 100644index 000000000..ecaf39977--- /dev/null+++ b/frontend/src/routes/(app)/settings/backups/s3/+page.ts@@ -0,0 +1,13 @@+import type { PageLoad } from './$types';+import { s3DestinationService } from '#lib/services/s3-destination-service';+import { resolveInitialTableRequest } from '#lib/utils/tables';+import type { SearchPaginationSortRequest } from '#lib/types/shared';++export const load: PageLoad = async () => {+ const requestOptions = resolveInitialTableRequest('arcane-s3-destinations-table', {+ pagination: { page: 1, limit: 20 },+ sort: { column: 'name', direction: 'asc' }+ } satisfies SearchPaginationSortRequest);+ const destinations = await s3DestinationService.list(requestOptions);+ return { destinations, requestOptions };+};diff --git a/frontend/src/routes/(app)/settings/backups/s3/s3-destination-dialog.svelte b/frontend/src/routes/(app)/settings/backups/s3/s3-destination-dialog.sveltenew file mode 100644index 000000000..03a530666--- /dev/null+++ b/frontend/src/routes/(app)/settings/backups/s3/s3-destination-dialog.svelte@@ -0,0 +1,195 @@+<script lang="ts">+ import { ResponsiveDialog } from '#lib/components/ui/responsive-dialog';+ import SheetFooterActions from '#lib/components/sheets/sheet-footer-actions.svelte';+ import FormInput from '#lib/components/form/form-input.svelte';+ import LabeledSwitch from '#lib/components/form/labeled-switch.svelte';+ import type { CreateS3Destination, S3Destination } from '#lib/types/s3-destination';+ import { createForm, preventDefault } from '#lib/utils/settings';+ import { s3DestinationService } from '#lib/services/s3-destination-service';+ import { toast } from 'svelte-sonner';+ import { z } from 'zod/v4';+ import * as m from '#lib/paraglide/messages.js';++ let {+ open = $bindable(),+ destination,+ saving,+ onSubmit+ }: {+ open: boolean;+ destination: S3Destination | null;+ saving: boolean;+ onSubmit: (input: CreateS3Destination) => Promise<void>;+ } = $props();++ const formSchema = z+ .object({+ name: z.string().trim().min(1, m.common_name_required()),+ endpoint: z.string(),+ bucket: z.string().trim().min(1, m.backups_s3_bucket_required()),+ region: z.string(),+ accessKeyId: z.string().trim().min(1, m.backups_s3_access_key_required()),+ secretAccessKey: z.string(),+ prefix: z.string(),+ useSsl: z.boolean(),+ forcePathStyle: z.boolean()+ })+ .superRefine((data, ctx) => {+ if (!data.endpoint.trim() && !data.region.trim()) {+ ctx.addIssue({+ code: z.ZodIssueCode.custom,+ message: m.backups_s3_region_required(),+ path: ['region']+ });+ }+ if (!destination?.secretConfigured && !data.secretAccessKey.trim()) {+ ctx.addIssue({+ code: z.ZodIssueCode.custom,+ message: m.backups_s3_secret_key_required(),+ path: ['secretAccessKey']+ });+ }+ });++ let formData = $derived<CreateS3Destination>({+ name: open && destination ? destination.name : '',+ endpoint: open && destination ? (destination.endpoint ?? '') : '',+ bucket: open && destination ? destination.bucket : '',+ region: open && destination ? destination.region : 'us-east-1',+ accessKeyId: open && destination ? destination.accessKeyId : '',+ secretAccessKey: '',+ prefix: open && destination ? (destination.prefix ?? '') : '',+ useSsl: open && destination ? destination.useSsl : true,+ forcePathStyle: open && destination ? destination.forcePathStyle : true+ });++ let { inputs, ...form } = $derived(createForm<typeof formSchema>(formSchema, formData));+ let testing = $state(false);+ let testedConfiguration = $state<string | null>(null);+ const currentConfiguration = $derived(+ JSON.stringify({+ endpoint: $inputs.endpoint.value.trim(),+ bucket: $inputs.bucket.value.trim(),+ region: $inputs.region.value.trim(),+ accessKeyId: $inputs.accessKeyId.value.trim(),+ secretAccessKey: $inputs.secretAccessKey.value.trim(),+ prefix: $inputs.prefix.value.trim().replace(/^\/+|\/+$/g, ''),+ useSsl: $inputs.useSsl.value,+ forcePathStyle: $inputs.forcePathStyle.value+ })+ );+ const connectionVerified = $derived(testedConfiguration === currentConfiguration);++ $effect(() => {+ open;+ destination?.id;+ testedConfiguration = null;+ });++ function handleSubmit() {+ const data = form.validate();+ if (!data) return;+ if (!connectionVerified) {+ toast.error(m.s3_destination_test_required());+ return;+ }+ onSubmit(data);+ }++ async function handleTest() {+ const data = form.validate();+ if (!data) return;+ const testedCandidate = currentConfiguration;+ testedConfiguration = null;+ testing = true;+ try {+ if (destination) {+ await s3DestinationService.test(destination.id, data);+ } else {+ await s3DestinationService.testConfiguration(data);+ }+ testedConfiguration = testedCandidate;+ toast.success(m.s3_destination_test_success({ name: data.name }));+ } catch (error) {+ toast.error(error instanceof Error ? error.message : m.s3_destination_test_failed({ name: data.name }));+ } finally {+ testing = false;+ }+ }+</script>++{#snippet destinationFields()}+ <FormInput bind:input={$inputs.name} label={m.common_name()} />+ <FormInput+ bind:input={$inputs.endpoint}+ label={m.backups_s3_endpoint_label()}+ description={m.backups_s3_endpoint_description()}+ />+ <FormInput bind:input={$inputs.bucket} label={m.backups_s3_bucket_label()} />+ <FormInput bind:input={$inputs.region} label={m.backups_s3_region_label()} description={m.backups_s3_region_description()} />+{/snippet}++{#snippet credentialFields()}+ <FormInput bind:input={$inputs.accessKeyId} label={m.backups_s3_access_key_label()} autocomplete="off" />+ <FormInput+ bind:input={$inputs.secretAccessKey}+ label={m.backups_s3_secret_key_label()}+ description={destination?.secretConfigured ? m.s3_destination_secret_keep_description() : undefined}+ type="password"+ autocomplete="new-password"+ />+{/snippet}++{#snippet connectionOptions()}+ <FormInput bind:input={$inputs.prefix} label={m.backups_s3_prefix_label()} description={m.backups_s3_prefix_description()} />+ <div class="mt-1 grid gap-4 border-t pt-5 sm:grid-cols-2">+ <LabeledSwitch+ id="s3-destination-ssl"+ bind:checked={$inputs.useSsl.value}+ label={m.backups_s3_ssl_label()}+ description={m.backups_s3_ssl_description()}+ />+ <LabeledSwitch+ id="s3-destination-path-style"+ bind:checked={$inputs.forcePathStyle.value}+ label={m.backups_s3_path_style_label()}+ description={m.backups_s3_path_style_description()}+ />+ </div>+{/snippet}++<ResponsiveDialog+ bind:open+ variant="sheet"+ title={destination ? m.s3_destination_edit_title() : m.s3_destination_add_title()}+ description={m.s3_destination_dialog_description()}+ contentClass="sm:max-w-[640px]"+>+ {#snippet children()}+ <form onsubmit={preventDefault(handleSubmit)} class="grid gap-4 py-6">+ {@render destinationFields()}+ {@render credentialFields()}+ {@render connectionOptions()}+ </form>+ {/snippet}+ {#snippet footer()}+ <div class="flex w-full flex-col gap-2">+ <p class={connectionVerified ? 'text-xs text-green-600' : 'text-xs text-muted-foreground'}>+ {connectionVerified ? m.s3_destination_test_verified() : m.s3_destination_test_required()}+ </p>+ <SheetFooterActions+ bind:open+ cancelDisabled={saving || testing}+ submitAction={connectionVerified ? (destination ? 'save' : 'create') : 'test'}+ submitDisabled={saving || testing}+ submitLoading={saving || testing}+ onSubmit={() => (connectionVerified ? handleSubmit() : handleTest())}+ submitLabel={connectionVerified+ ? destination+ ? m.common_save_changes()+ : m.s3_destination_add_title()+ : m.test_connection()}+ />+ </div>+ {/snippet}+</ResponsiveDialog>diff --git a/frontend/src/routes/(app)/settings/backups/s3/s3-destination-table.svelte b/frontend/src/routes/(app)/settings/backups/s3/s3-destination-table.sveltenew file mode 100644index 000000000..729588616--- /dev/null+++ b/frontend/src/routes/(app)/settings/backups/s3/s3-destination-table.svelte@@ -0,0 +1,164 @@+<script lang="ts">+ import ArcaneTable from '#lib/components/arcane-table/arcane-table.svelte';+ import RowActionsMenu from '#lib/components/arcane-table/row-actions-menu.svelte';+ import { UniversalMobileCard } from '#lib/components/arcane-table';+ import type { ColumnSpec, MobileFieldVisibility } from '#lib/components/arcane-table';+ import * as DropdownMenu from '#lib/components/ui/dropdown-menu';+ import type { S3Destination } from '#lib/types/s3-destination';+ import type { Paginated, SearchPaginationSortRequest } from '#lib/types/shared';+ import { EditIcon, RemoteEnvironmentIcon, TrashIcon, ClockIcon, GlobeIcon, TestIcon } from '#lib/icons';+ import { formatOptionalDateTime } from '#lib/utils/formatting';+ import * as m from '#lib/paraglide/messages.js';+ import IfPermitted from '#lib/components/if-permitted.svelte';+ import { Spinner } from '#lib/components/ui/spinner';+ import { toast } from 'svelte-sonner';+ import { s3DestinationService } from '#lib/services/s3-destination-service';++ let {+ destinations = $bindable(),+ requestOptions = $bindable(),+ onDestinationsChanged,+ onEdit,+ onDelete+ }: {+ destinations: Paginated<S3Destination>;+ requestOptions: SearchPaginationSortRequest;+ onDestinationsChanged: (options: SearchPaginationSortRequest) => Promise<Paginated<S3Destination>>;+ onEdit: (destination: S3Destination) => void;+ onDelete: (destination: S3Destination) => void;+ } = $props();++ const columns = [+ { accessorKey: 'name', title: m.common_name(), sortable: true, cell: NameCell },+ { accessorKey: 'bucket', title: m.backups_s3_bucket_label(), sortable: true },+ { accessorKey: 'endpoint', title: m.backups_s3_endpoint_label(), sortable: true, cell: EndpointCell },+ { accessorKey: 'region', title: m.backups_s3_region_label(), sortable: true },+ { accessorKey: 'prefix', title: m.backups_s3_prefix_label(), sortable: true, cell: PrefixCell },+ { accessorKey: 'updatedAt', title: m.common_updated(), sortable: true, cell: UpdatedCell }+ ] satisfies ColumnSpec<S3Destination>[];++ const mobileFields = [+ { id: 'bucket', label: m.backups_s3_bucket_label(), defaultVisible: true },+ { id: 'endpoint', label: m.backups_s3_endpoint_label(), defaultVisible: true },+ { id: 'region', label: m.backups_s3_region_label(), defaultVisible: true },+ { id: 'prefix', label: m.backups_s3_prefix_label(), defaultVisible: false }+ ];+ let mobileFieldVisibility = $state<Record<string, boolean>>({});+ let testingId = $state<string | null>(null);++ async function testDestination(destination: S3Destination) {+ testingId = destination.id;+ try {+ await s3DestinationService.test(destination.id);+ toast.success(m.s3_destination_test_success({ name: destination.name }));+ } catch (error) {+ toast.error(error instanceof Error ? error.message : m.s3_destination_test_failed({ name: destination.name }));+ } finally {+ testingId = null;+ }+ }+</script>++{#snippet NameCell({ item }: { item: S3Destination })}+ <span class="font-medium">{item.name}</span>+{/snippet}++{#snippet EndpointCell({ item }: { item: S3Destination })}+ <span class="text-muted-foreground">{item.endpoint || m.s3_destination_aws_default()}</span>+{/snippet}++{#snippet PrefixCell({ item }: { item: S3Destination })}+ <code class="text-xs">{item.prefix || '-'}</code>+{/snippet}++{#snippet UpdatedCell({ item }: { item: S3Destination })}+ {formatOptionalDateTime(item.updatedAt ?? item.createdAt)}+{/snippet}++{#snippet RowActions({ item }: { item: S3Destination })}+ <RowActionsMenu>+ <IfPermitted perm="s3-destinations:test">+ <DropdownMenu.Item onclick={() => testDestination(item)} disabled={testingId === item.id}>+ {#if testingId === item.id}+ <Spinner class="size-4" />+ {:else}+ <TestIcon class="size-4" />+ {/if}+ {m.test_connection()}+ </DropdownMenu.Item>+ </IfPermitted>+ <IfPermitted perm="s3-destinations:update">+ <DropdownMenu.Item onclick={() => onEdit(item)}>+ <EditIcon class="size-4" />+ {m.common_edit()}+ </DropdownMenu.Item>+ </IfPermitted>+ <IfPermitted perm="s3-destinations:delete">+ <DropdownMenu.Separator />+ <DropdownMenu.Item variant="destructive" onclick={() => onDelete(item)}>+ <TrashIcon class="size-4" />+ {m.common_delete()}+ </DropdownMenu.Item>+ </IfPermitted>+ </RowActionsMenu>+{/snippet}++{#snippet MobileCard({ item, mobileFieldVisibility }: { item: S3Destination; mobileFieldVisibility: MobileFieldVisibility })}+ <UniversalMobileCard+ {item}+ icon={{ component: RemoteEnvironmentIcon, variant: 'blue' }}+ title={(item) => item.name}+ fields={[+ {+ label: m.backups_s3_bucket_label(),+ getValue: (item) => item.bucket,+ icon: RemoteEnvironmentIcon,+ iconVariant: 'gray',+ show: mobileFieldVisibility['bucket'] ?? true+ },+ {+ label: m.backups_s3_endpoint_label(),+ getValue: (item) => item.endpoint || m.s3_destination_aws_default(),+ icon: GlobeIcon,+ iconVariant: 'gray',+ show: mobileFieldVisibility['endpoint'] ?? true+ },+ {+ label: m.backups_s3_region_label(),+ getValue: (item) => item.region,+ icon: GlobeIcon,+ iconVariant: 'gray',+ show: mobileFieldVisibility['region'] ?? true+ },+ {+ label: m.backups_s3_prefix_label(),+ getValue: (item) => item.prefix || '-',+ icon: RemoteEnvironmentIcon,+ iconVariant: 'gray',+ show: mobileFieldVisibility['prefix'] ?? false+ }+ ]}+ footer={{+ label: m.common_updated(),+ getValue: (item) => formatOptionalDateTime(item.updatedAt ?? item.createdAt),+ icon: ClockIcon+ }}+ rowActions={RowActions}+ />+{/snippet}++<ArcaneTable+ persistKey="arcane-s3-destinations-table"+ items={destinations}+ bind:requestOptions+ bind:mobileFieldVisibility+ onRefresh={async (options) => {+ requestOptions = options;+ destinations = await onDestinationsChanged(options);+ return destinations;+ }}+ {columns}+ {mobileFields}+ rowActions={RowActions}+ mobileCard={MobileCard}+/>diff --git a/frontend/src/routes/(app)/settings/backups/system-backup-table.svelte b/frontend/src/routes/(app)/settings/backups/system-backup-table.sveltenew file mode 100644index 000000000..df82d1578--- /dev/null+++ b/frontend/src/routes/(app)/settings/backups/system-backup-table.svelte@@ -0,0 +1,122 @@+<script lang="ts">+ import ArcaneTable from '#lib/components/arcane-table/arcane-table.svelte';+ import RowActionsMenu from '#lib/components/arcane-table/row-actions-menu.svelte';+ import { UniversalMobileCard, type ColumnSpec, type MobileFieldVisibility } from '#lib/components/arcane-table';+ import * as DropdownMenu from '#lib/components/ui/dropdown-menu';+ import BackupStatusCell from '#lib/components/arcane-table/cells/backup-status-cell.svelte';+ import BackupTriggerCell from '#lib/components/arcane-table/cells/backup-trigger-cell.svelte';+ import BackupDestinationCell from '#lib/components/arcane-table/cells/backup-destination-cell.svelte';+ import BackupSizeCell from '#lib/components/arcane-table/cells/backup-size-cell.svelte';+ import CreatedAtCell from '#lib/components/arcane-table/cells/created-at-cell.svelte';+ import type { Paginated, SearchPaginationSortRequest } from '#lib/types/shared';+ import type { SystemBackupRun } from '#lib/types/system-backup';+ import { BackupIcon, RestartIcon, TrashIcon, UploadIcon, ClockIcon } from '#lib/icons';+ import { bytes, formatDateTimeShort } from '#lib/utils/formatting';+ import { backupDestinationDisplay, backupStatusLabel, backupStatusVariant, backupTriggerLabel } from '#lib/utils/backups';+ import * as m from '#lib/paraglide/messages.js';++ let {+ backups = $bindable(),+ requestOptions = $bindable(),+ onChanged,+ onRestore,+ onUpload,+ onDelete+ }: {+ backups: Paginated<SystemBackupRun>;+ requestOptions: SearchPaginationSortRequest;+ onChanged: (options: SearchPaginationSortRequest) => Promise<Paginated<SystemBackupRun>>;+ onRestore: (backup: SystemBackupRun) => void;+ onUpload: (backup: SystemBackupRun) => void;+ onDelete: (backup: SystemBackupRun) => void;+ } = $props();++ let mobileFieldVisibility = $state<Record<string, boolean>>({});+ const columns = [+ { accessorKey: 'id', title: m.system_backups_id(), sortable: false, cell: IdCell },+ { accessorKey: 'status', title: m.common_status(), sortable: true, cell: StatusCell },+ { accessorKey: 'trigger', title: m.volume_backup_trigger(), sortable: true, cell: TriggerCell },+ { accessorKey: 'destination', title: m.backups_destination_label(), sortable: true, cell: DestinationCell },+ { accessorKey: 'size', title: m.common_size(), sortable: true, cell: SizeCell },+ { accessorKey: 'createdAt', title: m.common_created(), sortable: true, cell: CreatedCell },+ { accessorKey: 'error', title: m.common_error(), sortable: false, cell: ErrorCell }+ ] satisfies ColumnSpec<SystemBackupRun>[];+ const mobileFields = [+ { id: 'status', label: m.common_status(), defaultVisible: true },+ { id: 'trigger', label: m.volume_backup_trigger(), defaultVisible: true },+ { id: 'destination', label: m.backups_destination_label(), defaultVisible: true },+ { id: 'size', label: m.common_size(), defaultVisible: true }+ ];+</script>++{#snippet IdCell({ item }: { item: SystemBackupRun })}<code class="text-xs">{item.id.slice(0, 18)}…</code>{/snippet}+{#snippet StatusCell({ item }: { item: SystemBackupRun })}<BackupStatusCell status={item.status} />{/snippet}+{#snippet TriggerCell({ item }: { item: SystemBackupRun })}<BackupTriggerCell trigger={item.trigger} />{/snippet}+{#snippet DestinationCell({ item }: { item: SystemBackupRun })}<BackupDestinationCell {item} />{/snippet}+{#snippet SizeCell({ item }: { item: SystemBackupRun })}<BackupSizeCell size={item.size} />{/snippet}+{#snippet CreatedCell({ item }: { item: SystemBackupRun })}<CreatedAtCell value={item.createdAt} />{/snippet}+{#snippet ErrorCell({ item }: { item: SystemBackupRun })}<span class="max-w-72 truncate text-red-500">{item.error || '-'}</span+ >{/snippet}++{#snippet RowActions({ item }: { item: SystemBackupRun })}+ <RowActionsMenu>+ <DropdownMenu.Item onclick={() => onRestore(item)} disabled={item.status !== 'succeeded'}+ ><RestartIcon class="size-4" />{m.volumes_backups_restore()}</DropdownMenu.Item+ >+ {#if item.localSnapshotId && !item.remoteSnapshotId}+ <DropdownMenu.Item onclick={() => onUpload(item)}><UploadIcon class="size-4" />{m.backups_upload_s3()}</DropdownMenu.Item>+ {/if}+ <DropdownMenu.Separator />+ <DropdownMenu.Item variant="destructive" onclick={() => onDelete(item)}+ ><TrashIcon class="size-4" />{m.common_delete()}</DropdownMenu.Item+ >+ </RowActionsMenu>+{/snippet}++{#snippet MobileCard({ item, mobileFieldVisibility }: { item: SystemBackupRun; mobileFieldVisibility: MobileFieldVisibility })}+ <UniversalMobileCard+ {item}+ icon={{ component: BackupIcon, variant: 'blue' }}+ title={(item) => backupDestinationDisplay(item)}+ badges={[+ (item) => ({+ variant: backupStatusVariant(item.status),+ text: backupStatusLabel(item.status)+ })+ ]}+ fields={[+ {+ label: m.volume_backup_trigger(),+ getValue: (item) => backupTriggerLabel(item.trigger),+ icon: BackupIcon,+ iconVariant: 'gray',+ show: mobileFieldVisibility['trigger'] ?? true+ },+ {+ label: m.common_size(),+ getValue: (item) => bytes(item.size),+ icon: BackupIcon,+ iconVariant: 'gray',+ show: mobileFieldVisibility['size'] ?? true+ }+ ]}+ footer={{ label: m.common_created(), getValue: (item) => formatDateTimeShort(item.createdAt), icon: ClockIcon }}+ rowActions={RowActions}+ />+{/snippet}++<ArcaneTable+ persistKey="arcane-system-backups-table"+ items={backups}+ bind:requestOptions+ bind:mobileFieldVisibility+ onRefresh={async (options) => {+ requestOptions = options;+ backups = await onChanged(options);+ return backups;+ }}+ {columns}+ {mobileFields}+ rowActions={RowActions}+ mobileCard={MobileCard}+/>diff --git a/frontend/src/routes/(app)/volumes/components/volume-backup-table.svelte b/frontend/src/routes/(app)/volumes/components/volume-backup-table.svelteindex 8ed4bf8b1..7fee1b6f6 100644--- a/frontend/src/routes/(app)/volumes/components/volume-backup-table.svelte+++ b/frontend/src/routes/(app)/volumes/components/volume-backup-table.svelte@@ -1,8 +1,10 @@<script lang="ts">import { m } from '#lib/paraglide/messages';import { volumeBackupService, type VolumeBackupListResponse } from '#lib/services/volume-backup-service';+ import { s3DestinationService } from '#lib/services/s3-destination-service';import { volumeService } from '#lib/services/volume-service';- import type { BackupEntry } from '#lib/types/shared';+ import type { BackupEntry, CreateVolumeBackupRequest, VolumeBackupPolicy } from '#lib/types/shared';+ import type { S3Destination } from '#lib/types/s3-destination';import { onMount } from 'svelte';import {LoadingSpinnerIcon,@@ -14,14 +16,22 @@DownloadIcon,RestartIcon,FileTextIcon,- AlertIcon+ AlertIcon,+ UploadIcon,+ ArrowDownIcon} from '#lib/icons';- import { ArcaneButton } from '#lib/components/arcane-button';+ import { ArcaneButton, arcaneButtonVariants } from '#lib/components/arcane-button';+ import * as ButtonGroup from '#lib/components/ui/button-group';import { toast } from 'svelte-sonner';import { bytes, formatDateTimeShort } from '#lib/utils/formatting';import ArcaneTable from '#lib/components/arcane-table/arcane-table.svelte';import type { SearchPaginationSortRequest } from '#lib/types/shared';- import { UniversalMobileCard, type ColumnSpec, type MobileFieldVisibility } from '#lib/components/arcane-table';+ import {+ UniversalMobileCard,+ type BulkAction,+ type ColumnSpec,+ type MobileFieldVisibility+ } from '#lib/components/arcane-table';import * as DropdownMenu from '#lib/components/ui/dropdown-menu';import RowActionsMenu from '#lib/components/file-browser/row-actions-menu.svelte';import { openConfirmDialog } from '#lib/components/confirm-dialog';@@ -34,11 +44,28 @@import { hasPermission } from '#lib/utils/auth';import IfPermitted from '#lib/components/if-permitted.svelte';import { activityToastOptions, extractActivityId } from '#lib/utils/activity-toast';+ import BackupPolicyDialog from '#lib/components/backup-policy-dialog.svelte';+ import BackupPolicyCard from '#lib/components/backup-policy-card.svelte';+ import BackupStatusCell from '#lib/components/arcane-table/cells/backup-status-cell.svelte';+ import BackupTriggerCell from '#lib/components/arcane-table/cells/backup-trigger-cell.svelte';+ import BackupDestinationCell from '#lib/components/arcane-table/cells/backup-destination-cell.svelte';+ import BackupSizeCell from '#lib/components/arcane-table/cells/backup-size-cell.svelte';+ import CreatedAtCell from '#lib/components/arcane-table/cells/created-at-cell.svelte';+ import { cn } from '#lib/utils';+ import { bulkConfirmAndRun } from '#lib/utils/bulk-actions';+ import { extractApiErrorMessage } from '#lib/utils/api';+ import {+ backupDestinationDisplay,+ backupTriggerLabel,+ s3DestinationOptions as buildS3DestinationOptions+ } from '#lib/utils/backups';+ import SelectWithLabel from '#lib/components/form/select-with-label.svelte';let { volumeName }: { volumeName: string } = $props();const currentEnvId = $derived(environmentStore.selected?.id || '0');const canBackupVolume = $derived(hasPermission('volumes:backup', currentEnvId));+ const canDeleteBackup = $derived(hasPermission('volumes:backup', currentEnvId));let backupsPaginated = $state<VolumeBackupListResponse>({data: [],@@ -50,6 +77,14 @@}});let backupWarnings = $state<string[]>([]);+ let backupPolicies = $state<VolumeBackupPolicy[]>([]);+ let s3Destinations = $state<S3Destination[]>([]);+ let showBackupPolicy = $state(false);+ let editingBackupPolicyId = $state<string | undefined>();+ let showS3DestinationDialog = $state(false);+ let onDemandDestination = $state<'s3' | 'local_s3'>('s3');+ let onDemandS3DestinationId = $state('');+ const s3DestinationOptions = $derived(buildS3DestinationOptions(s3Destinations));let requestOptions = $state<SearchPaginationSortRequest>({pagination: { page: 1, limit: 10 },@@ -57,6 +92,9 @@});let creating = $state(false);+ let deletingSelected = $state(false);+ let selectedIds = $state<string[]>([]);+ let uploadingBackupId = $state<string | null>(null);let restoringFiles = $state(false);let showRestoreFiles = $state(false);let restoreTarget = $state<BackupEntry | null>(null);@@ -76,28 +114,48 @@backupsPaginated = result;backupWarnings = result.warnings ?? [];return result;- } catch (e: any) {- toast.error(e.message || 'Failed to load backups');+ } catch (error) {+ toast.error(error instanceof Error ? error.message : m.volumes_backup_load_failed());return backupsPaginated;}}- async function handleCreate() {+ async function handleCreate(request?: CreateVolumeBackupRequest) {creating = true;try {- const result = await volumeBackupService.createBackup(volumeName);+ const result = await volumeBackupService.createBackup(volumeName, request);toast.success(m.common_success(), activityToastOptions(extractActivityId(result)));await loadData(requestOptions);- } catch (e: any) {- toast.error(e.message || m.common_failed());+ return true;+ } catch (error) {+ toast.error(error instanceof Error ? error.message : m.common_failed());+ return false;} finally {creating = false;}}+ function openS3DestinationDialog(destination: 's3' | 'local_s3') {+ onDemandDestination = destination;+ onDemandS3DestinationId = s3Destinations.length === 1 ? (s3Destinations[0]?.id ?? '') : '';+ showS3DestinationDialog = true;+ }++ async function createS3Backup() {+ if (!onDemandS3DestinationId) return;+ const created = await handleCreate({+ destination: onDemandDestination,+ s3DestinationId: onDemandS3DestinationId+ });+ if (created) {+ showS3DestinationDialog = false;+ onDemandS3DestinationId = '';+ }+ }+async function handleDelete(backup: BackupEntry) {openConfirmDialog({- title: m.common_remove_title({ resource: 'Backup' }),+ title: m.common_remove_title({ resource: m.volumes_workspace_backup() }),message: m.volumes_backup_delete_confirm(),confirm: {label: m.common_remove(),@@ -105,16 +163,54 @@action: async () => {try {const result = await volumeBackupService.deleteBackup(backup.id);- toast.success(m.common_delete_success({ resource: 'Backup' }), activityToastOptions(extractActivityId(result)));+ toast.success(+ m.common_delete_success({ resource: m.volumes_workspace_backup() }),+ activityToastOptions(extractActivityId(result))+ );+ await loadData(requestOptions);+ } catch (error) {+ toast.error(extractApiErrorMessage(error));await loadData(requestOptions);- } catch (e: any) {- toast.error(e.message || m.common_delete_failed({ resource: 'Backup' }));}}}});}+ function handleDeleteSelected(ids: string[]) {+ bulkConfirmAndRun({+ ids,+ title: m.volume_backups_remove_selected_title({ count: ids.length }),+ message: m.volume_backups_remove_selected_message({ count: ids.length }),+ confirmLabel: m.common_remove(),+ destructive: true,+ run: (id) => volumeBackupService.deleteBackup(id),+ messages: {+ success: (count) => m.common_bulk_remove_success({ count, resource: m.volumes_backups_title() }),+ partial: (success, total, failed) =>+ m.common_bulk_remove_partial({ success, total, failed, resource: m.volumes_backups_title() }),+ failure: () => m.common_bulk_remove_failed({ count: ids.length, resource: m.volumes_backups_title() })+ },+ setLoading: (loading) => (deletingSelected = loading),+ onItemFailure: (_id, error) => toast.error(extractApiErrorMessage(error)),+ onComplete: () => loadData(requestOptions),+ clearSelection: () => (selectedIds = [])+ });+ }++ async function handleUpload(backup: BackupEntry, s3DestinationId: string) {+ uploadingBackupId = backup.id;+ try {+ const result = await volumeBackupService.uploadBackup(backup.id, s3DestinationId);+ toast.success(m.backups_upload_s3_success(), activityToastOptions(extractActivityId(result)));+ await loadData(requestOptions);+ } catch (error) {+ toast.error(error instanceof Error ? error.message : m.backups_upload_s3_failed());+ } finally {+ uploadingBackupId = null;+ }+ }+async function openRestoreFilesDialog(backup: BackupEntry) {restoreTarget = backup;selectedPaths = [];@@ -124,8 +220,8 @@backupFilesLoading = true;try {backupFiles = await volumeBackupService.listBackupFiles(backup.id);- } catch (e: any) {- toast.error(e.message || m.common_failed());+ } catch (error) {+ toast.error(error instanceof Error ? error.message : m.common_failed());} finally {backupFilesLoading = false;}@@ -176,8 +272,8 @@const result = await volumeBackupService.restoreBackup(volumeName, backup.id);toast.success(m.volumes_backup_restore_success(), activityToastOptions(extractActivityId(result)));await loadData(requestOptions);- } catch (e: any) {- toast.error(e.message || m.common_failed());+ } catch (error) {+ toast.error(error instanceof Error ? error.message : m.common_failed());}}}@@ -196,8 +292,8 @@activityToastOptions(extractActivityId(result)));showRestoreFiles = false;- } catch (e: any) {- toast.error(e.message || m.common_failed());+ } catch (error) {+ toast.error(error instanceof Error ? error.message : m.common_failed());} finally {restoringFiles = false;}@@ -208,30 +304,85 @@}onMount(async () => {- await loadData(requestOptions);+ const [collection, destinations] = await Promise.all([+ volumeBackupService.getPolicies(volumeName),+ canBackupVolume ? s3DestinationService.listAll().catch(() => []) : Promise.resolve([]),+ loadData(requestOptions)+ ]);+ backupPolicies = collection.policies;+ s3Destinations = destinations;});const columns = [{ accessorKey: 'id', title: m.common_id(), sortable: true, cell: IdCell },+ { accessorKey: 'status', title: m.common_status(), sortable: true, cell: StatusCell },+ { accessorKey: 'trigger', title: m.volume_backup_trigger(), sortable: true, cell: TriggerCell },+ { accessorKey: 'destination', title: m.backups_destination_label(), sortable: true, cell: DestinationCell },{ accessorKey: 'size', title: m.common_size(), sortable: true, cell: SizeCell },- { accessorKey: 'createdAt', title: m.common_created(), sortable: true, cell: CreatedCell }+ { accessorKey: 'createdAt', title: m.common_created(), sortable: true, cell: CreatedCell },+ {+ accessorKey: 'remoteSnapshotId',+ title: m.volume_backup_remote_snapshot(),+ sortable: true,+ cell: RemoteSnapshotCell,+ hidden: true+ },+ { accessorKey: 'error', title: m.common_error(), sortable: false, cell: ErrorCell, hidden: true }] satisfies ColumnSpec<BackupEntry>[];- const mobileFields = [{ id: 'size', label: m.common_size(), defaultVisible: true }];+ const mobileFields = [+ { id: 'status', label: m.common_status(), defaultVisible: true },+ { id: 'trigger', label: m.volume_backup_trigger(), defaultVisible: true },+ { id: 'destination', label: m.backups_destination_label(), defaultVisible: true },+ { id: 'size', label: m.common_size(), defaultVisible: true },+ { id: 'remoteSnapshotId', label: m.volume_backup_remote_snapshot(), defaultVisible: false }+ ];++ const bulkActions = $derived.by<BulkAction[]>(() => [+ {+ id: 'remove',+ label: m.common_remove_selected_count({ count: selectedIds.length }),+ action: 'remove',+ onClick: handleDeleteSelected,+ loading: deletingSelected,+ disabled: !canDeleteBackup || deletingSelected || selectedIds.length === 0,+ icon: TrashIcon+ }+ ]);let mobileFieldVisibility = $state<Record<string, boolean>>({});</script>-{#snippet IdCell({ value }: { value: any })}- <code class="font-mono text-xs font-medium">{value}</code>+{#snippet IdCell({ item }: { item: BackupEntry })}+ <code class="font-mono text-xs font-medium">{item.id}</code>+{/snippet}++{#snippet StatusCell({ item }: { item: BackupEntry })}+ <BackupStatusCell status={item.status} />+{/snippet}++{#snippet TriggerCell({ item }: { item: BackupEntry })}+ <BackupTriggerCell trigger={item.trigger} />+{/snippet}++{#snippet DestinationCell({ item }: { item: BackupEntry })}+ <BackupDestinationCell {item} />{/snippet}-{#snippet SizeCell({ value }: { value: any })}- {formatBytes(Number(value))}+{#snippet SizeCell({ item }: { item: BackupEntry })}+ <BackupSizeCell size={item.size} />{/snippet}-{#snippet CreatedCell({ value }: { value: any })}- {formatDateTimeShort(String(value))}+{#snippet CreatedCell({ item }: { item: BackupEntry })}+ <CreatedAtCell value={item.createdAt} />+{/snippet}++{#snippet RemoteSnapshotCell({ item }: { item: BackupEntry })}+ <code class="text-xs">{item.remoteSnapshotId || m.volume_backup_not_uploaded()}</code>+{/snippet}++{#snippet ErrorCell({ item }: { item: BackupEntry })}+ <span class="line-clamp-2 max-w-80 text-xs text-destructive">{item.error || '-'}</span>{/snippet}{#snippet RowActions({ item }: { item: BackupEntry })}@@ -239,18 +390,31 @@{#if canBackupVolume}<DropdownMenu.Item onclick={() => handleRestore(item)}><RestartIcon class="size-4" />- Restore+ {m.volumes_backups_restore()}</DropdownMenu.Item><DropdownMenu.Item onclick={() => openRestoreFilesDialog(item)}><FileTextIcon class="size-4" />- Restore files+ {m.volume_restore_files()}</DropdownMenu.Item>{/if}- <DropdownMenu.Item onclick={() => volumeBackupService.downloadBackup(item.id)}>- <DownloadIcon class="size-4" />- {m.templates_download()}- </DropdownMenu.Item>- <IfPermitted perm="volumes:delete">+ {#if item.format === 'archive' || item.localSnapshotId}+ <DropdownMenu.Item onclick={() => volumeBackupService.downloadBackup(item.id)}>+ <DownloadIcon class="size-4" />+ {m.templates_download()}+ </DropdownMenu.Item>+ {/if}+ {#if canBackupVolume && s3Destinations.length}+ {#each s3Destinations as destination (destination.id)}+ <DropdownMenu.Item+ disabled={item.status !== 'succeeded' || Boolean(item.remoteSnapshotId) || uploadingBackupId === item.id}+ onclick={() => handleUpload(item, destination.id)}+ >+ <UploadIcon class="size-4" />+ {m.backups_upload_s3()} · {destination.name}+ </DropdownMenu.Item>+ {/each}+ {/if}+ <IfPermitted perm="volumes:backup"><DropdownMenu.Separator /><DropdownMenu.Item variant="destructive" onclick={() => handleDelete(item)}><TrashIcon class="size-4" />@@ -262,15 +426,50 @@{#snippet ToolbarActions()}{#if canBackupVolume}- <ArcaneButton- action="create"- customLabel={m.volumes_backup_create()}- loading={creating}- disabled={creating}- onclick={handleCreate}- size="sm"- icon={AddIcon}- />+ <div class="flex items-center gap-2">+ <ArcaneButton+ action="create"+ customLabel={m.volume_backup_add_schedule()}+ onclick={() => {+ editingBackupPolicyId = undefined;+ showBackupPolicy = true;+ }}+ size="sm"+ icon={AddIcon}+ />+ <ButtonGroup.Root>+ <ArcaneButton+ action="create"+ customLabel={m.volumes_backup_create()}+ loading={creating}+ disabled={creating}+ onclick={() => handleCreate()}+ size="sm"+ icon={AddIcon}+ />+ <DropdownMenu.Root>+ <DropdownMenu.Trigger+ class={cn(arcaneButtonVariants({ tone: 'outline-primary', size: 'icon' }), 'size-8 rounded-md')}+ aria-label={m.common_open_menu()}+ disabled={creating}+ >+ <ArrowDownIcon class="size-4" />+ </DropdownMenu.Trigger>+ <DropdownMenu.Content align="end" class="w-64">+ <DropdownMenu.Label>{m.backups_destination_label()}</DropdownMenu.Label>+ <DropdownMenu.Item onclick={() => handleCreate({ destination: 'local' })}>+ {m.local()}+ </DropdownMenu.Item>+ <DropdownMenu.Item onclick={() => openS3DestinationDialog('s3')}>+ {m.backups_destination_s3()}+ </DropdownMenu.Item>+ <DropdownMenu.Item onclick={() => openS3DestinationDialog('local_s3')}>+ {m.backups_destination_local_s3()}+ </DropdownMenu.Item>+ </DropdownMenu.Content>+ </DropdownMenu.Root>+ </ButtonGroup.Root>+ </div>{/if}{/snippet}@@ -286,12 +485,33 @@icon={{ component: VolumesIcon, variant: 'blue' }}title={(item) => item.id}fields={[+ {+ label: m.volume_backup_trigger(),+ getValue: (item) => backupTriggerLabel(item.trigger),+ icon: ClockIcon,+ iconVariant: 'gray',+ show: mobileFieldVisibility['trigger'] ?? true+ },{label: m.common_size(),getValue: (item) => formatBytes(item.size),icon: InfoIcon,iconVariant: 'gray',show: mobileFieldVisibility['size'] ?? true+ },+ {+ label: m.backups_destination_label(),+ getValue: (item) => backupDestinationDisplay(item),+ icon: DownloadIcon,+ iconVariant: 'gray',+ show: mobileFieldVisibility['destination'] ?? true+ },+ {+ label: m.volume_backup_remote_snapshot(),+ getValue: (item) => item.remoteSnapshotId || m.volume_backup_not_uploaded(),+ icon: DownloadIcon,+ iconVariant: 'gray',+ show: mobileFieldVisibility['remoteSnapshotId'] ?? false}]}footer={{@@ -307,6 +527,27 @@<div class="flex items-center justify-between"><h2 class="text-lg font-semibold">{m.volumes_backups_title()}</h2></div>+ <div class="grid grid-cols-1 gap-1.5 text-xs text-muted-foreground sm:grid-cols-2 xl:grid-cols-3">+ {#each backupPolicies as policy (policy.id)}+ <BackupPolicyCard+ {policy}+ showStopContainers+ onEdit={canBackupVolume+ ? () => {+ editingBackupPolicyId = policy.id;+ showBackupPolicy = true;+ }+ : undefined}+ />+ {/each}+ </div>++ <Alert.Root class="py-2 [&>svg]:top-2">+ <InfoIcon class="size-4" />+ <Alert.Description class="text-xs">+ {m.volume_backup_encryption_note()}+ </Alert.Description>+ </Alert.Root>{#if backupWarnings.length > 0}<Alert.Root variant="warning" class="py-2 [&>svg]:top-2">@@ -320,11 +561,13 @@<ArcaneTablepersistKey="arcane-volume-backup-table"items={backupsPaginated}+ bind:selectedIdsbind:requestOptionsbind:mobileFieldVisibilityonRefresh={loadData}{columns}{mobileFields}+ {bulkActions}rowActions={RowActions}mobileCard={BackupMobileCardSnippet}customToolbarActions={ToolbarActions}@@ -342,7 +585,7 @@<Alert.Root class="py-2 [&>svg]:top-2"><InfoIcon class="size-4" /><Alert.Description class="text-xs">- {m.volumes_backup_safety_info()}+ {m.volume_backup_restore_files_lifecycle_info()}</Alert.Description></Alert.Root>@@ -405,3 +648,55 @@{/if}{/snippet}</ResponsiveDialog>++<ResponsiveDialog+ bind:open={showS3DestinationDialog}+ title={m.volume_backup_choose_s3_destination()}+ description={m.volume_backup_choose_s3_destination_description()}+ contentClass="sm:max-w-[520px]"+>+ {#snippet children()}+ <div class="py-2">+ <SelectWithLabel+ id="on-demand-volume-backup-s3-destination"+ value={onDemandS3DestinationId}+ onValueChange={(value) => (onDemandS3DestinationId = value)}+ label={m.volume_backup_s3_destination_label()}+ description={m.volume_backup_s3_destination_description()}+ options={s3DestinationOptions}+ />+ {#if s3Destinations.length === 0}+ <p class="mt-2 text-xs text-muted-foreground">{m.volume_backup_no_s3_destinations()}</p>+ {/if}+ </div>+ {/snippet}+ {#snippet footer()}+ <ArcaneButton action="cancel" onclick={() => (showS3DestinationDialog = false)} disabled={creating} />+ <ArcaneButton+ action="create"+ customLabel={m.volumes_backup_create()}+ onclick={createS3Backup}+ loading={creating}+ disabled={creating || !onDemandS3DestinationId}+ />+ {/snippet}+</ResponsiveDialog>++<BackupPolicyDialog+ bind:open={showBackupPolicy}+ idPrefix="volume-backup-policy"+ policies={backupPolicies}+ policyId={editingBackupPolicyId}+ addTitle={m.volume_backup_add_schedule()}+ description={m.volume_backup_policy_description()}+ enabledDescription={m.volume_backup_policy_enabled_description()}+ defaultSchedule="0 0 2 * * *"+ showStopContainers+ updatePolicies={async (policies) => (await volumeBackupService.updatePolicies(volumeName, policies)).policies}+ messages={{+ saved: m.volume_backup_policy_saved(),+ saveFailed: m.volume_backup_policy_save_failed(),+ removed: m.volume_backup_schedule_removed()+ }}+ onSaved={(policies) => (backupPolicies = policies)}+/>diff --git a/go.work.sum b/go.work.sumindex d6a9542e7..fbe20fb65 100644--- a/go.work.sum+++ b/go.work.sum@@ -472,6 +472,7 @@ github.com/aws/aws-sdk-go-v2 v1.39.6/go.mod h1:c9pm7VwuW0UPxAEYGyTmyurVcNrbF6Rt/github.com/aws/aws-sdk-go-v2 v1.41.1 h1:ABlyEARCDLN034NhxlRUSZr4l71mh+T5KAeGh6cerhU=github.com/aws/aws-sdk-go-v2 v1.41.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=github.com/aws/aws-sdk-go-v2 v1.42.0/go.mod h1:27+ACypSLljLAEKsCYOmrjKh83vuTRkuAe9Uv/3A4bg=+github.com/aws/aws-sdk-go-v2 v1.43.0/go.mod h1:5pKeft2eJj+gElQ38Jqg4ibCqh+/AK33/0X3hip7IjM=github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0 h1:6GMWV6CNpA/6fbFHnoAjrv4+LGfyTqZz2LtCHnspgDg=github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0/go.mod h1:/mXlTIVG9jbxkqDnr5UQNQxW1HRYxeGklkM9vAFeabg=github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 h1:t9yYsydLYNBk9cJ73rgPhPWqOh/52fcWDQB5b1JsKSY=@@ -480,10 +481,6 @@ github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaXgithub.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5 h1:zWFmPmgw4sveAYi1mRqG+E/g0461cJ5M4bJ8/nc6d3Q=github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5/go.mod h1:nVUlMLVV8ycXSb7mSkcNu9e3v/1TJq2RTlrPwhYWr5c=-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.13 h1:p1BBrg/Hhp6uK7zpejeI8QFXHJeC/mynzi04Sl03k9g=-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.13/go.mod h1:8cIfkE9MDhkRZGpQ22aV6/lkYeYSozpz16Smrs5x4Ls=-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 h1:3IZY0XAJquT3aHzbkHfPzy4ACPcEjVG0x87KOwtpqGY=-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14/go.mod h1:zwM6veDkhGgQFqkBy+uT28AAYpLu+uFMlPl+rCg/73E=github.com/aws/aws-sdk-go-v2/config v1.31.3 h1:RIb3yr/+PZ18YYNe6MDiG/3jVoJrPmdoCARwNkMGvco=github.com/aws/aws-sdk-go-v2/config v1.31.3/go.mod h1:jjgx1n7x0FAKl6TnakqrpkHWWKcX3xfWtdnIJs5K9CE=github.com/aws/aws-sdk-go-v2/config v1.31.20 h1:/jWF4Wu90EhKCgjTdy1DGxcbcbNrjfBHvksEL79tfQc=@@ -520,6 +517,7 @@ github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13/go.mod h1:oGnKwIYZ4Xgithub.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 h1:xOLELNKGp2vsiteLsvLPwxC+mYmO6OZ8PYgiuPJzF8U=github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17/go.mod h1:5M5CI3D12dNOtH3/mk6minaRwI2/37ifCURZISxA/IQ=github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.29/go.mod h1:MzoLFUArKGpGD+ukmPiTPG1X5x4o6M2kq4v2dr1FiEc=+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.31/go.mod h1:aVyUoytEyOViR6jhq6jula0xkc5NfBE2hgeF6BvOrao=github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.4 h1:j7vjtr1YIssWQOMeOWRbh3z8g2oY/xPjnZH2gLY4sGw=github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.4/go.mod h1:yDmJgqOiH4EA8Hndnv4KwAo8jCGTSnM5ASG1nBI+toA=github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13 h1:HBSI2kDkMdWz4ZM7FjwE7e/pWDEZ+nR95x8Ztet1ooY=@@ -527,6 +525,7 @@ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13/go.mod h1:YE94ZoDArI7github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 h1:WWLqlh79iO48yLkj1v3ISRNiv+3KdQoZ6JWyfcsyQik=github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17/go.mod h1:EhG22vHRrvF8oXSTYStZhJc1aUgKtnJe+aOiFEV90cM=github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.29/go.mod h1:71wt8W2EgswdZy9Mf9KNnzxZ3TiZlv4caKghPktDOkA=+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.31/go.mod h1:OERqI9k0draSLB8O8woxY3q25ZWTELRK4RRoLMuMZFo=github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=@@ -538,6 +537,7 @@ github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.4/go.mod h1:SPBBhkJxjcrzJBc+qY85egithub.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12 h1:itu4KHu8JK/N6NcLIISlf3LL1LccMqruLUXZ9y7yBZw=github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12/go.mod h1:i+6vTU3xziikTY3vcox23X8pPGW5X3wVgd1VZ7ha+x8=github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.30/go.mod h1:AS0HycUvJRFvTt613AYDOgO2jzw+00cVSMny8XB3yMY=+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.32/go.mod h1:2tNZkuWz54arj8mHVf+8Y7cKkcD8Wr/fBpENgEXpjLc=github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.0 h1:6+lZi2JeGKtCraAj1rpoZfKqnQ9SptseRZioejfUOLM=github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.0/go.mod h1:eb3gfbVIxIoGgJsi9pGne19dhCBpK6opTYpQqAmdy44=github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 h1:x2Ibm/Af8Fi+BH+Hsn9TXGdT+hKbDd5XOTZxTMxDk7o=@@ -549,10 +549,6 @@ github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.4 h1:Beh9oVgtQnBgR4sgithub.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.4/go.mod h1:b17At0o8inygF+c6FOD3rNyYZufPw62o9XJbSfQPgbo=github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3 h1:NEe7FaViguRQEm8zl8Ay/kC/QRsMtWUiCGZajQIsLdc=github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3/go.mod h1:JLuCKu5VfiLBBBl/5IzZILU7rxS0koQpHzMOCzycOJU=-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.22 h1:V51LGlOq/1VsDsHUdoklAQi7rMmx4qQubvFYAlP2254=-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.22/go.mod h1:4Pzhyz8hJOm2bepgl+NjvRx8vlUFAIIvJnZ/MkcNPpU=-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.24 h1:mdPwDQPqxlw9Sc62Nt15yjEcARaDbPXkjRYtXsUripo=-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.24/go.mod h1:ls5ytnwLTcQaUu32fMYXFI3MjpKuTwL840PAm9iqyEg=github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.4 h1:ueB2Te0NacDMnaC+68za9jLwkjzxGWm0KB5HTUHjLTI=github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.4/go.mod h1:nLEfLnVMmLvyIG58/6gsSA03F1voKGaCfHV7+lR8S7s=github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13 h1:kDqdFvMY4AtKoACfzIGD8A0+hbT41KTKF//gq7jITfM=@@ -560,22 +556,15 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13/go.mod h1:lgithub.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 h1:RuNSMoozM8oXlgLG/n6WLaFGoea7/CddrCfIiSA+xdY=github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17/go.mod h1:F2xxQ9TZz5gDWsclCtPQscGpP0VUOc8RqgFM3vDENmU=github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.29/go.mod h1:LfRkPCD8YHDM2E5eTkos2UpwYeZnBcVarTa8L59bJHA=+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.31/go.mod h1:wAhpCQbkov+IcvjozJbd2xRCoZybUEHNkcFunssNACg=github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.4 h1:HVSeukL40rHclNcUqVcBwE1YoZhOkoLeBfhUqR3tjIU=github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.4/go.mod h1:DnbBOv4FlIXHj2/xmrUQYtawRFC9L9ZmQPz+DBc6X5I=github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12 h1:R3uW0iKl8rgNEXNjVGliW/oMEh9fO/LlUEV8RvIFr1I=github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12/go.mod h1:XEttbEr5yqsw8ebi7vlDoGJJjMXRez4/s9pibpJyL5s=-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.29 h1:hiME6pBzC7OTl9LMtlyTWBuEl1f4QBcUmFDKC7MLXtc=-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.29/go.mod h1:G7RP+uhagpKtKhd1BM9N6JQqjCcGEU47K5lBVZQyRQw=-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.32 h1:jWXtZdCnhXa9sGFixRaU2AxT4DIVse9HS4E2f+/KwV0=-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.32/go.mod h1:9JS1UpfVvyD/ZPX8GsKb/Pq8scEM+7GP5fqh9SwH7po=github.com/aws/aws-sdk-go-v2/service/s3 v1.87.1 h1:2n6Pd67eJwAb/5KCX62/8RTU0aFAAW7V5XIGSghiHrw=github.com/aws/aws-sdk-go-v2/service/s3 v1.87.1/go.mod h1:w5PC+6GHLkvMJKasYGVloB3TduOtROEMqm15HSuIbw4=github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1 h1:Dq82AV+Qxpno/fG162eAhnD8d48t9S+GZCfz7yv1VeA=github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1/go.mod h1:MbKLznDKpf7PnSonNRUVYZzfP0CeLkRIUexeblgKcU4=-github.com/aws/aws-sdk-go-v2/service/s3 v1.103.3 h1:JRseEu/vIDMaWis4bSw0QbXL+cvIGc1XnX076H5ZXLE=-github.com/aws/aws-sdk-go-v2/service/s3 v1.103.3/go.mod h1:77ZAgynvx1txMvDG8gGWoWkO1augYDxkp9JElWFgjQU=-github.com/aws/aws-sdk-go-v2/service/s3 v1.106.0 h1:7QZWVJZWzHivHWIa+5TELLaBBkbuoj0GPwQtMlJ0sqk=-github.com/aws/aws-sdk-go-v2/service/s3 v1.106.0/go.mod h1:fcvq5L7dK+5cQFicEJwpI6e6Wn8NY2i6yT5wRLYVc7s=github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 h1:VrhDvQib/i0lxvr3zqlUwLwJP4fpmpyD9wYG1vfSu+Y=github.com/aws/aws-sdk-go-v2/service/signin v1.0.5/go.mod h1:k029+U8SY30/3/ras4G/Fnv/b88N4mAfliNn08Dem4M=github.com/aws/aws-sdk-go-v2/service/signin v1.1.5/go.mod h1:LxYujSTLPRlp2vTtcUO/+1ilrew8ytt6SvQyOgejzFQ=@@ -884,6 +873,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cugithub.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=+github.com/docker/buildx v0.36.0 h1:5UYbvkIXhkjiRZG15ZoWuA1BIuFatexA9yeL+b72MpM=+github.com/docker/buildx v0.36.0/go.mod h1:APOldMT9ejTIiYJ44OxhFX1zt+MM1cR9zz4DQLq8qLw=github.com/docker/cli v29.0.3+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=github.com/docker/cli v29.1.4+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=github.com/docker/cli v29.1.5+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=@@ -894,6 +885,8 @@ github.com/docker/cli-docs-tool v0.10.0/go.mod h1:5EM5zPnT2E7yCLERZmrDA234Vwn09fgithub.com/docker/cli-docs-tool v0.11.0 h1:7d8QARFb7QEobizqxmEM7fOteZEHwH/zWgHQtHZEcfE=github.com/docker/cli-docs-tool v0.11.0/go.mod h1:ma8BKiisUo8D6W05XEYIh3oa1UbgrZhi1nowyKFJa8Q=github.com/docker/compose/v5 v5.0.0/go.mod h1:BurapGv8zmYnsbSmlpCz5EU2Pi3YFV/PjeUnoFpcw64=+github.com/docker/compose/v5 v5.4.0 h1:i0xFmVvWv9QILfRE2PD07GmQqJYjpYAGutafIrJLURE=+github.com/docker/compose/v5 v5.4.0/go.mod h1:bNeEncwZhG0F4MN+0zgln4EA2ILqGZC7knRI8QYJz9U=github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0=github.com/docker/docker-credential-helpers v0.9.3/go.mod h1:x+4Gbw9aGmChi3qTLZj8Dfn0TD20M/fuWy0E5+WDeCo=@@ -1525,6 +1518,8 @@ github.com/moby/buildkit v0.26.2/go.mod h1:ylDa7IqzVJgLdi/wO7H1qLREFQpmhFbw2fbn4github.com/moby/buildkit v0.26.3/go.mod h1:4T4wJzQS4kYWIfFRjsbJry4QoxDBjK+UGOEOs1izL7w=github.com/moby/buildkit v0.27.0/go.mod h1:4STUkNc5t1nf03HS+01UmI2X6FdfOI3XaKt9QNoTsms=github.com/moby/go-archive v0.1.0/go.mod h1:G9B+YoujNohJmrIYFBpSd54GTUB4lt9S+xVQvsJyFuo=+github.com/moby/go-archive v0.3.2 h1:x893kC3zRygv2C+k4Y9kMxYRPLCj4XEJB0srbAP06Hw=+github.com/moby/go-archive v0.3.2/go.mod h1:Npdv43fFqlhZW7Xo8fbm3ZMYFvAGNviUPqX21VERbcE=github.com/moby/moby/api v1.52.0/go.mod h1:8mb+ReTlisw4pS6BRzCMts5M49W5M7bKt1cJy/YbAqc=github.com/moby/moby/api v1.54.2/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs=github.com/moby/moby/client v0.2.1/go.mod h1:O+/tw5d4a1Ha/ZA/tPxIZJapJRUS6LNZ1wiVRxYHyUE=diff --git a/tests/setup/compose-postgres.yaml b/tests/setup/compose-postgres.yamlindex 678a52872..7a6ae1d5a 100644--- a/tests/setup/compose-postgres.yaml+++ b/tests/setup/compose-postgres.yaml@@ -1,4 +1,19 @@services:+ minio:+ image: quay.io/minio/minio:RELEASE.2025-09-07T16-13-09Z+ # A directory created under the data dir is served as a bucket, so backups+ # have somewhere to write without a separate provisioning container.+ entrypoint:+ ['sh', '-c', 'mkdir -p /data/arcane-backups && exec minio server /data --address :9000']+ environment:+ - MINIO_ROOT_USER=arcane-test+ - MINIO_ROOT_PASSWORD=arcane-test-secret+ healthcheck:+ test: ['CMD', 'curl', '-fs', 'http://localhost:9000/minio/health/live']+ interval: 2s+ timeout: 5s+ retries: 15+postgres:image: postgres:18-alpineenvironment:@@ -33,6 +48,8 @@ services:depends_on:postgres:condition: service_healthy+ minio:+ condition: service_healthybuild:args:- BUILD_TAGS=playwrightdiff --git a/tests/setup/compose-proxy.yaml b/tests/setup/compose-proxy.yamlindex ab944bcf5..b308392a7 100644--- a/tests/setup/compose-proxy.yaml+++ b/tests/setup/compose-proxy.yaml@@ -1,4 +1,19 @@services:+ minio:+ image: quay.io/minio/minio:RELEASE.2025-09-07T16-13-09Z+ # A directory created under the data dir is served as a bucket, so backups+ # have somewhere to write without a separate provisioning container.+ entrypoint:+ ['sh', '-c', 'mkdir -p /data/arcane-backups && exec minio server /data --address :9000']+ environment:+ - MINIO_ROOT_USER=arcane-test+ - MINIO_ROOT_PASSWORD=arcane-test-secret+ healthcheck:+ test: ['CMD', 'curl', '-fs', 'http://localhost:9000/minio/health/live']+ interval: 2s+ timeout: 5s+ retries: 15+docker-socket-proxy:image: tecnativa/docker-socket-proxy:latestcontainer_name: arcane-docker-proxy-test@@ -49,7 +64,10 @@ services:- arcane_data:/app/data- ./projects:/app/data/projectsdepends_on:- - docker-socket-proxy+ docker-socket-proxy:+ condition: service_started+ minio:+ condition: service_healthybuild:args:- BUILD_TAGS=playwrightdiff --git a/tests/setup/compose.yaml b/tests/setup/compose.yamlindex 88c94d3bf..7f7ab14a7 100644--- a/tests/setup/compose.yaml+++ b/tests/setup/compose.yaml@@ -1,4 +1,19 @@services:+ minio:+ image: quay.io/minio/minio:RELEASE.2025-09-07T16-13-09Z+ # A directory created under the data dir is served as a bucket, so backups+ # have somewhere to write without a separate provisioning container.+ entrypoint:+ ['sh', '-c', 'mkdir -p /data/arcane-backups && exec minio server /data --address :9000']+ environment:+ - MINIO_ROOT_USER=arcane-test+ - MINIO_ROOT_PASSWORD=arcane-test-secret+ healthcheck:+ test: ['CMD', 'curl', '-fs', 'http://localhost:9000/minio/health/live']+ interval: 2s+ timeout: 5s+ retries: 15+arcane:image: arcane:playwright-testsports:@@ -15,6 +30,9 @@ services:- /var/run/docker.sock:/var/run/docker.sock- arcane_data:/app/data- ./projects:/app/data/projects+ depends_on:+ minio:+ condition: service_healthybuild:args:- BUILD_TAGS=playwrightdiff --git a/tests/spec/s3-backups.spec.ts b/tests/spec/s3-backups.spec.tsnew file mode 100644index 000000000..2af32afaa--- /dev/null+++ b/tests/spec/s3-backups.spec.ts@@ -0,0 +1,325 @@+import { test, expect, type Page } from '@playwright/test';++// The MinIO sidecar in the compose stack. Arcane reaches it over the compose+// network, and so must the Rustic helper container it starts for each backup.+const S3_ENDPOINT = process.env.E2E_S3_ENDPOINT ?? 'http://minio:9000';+const S3_BUCKET = process.env.E2E_S3_BUCKET ?? 'arcane-backups';+const S3_ACCESS_KEY_ID = process.env.E2E_S3_ACCESS_KEY_ID ?? 'arcane-test';+const S3_SECRET_ACCESS_KEY = process.env.E2E_S3_SECRET_ACCESS_KEY ?? 'arcane-test-secret';++type S3Destination = { id: string; name: string; bucket: string; secretConfigured: boolean };+type BackupEntry = {+ id: string;+ status: string;+ destination: string;+ format: string;+ remoteSnapshotId?: string;+ localSnapshotId?: string;+ error?: string;+};++function destinationPayload(overrides: Record<string, unknown> = {}) {+ return {+ name: `e2e-minio-${Date.now()}`,+ endpoint: S3_ENDPOINT,+ bucket: S3_BUCKET,+ region: '',+ accessKeyId: S3_ACCESS_KEY_ID,+ secretAccessKey: S3_SECRET_ACCESS_KEY,+ prefix: `e2e/${Date.now()}`,+ useSsl: false,+ forcePathStyle: true,+ ...overrides+ };+}++async function createDestinationViaApi(+ page: Page,+ overrides: Record<string, unknown> = {}+): Promise<S3Destination> {+ const response = await page.request.post('/api/backups/s3', {+ data: destinationPayload(overrides)+ });+ if (!response.ok()) {+ throw new Error(+ `Failed to create S3 destination: ${response.status()} ${await response.text()}`+ );+ }+ return (await response.json()) as S3Destination;+}++async function deleteDestinationViaApi(page: Page, destinationId: string) {+ await page.request.delete(`/api/backups/s3/${destinationId}`).catch(() => undefined);+}++async function createVolumeViaApi(page: Page, volumeName: string) {+ const response = await page.request.post('/api/environments/0/volumes', {+ data: { name: volumeName, driver: 'local' }+ });+ if (!response.ok()) {+ throw new Error(`Failed to create volume ${volumeName}: ${response.status()}`);+ }+}++async function removeVolumeViaApi(page: Page, volumeName: string) {+ await page.request+ .delete(`/api/environments/0/volumes/${encodeURIComponent(volumeName)}?force=true`)+ .catch(() => undefined);+}++async function getVolumeWorkspaceRevision(page: Page, volumeName: string) {+ const response = await page.request.get(+ `/api/environments/0/volumes/${encodeURIComponent(volumeName)}/workspace`+ );+ if (!response.ok()) {+ throw new Error(+ `Failed to read ${volumeName} workspace: ${response.status()} ${await response.text()}`+ );+ }+ const body = await response.json();+ return body.data.fileTreeRevision as string;+}++async function writeVolumeFile(page: Page, volumeName: string, fileName: string, content: string) {+ const fileTreeRevision = await getVolumeWorkspaceRevision(page, volumeName);+ const response = await page.request.put(+ `/api/environments/0/volumes/${encodeURIComponent(volumeName)}/workspace`,+ {+ multipart: {+ manifest: JSON.stringify({+ fileTreeRevision,+ fileChanges: [{ operation: 'create_file', relativePath: fileName, uploadIndex: 0 }]+ }),+ files: { name: fileName, mimeType: 'text/plain', buffer: Buffer.from(content) }+ }+ }+ );+ if (!response.ok()) {+ throw new Error(`Failed to upload ${fileName}: ${response.status()} ${await response.text()}`);+ }+}++async function readVolumeFile(page: Page, volumeName: string, filePath: string) {+ const relativePath = filePath.replace(/^\/+/, '');+ const response = await page.request.get(+ `/api/environments/0/volumes/${encodeURIComponent(volumeName)}/workspace/file?relativePath=${encodeURIComponent(relativePath)}`+ );+ if (!response.ok()) {+ return null;+ }+ const body = await response.json();+ const content = body?.data?.content;+ return typeof content === 'string' ? content : null;+}++async function deleteVolumeFile(page: Page, volumeName: string, filePath: string) {+ const relativePath = filePath.replace(/^\/+/, '');+ const fileTreeRevision = await getVolumeWorkspaceRevision(page, volumeName);+ const response = await page.request.put(+ `/api/environments/0/volumes/${encodeURIComponent(volumeName)}/workspace`,+ {+ multipart: {+ manifest: JSON.stringify({+ fileTreeRevision,+ fileChanges: [{ operation: 'delete', relativePath }]+ })+ }+ }+ );+ if (!response.ok()) {+ throw new Error(`Failed to delete ${filePath}: ${response.status()} ${await response.text()}`);+ }+}++async function listBackups(page: Page, volumeName: string): Promise<BackupEntry[]> {+ const response = await page.request.get(+ `/api/environments/0/volumes/${encodeURIComponent(volumeName)}/backups`+ );+ if (!response.ok()) {+ return [];+ }+ const body = await response.json();+ return (body?.data ?? []) as BackupEntry[];+}++// Backups run Rustic in a helper container, so the request returns before the+// snapshot finishes. Poll the record until it leaves the running state.+async function waitForBackup(page: Page, volumeName: string, backupId: string) {+ let last: BackupEntry | undefined;+ await expect+ .poll(+ async () => {+ last = (await listBackups(page, volumeName)).find((entry) => entry.id === backupId);+ return last?.status ?? 'missing';+ },+ { timeout: 180000, intervals: [2000] }+ )+ .not.toBe('running');+ expect(last?.status, `backup failed: ${last?.error ?? 'unknown error'}`).toBe('succeeded');+ return last as BackupEntry;+}++// Deleting a destination also verifies that no managed environment still+// references it, and is refused when one cannot be reached. Other specs create+// environments, so the release path depends on what is registered right now.+async function countRemoteEnvironments(page: Page) {+ const response = await page.request.get('/api/environments');+ if (!response.ok()) {+ return 0;+ }+ const body = await response.json();+ const environments = (body?.data ?? []) as { id: string }[];+ return environments.filter((environment) => environment.id !== '0').length;+}++async function createBackupViaApi(+ page: Page,+ volumeName: string,+ data: Record<string, unknown>+): Promise<BackupEntry> {+ const response = await page.request.post(+ `/api/environments/0/volumes/${encodeURIComponent(volumeName)}/backups`,+ { data, timeout: 180000 }+ );+ if (!response.ok()) {+ throw new Error(`Failed to create backup: ${response.status()} ${await response.text()}`);+ }+ const body = await response.json();+ return body.data as BackupEntry;+}++test.describe('S3 Backups', () => {+ test('creates a destination and verifies connectivity against MinIO', async ({ page }) => {+ const destination = await createDestinationViaApi(page);+ try {+ expect(destination.bucket).toBe(S3_BUCKET);+ expect(destination.secretConfigured).toBe(true);++ // Uploads, downloads, and deletes a probe object in the bucket.+ const test = await page.request.post(`/api/backups/s3/${destination.id}/test`);+ expect(test.status(), await test.text()).toBe(200);+ } finally {+ await deleteDestinationViaApi(page, destination.id);+ }+ });++ test('refuses the stored secret once connection settings change', async ({ page }) => {+ const destination = await createDestinationViaApi(page);+ try {+ const withoutSecret = await page.request.put(`/api/backups/s3/${destination.id}`, {+ data: destinationPayload({+ name: destination.name,+ bucket: 'someone-elses-bucket',+ secretAccessKey: ''+ })+ });+ expect(withoutSecret.status()).toBe(400);+ expect(await withoutSecret.text()).toContain('re-enter the secret access key');++ // The same edit is accepted once the secret is supplied again.+ const withSecret = await page.request.put(`/api/backups/s3/${destination.id}`, {+ data: destinationPayload({ name: destination.name, prefix: `e2e/${Date.now()}-moved` })+ });+ expect(withSecret.status(), await withSecret.text()).toBe(200);+ } finally {+ await deleteDestinationViaApi(page, destination.id);+ }+ });++ test('backs a volume up to S3 and restores it from the bucket', async ({ page }) => {+ test.setTimeout(300000);++ const volumeName = `e2e-s3-backup-${Date.now()}`;+ const fileContent = `arcane-e2e-${Date.now()}`;+ const destination = await createDestinationViaApi(page);+ let backupId: string | undefined;++ try {+ await createVolumeViaApi(page, volumeName);+ await writeVolumeFile(page, volumeName, 'payload.txt', fileContent);+ expect(await readVolumeFile(page, volumeName, '/payload.txt')).toBe(fileContent);++ const created = await createBackupViaApi(page, volumeName, {+ destination: 's3',+ s3DestinationId: destination.id+ });+ backupId = created.id;+ const backup = await waitForBackup(page, volumeName, created.id);++ expect(backup.destination).toBe('s3');+ expect(backup.format).toBe('rustic');+ expect(backup.remoteSnapshotId, 'no snapshot was recorded in S3').toBeTruthy();+ expect(backup.localSnapshotId ?? '').toBe('');++ // Rustic can only list these paths by reading the repository back out+ // of the bucket, so this proves the snapshot really landed in MinIO.+ const files = await page.request.get(+ `/api/environments/0/volumes/backups/${created.id}/files`+ );+ expect(files.status(), await files.text()).toBe(200);+ expect(JSON.stringify((await files.json())?.data ?? [])).toContain('payload.txt');++ await deleteVolumeFile(page, volumeName, '/payload.txt');+ expect(await readVolumeFile(page, volumeName, '/payload.txt')).toBeNull();++ const restore = await page.request.post(+ `/api/environments/0/volumes/${encodeURIComponent(volumeName)}/backups/${created.id}/restore`,+ { data: {}, timeout: 180000 }+ );+ expect(restore.status(), await restore.text()).toBe(200);+ expect(await readVolumeFile(page, volumeName, '/payload.txt')).toBe(fileContent);+ } finally {+ if (backupId) {+ await page.request+ .delete(`/api/environments/0/volumes/backups/${backupId}`)+ .catch(() => undefined);+ }+ await removeVolumeViaApi(page, volumeName);+ await deleteDestinationViaApi(page, destination.id);+ }+ });++ test('blocks deleting a destination a backup still references', async ({ page }) => {+ test.setTimeout(300000);++ const volumeName = `e2e-s3-inuse-${Date.now()}`;+ const destination = await createDestinationViaApi(page);+ let backupId: string | undefined;++ try {+ await createVolumeViaApi(page, volumeName);+ await writeVolumeFile(page, volumeName, 'payload.txt', 'in-use');++ const created = await createBackupViaApi(page, volumeName, {+ destination: 's3',+ s3DestinationId: destination.id+ });+ backupId = created.id;+ await waitForBackup(page, volumeName, created.id);++ const blocked = await page.request.delete(`/api/backups/s3/${destination.id}`);+ expect(blocked.status()).toBe(409);++ await page.request.delete(`/api/environments/0/volumes/backups/${created.id}`);+ backupId = undefined;++ const remoteEnvironments = await countRemoteEnvironments(page);+ const allowed = await page.request.delete(`/api/backups/s3/${destination.id}`);+ if (remoteEnvironments === 0) {+ expect(allowed.status(), await allowed.text()).toBe(200);+ } else {+ // A registered environment that cannot be checked keeps the+ // destination, so the local reference is no longer the blocker.+ expect([200, 409]).toContain(allowed.status());+ }+ } finally {+ if (backupId) {+ await page.request+ .delete(`/api/environments/0/volumes/backups/${backupId}`)+ .catch(() => undefined);+ }+ await removeVolumeViaApi(page, volumeName);+ await deleteDestinationViaApi(page, destination.id);+ }+ });+});diff --git a/types/backup/s3.go b/types/backup/s3.gonew file mode 100644index 000000000..e4f5dc5fa--- /dev/null+++ b/types/backup/s3.go@@ -0,0 +1,46 @@+package backup++import "time"++type S3Destination struct {+ ID string `json:"id"`+ Name string `json:"name"`+ Endpoint string `json:"endpoint,omitempty"`+ Bucket string `json:"bucket"`+ Region string `json:"region"`+ AccessKeyID string `json:"accessKeyId"`+ Prefix string `json:"prefix,omitempty"`+ UseSSL bool `json:"useSsl"`+ ForcePathStyle bool `json:"forcePathStyle"`+ SecretConfigured bool `json:"secretConfigured"`+ CreatedAt time.Time `json:"createdAt"`+ UpdatedAt *time.Time `json:"updatedAt,omitempty"`+}++type CreateS3Destination struct {+ Name string `json:"name"`+ Endpoint string `json:"endpoint,omitempty"`+ Bucket string `json:"bucket"`+ Region string `json:"region"`+ AccessKeyID string `json:"accessKeyId"`+ SecretAccessKey string `json:"secretAccessKey"`+ Prefix string `json:"prefix,omitempty"`+ UseSSL bool `json:"useSsl"`+ ForcePathStyle bool `json:"forcePathStyle"`+}++// UpdateS3Destination carries the same connection fields as create; an empty+// SecretAccessKey keeps the stored secret.+type UpdateS3Destination = CreateS3Destination++type S3DestinationSync struct {+ CreateS3Destination++ ID string `json:"id"`+ CreatedAt time.Time `json:"createdAt"`+ UpdatedAt *time.Time `json:"updatedAt,omitempty"`+}++type S3DestinationSyncRequest struct {+ Destinations []S3DestinationSync `json:"destinations"`+}diff --git a/types/backup/system_backup.go b/types/backup/system_backup.gonew file mode 100644index 000000000..2a563f7f0--- /dev/null+++ b/types/backup/system_backup.go@@ -0,0 +1,95 @@+package backup++import "time"++type SystemBackupDestination string++const (+ SystemBackupDestinationLocal SystemBackupDestination = "local"+ SystemBackupDestinationS3 SystemBackupDestination = "s3"+ SystemBackupDestinationLocalS3 SystemBackupDestination = "local_s3"+)++type SystemBackupRun struct {+ ID string `json:"id"`+ Size int64 `json:"size"`+ CreatedAt time.Time `json:"createdAt"`+ Status string `json:"status"`+ Trigger string `json:"trigger"`+ Destination SystemBackupDestination `json:"destination"`+ LocalSnapshotID string `json:"localSnapshotId,omitempty"`+ RemoteSnapshotID string `json:"remoteSnapshotId,omitempty"`+ S3DestinationID string `json:"s3DestinationId,omitempty"`+ S3DestinationName string `json:"s3DestinationName,omitempty"`+ PolicyID string `json:"policyId,omitempty"`+ Error string `json:"error,omitempty"`+}++type SystemBackupPolicy struct {+ ID string `json:"id"`+ Enabled bool `json:"enabled"`+ Schedule string `json:"schedule"`+ RetentionCount int `json:"retentionCount"`+ LocalEnabled bool `json:"localEnabled"`+ S3Enabled bool `json:"s3Enabled"`+ S3DestinationID string `json:"s3DestinationId,omitempty"`+ S3DestinationName string `json:"s3DestinationName,omitempty"`+ LastRun *SystemBackupRun `json:"lastRun,omitempty"`+}++// UpdateBackupPolicy is the shared policy-update payload for every backup+// domain; StopContainers only applies to volume backups.+type UpdateBackupPolicy struct {+ ID string `json:"id,omitempty"`+ Enabled bool `json:"enabled"`+ Schedule string `json:"schedule"`+ RetentionCount int `json:"retentionCount"`+ StopContainers bool `json:"stopContainers,omitempty"`+ LocalEnabled bool `json:"localEnabled"`+ S3Enabled bool `json:"s3Enabled"`+ S3DestinationID string `json:"s3DestinationId,omitempty"`+}++type UpdateSystemBackupPolicy = UpdateBackupPolicy++type SystemBackupPolicyCollection struct {+ Policies []SystemBackupPolicy `json:"policies"`+ RecoveryKeyStored bool `json:"recoveryKeyStored"`+}++type UpdateSystemBackupPolicies struct {+ Policies []UpdateSystemBackupPolicy `json:"policies"`+}++type SystemBackupRecoveryKeyStatus struct {+ Configured bool `json:"configured"`+}++type SystemBackupRecoveryKey struct {+ RecoveryKey string `json:"recoveryKey"`+}++type CreateSystemBackupRequest struct {+ Destination SystemBackupDestination `json:"destination,omitempty"`+ S3DestinationID string `json:"s3DestinationId,omitempty"`+ RecoveryKey string `json:"recoveryKey,omitempty"`+ PolicyID string `json:"policyId,omitempty"`+}++type RestoreSystemBackupRequest struct {+ RecoveryKey string `json:"recoveryKey"`+}++type UploadSystemBackupRequest struct {+ S3DestinationID string `json:"s3DestinationId"`+ RecoveryKey string `json:"recoveryKey"`+}++type DeleteSystemBackupRequest struct {+ RecoveryKey string `json:"recoveryKey,omitempty"`+}++type DiscoverSystemBackupsRequest struct {+ S3DestinationID string `json:"s3DestinationId"`+ RecoveryKey string `json:"recoveryKey"`+}diff --git a/types/volume/backup.go b/types/volume/backup.goindex e9f9aca8f..90ec5688e 100644--- a/types/volume/backup.go+++ b/types/volume/backup.go@@ -1,8 +1,72 @@package volume+import "github.com/getarcaneapp/arcane/types/v2/backup"++type BackupDestination string++const (+ BackupDestinationLocal BackupDestination = "local"+ BackupDestinationS3 BackupDestination = "s3"+ BackupDestinationLocalS3 BackupDestination = "local_s3"+)++type CreateBackupRequest struct {+ Destination BackupDestination `json:"destination,omitempty" doc:"Optional destination override for this manual backup"`+ PolicyID string `json:"policyId,omitempty" doc:"Optional backup policy whose settings should be used"`+ S3DestinationID string `json:"s3DestinationId,omitempty" doc:"Saved S3 destination for a manual remote backup"`+}++type UploadBackupRequest struct {+ S3DestinationID string `json:"s3DestinationId" doc:"S3 destination for the uploaded backup"`+}++type BackupFormat string++const (+ BackupFormatArchive BackupFormat = "archive"+ BackupFormatRustic BackupFormat = "rustic"+)+type BackupEntry struct {- ID string `json:"id" doc:"Unique identifier of the backup"`- VolumeName string `json:"volumeName" doc:"Name of the volume"`- Size int64 `json:"size" doc:"Size of the backup archive in bytes"`- CreatedAt string `json:"createdAt" doc:"When the backup was created"`+ ID string `json:"id" doc:"Unique identifier of the backup"`+ VolumeName string `json:"volumeName" doc:"Name of the volume"`+ Size int64 `json:"size" doc:"Total size of the backup contents"`+ CreatedAt string `json:"createdAt" doc:"When the backup was created"`+ Status string `json:"status" doc:"Backup result status"`+ Trigger string `json:"trigger" doc:"How the backup was started"`+ Destination BackupDestination `json:"destination" doc:"Requested backup storage target"`+ Format BackupFormat `json:"format" doc:"Storage format of the backup: legacy tar.gz archive or Rustic snapshot"`+ LocalSnapshotID string `json:"localSnapshotId,omitempty" doc:"Snapshot ID in the local Rustic repository"`+ RemoteSnapshotID string `json:"remoteSnapshotId,omitempty" doc:"Snapshot ID in the S3 Rustic repository"`+ S3DestinationID string `json:"s3DestinationId,omitempty" doc:"S3 destination used by the backup"`+ S3DestinationName string `json:"s3DestinationName,omitempty" doc:"Name of the S3 destination used by the backup"`+ PolicyID string `json:"policyId,omitempty" doc:"Backup policy that created the backup"`+ Error string `json:"error,omitempty" doc:"Backup error when the run failed"`+}++type BackupPolicy struct {+ ID string `json:"id"`+ VolumeName string `json:"volumeName"`+ Enabled bool `json:"enabled"`+ Schedule string `json:"schedule"`+ RetentionCount int `json:"retentionCount"`+ StopContainers bool `json:"stopContainers"`+ LocalEnabled bool `json:"localEnabled"`+ S3Enabled bool `json:"s3Enabled"`+ S3DestinationID string `json:"s3DestinationId,omitempty"`+ S3DestinationName string `json:"s3DestinationName,omitempty"`+ S3Available bool `json:"s3Available"`+ S3Bucket string `json:"s3Bucket,omitempty"`+ LastRun *BackupEntry `json:"lastRun,omitempty"`+}++type UpdateBackupPolicy = backup.UpdateBackupPolicy++type BackupPolicyCollection struct {+ Policies []BackupPolicy `json:"policies"`+ S3Available bool `json:"s3Available"`+}++type UpdateBackupPolicies struct {+ Policies []UpdateBackupPolicy `json:"policies"`}