Skip to content

Modern Docker Management, Designed for Everyone

feat: automated s3 backup (#3459)

affeldt28555c5dfbac47
diff --git a/.devcontainer/devcontainer-lock.json b/.devcontainer/devcontainer-lock.json
new file mode 100644
index 000000000..dab65ac66
--- /dev/null
+++ b/.devcontainer/devcontainer-lock.json
@@ -0,0 +1,24 @@
+{
+ "features": {
+ "ghcr.io/devcontainers/features/common-utils:2": {
+ "version": "2.5.9",
+ "resolved": "ghcr.io/devcontainers/features/common-utils@sha256:cb0c4d3c276f157eed17935747e364178d75fee17f55c4e129966f64633deb3a",
+ "integrity": "sha256:cb0c4d3c276f157eed17935747e364178d75fee17f55c4e129966f64633deb3a"
+ },
+ "ghcr.io/devcontainers/features/docker-outside-of-docker:1": {
+ "version": "1.10.0",
+ "resolved": "ghcr.io/devcontainers/features/docker-outside-of-docker@sha256:c2c2cf829505ead8e4892c88c31b6594ae94a2bbb209e16e1fac456c1a3a624e",
+ "integrity": "sha256:c2c2cf829505ead8e4892c88c31b6594ae94a2bbb209e16e1fac456c1a3a624e"
+ },
+ "ghcr.io/devcontainers/features/go:1": {
+ "version": "1.3.4",
+ "resolved": "ghcr.io/devcontainers/features/go@sha256:d85e921f91b41340055bb12b325d9d551170ed04b3b832e33530bf42f167c032",
+ "integrity": "sha256:d85e921f91b41340055bb12b325d9d551170ed04b3b832e33530bf42f167c032"
+ },
+ "ghcr.io/devcontainers/features/node:2": {
+ "version": "2.1.0",
+ "resolved": "ghcr.io/devcontainers/features/node@sha256:586c9a6f7dd40bd3ba2cd41e7f2f88dcc31fbe5d1442afcbf07ffbc66b686857",
+ "integrity": "sha256:586c9a6f7dd40bd3ba2cd41e7f2f88dcc31fbe5d1442afcbf07ffbc66b686857"
+ }
+ }
+}
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 4b3c997c3..c41bea684 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -271,6 +271,8 @@ jobs:
run: |
docker pull public.ecr.aws/nginx/nginx:stable-alpine
docker pull ghcr.io/linuxserver/radarr:nightly
+ docker pull quay.io/minio/minio:RELEASE.2025-09-07T16-13-09Z
+ docker pull ghcr.io/rustic-rs/rustic:v0.11.2
docker save public.ecr.aws/nginx/nginx:stable-alpine ghcr.io/linuxserver/radarr:nightly > /tmp/test-images.tar
- name: Setup and Install Playwright Dependencies
diff --git a/backend/api/api.go b/backend/api/api.go
index 440e3ac23..b80e8d564 100644
--- a/backend/api/api.go
+++ b/backend/api/api.go
@@ -33,9 +33,11 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/project"
"github.com/getarcaneapp/arcane/backend/v2/internal/registry"
"github.com/getarcaneapp/arcane/backend/v2/internal/role"
+ s3domain "github.com/getarcaneapp/arcane/backend/v2/internal/s3"
"github.com/getarcaneapp/arcane/backend/v2/internal/search"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/internal/swarm"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/systembackup"
"github.com/getarcaneapp/arcane/backend/v2/internal/template"
"github.com/getarcaneapp/arcane/backend/v2/internal/user"
"github.com/getarcaneapp/arcane/backend/v2/internal/variable"
@@ -227,6 +229,8 @@ type HandlerDeps struct {
Build *build.BuildService
BuildWorkspace *build.BuildWorkspaceService
Volume *volume.Module
+ S3Destination *s3domain.Module
+ SystemBackup *systembackup.Module
Network *network.NetworkService
Port *port.PortService
Swarm *swarm.Module
@@ -419,6 +423,8 @@ func registerHandlersInternal(api huma.API, deps HandlerDeps, handlerAppCtx hand
build.RegisterBuildWorkspaces(api, deps.BuildWorkspace, deps.Upload.Service())
deps.ImageUpdate.RegisterRoutes(api, handlerAppCtx)
deps.Settings.RegisterRoutes(api)
+ deps.S3Destination.RegisterRoutes(api)
+ deps.SystemBackup.RegisterRoutes(api, handlerAppCtx)
deps.JobSchedule.RegisterRoutes(api)
deps.Volume.RegisterRoutes(api, handlerAppCtx)
deps.Container.RegisterRoutes(api, handlerAppCtx)
diff --git a/backend/cli/recovery/recovery.go b/backend/cli/recovery/recovery.go
new file mode 100644
index 000000000..06d3d9a14
--- /dev/null
+++ b/backend/cli/recovery/recovery.go
@@ -0,0 +1,241 @@
+package recovery
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "log/slog"
+ "os"
+ "strings"
+ "time"
+
+ "emperror.dev/errors"
+ "gorm.io/gorm"
+ "gorm.io/gorm/clause"
+
+ "github.com/getarcaneapp/arcane/backend/v2/cli/upgrade"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/activity"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"
+ recoverytypes "github.com/getarcaneapp/arcane/backend/v2/internal/recovery"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/systembackup"
+ dockerutil "github.com/getarcaneapp/arcane/backend/v2/pkg/dockerutil"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"
+ rusticruntime "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/rustic"
+ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
+ backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"
+ "github.com/libtnb/sqlite"
+ "github.com/moby/moby/api/types/container"
+ "github.com/moby/moby/api/types/mount"
+ "github.com/moby/moby/client"
+ "github.com/spf13/cobra"
+)
+
+const rusticImage = rusticruntime.DefaultImage
+
+var requestPath string
+
+var RestoreCmd = &cobra.Command{
+ Use: "recovery-restore",
+ Short: "Apply a prepared Arcane recovery snapshot",
+ Hidden: true,
+ RunE: runRestoreInternal,
+}
+
+func init() {
+ RestoreCmd.Flags().StringVar(&requestPath, "request", "/app/data/.arcane-recovery-request.json", "Prepared recovery request")
+}
+
+func runRestoreInternal(_ *cobra.Command, _ []string) error {
+ ctx := context.Background()
+ data, err := os.ReadFile(requestPath)
+ if err != nil {
+ return fmt.Errorf("read recovery request: %w", err)
+ }
+ var request recoverytypes.RestoreRequest
+ if err := json.Unmarshal(data, &request); err != nil {
+ return fmt.Errorf("decode recovery request: %w", err)
+ }
+ _ = os.Remove(requestPath)
+ dockerClient, err := client.New(client.FromEnv)
+ if err != nil {
+ return fmt.Errorf("connect to Docker: %w", err)
+ }
+ defer func() { _ = dockerClient.Close() }()
+ inspect, err := libarcane.ContainerInspectWithCompatibility(ctx, dockerClient, request.ContainerID, client.ContainerInspectOptions{})
+ if err != nil {
+ return fmt.Errorf("inspect Arcane container: %w", err)
+ }
+ if _, err := dockerClient.ContainerStop(ctx, request.ContainerID, client.ContainerStopOptions{Timeout: new(30)}); err != nil {
+ return fmt.Errorf("stop Arcane container: %w", err)
+ }
+ if err := restoreSnapshotInternal(ctx, dockerClient, request); err != nil {
+ _, _ = dockerClient.ContainerStart(ctx, request.ContainerID, client.ContainerStartOptions{})
+ return err
+ }
+ manifestData, err := os.ReadFile("/app/data/.arcane-recovery.json")
+ if err != nil {
+ _, _ = dockerClient.ContainerStart(ctx, request.ContainerID, client.ContainerStartOptions{})
+ return fmt.Errorf("read restored recovery manifest: %w", err)
+ }
+ var manifest recoverytypes.Manifest
+ if err := json.Unmarshal(manifestData, &manifest); err != nil {
+ _, _ = dockerClient.ContainerStart(ctx, request.ContainerID, client.ContainerStartOptions{})
+ return fmt.Errorf("decode restored recovery manifest: %w", err)
+ }
+ _ = os.Remove("/app/data/.arcane-recovery.json")
+ if manifest.FormatVersion != 1 || len(manifest.Environment) == 0 {
+ _, _ = dockerClient.ContainerStart(ctx, request.ContainerID, client.ContainerStartOptions{})
+ return errors.New("unsupported or incomplete Arcane recovery manifest")
+ }
+ if err := finalizeRestoredBackupInternal(ctx, manifest.Environment["DATABASE_URL"], manifest.BackupID, manifest.ActivityID, request); err != nil {
+ _, _ = dockerClient.ContainerStart(ctx, request.ContainerID, client.ContainerStartOptions{})
+ return fmt.Errorf("finalize restored system backup: %w", err)
+ }
+ if err := upgrade.UpgradeContainer(ctx, dockerClient, inspect.Container, request.ContainerImage, manifest.Environment); err != nil {
+ return fmt.Errorf("recreate Arcane container with recovered configuration: %w", err)
+ }
+ return nil
+}
+
+func finalizeRestoredBackupInternal(ctx context.Context, databaseURL, manifestBackupID, manifestActivityID string, request recoverytypes.RestoreRequest) error {
+ if !strings.HasPrefix(databaseURL, "file:") {
+ return errors.New("restored Arcane database is not SQLite")
+ }
+ dsn, err := database.ParseSQLiteConnectionString(databaseURL)
+ if err != nil {
+ return err
+ }
+ db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{})
+ if err != nil {
+ return err
+ }
+ defer func() {
+ if sqlDB, dbErr := db.DB(); dbErr == nil {
+ _ = sqlDB.Close()
+ }
+ }()
+ db = db.WithContext(ctx)
+ if err := finalizeRestoredRunInternal(db, manifestBackupID, request); err != nil {
+ return err
+ }
+ if err := preserveSafetyBackupInternal(db, request.SafetyBackup); err != nil {
+ return err
+ }
+ return finalizeRestoredActivityInternal(db, manifestActivityID)
+}
+
+// finalizeRestoredRunInternal marks the restored run succeeded, filling only
+// the fields the snapshot's database copy did not already carry. Updates use
+// explicit column maps because the restored schema may predate the binary's.
+func finalizeRestoredRunInternal(db *gorm.DB, manifestBackupID string, request recoverytypes.RestoreRequest) error {
+ var run systembackup.SystemBackupRun
+ found := false
+ for _, backupID := range []string{manifestBackupID, request.BackupID} {
+ if strings.TrimSpace(backupID) == "" {
+ continue
+ }
+ err := db.Where("id = ? AND status = ?", backupID, systembackup.SystemBackupStatusRunning).First(&run).Error
+ if err == nil {
+ found = true
+ break
+ }
+ if !errors.Is(err, gorm.ErrRecordNotFound) {
+ return err
+ }
+ }
+ if !found {
+ err := db.Where("status = ?", systembackup.SystemBackupStatusRunning).Order("created_at DESC").First(&run).Error
+ if errors.Is(err, gorm.ErrRecordNotFound) {
+ return nil
+ }
+ if err != nil {
+ return err
+ }
+ }
+ values := map[string]any{"status": systembackup.SystemBackupStatusSucceeded, "error": ""}
+ if run.Size == 0 {
+ values["size"] = request.Size
+ }
+ if run.LocalSnapshotID == "" {
+ values["local_snapshot_id"] = request.LocalSnapshotID
+ }
+ if run.RemoteSnapshotID == "" {
+ values["remote_snapshot_id"] = request.RemoteSnapshotID
+ }
+ if run.S3DestinationID == "" {
+ values["s3_destination_id"] = request.S3DestinationID
+ }
+ return db.Model(&systembackup.SystemBackupRun{}).Where("id = ?", run.ID).Updates(values).Error
+}
+
+func preserveSafetyBackupInternal(db *gorm.DB, backup *recoverytypes.SafetyBackup) error {
+ if backup == nil || strings.TrimSpace(backup.ID) == "" || strings.TrimSpace(backup.LocalSnapshotID) == "" {
+ return nil
+ }
+ onConflict := map[string]any{
+ "size": backup.Size, "updated_at": time.Now().UTC(), "status": systembackup.SystemBackupStatusSucceeded,
+ "trigger": systembackup.SystemBackupTriggerSafety, "destination": backuptypes.SystemBackupDestinationLocal,
+ "local_snapshot_id": backup.LocalSnapshotID, "error": "",
+ }
+ return db.Model(&systembackup.SystemBackupRun{}).
+ Clauses(clause.OnConflict{Columns: []clause.Column{{Name: "id"}}, DoUpdates: clause.Assignments(onConflict)}).
+ Create(map[string]any{
+ "id": backup.ID, "size": backup.Size, "created_at": backup.CreatedAt, "updated_at": time.Now().UTC(),
+ "status": systembackup.SystemBackupStatusSucceeded,
+ "trigger": systembackup.SystemBackupTriggerSafety,
+ "destination": backuptypes.SystemBackupDestinationLocal,
+ "local_snapshot_id": backup.LocalSnapshotID, "remote_snapshot_id": "", "s3_destination_id": "",
+ "policy_id": "", "error": "",
+ }).Error
+}
+
+func finalizeRestoredActivityInternal(db *gorm.DB, activityID string) error {
+ var entry activity.Activity
+ query := db.Where("status = ?", activitytypes.StatusRunning)
+ if strings.TrimSpace(activityID) != "" {
+ query = query.Where("id = ?", activityID)
+ } else {
+ query = query.Where("resource_type = ?", "system_backup").Order("started_at DESC")
+ }
+ err := query.First(&entry).Error
+ if errors.Is(err, gorm.ErrRecordNotFound) {
+ return nil
+ }
+ if err != nil {
+ return err
+ }
+ now := time.Now().UTC()
+ return db.Model(&activity.Activity{}).Where("id = ?", entry.ID).Updates(map[string]any{
+ "status": activitytypes.StatusSuccess, "progress": 100, "step": "System backup completed",
+ "latest_message": "Arcane system backup created successfully", "error": nil,
+ "ended_at": now, "duration_ms": now.Sub(entry.StartedAt).Milliseconds(),
+ }).Error
+}
+
+func restoreSnapshotInternal(ctx context.Context, dockerClient *client.Client, request recoverytypes.RestoreRequest) error {
+ if _, err := dockerClient.ImageInspect(ctx, rusticImage); err != nil {
+ reader, pullErr := dockerClient.ImagePull(ctx, rusticImage, client.ImagePullOptions{})
+ if pullErr != nil {
+ return fmt.Errorf("pull Arcane tools image for Rustic: %w", pullErr)
+ }
+ if pullErr = dockerutil.RenderJSONMessageStream(reader, nil); pullErr != nil {
+ _ = reader.Close()
+ return fmt.Errorf("pull Arcane tools image for Rustic: %w", pullErr)
+ }
+ _ = reader.Close()
+ }
+ mounts := append([]mount.Mount{}, request.RepositoryMounts...)
+ request.AppDataMount.Target = "/restore"
+ request.AppDataMount.ReadOnly = false
+ mounts = append(mounts, request.AppDataMount)
+ snapshotPath := strings.TrimSpace(request.SnapshotPath)
+ if snapshotPath == "" {
+ snapshotPath = "/"
+ }
+ command := []string{"restore", "--delete", request.SnapshotID + ":" + snapshotPath, "/restore"}
+ if _, err := rusticruntime.Run(ctx, dockerClient, request.RecoveryKey, command, request.RepositoryEnvironment, mounts, container.NetworkMode(request.NetworkMode)); err != nil {
+ slog.Error("Rustic system restore failed", "error", err)
+ return fmt.Errorf("rustic system restore failed: %w", err)
+ }
+ return nil
+}
diff --git a/backend/cli/recovery/recovery_test.go b/backend/cli/recovery/recovery_test.go
new file mode 100644
index 000000000..ca9756524
--- /dev/null
+++ b/backend/cli/recovery/recovery_test.go
@@ -0,0 +1,89 @@
+package recovery
+
+import (
+ "context"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/getarcaneapp/arcane/backend/v2/internal/activity"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"
+ recoverytypes "github.com/getarcaneapp/arcane/backend/v2/internal/recovery"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/systembackup"
+ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
+ "github.com/libtnb/sqlite"
+ "github.com/stretchr/testify/require"
+ "gorm.io/gorm"
+)
+
+func newRestoredDatabaseForTestInternal(t *testing.T, runIDs ...string) (string, string) {
+ t.Helper()
+ databaseURL := "file:" + filepath.Join(t.TempDir(), "arcane.db")
+ dsn, err := database.ParseSQLiteConnectionString(databaseURL)
+ require.NoError(t, err)
+ db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{})
+ require.NoError(t, err)
+ require.NoError(t, db.AutoMigrate(&systembackup.SystemBackupRun{}, &activity.Activity{}))
+ for i, id := range runIDs {
+ require.NoError(t, db.Create(&systembackup.SystemBackupRun{
+ ID: id, CreatedAt: time.Date(2026, 1, 1+i, 0, 0, 0, 0, time.UTC),
+ Status: systembackup.SystemBackupStatusRunning,
+ }).Error)
+ }
+ resourceType := "system_backup"
+ require.NoError(t, db.Create(&activity.Activity{
+ ID: "activity-1", EnvironmentID: "0", Type: activitytypes.TypeResourceAction,
+ Status: activitytypes.StatusRunning, ResourceType: &resourceType,
+ StartedAt: time.Date(2026, 1, 2, 0, 0, 0, 0, time.UTC),
+ }).Error)
+ sqlDB, err := db.DB()
+ require.NoError(t, err)
+ require.NoError(t, sqlDB.Close())
+ return databaseURL, dsn
+}
+
+func TestFinalizeRestoredBackupInternal(t *testing.T) {
+ databaseURL, dsn := newRestoredDatabaseForTestInternal(t, "older", "selected")
+
+ require.NoError(t, finalizeRestoredBackupInternal(context.Background(), databaseURL, "selected", "activity-1", recoverytypes.RestoreRequest{
+ BackupID: "request-id", RemoteSnapshotID: "snapshot-1", S3DestinationID: "destination-1", Size: 1234,
+ SafetyBackup: &recoverytypes.SafetyBackup{
+ ID: "safety", LocalSnapshotID: "safety-snapshot", Size: 4321,
+ CreatedAt: time.Date(2026, 1, 3, 0, 0, 0, 0, time.UTC),
+ },
+ }))
+
+ db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{})
+ require.NoError(t, err)
+ var selected, older, safety systembackup.SystemBackupRun
+ require.NoError(t, db.First(&selected, "id = ?", "selected").Error)
+ require.Equal(t, systembackup.SystemBackupStatusSucceeded, selected.Status)
+ require.EqualValues(t, 1234, selected.Size)
+ require.Equal(t, "snapshot-1", selected.RemoteSnapshotID)
+ require.Equal(t, "destination-1", selected.S3DestinationID)
+ require.NoError(t, db.First(&older, "id = ?", "older").Error)
+ require.Equal(t, systembackup.SystemBackupStatusRunning, older.Status)
+ require.NoError(t, db.First(&safety, "id = ?", "safety").Error)
+ require.Equal(t, systembackup.SystemBackupStatusSucceeded, safety.Status)
+ require.Equal(t, systembackup.SystemBackupTriggerSafety, safety.Trigger)
+ require.Equal(t, "safety-snapshot", safety.LocalSnapshotID)
+ require.EqualValues(t, 4321, safety.Size)
+ var entry activity.Activity
+ require.NoError(t, db.First(&entry, "id = ?", "activity-1").Error)
+ require.Equal(t, activitytypes.StatusSuccess, entry.Status)
+}
+
+func TestFinalizeRestoredBackupInternalFallsBackForLegacyManifest(t *testing.T) {
+ databaseURL, dsn := newRestoredDatabaseForTestInternal(t, "older", "newest")
+
+ require.NoError(t, finalizeRestoredBackupInternal(context.Background(), databaseURL, "", "", recoverytypes.RestoreRequest{BackupID: "missing-discovery-id"}))
+
+ db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{})
+ require.NoError(t, err)
+ var newest systembackup.SystemBackupRun
+ require.NoError(t, db.First(&newest, "id = ?", "newest").Error)
+ require.Equal(t, systembackup.SystemBackupStatusSucceeded, newest.Status)
+ var entry activity.Activity
+ require.NoError(t, db.First(&entry, "id = ?", "activity-1").Error)
+ require.Equal(t, activitytypes.StatusSuccess, entry.Status)
+}
diff --git a/backend/cli/root.go b/backend/cli/root.go
index 9cc851084..fa8083cc7 100644
--- a/backend/cli/root.go
+++ b/backend/cli/root.go
@@ -9,6 +9,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/cli/admin"
"github.com/getarcaneapp/arcane/backend/v2/cli/generate"
+ "github.com/getarcaneapp/arcane/backend/v2/cli/recovery"
"github.com/getarcaneapp/arcane/backend/v2/cli/upgrade"
"github.com/getarcaneapp/arcane/backend/v2/internal/bootstrap"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
@@ -41,6 +42,7 @@ func Execute() {
func init() {
rootCmd.AddCommand(admin.AdminCmd)
rootCmd.AddCommand(upgrade.UpgradeCmd)
+ rootCmd.AddCommand(recovery.RestoreCmd)
rootCmd.AddCommand(generate.GenerateCmd)
}
diff --git a/backend/cli/upgrade/recovery_environment_test.go b/backend/cli/upgrade/recovery_environment_test.go
new file mode 100644
index 000000000..1689c4c7f
--- /dev/null
+++ b/backend/cli/upgrade/recovery_environment_test.go
@@ -0,0 +1,22 @@
+package upgrade
+
+import (
+ "slices"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestApplyRecoveredEnvironmentInternal(t *testing.T) {
+ result := applyRecoveredEnvironmentInternal([]string{
+ "JWT_SECRET=fresh", "JWT_SECRET_FILE=/run/secrets/jwt", "ENCRYPTION_KEY__FILE=/run/secrets/key", "KEEP=value",
+ }, map[string]string{"JWT_SECRET": "recovered-jwt", "ENCRYPTION_KEY": "recovered-key", "NEW_SETTING": "new"})
+
+ require.Contains(t, result, "JWT_SECRET=recovered-jwt")
+ require.Contains(t, result, "ENCRYPTION_KEY=recovered-key")
+ require.Contains(t, result, "NEW_SETTING=new")
+ require.Contains(t, result, "KEEP=value")
+ require.False(t, slices.ContainsFunc(result, func(value string) bool {
+ return value == "JWT_SECRET_FILE=/run/secrets/jwt" || value == "ENCRYPTION_KEY__FILE=/run/secrets/key"
+ }))
+}
diff --git a/backend/cli/upgrade/upgrade.go b/backend/cli/upgrade/upgrade.go
index 186a99cf1..b401902dc 100644
--- a/backend/cli/upgrade/upgrade.go
+++ b/backend/cli/upgrade/upgrade.go
@@ -117,7 +117,7 @@ func runUpgrade(cmd *cobra.Command, args []string) error {
// Perform the upgrade
slog.Info("Starting container upgrade", "container", containerName)
- if err := upgradeContainer(ctx, dockerClient, targetContainer, imageToPull); err != nil {
+ if err := UpgradeContainer(ctx, dockerClient, targetContainer, imageToPull, nil); err != nil {
return errors.WrapIf(err, "failed to upgrade container")
}
@@ -488,7 +488,10 @@ func pullImage(ctx context.Context, dockerClient *client.Client, imageName strin
return docker.RenderJSONMessageStream(reader, os.Stdout)
}
-func upgradeContainer(ctx context.Context, dockerClient *client.Client, oldContainer container.InspectResponse, newImage string) error {
+// UpgradeContainer recreates an Arcane container with the supplied image and
+// optional environment replacements. It is invoked from detached maintenance
+// containers so stopping the target cannot interrupt the operation.
+func UpgradeContainer(ctx context.Context, dockerClient *client.Client, oldContainer container.InspectResponse, newImage string, environment map[string]string) error {
originalName := strings.TrimPrefix(oldContainer.Name, "/")
oldName := fmt.Sprintf("%s-old-%d", originalName, time.Now().UnixNano())
@@ -496,6 +499,9 @@ func upgradeContainer(ctx context.Context, dockerClient *client.Client, oldConta
config := *oldContainer.Config
config.Image = newImage
config.Labels = normalizeRecreatedArcaneLabelsInternal(config.Labels)
+ if len(environment) > 0 {
+ config.Env = applyRecoveredEnvironmentInternal(config.Env, environment)
+ }
config.Labels = refreshRecreatedContainerLabelsInternal(ctx, dockerClient, config.Labels, oldContainer.Image, newImage)
@@ -619,6 +625,40 @@ func upgradeContainer(ctx context.Context, dockerClient *client.Client, oldConta
return nil
}
+func applyRecoveredEnvironmentInternal(current []string, recovered map[string]string) []string {
+ seen := make(map[string]struct{}, len(recovered))
+ updated := make([]string, 0, len(current)+len(recovered))
+ for _, entry := range current {
+ name, _, found := strings.Cut(entry, "=")
+ if !found {
+ updated = append(updated, entry)
+ continue
+ }
+ if baseName, ok := strings.CutSuffix(name, "__FILE"); ok {
+ if _, ok := recovered[baseName]; ok {
+ continue
+ }
+ }
+ if baseName, ok := strings.CutSuffix(name, "_FILE"); ok {
+ if _, ok := recovered[baseName]; ok {
+ continue
+ }
+ }
+ if replacement, ok := recovered[name]; ok {
+ updated = append(updated, name+"="+replacement)
+ seen[name] = struct{}{}
+ } else {
+ updated = append(updated, entry)
+ }
+ }
+ for name, value := range recovered {
+ if _, ok := seen[name]; !ok {
+ updated = append(updated, name+"="+value)
+ }
+ }
+ return updated
+}
+
// looksLikeContainerID checks if a string looks like a Docker container ID
// (12 or 64 lowercase hex characters, which Docker auto-generates as hostnames)
func looksLikeContainerID(s string) bool {
diff --git a/backend/go.mod b/backend/go.mod
index 09d2d3457..88cbc4f4a 100644
--- a/backend/go.mod
+++ b/backend/go.mod
@@ -11,9 +11,11 @@ require (
emperror.dev/emperror v0.33.0
emperror.dev/errors v0.8.1
github.com/anthdm/hollywood v1.0.5
+ github.com/aws/aws-sdk-go-v2 v1.43.6
github.com/aws/aws-sdk-go-v2/config v1.32.37
github.com/aws/aws-sdk-go-v2/credentials v1.19.36
github.com/aws/aws-sdk-go-v2/service/ecr v1.60.6
+ github.com/aws/aws-sdk-go-v2/service/s3 v1.106.0
github.com/cenkalti/backoff/v5 v5.0.3
github.com/coder/websocket v1.8.15
github.com/compose-spec/compose-go/v2 v2.14.0
@@ -33,6 +35,7 @@ require (
github.com/gofrs/flock v0.13.0
github.com/golang-jwt/jwt/v5 v5.3.1
github.com/google/go-containerregistry v0.21.9
+ github.com/google/uuid v1.6.0
github.com/jinzhu/copier v0.4.0
github.com/klauspost/compress v1.19.2
github.com/labstack/echo/v5 v5.3.1
@@ -91,13 +94,15 @@ require (
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
github.com/agnivade/levenshtein v1.2.1 // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
- github.com/aws/aws-sdk-go-v2 v1.43.6 // indirect
+ github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.37 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.37 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.37 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.38 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.17 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.24 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.37 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.32 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.5.6 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.33.6 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.6 // indirect
@@ -175,7 +180,6 @@ require (
github.com/google/go-tpm v0.9.8 // indirect
github.com/google/go-tpm-tools v0.4.8 // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
- github.com/google/uuid v1.6.0 // indirect
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
diff --git a/backend/go.sum b/backend/go.sum
index 36a1b844b..3c1fd298c 100644
--- a/backend/go.sum
+++ b/backend/go.sum
@@ -81,6 +81,8 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3d
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
github.com/aws/aws-sdk-go-v2 v1.43.6 h1:RrmFcqCBxkJuf7g1axVo5krB4jM/AO8r5e5oujrgdoQ=
github.com/aws/aws-sdk-go-v2 v1.43.6/go.mod h1:tXpPM+v0D1lndmga+HqqLDIzUFJlEeR21aspVklHF00=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 h1:3IZY0XAJquT3aHzbkHfPzy4ACPcEjVG0x87KOwtpqGY=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14/go.mod h1:zwM6veDkhGgQFqkBy+uT28AAYpLu+uFMlPl+rCg/73E=
github.com/aws/aws-sdk-go-v2/config v1.32.37 h1:Ljl7LOJB6ym0liuEl0+TZ3d7f5I8MEZN1Cj9PINlj/g=
github.com/aws/aws-sdk-go-v2/config v1.32.37/go.mod h1:WJ7pe7ZPpmG8Q5kKS53zeypIV4FBGACxmte8Uc6SgUc=
github.com/aws/aws-sdk-go-v2/credentials v1.19.36 h1:84s5xMme6ENYEdKG8rsbSFFg/8+lbHBeM9QYSO0gnDk=
@@ -97,10 +99,16 @@ github.com/aws/aws-sdk-go-v2/service/ecr v1.60.6 h1:RbjO6G1wu+q43r0322sDABXi9vq/
github.com/aws/aws-sdk-go-v2/service/ecr v1.60.6/go.mod h1:snsosIuclt9tpFKzldCnu1ykT5SYEND/bK9qTJmoJ+o=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.17 h1:OvYZOB3qA6zvfdRFiRFRzVSiElMYrz3GdntkXZxlp1o=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.17/go.mod h1:JgR/2Ew50ACfIWau1oeMRX59tMtC0kM+PYQGEaT04cY=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.24 h1:mdPwDQPqxlw9Sc62Nt15yjEcARaDbPXkjRYtXsUripo=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.24/go.mod h1:ls5ytnwLTcQaUu32fMYXFI3MjpKuTwL840PAm9iqyEg=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.37 h1:a3D4AjrOrTrP8+d9ILBthqrElf0z1JNol09Xvnwcys8=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.37/go.mod h1:ky0gTu+ukvUTuUKFIpp6Wid4oninrkCyvbFkVs0kpHM=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.32 h1:jWXtZdCnhXa9sGFixRaU2AxT4DIVse9HS4E2f+/KwV0=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.32/go.mod h1:9JS1UpfVvyD/ZPX8GsKb/Pq8scEM+7GP5fqh9SwH7po=
github.com/aws/aws-sdk-go-v2/service/kms v1.52.0 h1:QNtg+Mtj1zmepk568+UKBD5DFfqh+ESTUUqQT27JkQc=
github.com/aws/aws-sdk-go-v2/service/kms v1.52.0/go.mod h1:Y0+uxvxz6ib4KktRdK0V4X45Vcs/JyYoz8H71pO8xeI=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.106.0 h1:7QZWVJZWzHivHWIa+5TELLaBBkbuoj0GPwQtMlJ0sqk=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.106.0/go.mod h1:fcvq5L7dK+5cQFicEJwpI6e6Wn8NY2i6yT5wRLYVc7s=
github.com/aws/aws-sdk-go-v2/service/signin v1.5.6 h1:i68sFvXidKlkiSvI7d7Ilc1/UvW4CtBOaivH7jhG4fs=
github.com/aws/aws-sdk-go-v2/service/signin v1.5.6/go.mod h1:/h7Obr9WTtzbjTHGASRQwLN7Bupw+TC3x8x7fyx39hE=
github.com/aws/aws-sdk-go-v2/service/sso v1.33.6 h1:tpfGChmjUmv3W9WlRvy+stwKDTbFFdq8Zk9DbFPrfMU=
diff --git a/backend/internal/backup/engine.go b/backend/internal/backup/engine.go
new file mode 100644
index 000000000..046fb7329
--- /dev/null
+++ b/backend/internal/backup/engine.go
@@ -0,0 +1,296 @@
+// Package backup owns the shared Rustic backup engine used by volume and
+// system backups: typed repository operations, per-repository serialization
+// through the actor runtime, and run admission.
+package backup
+
+import (
+ "context"
+ "encoding/json/v2"
+ "fmt"
+ "io"
+ "log/slog"
+ "strings"
+ "time"
+
+ "emperror.dev/errors"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/common"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/image"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"
+ rusticruntime "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/rustic"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/volumehelper"
+ "github.com/google/uuid"
+ "github.com/moby/moby/api/types/container"
+ "github.com/moby/moby/api/types/mount"
+ "github.com/moby/moby/client"
+)
+
+// Admission scopes shared by the backup engine and the per-policy job
+// registries so scheduled and manual runs contend on the same leases.
+const (
+ VolumeAdmissionScope = "volume-backup"
+ SystemAdmissionScope = "system-backup"
+)
+
+// Repository addresses one Rustic repository. ID is the stable serialization
+// key: operations against the same ID run serially, different IDs concurrently.
+type Repository struct {
+ ID string
+ Environment []string
+ Mounts []mount.Mount
+}
+
+// Snapshot is the result of a completed Rustic backup operation.
+type Snapshot struct {
+ ID string `json:"id"`
+ Size int64 `json:"-"`
+}
+
+type rusticSnapshotOutputInternal struct {
+ ID string `json:"id"`
+ Summary struct {
+ TotalBytesProcessed int64 `json:"total_bytes_processed"`
+ } `json:"summary"`
+}
+
+// RestoreOptions selects what a snapshot restore writes and where.
+type RestoreOptions struct {
+ // DeleteExtra removes files in the target that are absent from the snapshot.
+ DeleteExtra bool
+ // SourcePath restores only this path inside the snapshot when non-empty.
+ SourcePath string
+ // DestinationPath overrides the restore target path inside the helper
+ // container; the target mount's path is used when empty.
+ DestinationPath string
+}
+
+// Engine executes typed Rustic operations through the official Rustic image.
+// Concurrency is actor-owned: one executor per repository ID plus the shared
+// application admission gate for run exclusivity.
+type Engine struct {
+ imageService *image.ImageService
+ runtime *actors.Runtime
+ admission *actors.Gate[actors.AdmissionKey]
+ lifecycleCtx context.Context
+ executors *actors.StateMap[string, *actors.Executor]
+}
+
+// NewEngine creates the shared backup engine on the actor runtime. ctx is the
+// application lifecycle context repository executors are spawned on.
+func NewEngine(ctx context.Context, runtime *actors.Runtime, admission *actors.Gate[actors.AdmissionKey], imageService *image.ImageService) *Engine {
+ return &Engine{
+ imageService: imageService,
+ runtime: runtime,
+ admission: admission,
+ lifecycleCtx: ctx,
+ executors: actors.NewStateMap[string, *actors.Executor](),
+ }
+}
+
+// TryAcquireRun admits at most one in-flight backup run per (scope, id),
+// shared between scheduled jobs and manual API triggers.
+func (e *Engine) TryAcquireRun(ctx context.Context, scope, id string) (*actors.Lease[actors.AdmissionKey], bool, error) {
+ if e == nil {
+ return nil, false, errors.New("backup engine is unavailable")
+ }
+ return e.admission.TryAcquire(ctx, actors.AdmissionKey{Scope: scope, ID: id})
+}
+
+// Stop joins every repository executor.
+func (e *Engine) Stop(ctx context.Context) error {
+ if e == nil {
+ return nil
+ }
+ executors, err := e.executors.Drain(ctx, "stop backup repository executors")
+ if err != nil {
+ return err
+ }
+ var stopErr error
+ for _, executor := range executors {
+ stopErr = errors.Combine(stopErr, executor.Stop(ctx))
+ }
+ return stopErr
+}
+
+// CreateSnapshot backs the source mount up into the repository and returns the
+// created snapshot. The repository is initialized on first use.
+func (e *Engine) CreateSnapshot(ctx context.Context, dockerClient *client.Client, repository Repository, password, label string, source mount.Mount) (Snapshot, error) {
+ output, err := e.runInternal(ctx, dockerClient, repository, password,
+ []string{"backup", "--init", "--json", "--as-path", "/", "--host", "arcane", "--label", label, "--", source.Target},
+ source,
+ )
+ if err != nil {
+ return Snapshot{}, err
+ }
+ var decoded rusticSnapshotOutputInternal
+ if err := json.Unmarshal([]byte(output), &decoded); err != nil {
+ return Snapshot{}, fmt.Errorf("failed to decode Rustic snapshot: %w", err)
+ }
+ if decoded.ID == "" {
+ return Snapshot{}, errors.New("rustic did not return a snapshot ID")
+ }
+ return Snapshot{ID: decoded.ID, Size: decoded.Summary.TotalBytesProcessed}, nil
+}
+
+// RestoreSnapshot restores a snapshot (or one path inside it) onto the target mount.
+func (e *Engine) RestoreSnapshot(ctx context.Context, dockerClient *client.Client, repository Repository, password, snapshotID string, target mount.Mount, options RestoreOptions) error {
+ command := []string{"restore"}
+ if options.DeleteExtra {
+ command = append(command, "--delete")
+ }
+ source := snapshotID + ":/"
+ if options.SourcePath != "" {
+ source = snapshotID + ":/" + strings.TrimPrefix(options.SourcePath, "/")
+ }
+ destination := options.DestinationPath
+ if destination == "" {
+ destination = target.Target
+ }
+ command = append(command, "--", source, destination)
+ _, err := e.runInternal(ctx, dockerClient, repository, password, command, target)
+ return err
+}
+
+// ListSnapshotFiles returns every path recorded in the snapshot.
+func (e *Engine) ListSnapshotFiles(ctx context.Context, dockerClient *client.Client, repository Repository, password, snapshotID string) ([]string, error) {
+ output, err := e.runInternal(ctx, dockerClient, repository, password, []string{"ls", "--json", "--recursive", "--", snapshotID + ":/"})
+ if err != nil {
+ return nil, fmt.Errorf("failed to list Rustic snapshot: %w", err)
+ }
+ var files []string
+ if err := json.Unmarshal([]byte(output), &files); err != nil {
+ return nil, fmt.Errorf("failed to decode Rustic file list: %w", err)
+ }
+ return files, nil
+}
+
+// DiscoveredSnapshot describes one snapshot found in a repository.
+type DiscoveredSnapshot struct {
+ ID string `json:"id"`
+ Time time.Time `json:"time"`
+ Summary struct {
+ TotalBytesProcessed int64 `json:"total_bytes_processed"`
+ } `json:"summary"`
+}
+
+// ListSnapshots enumerates every snapshot in the repository.
+func (e *Engine) ListSnapshots(ctx context.Context, dockerClient *client.Client, repository Repository, password string) ([]DiscoveredSnapshot, error) {
+ output, err := e.runInternal(ctx, dockerClient, repository, password, []string{"snapshots", "--json"})
+ if err != nil {
+ return nil, err
+ }
+ var snapshots []DiscoveredSnapshot
+ if err := json.Unmarshal([]byte(output), &snapshots); err != nil {
+ return nil, fmt.Errorf("failed to decode Rustic snapshots: %w", err)
+ }
+ if len(snapshots) == 0 || snapshots[0].ID != "" {
+ return snapshots, nil
+ }
+ // Newer Rustic versions group snapshots by (host, label).
+ var groups []struct {
+ Snapshots []DiscoveredSnapshot `json:"snapshots"`
+ }
+ if err := json.Unmarshal([]byte(output), &groups); err != nil {
+ return nil, fmt.Errorf("failed to decode grouped Rustic snapshots: %w", err)
+ }
+ snapshots = snapshots[:0]
+ for _, group := range groups {
+ snapshots = append(snapshots, group.Snapshots...)
+ }
+ return snapshots, nil
+}
+
+// ForgetSnapshot removes the snapshot from the repository and prunes its data.
+func (e *Engine) ForgetSnapshot(ctx context.Context, dockerClient *client.Client, repository Repository, password, snapshotID string) error {
+ _, err := e.runInternal(ctx, dockerClient, repository, password, []string{"forget", "--prune", "--", snapshotID})
+ return err
+}
+
+// Replicate copies one snapshot between repositories by restoring it into a
+// temporary volume and backing that volume up into the target repository. The
+// source is read once from the repository, never from the live data. Rustic's
+// native `copy` would move only missing packs, but it addresses the target via
+// a TOML config profile, which the env-only Repository cannot express yet —
+// the materialize-and-rebackup here trades disk and I/O for that simplicity.
+func (e *Engine) Replicate(ctx context.Context, dockerClient *client.Client, from Repository, fromSnapshotID string, to Repository, password, label string) (Snapshot, error) {
+ temporaryVolume := "arcane-rustic-copy-" + uuid.NewString()
+ if _, err := dockerClient.VolumeCreate(ctx, client.VolumeCreateOptions{Name: temporaryVolume, Labels: volumehelper.Labels()}); err != nil {
+ return Snapshot{}, fmt.Errorf("failed to create temporary Rustic copy volume: %w", err)
+ }
+ defer func() {
+ _, _ = dockerClient.VolumeRemove(context.WithoutCancel(ctx), temporaryVolume, client.VolumeRemoveOptions{Force: true})
+ }()
+ copyMount := mount.Mount{Type: mount.TypeVolume, Source: temporaryVolume, Target: "/volume"}
+ if err := e.RestoreSnapshot(ctx, dockerClient, from, password, fromSnapshotID, copyMount, RestoreOptions{DeleteExtra: true}); err != nil {
+ return Snapshot{}, fmt.Errorf("failed to load Rustic snapshot for replication: %w", err)
+ }
+ copyMount.ReadOnly = true
+ snapshot, err := e.CreateSnapshot(ctx, dockerClient, to, password, label, copyMount)
+ if err != nil {
+ return Snapshot{}, fmt.Errorf("failed to replicate Rustic snapshot: %w", err)
+ }
+ return snapshot, nil
+}
+
+func (e *Engine) executorForInternal(repositoryID string) (*actors.Executor, error) {
+ if executor, ok := e.executors.Get(repositoryID); ok {
+ return executor, nil
+ }
+ return actors.ApplyStateMap(e.lifecycleCtx, e.executors, "backup repository executor", func(values map[string]*actors.Executor) (*actors.Executor, bool, error) {
+ if executor, ok := values[repositoryID]; ok {
+ return executor, false, nil
+ }
+ executor, err := actors.NewExecutor(e.lifecycleCtx, e.runtime, "backup-repository", repositoryID, 3)
+ if err != nil {
+ return nil, false, err
+ }
+ values[repositoryID] = executor
+ return executor, true, nil
+ })
+}
+
+func (e *Engine) runInternal(ctx context.Context, dockerClient *client.Client, repository Repository, password string, command []string, extraMounts ...mount.Mount) (string, error) {
+ if e == nil {
+ return "", errors.New("backup engine is unavailable")
+ }
+ if strings.TrimSpace(repository.ID) == "" {
+ return "", errors.New("backup repository ID is required")
+ }
+ executor, err := e.executorForInternal(repository.ID)
+ if err != nil {
+ return "", err
+ }
+ return actors.Execute(ctx, executor, "rustic "+command[0], func(workCtx context.Context) (string, error) {
+ return e.runContainerInternal(workCtx, dockerClient, repository, password, command, extraMounts...)
+ }, nil)
+}
+
+func (e *Engine) ensureImageInternal(ctx context.Context, dockerClient *client.Client) error {
+ if _, err := dockerClient.ImageInspect(ctx, rusticruntime.DefaultImage); err == nil {
+ return nil
+ }
+ if e.imageService == nil {
+ return errors.New("image service is unavailable")
+ }
+ if err := e.imageService.PullImage(ctx, rusticruntime.DefaultImage, io.Discard, common.SystemUser, nil); err != nil {
+ return fmt.Errorf("failed to pull official Rustic image: %w", err)
+ }
+ return nil
+}
+func arcaneNetworkModeInternal(ctx context.Context, dockerClient *client.Client) container.NetworkMode {
+ arcane, err := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient)
+ if err != nil || arcane == nil || arcane.ID == "" {
+ slog.DebugContext(ctx, "backup engine: running Rustic on the default network", "error", err)
+ return ""
+ }
+ return container.NetworkMode("container:" + arcane.ID)
+}
+
+func (e *Engine) runContainerInternal(ctx context.Context, dockerClient *client.Client, repository Repository, password string, command []string, extraMounts ...mount.Mount) (string, error) {
+ if err := e.ensureImageInternal(ctx, dockerClient); err != nil {
+ return "", err
+ }
+ mounts := append([]mount.Mount{}, repository.Mounts...)
+ mounts = append(mounts, extraMounts...)
+ return rusticruntime.Run(ctx, dockerClient, password, command, repository.Environment, mounts, arcaneNetworkModeInternal(ctx, dockerClient))
+}
diff --git a/backend/internal/backup/policies.go b/backend/internal/backup/policies.go
new file mode 100644
index 000000000..7fb668538
--- /dev/null
+++ b/backend/internal/backup/policies.go
@@ -0,0 +1,137 @@
+package backup
+
+import (
+ "context"
+ "fmt"
+ "strings"
+
+ "emperror.dev/errors"
+
+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/schedule"
+ backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"
+ "gorm.io/gorm"
+)
+
+// PolicyReconciliation reconciles a full set of policy updates against the
+// existing rows of one backup domain: shared validation, create/update/delete
+// in a single transaction, and the scheduled-job swap. Volume and system
+// backups both run through it so the flow cannot drift between them.
+type PolicyReconciliation[P any] struct {
+ // Domain names the backup flavor in validation errors ("volume", "system").
+ Domain string
+ DB *database.DB
+ // Existing is every policy currently persisted for the reconciled scope.
+ Existing []P
+ ID func(*P) string
+ // New returns the blank policy for created entries, pre-scoped by the
+ // caller (e.g. with the volume name set).
+ New func() P
+ Apply func(*P, backuptypes.UpdateBackupPolicy)
+ // ValidateUpdate adds per-domain checks on top of the shared ones; nil skips.
+ ValidateUpdate func(backuptypes.UpdateBackupPolicy) error
+ // S3Configured verifies the destination exists and is usable; its error is
+ // returned to the caller verbatim.
+ S3Configured func(ctx context.Context, destinationID string) error
+ Unregister func(ctx context.Context, policyID string)
+ Reschedule func(ctx context.Context, policy *P)
+}
+
+func (r PolicyReconciliation[P]) Run(ctx context.Context, updates []backuptypes.UpdateBackupPolicy) error {
+ if err := r.validateInternal(ctx, updates); err != nil {
+ return err
+ }
+ policies, kept, err := r.buildInternal(updates)
+ if err != nil {
+ return err
+ }
+ err = r.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ for i := range policies {
+ if saveErr := tx.Save(&policies[i]).Error; saveErr != nil {
+ return saveErr
+ }
+ }
+ for i := range r.Existing {
+ if _, ok := kept[r.ID(&r.Existing[i])]; !ok {
+ if deleteErr := tx.Delete(&r.Existing[i]).Error; deleteErr != nil {
+ return deleteErr
+ }
+ }
+ }
+ return nil
+ })
+ if err != nil {
+ return fmt.Errorf("failed to save %s backup policies: %w", r.Domain, err)
+ }
+ for i := range r.Existing {
+ if _, ok := kept[r.ID(&r.Existing[i])]; !ok {
+ r.Unregister(ctx, r.ID(&r.Existing[i]))
+ }
+ }
+ for i := range policies {
+ r.Reschedule(ctx, &policies[i])
+ }
+ return nil
+}
+
+// validateInternal checks the shared policy rules and normalizes the updates
+// in place (canonical schedule, cleared destination when S3 is off).
+func (r PolicyReconciliation[P]) validateInternal(ctx context.Context, updates []backuptypes.UpdateBackupPolicy) error {
+ for i := range updates {
+ normalized, err := schedule.NormalizeSixField(updates[i].Schedule, r.Domain+" backup")
+ if err != nil {
+ return err
+ }
+ updates[i].Schedule = normalized
+ if updates[i].RetentionCount < 0 || updates[i].RetentionCount > 3650 {
+ return errors.New("retentionCount must be between 0 and 3650")
+ }
+ if !updates[i].LocalEnabled && !updates[i].S3Enabled {
+ return fmt.Errorf("select at least one %s backup destination", r.Domain)
+ }
+ if updates[i].S3Enabled {
+ if strings.TrimSpace(updates[i].S3DestinationID) == "" {
+ return fmt.Errorf("select an S3 destination for %s backups", r.Domain)
+ }
+ if err := r.S3Configured(ctx, updates[i].S3DestinationID); err != nil {
+ return err
+ }
+ } else {
+ updates[i].S3DestinationID = ""
+ }
+ if r.ValidateUpdate != nil {
+ if err := r.ValidateUpdate(updates[i]); err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
+// buildInternal maps the updates onto existing or new policy rows and reports
+// which existing IDs survive the reconciliation.
+func (r PolicyReconciliation[P]) buildInternal(updates []backuptypes.UpdateBackupPolicy) ([]P, map[string]struct{}, error) {
+ byID := make(map[string]P, len(r.Existing))
+ for i := range r.Existing {
+ byID[r.ID(&r.Existing[i])] = r.Existing[i]
+ }
+ policies := make([]P, 0, len(updates))
+ kept := make(map[string]struct{}, len(updates))
+ for _, update := range updates {
+ policy := r.New()
+ if update.ID != "" {
+ var ok bool
+ policy, ok = byID[update.ID]
+ if !ok {
+ return nil, nil, fmt.Errorf("%s backup policy not found", r.Domain)
+ }
+ if _, duplicate := kept[update.ID]; duplicate {
+ return nil, nil, fmt.Errorf("duplicate %s backup policy", r.Domain)
+ }
+ kept[update.ID] = struct{}{}
+ }
+ r.Apply(&policy, update)
+ policies = append(policies, policy)
+ }
+ return policies, kept, nil
+}
diff --git a/backend/internal/backup/retention.go b/backend/internal/backup/retention.go
new file mode 100644
index 000000000..f7822147a
--- /dev/null
+++ b/backend/internal/backup/retention.go
@@ -0,0 +1,25 @@
+package backup
+
+import (
+ "context"
+
+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"
+)
+
+// ExpiredRunIDs returns the IDs of succeeded runs with snapshots that fall
+// outside the newest keep entries for one policy, oldest last. Callers delete
+// each run through their own delete path so snapshots are forgotten too.
+func ExpiredRunIDs(ctx context.Context, db *database.DB, table, policyID string, keep int) ([]string, error) {
+ var ids []string
+ err := db.WithContext(ctx).
+ Table(table).
+ Where(
+ "policy_id = ? AND status = ? AND (COALESCE(local_snapshot_id, '') <> '' OR COALESCE(remote_snapshot_id, '') <> '')",
+ policyID,
+ "succeeded",
+ ).
+ Order("created_at DESC").
+ Offset(keep).
+ Pluck("id", &ids).Error
+ return ids, err
+}
diff --git a/backend/internal/bootstrap/jobs_bootstrap.go b/backend/internal/bootstrap/jobs_bootstrap.go
index 71923e062..6a579a1f6 100644
--- a/backend/internal/bootstrap/jobs_bootstrap.go
+++ b/backend/internal/bootstrap/jobs_bootstrap.go
@@ -15,6 +15,8 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/job"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/internal/system"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/systembackup"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/volume"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler"
schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
@@ -70,12 +72,14 @@ type registerJobsParams struct {
Scheduler schedulertypes.JobScheduler
ActorRuntime *actors.Runtime
- Activity *activity.ActivityService
- GitOpsSync *gitops.GitOpsSyncService
- Environment *environment.EnvironmentService
- JobSchedule *job.JobService
- Settings *settings.SettingsService
- Admission *actors.Gate[actors.AdmissionKey]
+ Activity *activity.ActivityService
+ GitOpsSync *gitops.GitOpsSyncService
+ Environment *environment.EnvironmentService
+ JobSchedule *job.JobService
+ Settings *settings.SettingsService
+ Volume *volume.VolumeService
+ SystemBackup *systembackup.SystemBackupService
+ Admission *actors.Gate[actors.AdmissionKey]
AutoUpdate *scheduler.AutoUpdateJob
ImageUpdateWatcher *scheduler.ImageUpdateWatcher
@@ -144,13 +148,15 @@ func registerJobs(params registerJobsParams) error {
// GitOps sync and environment health are no longer single global jobs; each
// entity registers its own dynamic job.
if err := registerDynamicJobs(dynamicJobsParams{
- AppCtx: params.AppCtx,
- Config: params.Config,
- Scheduler: params.Scheduler,
- GitOpsSync: params.GitOpsSync,
- Environment: params.Environment,
- JobSchedule: params.JobSchedule,
- Admission: params.Admission,
+ AppCtx: params.AppCtx,
+ Config: params.Config,
+ Scheduler: params.Scheduler,
+ GitOpsSync: params.GitOpsSync,
+ Environment: params.Environment,
+ JobSchedule: params.JobSchedule,
+ Volume: params.Volume,
+ SystemBackup: params.SystemBackup,
+ Admission: params.Admission,
}); err != nil {
return err
}
@@ -176,19 +182,36 @@ func registerJobs(params registerJobsParams) error {
}
type dynamicJobsParams struct {
- AppCtx context.Context
- Config *config.Config
- Scheduler schedulertypes.JobScheduler
- GitOpsSync *gitops.GitOpsSyncService
- Environment *environment.EnvironmentService
- JobSchedule *job.JobService
- Admission *actors.Gate[actors.AdmissionKey]
+ AppCtx context.Context
+ Config *config.Config
+ Scheduler schedulertypes.JobScheduler
+ GitOpsSync *gitops.GitOpsSyncService
+ Environment *environment.EnvironmentService
+ JobSchedule *job.JobService
+ Volume *volume.VolumeService
+ SystemBackup *systembackup.SystemBackupService
+ Admission *actors.Gate[actors.AdmissionKey]
}
// registerDynamicJobs injects the scheduler into the services that own per-entity
// jobs and registers the jobs for already-existing entities at startup. AddJob is
// an idempotent upsert, so these run safely before the scheduler is started.
func registerDynamicJobs(params dynamicJobsParams) error {
+ // Volume backups run on the environment that owns the Docker volume. This is
+ // registered on managers and agents; environment proxying persists each policy
+ // in the correct Arcane database.
+ if params.Volume != nil {
+ if err := params.Volume.SetScheduler(params.AppCtx, params.Scheduler, params.Admission); err != nil {
+ return err
+ }
+ params.Volume.RegisterBackupJobsOnStartup(params.AppCtx)
+ }
+ if !params.Config.AgentMode && params.SystemBackup != nil {
+ if err := params.SystemBackup.SetScheduler(params.AppCtx, params.Scheduler, params.Admission); err != nil {
+ return err
+ }
+ params.SystemBackup.RegisterBackupJobOnStartup(params.AppCtx)
+ }
// GitOps: one job per auto-sync-enabled sync (runs on manager and agents).
if params.GitOpsSync != nil {
if err := params.GitOpsSync.SetScheduler(params.AppCtx, params.Scheduler, params.Admission); err != nil {
diff --git a/backend/internal/dashboard/service_test.go b/backend/internal/dashboard/service_test.go
index 7e287a48b..bdb8ec077 100644
--- a/backend/internal/dashboard/service_test.go
+++ b/backend/internal/dashboard/service_test.go
@@ -191,7 +191,7 @@ func TestDashboardService_GetSnapshot_ReturnsDashboardSnapshot(t *testing.T) {
Status: project.ProjectStatusStopped,
}).Error)
projectSvc := project.NewProjectService(db, settingsSvc, nil, image.NewImageService(db, nil, nil, nil, nil, nil), nil, nil, nil, nil, config.Load())
- svc := NewDashboardService(db, dockerSvc, nil, projectSvc, nil, settingsSvc, nil, nil, nil, volume.NewVolumeService(db, nil, nil, nil, nil, nil))
+ svc := NewDashboardService(db, dockerSvc, nil, projectSvc, nil, settingsSvc, nil, nil, nil, volume.NewVolumeService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil))
snapshot, err := svc.GetSnapshot(context.Background(), DashboardActionItemsOptions{}, true)
require.NoError(t, err)
diff --git a/backend/internal/database/database.go b/backend/internal/database/database.go
index 5ddad12e9..5ad07492b 100644
--- a/backend/internal/database/database.go
+++ b/backend/internal/database/database.go
@@ -120,7 +120,7 @@ func connectDatabaseInternal(ctx context.Context, databaseURL string) (*DB, erro
switch {
case strings.HasPrefix(databaseURL, "file:"):
- connString, err := parseSqliteConnectionStringInternal(databaseURL)
+ connString, err := ParseSQLiteConnectionString(databaseURL)
if err != nil {
return nil, errors.WrapIf(err, "failed to parse SQLite connection string")
}
@@ -608,7 +608,7 @@ func missingEmbeddedDowngradeMigrationsInternal(ctx context.Context, db *sql.DB,
return missing, nil
}
-func parseSqliteConnectionStringInternal(connString string) (string, error) {
+func ParseSQLiteConnectionString(connString string) (string, error) {
if !strings.HasPrefix(connString, "file:") {
connString = "file:" + connString
}
diff --git a/backend/internal/database/database_test.go b/backend/internal/database/database_test.go
index 5503173cb..811adfbe7 100644
--- a/backend/internal/database/database_test.go
+++ b/backend/internal/database/database_test.go
@@ -202,6 +202,41 @@ func TestMigration070_PasskeysAndMFA_UpAndDown(t *testing.T) {
assert.Zero(t, columnCount)
}
+// TestMigration073_BackupSupport_PreservesExistingBackups proves that
+// pre-existing volume backup rows survive the backup-support migration as
+// format=archive, and that downgrading is refused while Rustic rows exist.
+func TestMigration073_BackupSupport_PreservesExistingBackups(t *testing.T) {
+ ctx := context.Background()
+ rawDB, _ := newSQLiteSQLDBInternal(t, t.TempDir(), "arcane-backup-support.db")
+
+ require.NoError(t, migrateDatabaseToVersionInternal(ctx, rawDB, dbProviderSQLite, MigrationOptions{}, 72))
+ _, err := rawDB.ExecContext(ctx, `INSERT INTO volume_backups (id, volume_name, size, created_at) VALUES ('legacy-1', 'app-data', 42, CURRENT_TIMESTAMP)`)
+ require.NoError(t, err)
+
+ require.NoError(t, migrateDatabaseToVersionInternal(ctx, rawDB, dbProviderSQLite, MigrationOptions{}, 73))
+ var format, status, destination string
+ var size int64
+ require.NoError(t, rawDB.QueryRow(`SELECT format, status, destination, size FROM volume_backups WHERE id = 'legacy-1'`).Scan(&format, &status, &destination, &size))
+ assert.Equal(t, "archive", format)
+ assert.Equal(t, "succeeded", status)
+ assert.Equal(t, "local", destination)
+ assert.EqualValues(t, 42, size)
+
+ // Downgrade is refused while a Rustic-format row exists.
+ _, err = rawDB.ExecContext(ctx, `INSERT INTO volume_backups (id, volume_name, size, created_at, format, local_snapshot_id) VALUES ('rustic-1', 'app-data', 7, CURRENT_TIMESTAMP, 'rustic', 'snap-1')`)
+ require.NoError(t, err)
+ err = migrateDatabaseToVersionInternal(ctx, rawDB, dbProviderSQLite, MigrationOptions{AllowDowngrade: true}, 72)
+ require.Error(t, err)
+
+ // With only archive rows left, the downgrade succeeds and keeps the row.
+ _, err = rawDB.ExecContext(ctx, `DELETE FROM volume_backups WHERE id = 'rustic-1'`)
+ require.NoError(t, err)
+ require.NoError(t, migrateDatabaseToVersionInternal(ctx, rawDB, dbProviderSQLite, MigrationOptions{AllowDowngrade: true}, 72))
+ var count int
+ require.NoError(t, rawDB.QueryRow(`SELECT COUNT(*) FROM volume_backups WHERE id = 'legacy-1'`).Scan(&count))
+ assert.Equal(t, 1, count)
+}
+
func TestMigration072_ProjectTags_UpDownAndCascade(t *testing.T) {
ctx := context.Background()
rawDB, _ := newSQLiteSQLDBInternal(t, t.TempDir(), "arcane-project-tags.db")
diff --git a/backend/internal/di/di.go b/backend/internal/di/di.go
index bd7301ac8..941cb175d 100644
--- a/backend/internal/di/di.go
+++ b/backend/internal/di/di.go
@@ -21,6 +21,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/session"
"github.com/getarcaneapp/arcane/backend/v2/internal/swarm"
"github.com/getarcaneapp/arcane/backend/v2/internal/system"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/systembackup"
"github.com/getarcaneapp/arcane/backend/v2/internal/upload"
"github.com/getarcaneapp/arcane/backend/v2/internal/variable"
"github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability"
@@ -92,6 +93,9 @@ var ServiceOptions = fx.Options(
provideWebhookModuleInternal,
variable.NewVariableService,
variable.New,
+ provideS3ModuleInternal,
+ provideS3ServiceInternal,
+ provideBackupEngineInternal,
upload.NewUploadService,
upload.New,
@@ -100,6 +104,9 @@ var ServiceOptions = fx.Options(
provideGitRepositoryModuleInternal,
provideGitRepositoryServiceInternal,
provideVolumeModuleInternal,
+ provideVolumeServiceInternal,
+ provideSystemBackupServiceInternal,
+ systembackup.New,
provideAuthServiceInternal,
provideAuthModuleInternal,
provideContainerRegistryModuleInternal,
diff --git a/backend/internal/di/di_test.go b/backend/internal/di/di_test.go
index 7452eadb3..00e583336 100644
--- a/backend/internal/di/di_test.go
+++ b/backend/internal/di/di_test.go
@@ -8,6 +8,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
"github.com/getarcaneapp/arcane/backend/v2/internal/apikey"
"github.com/getarcaneapp/arcane/backend/v2/internal/auth"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/backup"
"github.com/getarcaneapp/arcane/backend/v2/internal/docker"
"github.com/getarcaneapp/arcane/backend/v2/internal/environment"
"github.com/getarcaneapp/arcane/backend/v2/internal/event"
@@ -22,6 +23,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/project"
"github.com/getarcaneapp/arcane/backend/v2/internal/registry"
"github.com/getarcaneapp/arcane/backend/v2/internal/role"
+ s3domain "github.com/getarcaneapp/arcane/backend/v2/internal/s3"
"github.com/getarcaneapp/arcane/backend/v2/internal/session"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/internal/swarm"
@@ -42,6 +44,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/port"
"github.com/getarcaneapp/arcane/backend/v2/internal/search"
"github.com/getarcaneapp/arcane/backend/v2/internal/system"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/systembackup"
"github.com/getarcaneapp/arcane/backend/v2/internal/updater"
"github.com/getarcaneapp/arcane/backend/v2/internal/version"
"github.com/getarcaneapp/arcane/backend/v2/internal/volume"
@@ -68,6 +71,9 @@ type graphParams struct {
BuildWorkspace *build.BuildWorkspaceService
Lifecycle *project.LifecycleService
Volume *volume.Module
+ BackupEngine *backup.Engine
+ S3Destination *s3domain.Module
+ SystemBackup *systembackup.Module
Network *network.NetworkService
Port *port.PortService
Swarm *swarm.SwarmService
diff --git a/backend/internal/di/providers.go b/backend/internal/di/providers.go
index 9a0cfebac..f846db47d 100644
--- a/backend/internal/di/providers.go
+++ b/backend/internal/di/providers.go
@@ -9,6 +9,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
"github.com/getarcaneapp/arcane/backend/v2/internal/apikey"
"github.com/getarcaneapp/arcane/backend/v2/internal/auth"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/backup"
"github.com/getarcaneapp/arcane/backend/v2/internal/build"
"github.com/getarcaneapp/arcane/backend/v2/internal/docker"
"github.com/getarcaneapp/arcane/backend/v2/internal/environment"
@@ -26,9 +27,11 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/project"
"github.com/getarcaneapp/arcane/backend/v2/internal/registry"
"github.com/getarcaneapp/arcane/backend/v2/internal/role"
+ s3domain "github.com/getarcaneapp/arcane/backend/v2/internal/s3"
"github.com/getarcaneapp/arcane/backend/v2/internal/session"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/internal/swarm"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/systembackup"
"github.com/getarcaneapp/arcane/backend/v2/internal/template"
"github.com/getarcaneapp/arcane/backend/v2/internal/user"
"github.com/getarcaneapp/arcane/backend/v2/internal/version"
@@ -153,6 +156,12 @@ func provideSettingsModuleInternal(service *settings.SettingsService, environmen
})
}
+func provideBackupEngineInternal(ctx context.Context, lc fx.Lifecycle, runtime *actors.Runtime, admission *actors.Gate[actors.AdmissionKey], imageService *image.ImageService) *backup.Engine {
+ engine := backup.NewEngine(ctx, runtime, admission, imageService)
+ lc.Append(fx.Hook{OnStop: engine.Stop})
+ return engine
+}
+
func provideAdmissionGateInternal(ctx context.Context, lc fx.Lifecycle, runtime *actors.Runtime) (*actors.Gate[actors.AdmissionKey], error) {
gate, err := actors.NewGate[actors.AdmissionKey](ctx, runtime, "admission", "application")
if err != nil {
@@ -210,16 +219,32 @@ func provideGitRepositoryServiceInternal(module *gitrepo.Module) *gitrepo.GitRep
return module.Service()
}
-func provideVolumeModuleInternal(lc fx.Lifecycle, db *database.DB, docker *docker.DockerClientService, event *event.EventService, settings *settings.SettingsService, image *image.ImageService, activity *activity.ActivityService, cfg *config.Config, uploadService *upload.UploadService) *volume.Module {
+func provideS3ModuleInternal(db *database.DB, environmentService *environment.EnvironmentService) *s3domain.Module {
+ return s3domain.New(s3domain.Dependencies{
+ DB: db,
+ SyncRemoteDestinations: environmentService.SyncS3DestinationsToRemoteEnvironments,
+ CheckRemoteReferences: environmentService.CheckS3DestinationReferences,
+ })
+}
+
+func provideS3ServiceInternal(module *s3domain.Module) *s3domain.S3DestinationService {
+ return module.Service()
+}
+
+func provideVolumeModuleInternal(lc fx.Lifecycle, db *database.DB, dockerService *docker.DockerClientService, eventService *event.EventService, settingsService *settings.SettingsService, imageService *image.ImageService, activityService *activity.ActivityService, containerModule *container.Module, engine *backup.Engine, s3Service *s3domain.S3DestinationService, environmentService *environment.EnvironmentService, cfg *config.Config, uploadService *upload.UploadService) *volume.Module {
module := volume.New(volume.Dependencies{
- DB: db,
- Docker: docker,
- Event: event,
- Settings: settings,
- Image: image,
- Activity: activity,
- Config: cfg,
- Upload: uploadService,
+ DB: db,
+ Docker: dockerService,
+ Event: eventService,
+ Settings: settingsService,
+ Image: imageService,
+ Activity: activityService,
+ Environment: environmentService,
+ Container: containerModule.Service(),
+ Engine: engine,
+ S3: s3Service,
+ Config: cfg,
+ Upload: uploadService,
})
lc.Append(fx.Hook{
OnStop: func(ctx context.Context) error {
@@ -230,6 +255,14 @@ func provideVolumeModuleInternal(lc fx.Lifecycle, db *database.DB, docker *docke
return module
}
+func provideVolumeServiceInternal(module *volume.Module) *volume.VolumeService {
+ return module.Service()
+}
+
+func provideSystemBackupServiceInternal(db *database.DB, dockerService *docker.DockerClientService, volumeModule *volume.Module, engine *backup.Engine, s3Service *s3domain.S3DestinationService, activityService *activity.ActivityService, cfg *config.Config) *systembackup.SystemBackupService {
+ return systembackup.NewSystemBackupService(db, dockerService, volumeModule.Service(), engine, s3Service, activityService, cfg)
+}
+
func provideContainerModuleInternal(ctx context.Context, event *event.EventService, docker *docker.DockerClientService, image *image.ImageService, settings *settings.SettingsService, project *project.ProjectService, activity *activity.ActivityService) *container.Module {
return container.New(ctx, container.Dependencies{
Event: event,
diff --git a/backend/internal/environment/environment_health.go b/backend/internal/environment/environment_health.go
index 8428e9cb9..5923475e1 100644
--- a/backend/internal/environment/environment_health.go
+++ b/backend/internal/environment/environment_health.go
@@ -140,6 +140,9 @@ func (s *EnvironmentService) runHealthCheckInternal(ctx context.Context, envID s
if err := s.SyncRegistriesToEnvironment(syncCtx, envID); err != nil {
slog.WarnContext(syncCtx, "failed to sync registries during health check", "environment_id", envID, "error", err)
}
+ if err := s.SyncS3DestinationsToEnvironment(syncCtx, envID); err != nil {
+ slog.WarnContext(syncCtx, "failed to sync S3 destinations during health check", "environment_id", envID, "error", err)
+ }
if err := s.SyncRepositoriesToEnvironment(syncCtx, envID); err != nil {
slog.WarnContext(syncCtx, "failed to sync git repositories during health check", "environment_id", envID, "error", err)
}
diff --git a/backend/internal/environment/environment_proxy.go b/backend/internal/environment/environment_proxy.go
index 29f184c3b..5de69179f 100644
--- a/backend/internal/environment/environment_proxy.go
+++ b/backend/internal/environment/environment_proxy.go
@@ -2,12 +2,14 @@ package environment
import (
"github.com/getarcaneapp/arcane/backend/v2/internal/gitrepo"
+ s3domain "github.com/getarcaneapp/arcane/backend/v2/internal/s3"
"context"
"encoding/json/v2"
"log/slog"
"maps"
"net/http"
+ "net/url"
"strings"
"time"
@@ -20,6 +22,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/remenv"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
httputils "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
+ backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"
"github.com/getarcaneapp/arcane/types/v2/containerregistry"
"github.com/getarcaneapp/arcane/types/v2/gitops"
"go.getarcane.app/sys/crypto"
@@ -62,6 +65,59 @@ func (s *EnvironmentService) SyncRegistriesToRemoteEnvironments(ctx context.Cont
return nil
}
+// SyncS3DestinationsToRemoteEnvironments refreshes the destination cache on every enabled remote environment.
+func (s *EnvironmentService) SyncS3DestinationsToRemoteEnvironments(ctx context.Context) error {
+ envs, err := s.ListRemoteEnvironments(ctx)
+ if err != nil {
+ return errors.WrapIf(err, "failed to list remote environments for S3 destination sync")
+ }
+
+ var failedCount int
+ for _, env := range envs {
+ if env.AccessToken == nil || strings.TrimSpace(*env.AccessToken) == "" {
+ slog.DebugContext(ctx, "Skipping S3 destination sync for environment without access token", "environmentID", env.ID, "environmentName", env.Name)
+ continue
+ }
+ if err := s.SyncS3DestinationsToEnvironment(ctx, env.ID); err != nil {
+ failedCount++
+ slog.WarnContext(ctx, "Failed to sync S3 destinations to remote environment", "environmentID", env.ID, "environmentName", env.Name, "error", err)
+ }
+ }
+
+ if failedCount > 0 {
+ return errors.Errorf("failed to sync S3 destinations to %d remote environment(s)", failedCount)
+ }
+ return nil
+}
+
+// CheckS3DestinationReferences returns an error while any managed environment
+// still references the destination, or when a synced environment cannot be
+// checked conclusively. Deleting credentials that a remote environment still
+// needs would strand its policies and retained backups.
+func (s *EnvironmentService) CheckS3DestinationReferences(ctx context.Context, destinationID string) error {
+ envs, err := s.ListRemoteEnvironments(ctx)
+ if err != nil {
+ return errors.WrapIf(err, "failed to list remote environments for S3 destination reference check")
+ }
+ for _, env := range envs {
+ if env.AccessToken == nil || strings.TrimSpace(*env.AccessToken) == "" {
+ // Environments without an access token never receive destination
+ // syncs, so they cannot hold a reference.
+ continue
+ }
+ var result struct {
+ InUse bool `json:"inUse"`
+ }
+ if err := s.ProxyJSONRequestForEnvironment(ctx, env, http.MethodGet, "/api/backups/s3/"+url.PathEscape(destinationID)+"/in-use", nil, &result); err != nil {
+ return errors.WrapIff(err, "cannot verify S3 destination references on environment %s; restore connectivity before deleting", env.Name)
+ }
+ if result.InUse {
+ return errors.Errorf("still referenced by environment %s", env.Name)
+ }
+ }
+ return nil
+}
+
type remoteEnvironmentTargetInternal struct {
ID string
Name string
@@ -322,6 +378,22 @@ func (s *EnvironmentService) SyncRegistriesToEnvironment(ctx context.Context, en
)
}
+// SyncS3DestinationsToEnvironment sends manager-owned destinations to one remote environment.
+func (s *EnvironmentService) SyncS3DestinationsToEnvironment(ctx context.Context, environmentID string) error {
+ return fanOutSyncToEnvironmentInternal(ctx, s, environmentID, "S3 destinations", "/api/backups/s3/sync",
+ func(_ context.Context, destination s3domain.S3Destination) (backuptypes.S3DestinationSync, bool, error) {
+ secret, err := crypto.Decrypt(destination.SecretAccessKey)
+ if err != nil {
+ return backuptypes.S3DestinationSync{}, false, errors.WrapIff(err, "failed to decrypt S3 destination %s for sync", destination.ID)
+ }
+ return destination.ToSync(secret), true, nil
+ },
+ func(items []backuptypes.S3DestinationSync) backuptypes.S3DestinationSyncRequest {
+ return backuptypes.S3DestinationSyncRequest{Destinations: items}
+ },
+ )
+}
+
// SyncRepositoriesToEnvironment syncs all git repositories from this manager to a remote environment
func (s *EnvironmentService) SyncRepositoriesToEnvironment(ctx context.Context, environmentID string) error {
return fanOutSyncToEnvironmentInternal(ctx, s, environmentID, "git repositories", "/api/git-repositories/sync",
diff --git a/backend/internal/environment/handler.go b/backend/internal/environment/handler.go
index 9feb0d177..1412689fd 100644
--- a/backend/internal/environment/handler.go
+++ b/backend/internal/environment/handler.go
@@ -908,13 +908,17 @@ func (h *EnvironmentHandler) PairAgent(ctx context.Context, input *PairAgentInpu
}, nil
}
-// SyncEnvironment syncs container registries and git repositories to an environment.
+// SyncEnvironment syncs manager-owned resources to an environment.
func (h *EnvironmentHandler) SyncEnvironment(ctx context.Context, input *SyncEnvironmentInput) (*SyncEnvironmentOutput, error) {
// Sync registries
if err := h.environmentService.SyncRegistriesToEnvironment(ctx, input.ID); err != nil {
slog.WarnContext(ctx, "Failed to sync registries", "environmentID", input.ID, "error", err.Error())
}
+ if err := h.environmentService.SyncS3DestinationsToEnvironment(ctx, input.ID); err != nil {
+ slog.WarnContext(ctx, "Failed to sync S3 destinations", "environmentID", input.ID, "error", err.Error())
+ }
+
// Sync git repositories
if err := h.environmentService.SyncRepositoriesToEnvironment(ctx, input.ID); err != nil {
slog.WarnContext(ctx, "Failed to sync git repositories", "environmentID", input.ID, "error", err.Error())
@@ -997,6 +1001,18 @@ func (h *EnvironmentHandler) triggerEnvironmentResourceSyncInternal(ctx context.
}
}(detachedCtx, environmentID, environmentName, reason)
+ go func(syncCtx context.Context, envID string, envName string, syncReason string) {
+ syncCtx, cancel := context.WithTimeout(syncCtx, edge.DefaultProxyTimeout)
+ defer cancel()
+ if err := h.environmentService.SyncS3DestinationsToEnvironment(syncCtx, envID); err != nil {
+ slog.WarnContext(syncCtx, "Failed to sync S3 destinations to environment",
+ "environmentID", envID,
+ "environmentName", envName,
+ "reason", syncReason,
+ "error", err.Error())
+ }
+ }(detachedCtx, environmentID, environmentName, reason)
+
go func(syncCtx context.Context, envID string, envName string, syncReason string) {
syncCtx, cancel := context.WithTimeout(syncCtx, edge.DefaultProxyTimeout)
defer cancel()
diff --git a/backend/internal/environment/service_test.go b/backend/internal/environment/service_test.go
index 6a522ec66..f5bc301c1 100644
--- a/backend/internal/environment/service_test.go
+++ b/backend/internal/environment/service_test.go
@@ -28,10 +28,12 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/registry"
+ s3domain "github.com/getarcaneapp/arcane/backend/v2/internal/s3"
"github.com/getarcaneapp/arcane/backend/v2/internal/user"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/entityjobs"
+ backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"
"github.com/getarcaneapp/arcane/types/v2/containerregistry"
"github.com/getarcaneapp/arcane/types/v2/environment"
"github.com/getarcaneapp/arcane/types/v2/gitops"
@@ -132,6 +134,7 @@ func setupEnvironmentServiceTestDB(t *testing.T) *database.DB {
require.NoError(t, db.AutoMigrate(
&Environment{},
&registry.ContainerRegistry{},
+ &s3domain.S3Destination{},
&settings.SettingVariable{},
&common.User{},
&apikey.ApiKey{},
@@ -450,6 +453,49 @@ func TestEnvironmentService_SyncRegistriesToEnvironment_IncludesECRFields(t *tes
require.NoError(t, err)
}
+func TestEnvironmentService_SyncS3DestinationsToEnvironment(t *testing.T) {
+ ctx := context.Background()
+ db := setupEnvironmentServiceTestDB(t)
+ svc := NewEnvironmentService(db, nil, nil, nil, nil, nil)
+
+ encryptedSecret, err := crypto.Encrypt("s3-secret")
+ require.NoError(t, err)
+ require.NoError(t, db.WithContext(ctx).Create(&s3domain.S3Destination{
+ BaseModel: database.BaseModel{ID: "s3-1"},
+ Name: "Offsite",
+ Endpoint: "https://s3.example.com",
+ Bucket: "volume-backups",
+ Region: "eu-central-1",
+ AccessKeyID: "s3-access-key",
+ SecretAccessKey: encryptedSecret,
+ Prefix: "agents",
+ UseSSL: true,
+ ForcePathStyle: true,
+ }).Error)
+
+ accessToken := "agent-token"
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ require.Equal(t, http.MethodPost, r.Method)
+ require.Equal(t, "/api/backups/s3/sync", r.URL.Path)
+ require.Equal(t, accessToken, r.Header.Get("X-API-Key"))
+ require.Equal(t, accessToken, r.Header.Get("X-Arcane-Agent-Token"))
+
+ var syncReq backuptypes.S3DestinationSyncRequest
+ require.NoError(t, json.NewDecoder(r.Body).Decode(&syncReq))
+ require.Len(t, syncReq.Destinations, 1)
+ require.Equal(t, "s3-1", syncReq.Destinations[0].ID)
+ require.Equal(t, "s3-secret", syncReq.Destinations[0].SecretAccessKey)
+ require.Equal(t, "volume-backups", syncReq.Destinations[0].Bucket)
+
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(`{"success":true,"data":{"message":"ok"}}`))
+ }))
+ defer server.Close()
+
+ createTestEnvironment(t, db, "env-s3", server.URL, &accessToken)
+ require.NoError(t, svc.SyncS3DestinationsToEnvironment(ctx, "env-s3"))
+}
+
func TestEnvironmentService_SyncRepositoriesToEnvironment_UsesAgentHeaders(t *testing.T) {
ctx := context.Background()
db := setupEnvironmentServiceTestDB(t)
diff --git a/backend/internal/middleware/environment_middleware.go b/backend/internal/middleware/environment_middleware.go
index a7bf271b1..bb2b14810 100644
--- a/backend/internal/middleware/environment_middleware.go
+++ b/backend/internal/middleware/environment_middleware.go
@@ -430,6 +430,9 @@ func isManagementPathInternal(method, suffix string) bool {
if isCentralSwarmManagementPathInternal(method, suffix) {
return true
}
+ if isCentralVolumeBackupPolicyPathInternal(method, suffix) {
+ return true
+ }
if suffix == "/activities" || strings.HasPrefix(suffix, "/activities/") {
return true
}
@@ -452,6 +455,14 @@ func isManagementPathInternal(method, suffix string) bool {
return isManagement
}
+func isCentralVolumeBackupPolicyPathInternal(method, suffix string) bool {
+ if method != http.MethodGet && method != http.MethodPut {
+ return false
+ }
+ parts := strings.Split(strings.Trim(suffix, "/"), "/")
+ return len(parts) == 3 && parts[0] == "volumes" && parts[1] != "" && parts[2] == "backup-policy"
+}
+
func isCentralSwarmManagementPathInternal(method, suffix string) bool {
if method == http.MethodGet && (suffix == "/swarm/join-candidates" || suffix == "/swarm/nodes") {
return true
diff --git a/backend/internal/middleware/environment_middleware_test.go b/backend/internal/middleware/environment_middleware_test.go
index 11da650d1..ee7e76a5e 100644
--- a/backend/internal/middleware/environment_middleware_test.go
+++ b/backend/internal/middleware/environment_middleware_test.go
@@ -396,3 +396,10 @@ func TestIsCentralSwarmManagementPathInternal_IsMethodAware(t *testing.T) {
assert.True(t, isCentralSwarmManagementPathInternal(http.MethodPut, "/swarm/nodes/node-1/agent/binding"))
assert.True(t, isCentralSwarmManagementPathInternal(http.MethodDelete, "/swarm/nodes/node-1/agent/binding"))
}
+
+func TestIsCentralVolumeBackupPolicyPathInternal_IsMethodAware(t *testing.T) {
+ assert.True(t, isCentralVolumeBackupPolicyPathInternal(http.MethodGet, "/volumes/app-data/backup-policy"))
+ assert.True(t, isCentralVolumeBackupPolicyPathInternal(http.MethodPut, "/volumes/app-data/backup-policy"))
+ assert.False(t, isCentralVolumeBackupPolicyPathInternal(http.MethodPost, "/volumes/app-data/backup-policy"))
+ assert.False(t, isCentralVolumeBackupPolicyPathInternal(http.MethodGet, "/volumes/app-data/backups"))
+}
diff --git a/backend/internal/recovery/request.go b/backend/internal/recovery/request.go
new file mode 100644
index 000000000..0b3905940
--- /dev/null
+++ b/backend/internal/recovery/request.go
@@ -0,0 +1,44 @@
+package recovery
+
+import (
+ "time"
+
+ "github.com/moby/moby/api/types/mount"
+)
+
+type RestoreRequest struct {
+ BackupID string `json:"backupId"`
+ ContainerID string `json:"containerId"`
+ ContainerImage string `json:"containerImage"`
+ SnapshotID string `json:"snapshotId"`
+ SnapshotPath string `json:"snapshotPath"`
+ LocalSnapshotID string `json:"localSnapshotId,omitempty"`
+ RemoteSnapshotID string `json:"remoteSnapshotId,omitempty"`
+ S3DestinationID string `json:"s3DestinationId,omitempty"`
+ Size int64 `json:"size,omitempty"`
+ SafetyBackup *SafetyBackup `json:"safetyBackup,omitempty"`
+ RecoveryKey string `json:"recoveryKey"`
+ RepositoryEnvironment []string `json:"repositoryEnvironment"`
+ RepositoryMounts []mount.Mount `json:"repositoryMounts"`
+ AppDataMount mount.Mount `json:"appDataMount"`
+ // NetworkMode attaches the detached Rustic restore container to one of
+ // Arcane's networks so network-local S3 endpoints stay reachable while
+ // Arcane itself is stopped.
+ NetworkMode string `json:"networkMode,omitempty"`
+}
+
+type SafetyBackup struct {
+ ID string `json:"id"`
+ LocalSnapshotID string `json:"localSnapshotId"`
+ Size int64 `json:"size"`
+ CreatedAt time.Time `json:"createdAt"`
+}
+
+type Manifest struct {
+ FormatVersion int `json:"formatVersion"`
+ ArcaneVersion string `json:"arcaneVersion"`
+ BackupID string `json:"backupId,omitempty"`
+ ActivityID string `json:"activityId,omitempty"`
+ CreatedAt time.Time `json:"createdAt"`
+ Environment map[string]string `json:"environment"`
+}
diff --git a/backend/internal/registry/handler.go b/backend/internal/registry/handler.go
index 3c8aec6d5..17ffb4a7b 100644
--- a/backend/internal/registry/handler.go
+++ b/backend/internal/registry/handler.go
@@ -18,6 +18,8 @@ import (
"go.getarcane.app/sys/crypto"
)
+const registryRemoteSyncFailureMessageInternal = "Failed to fan out registry sync to remote environments"
+
// ContainerRegistryHandler handles container registry management endpoints.
type ContainerRegistryHandler struct {
registryService *ContainerRegistryService
@@ -392,7 +394,7 @@ func (h *ContainerRegistryHandler) triggerRemoteRegistrySync(ctx context.Context
go func(syncCtx context.Context, syncReason string) {
if err := h.syncRemoteRegistries(syncCtx); err != nil {
- slog.WarnContext(syncCtx, "Failed to fan out registry sync to remote environments", "reason", syncReason, "error", err.Error())
+ slog.WarnContext(syncCtx, registryRemoteSyncFailureMessageInternal, "reason", syncReason, "error", err.Error())
}
}(detachedCtx, reason)
}
diff --git a/backend/internal/s3/handler.go b/backend/internal/s3/handler.go
new file mode 100644
index 000000000..38ebcc006
--- /dev/null
+++ b/backend/internal/s3/handler.go
@@ -0,0 +1,247 @@
+package s3
+
+import (
+ "context"
+ "log/slog"
+ "net/http"
+
+ "emperror.dev/errors"
+
+ "github.com/danielgtaylor/huma/v2"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/authz"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"
+ backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"
+ "github.com/getarcaneapp/arcane/types/v2/base"
+)
+
+const (
+ s3DestinationPathInternal = "/backups/s3"
+ s3DestinationTagInternal = "S3 Destinations"
+ s3ConnectionTestSuccessInternal = "S3 connection test succeeded"
+ s3RemoteSyncFailureMessageInternal = "Failed to fan out S3 destination sync to remote environments"
+)
+
+type s3DestinationHandlerInternal struct {
+ service *S3DestinationService
+ syncRemoteDestinations func(context.Context) error
+}
+
+type listS3DestinationsInputInternal struct {
+ Search string `query:"search" doc:"Search query"`
+ Sort string `query:"sort" doc:"Column to sort by"`
+ Order string `query:"order" default:"asc" doc:"Sort direction"`
+ Start int `query:"start" default:"0" doc:"Start index"`
+ Limit int `query:"limit" default:"20" doc:"Limit"`
+}
+
+type listS3DestinationsOutputInternal struct {
+ Body struct {
+ Data []backuptypes.S3Destination `json:"data"`
+ Pagination base.PaginationResponse `json:"pagination"`
+ }
+}
+
+type listAllS3DestinationsOutputInternal struct {
+ Body []backuptypes.S3Destination
+}
+
+type s3DestinationIDInputInternal struct {
+ ID string `path:"id" doc:"S3 destination ID"`
+}
+
+type createS3DestinationInputInternal struct {
+ Body backuptypes.CreateS3Destination
+}
+
+type updateS3DestinationInputInternal struct {
+ ID string `path:"id" doc:"S3 destination ID"`
+ Body backuptypes.UpdateS3Destination
+}
+
+type s3DestinationOutputInternal struct {
+ Body backuptypes.S3Destination
+}
+
+type s3DestinationMessageOutputInternal struct {
+ Body base.ApiResponse[base.MessageResponse]
+}
+
+type testS3DestinationInputInternal struct {
+ ID string `path:"id" doc:"S3 destination ID"`
+ Body *backuptypes.UpdateS3Destination `json:"body,omitempty"`
+}
+
+type testS3DestinationConfigurationInputInternal struct {
+ Body backuptypes.CreateS3Destination
+}
+
+type syncS3DestinationsInputInternal struct {
+ Body backuptypes.S3DestinationSyncRequest
+}
+
+func RegisterS3Destinations(api huma.API, service *S3DestinationService, syncRemoteDestinations func(context.Context) error) {
+ handler := &s3DestinationHandlerInternal{service: service, syncRemoteDestinations: syncRemoteDestinations}
+
+ handlerutil.RegisterSecured(api,
+ handlerutil.Operation("list-s3-destinations", http.MethodGet, s3DestinationPathInternal, "List S3 destinations", "List saved S3-compatible backup destinations with search, sorting, and pagination", s3DestinationTagInternal),
+ authz.PermS3DestinationsList, handler.listInternal)
+ handlerutil.RegisterSecured(api,
+ handlerutil.Operation("list-all-s3-destinations", http.MethodGet, s3DestinationPathInternal+"/options", "List all S3 destination options", "List saved S3-compatible destinations for backup configuration selectors", s3DestinationTagInternal),
+ authz.PermS3DestinationsList, handler.listAllInternal)
+ handlerutil.RegisterSecured(api,
+ handlerutil.Operation("sync-s3-destinations", http.MethodPost, s3DestinationPathInternal+"/sync", "Sync S3 destinations", "Synchronize manager-owned S3 destinations to an agent", s3DestinationTagInternal),
+ authz.PermS3DestinationsSync, handler.syncInternal)
+ handlerutil.RegisterSecured(api,
+ handlerutil.Operation("get-s3-destination", http.MethodGet, s3DestinationPathInternal+"/{id}", "Get S3 destination", "", s3DestinationTagInternal),
+ authz.PermS3DestinationsRead, handler.getInternal)
+ handlerutil.RegisterSecured(api,
+ handlerutil.Operation("create-s3-destination", http.MethodPost, s3DestinationPathInternal, "Create S3 destination", "", s3DestinationTagInternal),
+ authz.PermS3DestinationsCreate, handler.createInternal)
+ handlerutil.RegisterSecured(api,
+ handlerutil.Operation("update-s3-destination", http.MethodPut, s3DestinationPathInternal+"/{id}", "Update S3 destination", "", s3DestinationTagInternal),
+ authz.PermS3DestinationsUpdate, handler.updateInternal)
+ handlerutil.RegisterSecured(api,
+ handlerutil.Operation("test-s3-destination-configuration", http.MethodPost, s3DestinationPathInternal+"/test", "Test unsaved S3 destination configuration", "Verify upload, download, and delete access before saving an S3 destination", s3DestinationTagInternal),
+ authz.PermS3DestinationsTest, handler.testConfigurationInternal)
+ handlerutil.RegisterSecured(api,
+ handlerutil.Operation("test-s3-destination", http.MethodPost, s3DestinationPathInternal+"/{id}/test", "Test S3 destination", "Verify upload, download, and delete access using the saved or supplied S3 destination configuration", s3DestinationTagInternal),
+ authz.PermS3DestinationsTest, handler.testInternal)
+ handlerutil.RegisterSecured(api,
+ handlerutil.Operation("get-s3-destination-usage", http.MethodGet, s3DestinationPathInternal+"/{id}/in-use", "Check S3 destination references", "Report whether backup records, policies, or settings on this environment still reference the destination", s3DestinationTagInternal),
+ authz.PermS3DestinationsRead, handler.inUseInternal)
+ handlerutil.RegisterSecured(api,
+ handlerutil.Operation("delete-s3-destination", http.MethodDelete, s3DestinationPathInternal+"/{id}", "Delete S3 destination", "", s3DestinationTagInternal),
+ authz.PermS3DestinationsDelete, handler.deleteInternal)
+}
+
+type s3DestinationUsageOutputInternal struct {
+ Body struct {
+ InUse bool `json:"inUse"`
+ }
+}
+
+func (h *s3DestinationHandlerInternal) inUseInternal(ctx context.Context, input *s3DestinationIDInputInternal) (*s3DestinationUsageOutputInternal, error) {
+ inUse, err := h.service.DestinationInUse(ctx, input.ID)
+ if err != nil {
+ return nil, huma.Error500InternalServerError(err.Error())
+ }
+ output := &s3DestinationUsageOutputInternal{}
+ output.Body.InUse = inUse
+ return output, nil
+}
+
+func (h *s3DestinationHandlerInternal) listInternal(ctx context.Context, input *listS3DestinationsInputInternal) (*listS3DestinationsOutputInternal, error) {
+ params := handlerutil.PaginationParams(input.Start, input.Limit, input.Sort, input.Order, input.Search)
+ destinations, paginationResponse, err := h.service.ListS3Destinations(ctx, params)
+ if err != nil {
+ return nil, huma.Error500InternalServerError(err.Error())
+ }
+ output := &listS3DestinationsOutputInternal{}
+ output.Body.Data = destinations
+ output.Body.Pagination = handlerutil.PaginationResponse(paginationResponse)
+ return output, nil
+}
+
+func (h *s3DestinationHandlerInternal) listAllInternal(ctx context.Context, _ *struct{}) (*listAllS3DestinationsOutputInternal, error) {
+ destinations, err := h.service.ListAllS3Destinations(ctx)
+ if err != nil {
+ return nil, huma.Error500InternalServerError(err.Error())
+ }
+ return &listAllS3DestinationsOutputInternal{Body: destinations}, nil
+}
+
+func (h *s3DestinationHandlerInternal) getInternal(ctx context.Context, input *s3DestinationIDInputInternal) (*s3DestinationOutputInternal, error) {
+ destination, err := h.service.GetS3Destination(ctx, input.ID)
+ if errors.Is(err, ErrS3DestinationNotFound) {
+ return nil, huma.Error404NotFound(err.Error())
+ }
+ if err != nil {
+ return nil, huma.Error500InternalServerError(err.Error())
+ }
+ return &s3DestinationOutputInternal{Body: *destination}, nil
+}
+
+func (h *s3DestinationHandlerInternal) createInternal(ctx context.Context, input *createS3DestinationInputInternal) (*s3DestinationOutputInternal, error) {
+ if err := h.service.TestS3DestinationConfiguration(ctx, input.Body); err != nil {
+ return nil, huma.Error400BadRequest(err.Error())
+ }
+ destination, err := h.service.CreateS3Destination(ctx, input.Body)
+ if err != nil {
+ return nil, huma.Error400BadRequest(err.Error())
+ }
+ h.triggerRemoteSyncInternal(ctx, "S3 destination creation")
+ return &s3DestinationOutputInternal{Body: *destination}, nil
+}
+
+func (h *s3DestinationHandlerInternal) updateInternal(ctx context.Context, input *updateS3DestinationInputInternal) (*s3DestinationOutputInternal, error) {
+ if err := h.service.TestS3Destination(ctx, input.ID, &input.Body); err != nil {
+ if errors.Is(err, ErrS3DestinationNotFound) {
+ return nil, huma.Error404NotFound(err.Error())
+ }
+ return nil, huma.Error400BadRequest(err.Error())
+ }
+ destination, err := h.service.UpdateS3Destination(ctx, input.ID, input.Body)
+ if errors.Is(err, ErrS3DestinationNotFound) {
+ return nil, huma.Error404NotFound(err.Error())
+ }
+ if err != nil {
+ return nil, huma.Error400BadRequest(err.Error())
+ }
+ h.triggerRemoteSyncInternal(ctx, "S3 destination update")
+ return &s3DestinationOutputInternal{Body: *destination}, nil
+}
+
+func (h *s3DestinationHandlerInternal) deleteInternal(ctx context.Context, input *s3DestinationIDInputInternal) (*s3DestinationMessageOutputInternal, error) {
+ err := h.service.DeleteS3Destination(ctx, input.ID)
+ if errors.Is(err, ErrS3DestinationNotFound) {
+ return nil, huma.Error404NotFound(err.Error())
+ }
+ if errors.Is(err, ErrS3DestinationInUse) {
+ return nil, huma.Error409Conflict(err.Error())
+ }
+ if err != nil {
+ return nil, huma.Error500InternalServerError(err.Error())
+ }
+ h.triggerRemoteSyncInternal(ctx, "S3 destination deletion")
+ return &s3DestinationMessageOutputInternal{Body: successMessageResponseInternal("S3 destination deleted successfully")}, nil
+}
+
+func (h *s3DestinationHandlerInternal) testInternal(ctx context.Context, input *testS3DestinationInputInternal) (*s3DestinationMessageOutputInternal, error) {
+ if err := h.service.TestS3Destination(ctx, input.ID, input.Body); err != nil {
+ if errors.Is(err, ErrS3DestinationNotFound) {
+ return nil, huma.Error404NotFound(err.Error())
+ }
+ return nil, huma.Error400BadRequest(err.Error())
+ }
+ return &s3DestinationMessageOutputInternal{Body: successMessageResponseInternal(s3ConnectionTestSuccessInternal)}, nil
+}
+
+func successMessageResponseInternal(message string) base.ApiResponse[base.MessageResponse] {
+ return base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: message}}
+}
+
+func (h *s3DestinationHandlerInternal) triggerRemoteSyncInternal(ctx context.Context, reason string) {
+ if h.syncRemoteDestinations == nil {
+ return
+ }
+ detachedCtx := context.WithoutCancel(ctx)
+ go func(syncCtx context.Context, syncReason string) {
+ if err := h.syncRemoteDestinations(syncCtx); err != nil {
+ slog.WarnContext(syncCtx, s3RemoteSyncFailureMessageInternal, "reason", syncReason, "error", err.Error())
+ }
+ }(detachedCtx, reason)
+}
+
+func (h *s3DestinationHandlerInternal) testConfigurationInternal(ctx context.Context, input *testS3DestinationConfigurationInputInternal) (*s3DestinationMessageOutputInternal, error) {
+ if err := h.service.TestS3DestinationConfiguration(ctx, input.Body); err != nil {
+ return nil, huma.Error400BadRequest(err.Error())
+ }
+ return &s3DestinationMessageOutputInternal{Body: successMessageResponseInternal(s3ConnectionTestSuccessInternal)}, nil
+}
+
+func (h *s3DestinationHandlerInternal) syncInternal(ctx context.Context, input *syncS3DestinationsInputInternal) (*s3DestinationMessageOutputInternal, error) {
+ if err := h.service.SyncS3Destinations(ctx, input.Body.Destinations); err != nil {
+ return nil, huma.Error400BadRequest(err.Error())
+ }
+ return &s3DestinationMessageOutputInternal{Body: successMessageResponseInternal("S3 destinations synced successfully")}, nil
+}
diff --git a/backend/internal/s3/model.go b/backend/internal/s3/model.go
new file mode 100644
index 000000000..74fbf68a2
--- /dev/null
+++ b/backend/internal/s3/model.go
@@ -0,0 +1,60 @@
+package s3
+
+import (
+ "strings"
+
+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"
+ backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"
+)
+
+type S3Destination struct {
+ database.BaseModel
+
+ Name string `json:"name" gorm:"column:name;type:text;not null;uniqueIndex" sortable:"true"`
+ Endpoint string `json:"endpoint,omitempty" gorm:"column:endpoint;type:text" sortable:"true"`
+ Bucket string `json:"bucket" gorm:"column:bucket;type:text;not null" sortable:"true"`
+ Region string `json:"region" gorm:"column:region;type:text;not null" sortable:"true"`
+ AccessKeyID string `json:"accessKeyId" gorm:"column:access_key_id;type:text;not null" sortable:"true"`
+ SecretAccessKey string `json:"-" gorm:"column:secret_access_key;type:text;not null"`
+ Prefix string `json:"prefix,omitempty" gorm:"column:prefix;type:text" sortable:"true"`
+ UseSSL bool `json:"useSsl" gorm:"column:use_ssl;not null" sortable:"true"`
+ ForcePathStyle bool `json:"forcePathStyle" gorm:"column:force_path_style;not null" sortable:"true"`
+}
+
+func (S3Destination) TableName() string {
+ return "s3_destinations"
+}
+
+func (d S3Destination) ToDTO() backuptypes.S3Destination {
+ return backuptypes.S3Destination{
+ ID: d.ID,
+ Name: d.Name,
+ Endpoint: d.Endpoint,
+ Bucket: d.Bucket,
+ Region: d.Region,
+ AccessKeyID: d.AccessKeyID,
+ Prefix: d.Prefix,
+ UseSSL: d.UseSSL,
+ ForcePathStyle: d.ForcePathStyle,
+ SecretConfigured: strings.TrimSpace(d.SecretAccessKey) != "",
+ CreatedAt: d.CreatedAt,
+ UpdatedAt: d.UpdatedAt,
+ }
+}
+
+func (d S3Destination) ToSync(secretAccessKey string) backuptypes.S3DestinationSync {
+ return backuptypes.S3DestinationSync{
+ ID: d.ID,
+ Name: d.Name,
+ Endpoint: d.Endpoint,
+ Bucket: d.Bucket,
+ Region: d.Region,
+ AccessKeyID: d.AccessKeyID,
+ SecretAccessKey: secretAccessKey,
+ Prefix: d.Prefix,
+ UseSSL: d.UseSSL,
+ ForcePathStyle: d.ForcePathStyle,
+ CreatedAt: d.CreatedAt,
+ UpdatedAt: d.UpdatedAt,
+ }
+}
diff --git a/backend/internal/s3/module.go b/backend/internal/s3/module.go
new file mode 100644
index 000000000..f98a3738b
--- /dev/null
+++ b/backend/internal/s3/module.go
@@ -0,0 +1,41 @@
+// Package s3 owns reusable S3-compatible backup destinations and their HTTP surface.
+package s3
+
+import (
+ "context"
+
+ "github.com/danielgtaylor/huma/v2"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"
+)
+
+type Dependencies struct {
+ DB *database.DB
+ SyncRemoteDestinations func(context.Context) error
+ CheckRemoteReferences func(ctx context.Context, destinationID string) error
+}
+
+type Module struct {
+ service *S3DestinationService
+ deps Dependencies
+}
+
+func New(deps Dependencies) *Module {
+ service := NewS3DestinationService(deps.DB)
+ service.checkRemoteReferences = deps.CheckRemoteReferences
+ return &Module{service: service, deps: deps}
+}
+
+func (m *Module) Service() *S3DestinationService {
+ if m == nil {
+ return nil
+ }
+ return m.service
+}
+
+func (m *Module) RegisterRoutes(api huma.API) {
+ if m == nil {
+ RegisterS3Destinations(api, nil, nil)
+ return
+ }
+ RegisterS3Destinations(api, m.service, m.deps.SyncRemoteDestinations)
+}
diff --git a/backend/internal/s3/service.go b/backend/internal/s3/service.go
new file mode 100644
index 000000000..6cbf7ae11
--- /dev/null
+++ b/backend/internal/s3/service.go
@@ -0,0 +1,331 @@
+package s3
+
+import (
+ "context"
+ "fmt"
+ "strings"
+
+ "emperror.dev/errors"
+
+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
+ s3config "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/s3"
+ backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"
+ "go.getarcane.app/sys/crypto"
+ "gorm.io/gorm"
+)
+
+var (
+ ErrS3DestinationNotFound = errors.New("S3 destination not found")
+ ErrS3DestinationInUse = errors.New("S3 destination is in use")
+ // errS3SecretRequiredInternal enforces the fresh-secret rule: a stored
+ // secret is never combined with changed connection or trust settings.
+ errS3SecretRequiredInternal = errors.New("re-enter the secret access key when changing the endpoint, bucket, region, access key ID, SSL mode, or path style")
+)
+
+type S3DestinationService struct {
+ db *database.DB
+ // checkRemoteReferences rejects a deletion while any managed environment
+ // still references the destination (or cannot be checked conclusively).
+ // Nil outside the manager's module wiring.
+ checkRemoteReferences func(ctx context.Context, destinationID string) error
+}
+
+func NewS3DestinationService(db *database.DB) *S3DestinationService {
+ return &S3DestinationService{db: db}
+}
+
+func destinationConfigurationInternal(id string, input backuptypes.CreateS3Destination) s3config.Configuration {
+ return s3config.Configuration{
+ ID: id,
+ Name: input.Name,
+ Endpoint: input.Endpoint,
+ Bucket: input.Bucket,
+ Region: input.Region,
+ AccessKeyID: input.AccessKeyID,
+ SecretAccessKey: input.SecretAccessKey,
+ Prefix: input.Prefix,
+ UseSSL: input.UseSSL,
+ ForcePathStyle: input.ForcePathStyle,
+ }.Normalized()
+}
+
+func s3DestinationsToDTOsInternal(destinations []S3Destination) []backuptypes.S3Destination {
+ result := make([]backuptypes.S3Destination, len(destinations))
+ for i := range destinations {
+ result[i] = destinations[i].ToDTO()
+ }
+ return result
+}
+
+func (s *S3DestinationService) ListS3Destinations(ctx context.Context, params pagination.QueryParams) ([]backuptypes.S3Destination, pagination.Response, error) {
+ var destinations []S3Destination
+ query := s.db.WithContext(ctx).Model(&S3Destination{})
+ if term := strings.TrimSpace(params.Search); term != "" {
+ pattern := "%" + term + "%"
+ query = query.Where("name LIKE ? OR endpoint LIKE ? OR bucket LIKE ? OR region LIKE ? OR prefix LIKE ?", pattern, pattern, pattern, pattern, pattern)
+ }
+ response, err := pagination.PaginateAndSortDB(params, query, &destinations)
+ if err != nil {
+ return nil, pagination.Response{}, fmt.Errorf("failed to list S3 destinations: %w", err)
+ }
+ return s3DestinationsToDTOsInternal(destinations), response, nil
+}
+
+func (s *S3DestinationService) ListAllS3Destinations(ctx context.Context) ([]backuptypes.S3Destination, error) {
+ var destinations []S3Destination
+ if err := s.db.WithContext(ctx).Order("name ASC").Find(&destinations).Error; err != nil {
+ return nil, fmt.Errorf("failed to list S3 destinations: %w", err)
+ }
+ return s3DestinationsToDTOsInternal(destinations), nil
+}
+
+func (s *S3DestinationService) getS3DestinationModelInternal(ctx context.Context, id string) (*S3Destination, error) {
+ var destination S3Destination
+ if err := s.db.WithContext(ctx).Where("id = ?", strings.TrimSpace(id)).First(&destination).Error; err != nil {
+ if errors.Is(err, gorm.ErrRecordNotFound) {
+ return nil, ErrS3DestinationNotFound
+ }
+ return nil, fmt.Errorf("failed to load S3 destination: %w", err)
+ }
+ return &destination, nil
+}
+
+func (s *S3DestinationService) GetS3Destination(ctx context.Context, id string) (*backuptypes.S3Destination, error) {
+ destination, err := s.getS3DestinationModelInternal(ctx, id)
+ if err != nil {
+ return nil, err
+ }
+ dto := destination.ToDTO()
+ return &dto, nil
+}
+
+func applyS3ConfigurationInternal(destination *S3Destination, configuration s3config.Configuration, encryptedSecret string) {
+ destination.Name = configuration.Name
+ destination.Endpoint = configuration.Endpoint
+ destination.Bucket = configuration.Bucket
+ destination.Region = configuration.Region
+ destination.AccessKeyID = configuration.AccessKeyID
+ destination.SecretAccessKey = encryptedSecret
+ destination.Prefix = configuration.Prefix
+ destination.UseSSL = configuration.UseSSL
+ destination.ForcePathStyle = configuration.ForcePathStyle
+}
+
+func (s *S3DestinationService) CreateS3Destination(ctx context.Context, input backuptypes.CreateS3Destination) (*backuptypes.S3Destination, error) {
+ configuration := destinationConfigurationInternal("", input)
+ if err := configuration.Validate(true); err != nil {
+ return nil, err
+ }
+ encryptedSecret, err := crypto.Encrypt(configuration.SecretAccessKey)
+ if err != nil {
+ return nil, fmt.Errorf("failed to encrypt S3 secret access key: %w", err)
+ }
+ destination := &S3Destination{}
+ applyS3ConfigurationInternal(destination, configuration, encryptedSecret)
+ if err := s.db.WithContext(ctx).Create(destination).Error; err != nil {
+ return nil, fmt.Errorf("failed to create S3 destination: %w", err)
+ }
+ dto := destination.ToDTO()
+ return &dto, nil
+}
+
+// storedConfigurationInternal returns the persisted configuration without the
+// decrypted secret, for comparing connection fields against an update.
+func storedConfigurationInternal(destination *S3Destination) s3config.Configuration {
+ return s3config.Configuration{
+ ID: destination.ID,
+ Name: destination.Name,
+ Endpoint: destination.Endpoint,
+ Bucket: destination.Bucket,
+ Region: destination.Region,
+ AccessKeyID: destination.AccessKeyID,
+ Prefix: destination.Prefix,
+ UseSSL: destination.UseSSL,
+ ForcePathStyle: destination.ForcePathStyle,
+ }.Normalized()
+}
+
+func (s *S3DestinationService) UpdateS3Destination(ctx context.Context, id string, input backuptypes.UpdateS3Destination) (*backuptypes.S3Destination, error) {
+ configuration := destinationConfigurationInternal("", input)
+ if err := configuration.Validate(false); err != nil {
+ return nil, err
+ }
+ destination, err := s.getS3DestinationModelInternal(ctx, id)
+ if err != nil {
+ return nil, err
+ }
+ if configuration.SecretAccessKey == "" && !s3config.ConnectionFieldsEqual(storedConfigurationInternal(destination), configuration) {
+ return nil, errS3SecretRequiredInternal
+ }
+ encryptedSecret := destination.SecretAccessKey
+ if configuration.SecretAccessKey != "" {
+ encryptedSecret, err = crypto.Encrypt(configuration.SecretAccessKey)
+ if err != nil {
+ return nil, fmt.Errorf("failed to encrypt S3 secret access key: %w", err)
+ }
+ }
+ applyS3ConfigurationInternal(destination, configuration, encryptedSecret)
+ if err := s.db.WithContext(ctx).Save(destination).Error; err != nil {
+ return nil, fmt.Errorf("failed to update S3 destination: %w", err)
+ }
+ dto := destination.ToDTO()
+ return &dto, nil
+}
+
+func (s *S3DestinationService) DeleteS3Destination(ctx context.Context, id string) error {
+ destination, err := s.getS3DestinationModelInternal(ctx, id)
+ if err != nil {
+ return err
+ }
+ inUse, err := s.DestinationInUse(ctx, id)
+ if err != nil {
+ return err
+ }
+ if inUse {
+ return fmt.Errorf("%w: remove it from backup configurations and retained backups first", ErrS3DestinationInUse)
+ }
+ if s.checkRemoteReferences != nil {
+ if err := s.checkRemoteReferences(ctx, id); err != nil {
+ return fmt.Errorf("%w: %s", ErrS3DestinationInUse, err.Error())
+ }
+ }
+ if err := s.db.WithContext(ctx).Delete(destination).Error; err != nil {
+ return fmt.Errorf("failed to delete S3 destination: %w", err)
+ }
+ return nil
+}
+
+// DestinationInUse reports whether any local backup record or policy
+// references the destination.
+func (s *S3DestinationService) DestinationInUse(ctx context.Context, id string) (bool, error) {
+ db := s.db.WithContext(ctx)
+ counts := []*gorm.DB{
+ db.Table("system_backup_runs").Where("s3_destination_id = ? AND remote_snapshot_id <> ''", id),
+ db.Table("system_backup_policies").Where("s3_destination_id = ? AND s3_enabled = ?", id, true),
+ db.Table("volume_backups").Where("s3_destination_id = ? AND remote_snapshot_id <> ''", id),
+ db.Table("volume_backup_policies").Where("s3_destination_id = ? AND s3_enabled = ?", id, true),
+ }
+ for _, query := range counts {
+ var count int64
+ if err := query.Count(&count).Error; err != nil {
+ return false, fmt.Errorf("failed to check S3 destination references: %w", err)
+ }
+ if count > 0 {
+ return true, nil
+ }
+ }
+ return false, nil
+}
+
+func (s *S3DestinationService) S3DestinationExists(ctx context.Context, id string) bool {
+ if strings.TrimSpace(id) == "" {
+ return false
+ }
+ var count int64
+ return s.db.WithContext(ctx).Model(&S3Destination{}).Where("id = ?", id).Count(&count).Error == nil && count == 1
+}
+
+// SyncS3Destinations replaces an agent's destination cache with the manager-owned destinations.
+func (s *S3DestinationService) SyncS3Destinations(ctx context.Context, destinations []backuptypes.S3DestinationSync) error {
+ var existing []S3Destination
+ if err := s.db.WithContext(ctx).Find(&existing).Error; err != nil {
+ return fmt.Errorf("failed to load existing S3 destinations: %w", err)
+ }
+ existingByID := make(map[string]*S3Destination, len(existing))
+ for i := range existing {
+ existingByID[existing[i].ID] = &existing[i]
+ }
+
+ syncedIDs := make(map[string]struct{}, len(destinations))
+ for _, item := range destinations {
+ configuration := destinationConfigurationInternal(item.ID, item.CreateS3Destination)
+ if configuration.ID == "" {
+ return errors.New("S3 destination ID is required")
+ }
+ if err := configuration.Validate(true); err != nil {
+ return fmt.Errorf("invalid S3 destination %s: %w", configuration.ID, err)
+ }
+ encryptedSecret, err := crypto.Encrypt(configuration.SecretAccessKey)
+ if err != nil {
+ return fmt.Errorf("failed to encrypt S3 secret access key for %s: %w", configuration.ID, err)
+ }
+ destination, exists := existingByID[configuration.ID]
+ if !exists {
+ destination = &S3Destination{ID: configuration.ID}
+ }
+ applyS3ConfigurationInternal(destination, configuration, encryptedSecret)
+ if !item.CreatedAt.IsZero() {
+ destination.CreatedAt = item.CreatedAt
+ }
+ if item.UpdatedAt != nil {
+ destination.UpdatedAt = item.UpdatedAt
+ }
+ if err := s.db.WithContext(ctx).Save(destination).Error; err != nil {
+ return fmt.Errorf("failed to sync S3 destination %s: %w", configuration.ID, err)
+ }
+ syncedIDs[configuration.ID] = struct{}{}
+ }
+
+ // One undeletable destination (still referenced locally) must not abort
+ // the rest of the reconciliation; report every failure after the sweep.
+ var removeErr error
+ for i := range existing {
+ if _, ok := syncedIDs[existing[i].ID]; ok {
+ continue
+ }
+ if err := s.DeleteS3Destination(ctx, existing[i].ID); err != nil {
+ removeErr = errors.Combine(removeErr, fmt.Errorf("failed to remove unsynced S3 destination %s: %w", existing[i].ID, err))
+ }
+ }
+ return removeErr
+}
+
+// Configuration returns a destination's decrypted runtime configuration for backup domains.
+func (s *S3DestinationService) Configuration(ctx context.Context, id string) (s3config.Configuration, error) {
+ destination, err := s.getS3DestinationModelInternal(ctx, id)
+ if err != nil {
+ return s3config.Configuration{}, err
+ }
+ secret, err := crypto.Decrypt(destination.SecretAccessKey)
+ if err != nil {
+ return s3config.Configuration{}, fmt.Errorf("failed to decrypt S3 secret access key: %w", err)
+ }
+ return s3config.Configuration{
+ ID: destination.ID,
+ Name: destination.Name,
+ Endpoint: destination.Endpoint,
+ Bucket: destination.Bucket,
+ Region: destination.Region,
+ AccessKeyID: destination.AccessKeyID,
+ SecretAccessKey: secret,
+ Prefix: destination.Prefix,
+ UseSSL: destination.UseSSL,
+ ForcePathStyle: destination.ForcePathStyle,
+ }.Normalized(), nil
+}
+
+func (s *S3DestinationService) TestS3Destination(ctx context.Context, id string, input *backuptypes.UpdateS3Destination) error {
+ configuration, err := s.Configuration(ctx, id)
+ if err != nil {
+ return err
+ }
+ if input != nil {
+ updated := destinationConfigurationInternal("", *input)
+ updated.ID = configuration.ID
+ if updated.SecretAccessKey == "" {
+ // The stored secret may only sign requests against the stored
+ // connection settings; any change requires a fresh secret.
+ if !s3config.ConnectionFieldsEqual(configuration, updated) {
+ return errS3SecretRequiredInternal
+ }
+ updated.SecretAccessKey = configuration.SecretAccessKey
+ }
+ configuration = updated
+ }
+ return s3config.TestConnection(ctx, configuration)
+}
+
+func (s *S3DestinationService) TestS3DestinationConfiguration(ctx context.Context, input backuptypes.CreateS3Destination) error {
+ return s3config.TestConnection(ctx, destinationConfigurationInternal("", input))
+}
diff --git a/backend/internal/s3/service_test.go b/backend/internal/s3/service_test.go
new file mode 100644
index 000000000..727940f75
--- /dev/null
+++ b/backend/internal/s3/service_test.go
@@ -0,0 +1,274 @@
+package s3
+
+import (
+ "context"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strconv"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/settings"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
+ backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"
+ sqlite "github.com/libtnb/sqlite"
+ "github.com/stretchr/testify/require"
+ "go.getarcane.app/sys/crypto"
+ "gorm.io/gorm"
+)
+
+func setupS3DestinationServiceTestInternal(t *testing.T) (*S3DestinationService, *gorm.DB) {
+ t.Helper()
+
+ gormDB, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{})
+ require.NoError(t, err)
+ require.NoError(t, gormDB.AutoMigrate(
+ &S3Destination{},
+ &settings.SettingVariable{},
+ ))
+ require.NoError(t, gormDB.Exec("CREATE TABLE system_backup_runs (s3_destination_id text, remote_snapshot_id text)").Error)
+ require.NoError(t, gormDB.Exec("CREATE TABLE system_backup_policies (s3_destination_id text, s3_enabled numeric)").Error)
+ require.NoError(t, gormDB.Exec("CREATE TABLE volume_backups (s3_destination_id text, remote_snapshot_id text)").Error)
+ require.NoError(t, gormDB.Exec("CREATE TABLE volume_backup_policies (s3_destination_id text, s3_enabled numeric)").Error)
+ crypto.InitEncryption(&crypto.Config{
+ EncryptionKey: "test-encryption-key-for-s3-destination-32bytes",
+ Environment: "test",
+ })
+ return NewS3DestinationService(&database.DB{DB: gormDB}), gormDB
+}
+
+func TestS3DestinationService_CRUD(t *testing.T) {
+ service, gormDB := setupS3DestinationServiceTestInternal(t)
+ ctx := context.Background()
+
+ created, err := service.CreateS3Destination(ctx, backuptypes.CreateS3Destination{
+ Name: "Offsite",
+ Endpoint: "https://s3.example.com",
+ Bucket: "arcane-backups",
+ Region: "eu-central-1",
+ AccessKeyID: "access-key",
+ SecretAccessKey: "secret-key",
+ Prefix: "/production/",
+ UseSSL: true,
+ ForcePathStyle: true,
+ })
+ require.NoError(t, err)
+ require.NotEmpty(t, created.ID)
+ require.True(t, created.SecretConfigured)
+ require.Equal(t, "production", created.Prefix)
+
+ var stored S3Destination
+ require.NoError(t, gormDB.First(&stored, "id = ?", created.ID).Error)
+ require.NotEqual(t, "secret-key", stored.SecretAccessKey)
+
+ listed, page, err := service.ListS3Destinations(ctx, pagination.QueryParams{
+ SearchQuery: pagination.SearchQuery{Search: "arcane-backups"},
+ Params: pagination.Params{Limit: 20},
+ })
+ require.NoError(t, err)
+ require.EqualValues(t, 1, page.TotalItems)
+ require.Len(t, listed, 1)
+
+ updated, err := service.UpdateS3Destination(ctx, created.ID, backuptypes.UpdateS3Destination{
+ Name: "Primary offsite",
+ Endpoint: created.Endpoint,
+ Bucket: created.Bucket,
+ Region: created.Region,
+ AccessKeyID: created.AccessKeyID,
+ Prefix: created.Prefix,
+ UseSSL: created.UseSSL,
+ ForcePathStyle: created.ForcePathStyle,
+ })
+ require.NoError(t, err)
+ require.Equal(t, "Primary offsite", updated.Name)
+
+ var updatedStored S3Destination
+ require.NoError(t, gormDB.First(&updatedStored, "id = ?", created.ID).Error)
+ require.Equal(t, stored.SecretAccessKey, updatedStored.SecretAccessKey)
+
+ require.NoError(t, service.DeleteS3Destination(ctx, created.ID))
+ _, err = service.GetS3Destination(ctx, created.ID)
+ require.ErrorIs(t, err, ErrS3DestinationNotFound)
+}
+
+func TestS3DestinationService_DeleteRejectsConfiguredDestination(t *testing.T) {
+ service, gormDB := setupS3DestinationServiceTestInternal(t)
+ ctx := context.Background()
+ destination, err := service.CreateS3Destination(ctx, backuptypes.CreateS3Destination{
+ Name: "In use",
+ Bucket: "arcane-backups",
+ Region: "us-east-1",
+ AccessKeyID: "access-key",
+ SecretAccessKey: "secret-key",
+ UseSSL: true,
+ })
+ require.NoError(t, err)
+ require.NoError(t, gormDB.Exec("INSERT INTO volume_backup_policies (s3_destination_id, s3_enabled) VALUES (?, 1)", destination.ID).Error)
+
+ err = service.DeleteS3Destination(ctx, destination.ID)
+ require.ErrorIs(t, err, ErrS3DestinationInUse)
+}
+
+func TestS3DestinationService_SyncS3Destinations(t *testing.T) {
+ service, gormDB := setupS3DestinationServiceTestInternal(t)
+ ctx := context.Background()
+
+ legacy, err := service.CreateS3Destination(ctx, backuptypes.CreateS3Destination{
+ Name: "Legacy",
+ Bucket: "legacy-bucket",
+ Region: "us-east-1",
+ AccessKeyID: "legacy-access-key",
+ SecretAccessKey: "legacy-secret",
+ UseSSL: true,
+ })
+ require.NoError(t, err)
+
+ createdAt := time.Now().Add(-time.Hour).UTC().Truncate(time.Second)
+ require.NoError(t, service.SyncS3Destinations(ctx, []backuptypes.S3DestinationSync{
+ {
+ ID: "destination-1",
+ Name: "Remote primary",
+ Endpoint: "https://s3.example.com",
+ Bucket: "volume-backups",
+ Region: "eu-central-1",
+ AccessKeyID: "remote-access-key",
+ SecretAccessKey: "remote-secret",
+ Prefix: "/agents/",
+ UseSSL: true,
+ ForcePathStyle: true,
+ CreatedAt: createdAt,
+ },
+ }))
+
+ var synced S3Destination
+ require.NoError(t, gormDB.First(&synced, "id = ?", "destination-1").Error)
+ require.Equal(t, "Remote primary", synced.Name)
+ require.Equal(t, "agents", synced.Prefix)
+ require.NotEqual(t, "remote-secret", synced.SecretAccessKey)
+ secret, err := crypto.Decrypt(synced.SecretAccessKey)
+ require.NoError(t, err)
+ require.Equal(t, "remote-secret", secret)
+
+ _, err = service.GetS3Destination(ctx, legacy.ID)
+ require.ErrorIs(t, err, ErrS3DestinationNotFound)
+
+ require.NoError(t, service.SyncS3Destinations(ctx, []backuptypes.S3DestinationSync{
+ {
+ ID: "destination-1",
+ Name: "Remote primary updated",
+ Bucket: "updated-volume-backups",
+ Region: "eu-west-1",
+ AccessKeyID: "updated-access-key",
+ SecretAccessKey: "updated-secret",
+ UseSSL: true,
+ },
+ }))
+
+ require.NoError(t, gormDB.First(&synced, "id = ?", "destination-1").Error)
+ require.Equal(t, "Remote primary updated", synced.Name)
+ require.Equal(t, "updated-volume-backups", synced.Bucket)
+ secret, err = crypto.Decrypt(synced.SecretAccessKey)
+ require.NoError(t, err)
+ require.Equal(t, "updated-secret", secret)
+}
+
+func TestS3DestinationService_TestS3DestinationRoundTrip(t *testing.T) {
+ var object []byte
+ var requestMethods []string
+ expectedPathPrefix := "/arcane-backups/production/.arcane-connection-test-"
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ require.NotEmpty(t, r.Header.Get("Authorization"))
+ require.True(t, strings.HasPrefix(r.URL.Path, expectedPathPrefix))
+ requestMethods = append(requestMethods, r.Method)
+ switch r.Method {
+ case http.MethodPut:
+ var err error
+ object, err = io.ReadAll(r.Body)
+ require.NoError(t, err)
+ w.Header().Set("ETag", `"test-etag"`)
+ w.WriteHeader(http.StatusOK)
+ case http.MethodGet:
+ w.Header().Set("Content-Length", strconv.Itoa(len(object)))
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write(object)
+ case http.MethodDelete:
+ object = nil
+ w.WriteHeader(http.StatusNoContent)
+ default:
+ http.Error(w, "unsupported method", http.StatusMethodNotAllowed)
+ }
+ }))
+ defer server.Close()
+
+ service, _ := setupS3DestinationServiceTestInternal(t)
+ require.NoError(t, service.TestS3DestinationConfiguration(context.Background(), backuptypes.CreateS3Destination{
+ Name: "Unsaved destination",
+ Endpoint: server.URL,
+ Bucket: "arcane-backups",
+ Region: "",
+ AccessKeyID: "access-key",
+ SecretAccessKey: "secret-key",
+ Prefix: "production",
+ UseSSL: false,
+ ForcePathStyle: true,
+ }))
+ destination, err := service.CreateS3Destination(context.Background(), backuptypes.CreateS3Destination{
+ Name: "Test destination",
+ Endpoint: server.URL,
+ Bucket: "arcane-backups",
+ Region: "",
+ AccessKeyID: "access-key",
+ SecretAccessKey: "secret-key",
+ Prefix: "production",
+ UseSSL: false,
+ ForcePathStyle: true,
+ })
+ require.NoError(t, err)
+
+ require.NoError(t, service.TestS3Destination(context.Background(), destination.ID, nil))
+
+ // Changing a connection field without re-supplying the secret is rejected
+ // before any outbound request: the stored secret must never sign requests
+ // against caller-modified connection settings.
+ requestsBeforeRejection := len(requestMethods)
+ err = service.TestS3Destination(context.Background(), destination.ID, &backuptypes.UpdateS3Destination{
+ Name: destination.Name,
+ Endpoint: server.URL,
+ Bucket: "edited-bucket",
+ Region: "",
+ AccessKeyID: destination.AccessKeyID,
+ Prefix: "edited",
+ UseSSL: false,
+ ForcePathStyle: true,
+ })
+ require.ErrorContains(t, err, "re-enter the secret access key")
+ require.Len(t, requestMethods, requestsBeforeRejection)
+
+ expectedPathPrefix = "/edited-bucket/edited/.arcane-connection-test-"
+ require.NoError(t, service.TestS3Destination(context.Background(), destination.ID, &backuptypes.UpdateS3Destination{
+ Name: destination.Name,
+ Endpoint: server.URL,
+ Bucket: "edited-bucket",
+ Region: "",
+ AccessKeyID: destination.AccessKeyID,
+ SecretAccessKey: "secret-key",
+ Prefix: "edited",
+ UseSSL: false,
+ ForcePathStyle: true,
+ }))
+
+ require.Equal(t, []string{
+ http.MethodPut, http.MethodGet, http.MethodDelete,
+ http.MethodPut, http.MethodGet, http.MethodDelete,
+ http.MethodPut, http.MethodGet, http.MethodDelete,
+ }, requestMethods)
+ require.Empty(t, object)
+ persisted, err := service.GetS3Destination(context.Background(), destination.ID)
+ require.NoError(t, err)
+ require.Equal(t, "arcane-backups", persisted.Bucket)
+ require.Equal(t, "production", persisted.Prefix)
+ require.Empty(t, persisted.Region)
+}
diff --git a/backend/internal/settings/model.go b/backend/internal/settings/model.go
index ce107a503..12767d225 100644
--- a/backend/internal/settings/model.go
+++ b/backend/internal/settings/model.go
@@ -174,6 +174,10 @@ type Settings struct {
// API Keys category (admin management page - no actual settings)
ApiKeysCategoryPlaceholder SettingVariable `key:"apiKeysCategory,internal" meta:"label=API Keys;type=internal;keywords=api,keys,tokens,authentication,access,programmatic,integration;category=apikeys;description=Manage API keys for programmatic access" catmeta:"id=apikeys;title=API Keys;icon=apikey;url=/settings/api-keys;description=Create and manage API keys for programmatic access to Arcane"`
+ S3DestinationsCategoryPlaceholder SettingVariable `key:"s3DestinationsCategory,internal" meta:"label=S3 Destinations;type=internal;keywords=s3,storage,backup,aws,minio,backblaze,hetzner;category=s3destinations;description=Manage reusable S3-compatible storage destinations" catmeta:"id=s3destinations;title=S3 Destinations;icon=remoteenvironment;url=/settings/backups/s3;description=Create and manage reusable S3-compatible backup destinations"`
+
+ SystemBackupsCategoryPlaceholder SettingVariable `key:"systemBackupsCategory,internal" meta:"label=Backups;type=internal;keywords=backup,recovery,restore,configuration,secrets;category=systembackups;description=Back up and restore this Arcane instance" catmeta:"id=systembackups;title=Backups;icon=backup;url=/settings/backups;description=Create encrypted recovery backups of Arcane data and configuration"`
+
FederatedCredentialsCategoryPlaceholder SettingVariable `key:"federatedCredentialsCategory,internal" meta:"label=Federated Credentials;type=internal;keywords=federated,credentials,workload,identity,oidc,token exchange,ci,github,gitlab;category=authentication;description=Manage workload identity federation credentials"`
// Webhooks category (management page - no actual settings)
diff --git a/backend/internal/system/upgrade_service.go b/backend/internal/system/upgrade_service.go
index 2faabd008..f30aa32ed 100644
--- a/backend/internal/system/upgrade_service.go
+++ b/backend/internal/system/upgrade_service.go
@@ -50,12 +50,6 @@ type SystemUpgradeService struct {
settingsService *settings.SettingsService
}
-type upgraderRuntimeOptionsInternal struct {
- ContainerEnv []string
- Mounts []mount.Mount
- NetworkMode container.NetworkMode
-}
-
func NewSystemUpgradeService(
db *database.DB,
dockerService *docker.DockerClientService,
@@ -211,7 +205,7 @@ func (s *SystemUpgradeService) TriggerUpgradeViaCLI(ctx context.Context, user co
}
// Create the upgrader container config
- runtimeOptions, err := resolveSystemUpgraderRuntimeOptionsInternal(
+ containerEnv, runtimeMounts, networkMode, err := ResolveUpgraderRuntimeOptions(
ctx,
s.dockerService.DockerHost(),
&currentContainer,
@@ -242,14 +236,14 @@ func (s *SystemUpgradeService) TriggerUpgradeViaCLI(ctx context.Context, user co
// is short-lived and never goes through the runtime-identity drop, so
// don't rely on the image's default user.
User: "0:0",
- Env: runtimeOptions.ContainerEnv,
+ Env: containerEnv,
Labels: map[string]string{
"com.getarcaneapp.arcane.upgrader": "true",
"com.getarcaneapp.arcane": "true",
},
}
- mounts := append([]mount.Mount{}, runtimeOptions.Mounts...)
+ mounts := append([]mount.Mount{}, runtimeMounts...)
if appDataMount != nil {
mounts = append(mounts, *appDataMount)
}
@@ -262,7 +256,7 @@ func (s *SystemUpgradeService) TriggerUpgradeViaCLI(ctx context.Context, user co
hostConfig := &container.HostConfig{
AutoRemove: !keepUpgraderContainer, // default: clean up after completion
Mounts: mounts,
- NetworkMode: runtimeOptions.NetworkMode,
+ NetworkMode: networkMode,
}
// Inherit the security context that lets the running Arcane container reach
// the Docker socket (e.g. SELinux label=disable, privileged); the upgrader
@@ -328,32 +322,30 @@ func determineUpgradeBinaryPathInternal(containerLabels map[string]string) strin
return "/app/arcane"
}
-func resolveSystemUpgraderRuntimeOptionsInternal(
+// ResolveUpgraderRuntimeOptions determines how a helper container reaches the Docker daemon.
+func ResolveUpgraderRuntimeOptions(
ctx context.Context,
dockerHost string,
currentContainer *container.InspectResponse,
discoverHostPath func(context.Context, string) (string, error),
isRunningInDocker func() bool,
selectReachableNetwork func(context.Context, *container.InspectResponse, string) string,
-) (upgraderRuntimeOptionsInternal, error) {
- options := upgraderRuntimeOptionsInternal{
- ContainerEnv: vuln.BuildDockerHostEnv(dockerHost),
- }
+) ([]string, []mount.Mount, container.NetworkMode, error) {
+ containerEnv := vuln.BuildDockerHostEnv(dockerHost)
scheme, socketPath, err := vuln.ParseDockerHost(dockerHost)
if err != nil {
- return upgraderRuntimeOptionsInternal{}, errors.WrapIff(err, "resolve docker host %q", dockerHost)
+ return nil, nil, "", errors.WrapIff(err, "resolve docker host %q", dockerHost)
}
if scheme != "unix" {
// The upgrader must reach the tcp DOCKER_HOST, so prefer a network the
// daemon proxy is actually on over the plain auto heuristic (#3533).
+ networkMode := dockerutils.SelectAutoNetworkMode(currentContainer)
if selectReachableNetwork != nil {
- options.NetworkMode = container.NetworkMode(selectReachableNetwork(ctx, currentContainer, dockerHost))
- } else {
- options.NetworkMode = container.NetworkMode(dockerutils.SelectAutoNetworkMode(currentContainer))
+ networkMode = selectReachableNetwork(ctx, currentContainer, dockerHost)
}
- return options, nil
+ return containerEnv, nil, container.NetworkMode(networkMode), nil
}
socketSource, err := vuln.ResolveUnixSocketSource(
@@ -363,16 +355,16 @@ func resolveSystemUpgraderRuntimeOptionsInternal(
isRunningInDocker,
)
if err != nil {
- return upgraderRuntimeOptionsInternal{}, errors.WrapIf(err, "resolve unix socket source")
+ return nil, nil, "", errors.WrapIf(err, "resolve unix socket source")
}
- options.Mounts = append(options.Mounts, mount.Mount{
+ mounts := []mount.Mount{{
Type: mount.TypeBind,
Source: socketSource,
Target: socketPath,
- })
+ }}
- return options, nil
+ return containerEnv, mounts, "", nil
}
// getCurrentContainerID detects if we're running in Docker and returns container ID
diff --git a/backend/internal/system/upgrade_service_test.go b/backend/internal/system/upgrade_service_test.go
index cd3bd81b4..494a59f65 100644
--- a/backend/internal/system/upgrade_service_test.go
+++ b/backend/internal/system/upgrade_service_test.go
@@ -205,7 +205,7 @@ func TestResolveSystemUpgraderRuntimeOptionsInternal_TCPDockerHost(t *testing.T)
},
}
- options, err := resolveSystemUpgraderRuntimeOptionsInternal(
+ containerEnv, mounts, networkMode, err := ResolveUpgraderRuntimeOptions(
context.Background(),
"tcp://docker-socket-proxy:2375",
currentContainer,
@@ -214,13 +214,13 @@ func TestResolveSystemUpgraderRuntimeOptionsInternal_TCPDockerHost(t *testing.T)
nil,
)
require.NoError(t, err)
- require.Equal(t, []string{"DOCKER_HOST=tcp://docker-socket-proxy:2375"}, options.ContainerEnv)
- require.Empty(t, options.Mounts)
- require.Equal(t, containertypes.NetworkMode("arcane-test"), options.NetworkMode)
+ require.Equal(t, []string{"DOCKER_HOST=tcp://docker-socket-proxy:2375"}, containerEnv)
+ require.Empty(t, mounts)
+ require.Equal(t, containertypes.NetworkMode("arcane-test"), networkMode)
}
func TestResolveSystemUpgraderRuntimeOptionsInternal_UnixDockerHost(t *testing.T) {
- options, err := resolveSystemUpgraderRuntimeOptionsInternal(
+ containerEnv, mounts, networkMode, err := ResolveUpgraderRuntimeOptions(
context.Background(),
"unix:///var/run/docker.sock",
nil,
@@ -231,19 +231,19 @@ func TestResolveSystemUpgraderRuntimeOptionsInternal_UnixDockerHost(t *testing.T
nil,
)
require.NoError(t, err)
- require.Equal(t, []string{"DOCKER_HOST=unix:///var/run/docker.sock"}, options.ContainerEnv)
- require.Equal(t, containertypes.NetworkMode(""), options.NetworkMode)
+ require.Equal(t, []string{"DOCKER_HOST=unix:///var/run/docker.sock"}, containerEnv)
+ require.Equal(t, containertypes.NetworkMode(""), networkMode)
require.Equal(t, []mounttypes.Mount{
{
Type: mounttypes.TypeBind,
Source: "/host/run/docker.sock",
Target: "/var/run/docker.sock",
},
- }, options.Mounts)
+ }, mounts)
}
func TestResolveSystemUpgraderRuntimeOptionsInternal_DefaultDockerHost(t *testing.T) {
- options, err := resolveSystemUpgraderRuntimeOptionsInternal(
+ containerEnv, mounts, _, err := ResolveUpgraderRuntimeOptions(
context.Background(),
"",
nil,
@@ -254,18 +254,18 @@ func TestResolveSystemUpgraderRuntimeOptionsInternal_DefaultDockerHost(t *testin
nil,
)
require.NoError(t, err)
- require.Nil(t, options.ContainerEnv)
+ require.Nil(t, containerEnv)
require.Equal(t, []mounttypes.Mount{
{
Type: mounttypes.TypeBind,
Source: "/var/run/docker.sock",
Target: "/var/run/docker.sock",
},
- }, options.Mounts)
+ }, mounts)
}
func TestResolveSystemUpgraderRuntimeOptionsInternal_UnixDockerHostResolutionError(t *testing.T) {
- _, err := resolveSystemUpgraderRuntimeOptionsInternal(
+ _, _, _, err := ResolveUpgraderRuntimeOptions(
context.Background(),
"unix:///var/run/docker.sock",
nil,
diff --git a/backend/internal/systembackup/handler.go b/backend/internal/systembackup/handler.go
new file mode 100644
index 000000000..7e009e431
--- /dev/null
+++ b/backend/internal/systembackup/handler.go
@@ -0,0 +1,274 @@
+package systembackup
+
+import (
+ "context"
+ "net/http"
+ "strings"
+
+ "emperror.dev/errors"
+
+ "github.com/danielgtaylor/huma/v2"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/activity"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/authz"
+ activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"
+ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
+ backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"
+ "github.com/getarcaneapp/arcane/types/v2/base"
+ "github.com/samber/mo"
+)
+
+type SystemBackupHandler struct {
+ service *SystemBackupService
+ activity *activity.ActivityService
+ appCtx context.Context
+}
+
+type ListSystemBackupsInput struct {
+ Search string `query:"search"`
+ Sort string `query:"sort" default:"createdAt"`
+ Order string `query:"order" default:"desc"`
+ Start int `query:"start" default:"0"`
+ Limit int `query:"limit" default:"20"`
+}
+
+type ListSystemBackupsOutput struct {
+ Body struct {
+ Data []backuptypes.SystemBackupRun `json:"data"`
+ Pagination base.PaginationResponse `json:"pagination"`
+ }
+}
+
+type SystemBackupPoliciesOutput struct {
+ Body backuptypes.SystemBackupPolicyCollection
+}
+type UpdateSystemBackupPoliciesInput struct {
+ Body backuptypes.UpdateSystemBackupPolicies
+}
+type SetSystemBackupRecoveryKeyInput struct {
+ Body backuptypes.SystemBackupRecoveryKey
+}
+type SystemBackupRecoveryKeyOutput struct {
+ Body backuptypes.SystemBackupRecoveryKeyStatus
+}
+type GenerateSystemBackupRecoveryKeyOutput struct {
+ Body backuptypes.SystemBackupRecoveryKey
+}
+type CreateSystemBackupInput struct {
+ Body backuptypes.CreateSystemBackupRequest
+}
+type SystemBackupOutput struct{ Body backuptypes.SystemBackupRun }
+type RestoreSystemBackupInput struct {
+ ID string `path:"id"`
+ Body backuptypes.RestoreSystemBackupRequest
+}
+type UploadSystemBackupInput struct {
+ ID string `path:"id"`
+ Body backuptypes.UploadSystemBackupRequest
+}
+type DeleteSystemBackupInput struct {
+ ID string `path:"id"`
+ Body backuptypes.DeleteSystemBackupRequest
+}
+type DiscoverSystemBackupsInput struct {
+ Body backuptypes.DiscoverSystemBackupsRequest
+}
+type DiscoverSystemBackupsOutput struct{ Body base.ApiResponse[int] }
+type SystemBackupMessageOutput struct {
+ Body base.ApiResponse[base.MessageResponse]
+}
+
+func RegisterSystemBackups(api huma.API, service *SystemBackupService, activityService *activity.ActivityService, appCtx handlerutil.ActivityAppContext) {
+ h := &SystemBackupHandler{service: service, activity: activityService, appCtx: appCtx.Context()}
+ adminOnly := middleware.RequireGlobalAdmin(api)
+ middleware.RegisterWithPermission(api, huma.Operation{OperationID: "list-system-backups", Method: http.MethodGet, Path: "/backups", Summary: "List Arcane system backups", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsRead, h.List)
+ middleware.RegisterWithPermission(api, huma.Operation{OperationID: "get-system-backup-policies", Method: http.MethodGet, Path: "/backups/policies", Summary: "Get Arcane system backup policies", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsRead, h.GetPolicies)
+ middleware.RegisterWithPermission(api, huma.Operation{OperationID: "update-system-backup-policies", Method: http.MethodPut, Path: "/backups/policies", Summary: "Update Arcane system backup policies", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsManage, h.UpdatePolicies)
+ middleware.RegisterWithPermission(api, huma.Operation{OperationID: "generate-system-backup-recovery-key", Method: http.MethodPost, Path: "/backups/recovery-key/generate", Summary: "Generate an Arcane system backup recovery key", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsRecoveryKey, h.GenerateRecoveryKey)
+ middleware.RegisterWithPermission(api, huma.Operation{OperationID: "set-system-backup-recovery-key", Method: http.MethodPut, Path: "/backups/recovery-key", Summary: "Configure Arcane system backup recovery key", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsRecoveryKey, h.SetRecoveryKey)
+ middleware.RegisterWithPermission(api, huma.Operation{OperationID: "create-system-backup", Method: http.MethodPost, Path: "/backups", Summary: "Create Arcane system backup", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsManage, h.Create)
+ middleware.RegisterWithPermission(api, huma.Operation{OperationID: "discover-system-backups", Method: http.MethodPost, Path: "/backups/discover", Summary: "Discover Arcane system backups in S3", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsManage, h.Discover)
+ middleware.RegisterWithPermission(api, huma.Operation{OperationID: "restore-system-backup", Method: http.MethodPost, Path: "/backups/{id}/restore", Summary: "Restore Arcane system backup", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsRestore, h.Restore)
+ middleware.RegisterWithPermission(api, huma.Operation{OperationID: "upload-system-backup", Method: http.MethodPost, Path: "/backups/{id}/upload", Summary: "Upload Arcane system backup", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsManage, h.Upload)
+ middleware.RegisterWithPermission(api, huma.Operation{OperationID: "delete-system-backup", Method: http.MethodDelete, Path: "/backups/{id}", Summary: "Delete Arcane system backup", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsManage, h.Delete)
+}
+
+func (h *SystemBackupHandler) List(ctx context.Context, input *ListSystemBackupsInput) (*ListSystemBackupsOutput, error) {
+ params := handlerutil.PaginationParams(input.Start, input.Limit, input.Sort, input.Order, input.Search)
+ runs, page, err := h.service.ListBackups(ctx, params)
+ if err != nil {
+ return nil, huma.Error500InternalServerError(err.Error())
+ }
+ out := &ListSystemBackupsOutput{}
+ out.Body.Data, out.Body.Pagination = runs, handlerutil.PaginationResponse(page)
+ return out, nil
+}
+
+func (h *SystemBackupHandler) GetPolicies(ctx context.Context, _ *struct{}) (*SystemBackupPoliciesOutput, error) {
+ policies, err := h.service.GetPolicies(ctx)
+ if err != nil {
+ return nil, huma.Error500InternalServerError(err.Error())
+ }
+ return &SystemBackupPoliciesOutput{Body: *policies}, nil
+}
+
+func (h *SystemBackupHandler) UpdatePolicies(ctx context.Context, input *UpdateSystemBackupPoliciesInput) (*SystemBackupPoliciesOutput, error) {
+ user, err := handlerutil.RequireUser(ctx)
+ if err != nil {
+ return nil, err
+ }
+ var policies *backuptypes.SystemBackupPolicyCollection
+ _, err = activitylib.RunHandlerActivity(utils.ActivityRuntimeContext(ctx, h.appCtx), h.activity, activitylib.HandlerOptions{
+ EnvironmentID: "0", Type: activitytypes.TypeResourceAction, ResourceType: "system_backup", ResourceID: "policies", ResourceName: "Arcane", User: user,
+ Step: "Saving system backup schedules", Message: "Saving Arcane system backup schedules", SuccessMessage: "Arcane system backup schedules saved",
+ Metadata: database.JSON{"action": "update_system_backup_policies", "policyCount": len(input.Body.Policies)},
+ }, func(activityCtx context.Context) error {
+ var updateErr error
+ policies, updateErr = h.service.UpdatePolicies(activityCtx, input.Body.Policies)
+ return updateErr
+ })
+ if err != nil {
+ return nil, huma.Error400BadRequest(err.Error())
+ }
+ return &SystemBackupPoliciesOutput{Body: *policies}, nil
+}
+
+func (h *SystemBackupHandler) GenerateRecoveryKey(_ context.Context, _ *struct{}) (*GenerateSystemBackupRecoveryKeyOutput, error) {
+ recoveryKey, err := h.service.GenerateRecoveryKey()
+ if err != nil {
+ return nil, huma.Error500InternalServerError(err.Error())
+ }
+ return &GenerateSystemBackupRecoveryKeyOutput{Body: *recoveryKey}, nil
+}
+
+func (h *SystemBackupHandler) SetRecoveryKey(ctx context.Context, input *SetSystemBackupRecoveryKeyInput) (*SystemBackupRecoveryKeyOutput, error) {
+ user, err := handlerutil.RequireUser(ctx)
+ if err != nil {
+ return nil, err
+ }
+ var status *backuptypes.SystemBackupRecoveryKeyStatus
+ _, err = activitylib.RunHandlerActivity(utils.ActivityRuntimeContext(ctx, h.appCtx), h.activity, activitylib.HandlerOptions{
+ EnvironmentID: "0", Type: activitytypes.TypeResourceAction, ResourceType: "system_backup", ResourceID: "recovery-key", ResourceName: "Arcane", User: user,
+ Step: "Configuring recovery key", Message: "Configuring Arcane system backup recovery key", SuccessMessage: "Arcane system backup recovery key configured",
+ Metadata: database.JSON{"action": "set_system_backup_recovery_key"},
+ }, func(activityCtx context.Context) error {
+ var setErr error
+ status, setErr = h.service.SetRecoveryKey(activityCtx, input.Body.RecoveryKey)
+ return setErr
+ })
+ if err != nil {
+ return nil, huma.Error400BadRequest(err.Error())
+ }
+ return &SystemBackupRecoveryKeyOutput{Body: *status}, nil
+}
+
+func (h *SystemBackupHandler) Discover(ctx context.Context, input *DiscoverSystemBackupsInput) (*DiscoverSystemBackupsOutput, error) {
+ user, err := handlerutil.RequireUser(ctx)
+ if err != nil {
+ return nil, err
+ }
+ count := 0
+ _, err = activitylib.RunHandlerActivity(utils.ActivityRuntimeContext(ctx, h.appCtx), h.activity, activitylib.HandlerOptions{
+ EnvironmentID: "0", Type: activitytypes.TypeResourceAction, ResourceType: "system_backup", ResourceID: "s3", ResourceName: "Arcane", User: user,
+ Step: "Discovering system backups", Message: "Discovering Arcane system backups in S3", SuccessMessage: "Arcane system backup discovery completed",
+ Metadata: database.JSON{"action": "discover_system_backups", "s3DestinationId": input.Body.S3DestinationID},
+ }, func(activityCtx context.Context) error {
+ var discoverErr error
+ count, discoverErr = h.service.DiscoverRemoteBackups(activityCtx, input.Body)
+ return discoverErr
+ })
+ if err != nil {
+ return nil, huma.Error400BadRequest(err.Error())
+ }
+ return &DiscoverSystemBackupsOutput{Body: base.ApiResponse[int]{Success: true, Data: count}}, nil
+}
+
+func (h *SystemBackupHandler) Create(ctx context.Context, input *CreateSystemBackupInput) (*SystemBackupOutput, error) {
+ user, err := handlerutil.RequireUser(ctx)
+ if err != nil {
+ return nil, err
+ }
+ var run *SystemBackupRun
+ _, err = activitylib.RunHandlerActivity(utils.ActivityRuntimeContext(ctx, h.appCtx), h.activity, activitylib.HandlerOptions{
+ EnvironmentID: "0", Type: activitytypes.TypeResourceAction, ResourceType: "system_backup", ResourceID: "arcane", ResourceName: "Arcane", User: user,
+ Step: "Creating system backup", Message: "Creating Arcane system backup", SuccessMessage: "Arcane system backup created successfully",
+ Metadata: database.JSON{"action": "create_system_backup", "destination": input.Body.Destination, "s3DestinationId": input.Body.S3DestinationID, "policyId": input.Body.PolicyID},
+ }, func(activityCtx context.Context) error {
+ var e error
+ run, e = h.service.CreateBackup(activityCtx, *user, SystemBackupTriggerManual, input.Body)
+ return e
+ })
+ if errors.Is(err, ErrSystemBackupAlreadyRunning) {
+ return nil, huma.Error409Conflict(err.Error())
+ }
+ if err != nil {
+ return nil, huma.Error500InternalServerError(err.Error())
+ }
+ dto := run.ToDTO()
+ return &SystemBackupOutput{Body: dto}, nil
+}
+
+func (h *SystemBackupHandler) Restore(ctx context.Context, input *RestoreSystemBackupInput) (*SystemBackupMessageOutput, error) {
+ user, err := handlerutil.RequireUser(ctx)
+ if err != nil {
+ return nil, err
+ }
+ activityID, err := activitylib.RunHandlerActivity(utils.ActivityRuntimeContext(ctx, h.appCtx), h.activity, activitylib.HandlerOptions{
+ EnvironmentID: "0", Type: activitytypes.TypeResourceAction, ResourceType: "system_backup", ResourceID: input.ID, ResourceName: "Arcane", User: user,
+ Step: "Preparing system restore", Message: "Preparing Arcane system restore", SuccessMessage: "Arcane system restore started",
+ Metadata: database.JSON{"action": "restore_system_backup", "backupId": input.ID},
+ }, func(activityCtx context.Context) error {
+ return h.service.RestoreBackup(activityCtx, input.ID, input.Body.RecoveryKey, *user)
+ })
+ if err != nil {
+ return nil, huma.Error500InternalServerError(err.Error())
+ }
+ return messageOutputInternal("Arcane system restore started", activityID), nil
+}
+
+func (h *SystemBackupHandler) Upload(ctx context.Context, input *UploadSystemBackupInput) (*SystemBackupOutput, error) {
+ user, err := handlerutil.RequireUser(ctx)
+ if err != nil {
+ return nil, err
+ }
+ var run *SystemBackupRun
+ _, err = activitylib.RunHandlerActivity(utils.ActivityRuntimeContext(ctx, h.appCtx), h.activity, activitylib.HandlerOptions{
+ EnvironmentID: "0", Type: activitytypes.TypeResourceAction, ResourceType: "system_backup", ResourceID: input.ID, ResourceName: "Arcane", User: user,
+ Step: "Uploading system backup", Message: "Uploading Arcane system backup", SuccessMessage: "Arcane system backup uploaded successfully",
+ Metadata: database.JSON{"action": "upload_system_backup", "backupId": input.ID, "s3DestinationId": input.Body.S3DestinationID},
+ }, func(activityCtx context.Context) error {
+ var e error
+ run, e = h.service.UploadBackup(activityCtx, input.ID, input.Body)
+ return e
+ })
+ if err != nil {
+ return nil, huma.Error500InternalServerError(err.Error())
+ }
+ dto := run.ToDTO()
+ return &SystemBackupOutput{Body: dto}, nil
+}
+
+func (h *SystemBackupHandler) Delete(ctx context.Context, input *DeleteSystemBackupInput) (*SystemBackupMessageOutput, error) {
+ user, err := handlerutil.RequireUser(ctx)
+ if err != nil {
+ return nil, err
+ }
+ activityID, err := activitylib.RunHandlerActivity(utils.ActivityRuntimeContext(ctx, h.appCtx), h.activity, activitylib.HandlerOptions{
+ EnvironmentID: "0", Type: activitytypes.TypeResourceAction, ResourceType: "system_backup", ResourceID: input.ID, ResourceName: "Arcane", User: user,
+ Step: "Deleting system backup", Message: "Deleting Arcane system backup", SuccessMessage: "Arcane system backup deleted successfully",
+ Metadata: database.JSON{"action": "delete_system_backup", "backupId": input.ID},
+ }, func(activityCtx context.Context) error {
+ return h.service.DeleteBackup(activityCtx, input.ID, input.Body.RecoveryKey)
+ })
+ if err != nil {
+ return nil, huma.Error500InternalServerError(err.Error())
+ }
+ return messageOutputInternal("Arcane system backup deleted successfully", activityID), nil
+}
+
+func messageOutputInternal(message, activityID string) *SystemBackupMessageOutput {
+ return &SystemBackupMessageOutput{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: message, ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()}}}
+}
diff --git a/backend/internal/systembackup/model.go b/backend/internal/systembackup/model.go
new file mode 100644
index 000000000..810b98997
--- /dev/null
+++ b/backend/internal/systembackup/model.go
@@ -0,0 +1,83 @@
+package systembackup
+
+import (
+ "time"
+
+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"
+ backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"
+)
+
+type SystemBackupStatus string
+
+const (
+ SystemBackupStatusRunning SystemBackupStatus = "running"
+ SystemBackupStatusSucceeded SystemBackupStatus = "succeeded"
+ SystemBackupStatusFailed SystemBackupStatus = "failed"
+)
+
+type SystemBackupTrigger string
+
+const (
+ SystemBackupTriggerManual SystemBackupTrigger = "manual"
+ SystemBackupTriggerScheduled SystemBackupTrigger = "scheduled"
+ SystemBackupTriggerSafety SystemBackupTrigger = "safety"
+)
+
+type SystemBackupRun struct {
+ database.BaseModel
+
+ Size int64 `json:"size" gorm:"column:size" sortable:"true"`
+ CreatedAt time.Time `json:"createdAt" gorm:"column:created_at" sortable:"true"`
+ Status SystemBackupStatus `json:"status" gorm:"column:status;type:text;not null" sortable:"true"`
+ Trigger SystemBackupTrigger `json:"trigger" gorm:"column:trigger;type:text;not null" sortable:"true"`
+ Destination backuptypes.SystemBackupDestination `json:"destination" gorm:"column:destination;type:text;not null" sortable:"true"`
+ LocalSnapshotID string `json:"localSnapshotId,omitempty" gorm:"column:local_snapshot_id;type:text"`
+ RemoteSnapshotID string `json:"remoteSnapshotId,omitempty" gorm:"column:remote_snapshot_id;type:text"`
+ S3DestinationID string `json:"s3DestinationId,omitempty" gorm:"column:s3_destination_id;type:text;index"`
+ S3DestinationName string `json:"s3DestinationName,omitempty" gorm:"-"`
+ PolicyID string `json:"policyId,omitempty" gorm:"column:policy_id;type:text;index"`
+ Error string `json:"error,omitempty" gorm:"column:error;type:text"`
+}
+
+func (SystemBackupRun) TableName() string { return "system_backup_runs" }
+
+func (b SystemBackupRun) ToDTO() backuptypes.SystemBackupRun {
+ return backuptypes.SystemBackupRun{
+ ID: b.ID, Size: b.Size, CreatedAt: b.CreatedAt, Status: string(b.Status), Trigger: string(b.Trigger),
+ Destination: b.Destination, LocalSnapshotID: b.LocalSnapshotID, RemoteSnapshotID: b.RemoteSnapshotID,
+ S3DestinationID: b.S3DestinationID, S3DestinationName: b.S3DestinationName, PolicyID: b.PolicyID, Error: b.Error,
+ }
+}
+
+type SystemBackupPolicy struct {
+ database.BaseModel
+
+ Enabled bool `gorm:"column:enabled;not null;default:false"`
+ Schedule string `gorm:"column:schedule;type:text;not null"`
+ RetentionCount int `gorm:"column:retention_count;not null;default:7"`
+ LocalEnabled bool `gorm:"column:local_enabled;not null;default:true"`
+ S3Enabled bool `gorm:"column:s3_enabled;not null;default:false"`
+ S3DestinationID string `gorm:"column:s3_destination_id;type:text;index"`
+}
+
+func (SystemBackupPolicy) TableName() string { return "system_backup_policies" }
+
+func (p SystemBackupPolicy) ToDTO(lastRun *SystemBackupRun) backuptypes.SystemBackupPolicy {
+ dto := backuptypes.SystemBackupPolicy{
+ ID: p.ID, Enabled: p.Enabled, Schedule: p.Schedule, RetentionCount: p.RetentionCount,
+ LocalEnabled: p.LocalEnabled, S3Enabled: p.S3Enabled, S3DestinationID: p.S3DestinationID,
+ }
+ if lastRun != nil {
+ run := lastRun.ToDTO()
+ dto.LastRun = &run
+ }
+ return dto
+}
+
+type SystemBackupRecoveryConfig struct {
+ database.BaseModel
+
+ EncryptedRecoveryKey string `gorm:"column:encrypted_recovery_key;type:text;not null"`
+}
+
+func (SystemBackupRecoveryConfig) TableName() string { return "system_backup_recovery_config" }
diff --git a/backend/internal/systembackup/module.go b/backend/internal/systembackup/module.go
new file mode 100644
index 000000000..f30343640
--- /dev/null
+++ b/backend/internal/systembackup/module.go
@@ -0,0 +1,32 @@
+// Package systembackup owns Arcane system-recovery backup orchestration and its HTTP surface.
+package systembackup
+
+import (
+ "github.com/danielgtaylor/huma/v2"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/activity"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"
+)
+
+type Module struct {
+ service *SystemBackupService
+ activity *activity.ActivityService
+}
+
+func New(service *SystemBackupService, activityService *activity.ActivityService) *Module {
+ return &Module{service: service, activity: activityService}
+}
+
+func (m *Module) Service() *SystemBackupService {
+ if m == nil {
+ return nil
+ }
+ return m.service
+}
+
+func (m *Module) RegisterRoutes(api huma.API, appCtx handlerutil.ActivityAppContext) {
+ if m == nil {
+ RegisterSystemBackups(api, nil, nil, appCtx)
+ return
+ }
+ RegisterSystemBackups(api, m.service, m.activity, appCtx)
+}
diff --git a/backend/internal/systembackup/service.go b/backend/internal/systembackup/service.go
new file mode 100644
index 000000000..b525285c6
--- /dev/null
+++ b/backend/internal/systembackup/service.go
@@ -0,0 +1,1042 @@
+package systembackup
+
+import (
+ "context"
+ "crypto/rand"
+ "encoding/base32"
+ "encoding/json"
+ "fmt"
+ "log/slog"
+ "maps"
+ "net/url"
+ "os"
+ "path/filepath"
+ "reflect"
+ "regexp"
+ "slices"
+ "strings"
+ "time"
+
+ "emperror.dev/errors"
+
+ "github.com/getarcaneapp/arcane/backend/v2/internal/activity"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/backup"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/common"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/config"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/docker"
+ recoverytypes "github.com/getarcaneapp/arcane/backend/v2/internal/recovery"
+ s3domain "github.com/getarcaneapp/arcane/backend/v2/internal/s3"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/system"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/volume"
+ dockerutil "github.com/getarcaneapp/arcane/backend/v2/pkg/dockerutil"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"
+ activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/projects"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/entityjobs"
+ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
+ backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"
+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
+ "github.com/google/uuid"
+ containertypes "github.com/moby/moby/api/types/container"
+ "github.com/moby/moby/api/types/mount"
+ "github.com/moby/moby/client"
+ "go.getarcane.app/sys/cgroup"
+ "go.getarcane.app/sys/crypto"
+ "gorm.io/gorm"
+)
+
+const (
+ defaultSystemBackupSchedule = "0 0 3 * * *"
+ systemRecoveryManifestName = ".arcane-recovery.json"
+ systemRecoveryHelperPath = "/app/arcane-recovery-helper"
+ systemAdmissionID = "system"
+)
+
+var ErrSystemBackupAlreadyRunning = errors.New("an Arcane system backup is already running")
+
+type SystemBackupService struct {
+ db *database.DB
+ dockerService *docker.DockerClientService
+ volumeService *volume.VolumeService
+ engine *backup.Engine
+ s3Destinations *s3domain.S3DestinationService
+ activityService *activity.ActivityService
+ config *config.Config
+ jobs *entityjobs.Registry
+}
+
+func NewSystemBackupService(db *database.DB, dockerService *docker.DockerClientService, volumeService *volume.VolumeService, engine *backup.Engine, s3Destinations *s3domain.S3DestinationService, activityService *activity.ActivityService, cfg *config.Config) *SystemBackupService {
+ return &SystemBackupService{
+ db: db, dockerService: dockerService, volumeService: volumeService, engine: engine,
+ s3Destinations: s3Destinations, activityService: activityService, config: cfg,
+ jobs: entityjobs.New("system-backup:", backup.SystemAdmissionScope),
+ }
+}
+
+// SupportedDatabaseProvider reports whether system recovery works on the
+// configured database. Recovery is currently SQLite-only.
+func (s *SystemBackupService) SupportedDatabaseProvider() bool {
+ return strings.HasPrefix(s.config.DatabaseURL, "file:")
+}
+
+// recoveryKeyFormatInternal is the only accepted key shape, generated by
+// GenerateRecoveryKey: 48 base32 characters (~240 bits from crypto/rand) in 8
+// hyphenated groups for hand transcription. The key is the Rustic repository
+// password and is used exactly as displayed, hyphens included.
+var recoveryKeyFormatInternal = regexp.MustCompile(`^[A-Z2-7]{6}(-[A-Z2-7]{6}){7}$`)
+
+func validateRecoveryKeyInternal(key string) error {
+ if !recoveryKeyFormatInternal.MatchString(strings.TrimSpace(key)) {
+ return errors.New("enter the generated recovery key (8 groups of 6 characters)")
+ }
+ return nil
+}
+
+func recoveryHelperExecutableInternal(mounts []containertypes.MountPoint, executablePath string) (string, *mount.Mount, error) {
+ executablePath = strings.TrimSpace(executablePath)
+ if executablePath == "" {
+ return "", nil, errors.New("arcane executable path is empty")
+ }
+ executableMount := dockerutil.MountForSubpath(mounts, executablePath, systemRecoveryHelperPath)
+ if executableMount == nil {
+ return executablePath, nil, nil
+ }
+ if executableMount.Type != mount.TypeBind {
+ return "", nil, errors.New("the Arcane executable must come from the image or a bind mount")
+ }
+ executableMount.ReadOnly = true
+ return systemRecoveryHelperPath, executableMount, nil
+}
+
+func (s *SystemBackupService) recoveryEnvironmentInternal() map[string]string {
+ result := make(map[string]string)
+ var visit func(reflect.Value)
+ visit = func(value reflect.Value) {
+ if value.Kind() == reflect.Pointer {
+ if value.IsNil() {
+ return
+ }
+ value = value.Elem()
+ }
+ if value.Kind() != reflect.Struct {
+ return
+ }
+ typeOf := value.Type()
+ for i := range value.NumField() {
+ fieldType := typeOf.Field(i)
+ field := value.Field(i)
+ if fieldType.Anonymous {
+ visit(field)
+ continue
+ }
+ name := fieldType.Tag.Get("env")
+ if name == "" || !field.CanInterface() {
+ continue
+ }
+ if field.Type() == reflect.TypeFor[os.FileMode]() {
+ mode, ok := reflect.TypeAssert[os.FileMode](field)
+ if ok {
+ result[name] = fmt.Sprintf("%#o", uint32(mode))
+ }
+ } else {
+ result[name] = fmt.Sprint(field.Interface())
+ }
+ }
+ }
+ visit(reflect.ValueOf(s.config))
+ return result
+}
+
+// dataDirectoryInternal resolves the directory holding the SQLite database:
+// /app/data in the container, the local data directory in host development.
+// The recovery manifest must live inside it so every snapshot includes it.
+func (s *SystemBackupService) dataDirectoryInternal() (string, error) {
+ parsed, err := url.Parse(s.config.DatabaseURL)
+ if err != nil {
+ return "", fmt.Errorf("failed to parse database URL: %w", err)
+ }
+ databasePath := parsed.Opaque
+ if databasePath == "" {
+ databasePath = parsed.Path
+ }
+ if strings.TrimSpace(databasePath) == "" {
+ return "", errors.New("cannot resolve the Arcane data directory from the database URL")
+ }
+ absolute, err := filepath.Abs(databasePath)
+ if err != nil {
+ return "", fmt.Errorf("failed to resolve the Arcane data directory: %w", err)
+ }
+ return filepath.Dir(absolute), nil
+}
+
+func (s *SystemBackupService) manifestPathInternal() (string, error) {
+ directory, err := s.dataDirectoryInternal()
+ if err != nil {
+ return "", err
+ }
+ return filepath.Join(directory, systemRecoveryManifestName), nil
+}
+
+func (s *SystemBackupService) writeManifestInternal(ctx context.Context, backupID string) error {
+ manifest := recoverytypes.Manifest{FormatVersion: 1, ArcaneVersion: config.Version, BackupID: backupID, ActivityID: activitylib.IDFromContext(ctx), CreatedAt: time.Now().UTC(), Environment: s.recoveryEnvironmentInternal()}
+ data, err := json.MarshalIndent(manifest, "", " ")
+ if err != nil {
+ return fmt.Errorf("failed to encode recovery manifest: %w", err)
+ }
+ manifestPath, err := s.manifestPathInternal()
+ if err != nil {
+ return err
+ }
+ if err := os.WriteFile(manifestPath, data, 0o600); err != nil {
+ return fmt.Errorf("failed to write recovery manifest: %w", err)
+ }
+ return nil
+}
+
+func (s *SystemBackupService) appDataMountInternal(ctx context.Context, dockerClient *client.Client, target string, readOnly bool) (mount.Mount, error) {
+ if _, containerErr := cgroup.CurrentContainerID(); containerErr != nil {
+ // Host development: bind-mount the local data directory into the
+ // Rustic helper directly instead of re-sharing the container's mount.
+ directory, dirErr := s.dataDirectoryInternal()
+ if dirErr != nil {
+ return mount.Mount{}, dirErr
+ }
+ return mount.Mount{Type: mount.TypeBind, Source: directory, Target: target, ReadOnly: readOnly}, nil
+ }
+ dataMount, err := dockerutil.MountForCurrentContainerSubpath(ctx, dockerClient, "/app/data", target)
+ if err != nil {
+ return mount.Mount{}, fmt.Errorf("failed to inspect Arcane data mount: %w", err)
+ }
+ if dataMount == nil {
+ return mount.Mount{}, errors.New("arcane system backups require /app/data to be mounted into the Arcane container")
+ }
+ dataMount.ReadOnly = readOnly
+ return *dataMount, nil
+}
+
+func (s *SystemBackupService) localRepositoryInternal(ctx context.Context, dockerClient *client.Client, readOnly bool) (backup.Repository, error) {
+ storage, err := s.volumeService.BackupStorageMount(ctx, dockerClient, "/repository", readOnly)
+ if err != nil {
+ return backup.Repository{}, err
+ }
+ return backup.Repository{
+ ID: "system:local",
+ Environment: []string{"RUSTIC_REPOSITORY=/repository/system-recovery"},
+ Mounts: []mount.Mount{storage},
+ }, nil
+}
+
+func (s *SystemBackupService) remoteRepositoryInternal(ctx context.Context, destinationID string) (backup.Repository, error) {
+ configuration, err := s.s3Destinations.Configuration(ctx, destinationID)
+ if err != nil {
+ return backup.Repository{}, errors.New("the selected S3 backup destination is not configured")
+ }
+ return backup.Repository{
+ ID: "system:s3:" + destinationID,
+ Environment: configuration.RusticEnvironment("arcane-system-recovery"),
+ }, nil
+}
+
+func (s *SystemBackupService) createSnapshotInternal(ctx context.Context, dockerClient *client.Client, repository backup.Repository, recoveryKey string) (backup.Snapshot, error) {
+ dataMount, err := s.appDataMountInternal(ctx, dockerClient, "/arcane-data", true)
+ if err != nil {
+ return backup.Snapshot{}, err
+ }
+ return s.engine.CreateSnapshot(ctx, dockerClient, repository, recoveryKey, "arcane-system-recovery", dataMount)
+}
+
+func systemSnapshotPathFromFilesInternal(files []string) (string, error) {
+ for _, file := range files {
+ if strings.TrimPrefix(file, "/") == ".arcane-recovery.json" {
+ return "/", nil
+ }
+ }
+ for _, file := range files {
+ if strings.TrimPrefix(file, "/") == "app/data/.arcane-recovery.json" {
+ return "/app/data", nil
+ }
+ }
+ return "", errors.New("system recovery snapshot does not contain an Arcane recovery manifest")
+}
+
+func (s *SystemBackupService) snapshotPathInternal(ctx context.Context, dockerClient *client.Client, repository backup.Repository, recoveryKey, snapshotID string) (string, error) {
+ files, err := s.engine.ListSnapshotFiles(ctx, dockerClient, repository, recoveryKey, snapshotID)
+ if err != nil {
+ return "", fmt.Errorf("failed to inspect system recovery snapshot: %w", err)
+ }
+ return systemSnapshotPathFromFilesInternal(files)
+}
+
+func (s *SystemBackupService) recoveryKeyInternal(ctx context.Context, supplied string) (string, error) {
+ if strings.TrimSpace(supplied) != "" {
+ if err := validateRecoveryKeyInternal(supplied); err != nil {
+ return "", err
+ }
+ return supplied, nil
+ }
+ var recoveryConfig SystemBackupRecoveryConfig
+ err := s.db.WithContext(ctx).Where("id = ?", systemRecoveryConfigID).First(&recoveryConfig).Error
+ if errors.Is(err, gorm.ErrRecordNotFound) {
+ return "", errors.New("enter the recovery key or configure one in system backups")
+ }
+ if err != nil {
+ return "", fmt.Errorf("failed to load stored recovery key: %w", err)
+ }
+ key, err := crypto.Decrypt(recoveryConfig.EncryptedRecoveryKey)
+ if err != nil {
+ return "", fmt.Errorf("failed to decrypt stored recovery key: %w", err)
+ }
+ return key, validateRecoveryKeyInternal(key)
+}
+
+// resolveBackupPlanInternal resolves which destinations a run targets, from
+// either the policy or the request's destination enum.
+func (s *SystemBackupService) resolveBackupPlanInternal(ctx context.Context, request backuptypes.CreateSystemBackupRequest) (localEnabled, s3Enabled bool, destinationID string, destination backuptypes.SystemBackupDestination, err error) {
+ destinationID = strings.TrimSpace(request.S3DestinationID)
+ if request.PolicyID != "" {
+ policy, policyErr := s.loadPolicyInternal(ctx, request.PolicyID)
+ if policyErr != nil {
+ return false, false, "", "", policyErr
+ }
+ if policy == nil {
+ return false, false, "", "", errors.New("system backup policy not found")
+ }
+ localEnabled, s3Enabled, destinationID = policy.LocalEnabled, policy.S3Enabled, policy.S3DestinationID
+ } else {
+ switch request.Destination {
+ case backuptypes.SystemBackupDestinationLocal, "":
+ localEnabled = true
+ case backuptypes.SystemBackupDestinationS3:
+ s3Enabled = true
+ case backuptypes.SystemBackupDestinationLocalS3:
+ localEnabled, s3Enabled = true, true
+ default:
+ return false, false, "", "", errors.New("unknown system backup destination")
+ }
+ }
+ if !localEnabled && !s3Enabled {
+ return false, false, "", "", errors.New("select at least one system backup destination")
+ }
+ if s3Enabled && destinationID == "" {
+ return false, false, "", "", errors.New("select an S3 destination for the system backup")
+ }
+ destination = backuptypes.SystemBackupDestinationLocal
+ if localEnabled && s3Enabled {
+ destination = backuptypes.SystemBackupDestinationLocalS3
+ } else if s3Enabled {
+ destination = backuptypes.SystemBackupDestinationS3
+ }
+ return localEnabled, s3Enabled, destinationID, destination, nil
+}
+
+func (s *SystemBackupService) CreateBackup(ctx context.Context, user common.User, trigger SystemBackupTrigger, request backuptypes.CreateSystemBackupRequest) (_ *SystemBackupRun, err error) {
+ if !s.SupportedDatabaseProvider() {
+ return nil, errors.New("arcane system recovery currently requires the SQLite database provider")
+ }
+ recoveryKey, err := s.recoveryKeyInternal(ctx, request.RecoveryKey)
+ if err != nil {
+ return nil, err
+ }
+ localEnabled, s3Enabled, destinationID, destination, err := s.resolveBackupPlanInternal(ctx, request)
+ if err != nil {
+ return nil, err
+ }
+ lease, admitted, leaseErr := s.engine.TryAcquireRun(ctx, backup.SystemAdmissionScope, systemAdmissionID)
+ if leaseErr != nil {
+ return nil, leaseErr
+ }
+ if !admitted {
+ return nil, ErrSystemBackupAlreadyRunning
+ }
+ defer lease.Release()
+ run := &SystemBackupRun{CreatedAt: time.Now().UTC(), Status: SystemBackupStatusRunning, Trigger: trigger, Destination: destination, S3DestinationID: destinationID, PolicyID: request.PolicyID}
+ run.ID = "system-" + uuid.NewString()
+ if err := s.db.WithContext(ctx).Create(run).Error; err != nil {
+ return nil, err
+ }
+ defer func() {
+ if err != nil {
+ run.Status, run.Error = SystemBackupStatusFailed, err.Error()
+ } else {
+ run.Status, run.Error = SystemBackupStatusSucceeded, ""
+ }
+ if saveErr := s.db.WithContext(context.WithoutCancel(ctx)).Save(run).Error; saveErr != nil {
+ err = errors.Combine(err, fmt.Errorf("failed to save system backup result: %w", saveErr))
+ }
+ }()
+ dockerClient, err := s.dockerService.GetClient(ctx)
+ if err != nil {
+ return run, err
+ }
+ localRepository, err := s.localRepositoryInternal(ctx, dockerClient, false)
+ if err != nil {
+ return run, err
+ }
+ // The database is locked only while the local staging snapshot is taken.
+ // An S3 copy replicates that immutable snapshot afterwards, so the
+ // exclusive lock never spans a network upload and the source is read once.
+ stagedSnapshot, err := s.snapshotWithDatabaseLockInternal(ctx, dockerClient, localRepository, recoveryKey, run.ID)
+ if err != nil {
+ return run, err
+ }
+ // The staged snapshot is kept only for local-enabled runs. Forgetting it in
+ // a defer covers the failure paths too: a failed replication otherwise
+ // orphans one unreferenced snapshot in the staging repository per run.
+ defer func() {
+ if localEnabled {
+ return
+ }
+ cleanupCtx := context.WithoutCancel(ctx)
+ if forgetErr := s.engine.ForgetSnapshot(cleanupCtx, dockerClient, localRepository, recoveryKey, stagedSnapshot.ID); forgetErr != nil {
+ slog.WarnContext(cleanupCtx, "failed to remove staged system recovery snapshot", "snapshot_id", stagedSnapshot.ID, "error", forgetErr)
+ }
+ }()
+ if localEnabled {
+ run.LocalSnapshotID, run.Size = stagedSnapshot.ID, stagedSnapshot.Size
+ }
+ if s3Enabled {
+ remoteRepository, repoErr := s.remoteRepositoryInternal(ctx, destinationID)
+ if repoErr != nil {
+ return run, repoErr
+ }
+ remoteSnapshot, replicateErr := s.engine.Replicate(ctx, dockerClient, localRepository, stagedSnapshot.ID, remoteRepository, recoveryKey, "arcane-system-recovery")
+ if replicateErr != nil {
+ return run, fmt.Errorf("failed to create S3 system recovery snapshot: %w", replicateErr)
+ }
+ run.RemoteSnapshotID = remoteSnapshot.ID
+ if run.Size == 0 {
+ run.Size = remoteSnapshot.Size
+ }
+ }
+ return run, nil
+}
+
+// snapshotWithDatabaseLockInternal takes the local staging snapshot while the
+// SQLite database is checkpointed and exclusively locked, and releases the
+// lock before returning.
+func (s *SystemBackupService) snapshotWithDatabaseLockInternal(ctx context.Context, dockerClient *client.Client, repository backup.Repository, recoveryKey, backupID string) (backup.Snapshot, error) {
+ if err := s.writeManifestInternal(ctx, backupID); err != nil {
+ return backup.Snapshot{}, err
+ }
+ defer func() {
+ if manifestPath, pathErr := s.manifestPathInternal(); pathErr == nil {
+ _ = os.Remove(manifestPath)
+ }
+ }()
+ sqlDB, err := s.db.SQLDB()
+ if err != nil {
+ return backup.Snapshot{}, err
+ }
+ if _, err := sqlDB.ExecContext(ctx, "PRAGMA wal_checkpoint(FULL)"); err != nil {
+ return backup.Snapshot{}, fmt.Errorf("failed to checkpoint Arcane database: %w", err)
+ }
+ connection, err := sqlDB.Conn(ctx)
+ if err != nil {
+ return backup.Snapshot{}, fmt.Errorf("failed to reserve Arcane database connection: %w", err)
+ }
+ defer func() { _ = connection.Close() }()
+ if _, err := connection.ExecContext(ctx, "BEGIN EXCLUSIVE"); err != nil {
+ return backup.Snapshot{}, fmt.Errorf("failed to lock Arcane database for backup: %w", err)
+ }
+ defer func() { _, _ = connection.ExecContext(context.WithoutCancel(ctx), "ROLLBACK") }()
+ snapshot, err := s.createSnapshotInternal(ctx, dockerClient, repository, recoveryKey)
+ if err != nil {
+ return backup.Snapshot{}, fmt.Errorf("failed to create local system recovery snapshot: %w", err)
+ }
+ return snapshot, nil
+}
+
+func (s *SystemBackupService) ListBackups(ctx context.Context, params pagination.QueryParams) ([]backuptypes.SystemBackupRun, pagination.Response, error) {
+ var runs []SystemBackupRun
+ query := s.db.WithContext(ctx).Model(&SystemBackupRun{})
+ if term := strings.TrimSpace(params.Search); term != "" {
+ pattern := "%" + term + "%"
+ query = query.Where("status LIKE ? OR trigger LIKE ? OR destination LIKE ? OR error LIKE ?", pattern, pattern, pattern, pattern)
+ }
+ response, err := pagination.PaginateAndSortDB(params, query, &runs)
+ if err != nil {
+ return nil, pagination.Response{}, fmt.Errorf("failed to list system backups: %w", err)
+ }
+ names := make(map[string]string)
+ if s.s3Destinations != nil {
+ if destinations, listErr := s.s3Destinations.ListAllS3Destinations(ctx); listErr == nil {
+ for _, destination := range destinations {
+ names[destination.ID] = destination.Name
+ }
+ }
+ }
+ result := make([]backuptypes.SystemBackupRun, len(runs))
+ for i := range runs {
+ runs[i].S3DestinationName = names[runs[i].S3DestinationID]
+ result[i] = runs[i].ToDTO()
+ }
+ return result, response, nil
+}
+
+func (s *SystemBackupService) backupInternal(ctx context.Context, id string) (*SystemBackupRun, error) {
+ var run SystemBackupRun
+ if err := s.db.WithContext(ctx).Where("id = ?", id).First(&run).Error; err != nil {
+ return nil, err
+ }
+ return &run, nil
+}
+
+func (s *SystemBackupService) DeleteBackup(ctx context.Context, id, recoveryKey string) error {
+ // Deletes contend with create/restore/upload on the system run lease so a
+ // slow operation cannot resurrect or double-forget snapshots.
+ lease, admitted, err := s.engine.TryAcquireRun(ctx, backup.SystemAdmissionScope, systemAdmissionID)
+ if err != nil {
+ return err
+ }
+ if !admitted {
+ return ErrSystemBackupAlreadyRunning
+ }
+ defer lease.Release()
+ run, err := s.backupInternal(ctx, id)
+ if err != nil {
+ return err
+ }
+ // A run without snapshots (a failed attempt) is just a row; deleting it
+ // needs neither the key nor Rustic.
+ key := ""
+ if run.LocalSnapshotID != "" || run.RemoteSnapshotID != "" {
+ if key, err = s.recoveryKeyInternal(ctx, recoveryKey); err != nil {
+ return err
+ }
+ }
+ dockerClient, err := s.dockerService.GetClient(ctx)
+ if err != nil {
+ return err
+ }
+ var deleteErr error
+ if run.LocalSnapshotID != "" {
+ repository, repoErr := s.localRepositoryInternal(ctx, dockerClient, false)
+ if repoErr == nil {
+ repoErr = s.engine.ForgetSnapshot(ctx, dockerClient, repository, key, run.LocalSnapshotID)
+ }
+ if repoErr != nil {
+ deleteErr = errors.Combine(deleteErr, fmt.Errorf("failed to delete local snapshot: %w", repoErr))
+ } else {
+ run.LocalSnapshotID = ""
+ }
+ }
+ if run.RemoteSnapshotID != "" {
+ repository, repoErr := s.remoteRepositoryInternal(ctx, run.S3DestinationID)
+ if repoErr == nil {
+ repoErr = s.engine.ForgetSnapshot(ctx, dockerClient, repository, key, run.RemoteSnapshotID)
+ }
+ if repoErr != nil {
+ deleteErr = errors.Combine(deleteErr, fmt.Errorf("failed to delete S3 snapshot: %w", repoErr))
+ } else {
+ run.RemoteSnapshotID, run.S3DestinationID = "", ""
+ }
+ }
+ if deleteErr != nil {
+ run.Error = deleteErr.Error()
+ _ = s.db.WithContext(ctx).Save(run).Error
+ return deleteErr
+ }
+ return s.db.WithContext(ctx).Delete(run).Error
+}
+
+func (s *SystemBackupService) DiscoverRemoteBackups(ctx context.Context, request backuptypes.DiscoverSystemBackupsRequest) (int, error) {
+ recoveryKey, err := s.recoveryKeyInternal(ctx, request.RecoveryKey)
+ if err != nil {
+ return 0, err
+ }
+ repository, err := s.remoteRepositoryInternal(ctx, request.S3DestinationID)
+ if err != nil {
+ return 0, err
+ }
+ dockerClient, err := s.dockerService.GetClient(ctx)
+ if err != nil {
+ return 0, err
+ }
+ snapshots, err := s.engine.ListSnapshots(ctx, dockerClient, repository, recoveryKey)
+ if err != nil {
+ return 0, fmt.Errorf("failed to open system recovery repository: %w", err)
+ }
+ var knownIDs []string
+ if err := s.db.WithContext(ctx).Model(&SystemBackupRun{}).
+ Where("s3_destination_id = ? AND remote_snapshot_id <> ''", request.S3DestinationID).
+ Pluck("remote_snapshot_id", &knownIDs).Error; err != nil {
+ return 0, err
+ }
+ known := make(map[string]struct{}, len(knownIDs))
+ for _, id := range knownIDs {
+ known[id] = struct{}{}
+ }
+ created := 0
+ for _, snapshot := range snapshots {
+ if strings.TrimSpace(snapshot.ID) == "" {
+ continue
+ }
+ if _, exists := known[snapshot.ID]; exists {
+ continue
+ }
+ createdAt := snapshot.Time
+ if createdAt.IsZero() {
+ createdAt = time.Now().UTC()
+ }
+ run := &SystemBackupRun{
+ Size: snapshot.Summary.TotalBytesProcessed, CreatedAt: createdAt, Status: SystemBackupStatusSucceeded,
+ Trigger: SystemBackupTriggerManual, Destination: backuptypes.SystemBackupDestinationS3,
+ RemoteSnapshotID: snapshot.ID, S3DestinationID: request.S3DestinationID,
+ }
+ run.ID = fmt.Sprintf("remote-%s-%s", request.S3DestinationID, snapshot.ID)
+ if err := s.db.WithContext(ctx).Create(run).Error; err != nil {
+ return created, fmt.Errorf("failed to save discovered system backup: %w", err)
+ }
+ created++
+ }
+ return created, nil
+}
+
+func (s *SystemBackupService) UploadBackup(ctx context.Context, id string, request backuptypes.UploadSystemBackupRequest) (*SystemBackupRun, error) {
+ lease, admitted, err := s.engine.TryAcquireRun(ctx, backup.SystemAdmissionScope, systemAdmissionID)
+ if err != nil {
+ return nil, err
+ }
+ if !admitted {
+ return nil, ErrSystemBackupAlreadyRunning
+ }
+ defer lease.Release()
+ run, err := s.backupInternal(ctx, id)
+ if err != nil {
+ return nil, err
+ }
+ if run.Status != SystemBackupStatusSucceeded || run.LocalSnapshotID == "" {
+ return nil, errors.New("only successful local system backups can be uploaded")
+ }
+ if run.RemoteSnapshotID != "" {
+ return nil, errors.New("system backup has already been uploaded")
+ }
+ key, err := s.recoveryKeyInternal(ctx, request.RecoveryKey)
+ if err != nil {
+ return nil, err
+ }
+ if strings.TrimSpace(request.S3DestinationID) == "" {
+ return nil, errors.New("select an S3 destination for the upload")
+ }
+ dockerClient, err := s.dockerService.GetClient(ctx)
+ if err != nil {
+ return nil, err
+ }
+ localRepository, err := s.localRepositoryInternal(ctx, dockerClient, true)
+ if err != nil {
+ return nil, err
+ }
+ remoteRepository, err := s.remoteRepositoryInternal(ctx, request.S3DestinationID)
+ if err != nil {
+ return nil, err
+ }
+ snapshot, err := s.engine.Replicate(ctx, dockerClient, localRepository, run.LocalSnapshotID, remoteRepository, key, "arcane-system-recovery")
+ if err != nil {
+ return nil, fmt.Errorf("failed to upload system recovery snapshot: %w", err)
+ }
+ run.RemoteSnapshotID, run.S3DestinationID = snapshot.ID, request.S3DestinationID
+ run.Destination = backuptypes.SystemBackupDestinationLocalS3
+ if err := s.db.WithContext(ctx).Save(run).Error; err != nil {
+ return nil, fmt.Errorf("failed to save uploaded system backup: %w", err)
+ }
+ return run, nil
+}
+
+func (s *SystemBackupService) RestoreBackup(ctx context.Context, id, recoveryKey string, user common.User) error {
+ run, err := s.backupInternal(ctx, id)
+ if err != nil {
+ return err
+ }
+ if run.Status != SystemBackupStatusSucceeded {
+ return errors.New("only successful system backups can be restored")
+ }
+ key, err := s.recoveryKeyInternal(ctx, recoveryKey)
+ if err != nil {
+ return err
+ }
+ // A local safety snapshot is created before the detached helper is allowed to
+ // stop Arcane and replace its data.
+ safetyBackup, err := s.CreateBackup(ctx, user, SystemBackupTriggerSafety, backuptypes.CreateSystemBackupRequest{
+ Destination: backuptypes.SystemBackupDestinationLocal,
+ RecoveryKey: key,
+ })
+ if err != nil {
+ return fmt.Errorf("failed to create pre-restore system backup: %w", err)
+ }
+ dockerClient, err := s.dockerService.GetClient(ctx)
+ if err != nil {
+ return err
+ }
+ containerID, err := cgroup.CurrentContainerID()
+ if err != nil {
+ return errors.New("arcane system restore requires Arcane to run in Docker")
+ }
+ inspectResult, err := libarcane.ContainerInspectWithCompatibility(ctx, dockerClient, containerID, client.ContainerInspectOptions{})
+ if err != nil {
+ return fmt.Errorf("inspect Arcane container: %w", err)
+ }
+ current := inspectResult.Container
+ appDataMount := dockerutil.MountForDestination(current.Mounts, "/app/data", "/app/data")
+ if appDataMount == nil {
+ return errors.New("arcane system restore requires /app/data to be mounted")
+ }
+ var repository backup.Repository
+ var snapshotID string
+ switch {
+ case run.LocalSnapshotID != "":
+ repository, err = s.localRepositoryInternal(ctx, dockerClient, true)
+ snapshotID = run.LocalSnapshotID
+ case run.RemoteSnapshotID != "":
+ repository, err = s.remoteRepositoryInternal(ctx, run.S3DestinationID)
+ snapshotID = run.RemoteSnapshotID
+ default:
+ return errors.New("system backup has no Rustic snapshot")
+ }
+ if err != nil {
+ return err
+ }
+ snapshotPath, err := s.snapshotPathInternal(ctx, dockerClient, repository, key, snapshotID)
+ if err != nil {
+ return err
+ }
+ request := recoverytypes.RestoreRequest{
+ BackupID: run.ID, ContainerID: current.ID, ContainerImage: current.Config.Image, SnapshotID: snapshotID, SnapshotPath: snapshotPath, RecoveryKey: key,
+ LocalSnapshotID: run.LocalSnapshotID, RemoteSnapshotID: run.RemoteSnapshotID, S3DestinationID: run.S3DestinationID,
+ Size: run.Size, NetworkMode: rusticRestoreNetworkModeInternal(&current),
+ SafetyBackup: &recoverytypes.SafetyBackup{
+ ID: safetyBackup.ID, LocalSnapshotID: safetyBackup.LocalSnapshotID,
+ Size: safetyBackup.Size, CreatedAt: safetyBackup.CreatedAt,
+ },
+ RepositoryEnvironment: repository.Environment, RepositoryMounts: repository.Mounts, AppDataMount: *appDataMount,
+ }
+ requestData, err := json.Marshal(request)
+ if err != nil {
+ return fmt.Errorf("encode recovery request: %w", err)
+ }
+ requestFile := "/app/data/.arcane-recovery-request.json"
+ if err := os.WriteFile(requestFile, requestData, 0o600); err != nil {
+ return fmt.Errorf("write recovery request: %w", err)
+ }
+ cleanupRequest := true
+ defer func() {
+ if cleanupRequest {
+ _ = os.Remove(requestFile)
+ }
+ }()
+ containerEnv, runtimeMounts, networkMode, err := system.ResolveUpgraderRuntimeOptions(ctx, s.dockerService.DockerHost(), &current,
+ func(ctx context.Context, containerPath string) (string, error) {
+ return projects.GetHostPathForContainerPath(ctx, dockerClient, containerPath)
+ },
+ func() bool { _, err := cgroup.CurrentContainerID(); return err == nil },
+ func(ctx context.Context, inspect *containertypes.InspectResponse, dockerHost string) string {
+ return dockerutil.SelectDockerHostReachableNetworkMode(ctx, dockerClient, inspect, dockerHost)
+ },
+ )
+ if err != nil {
+ return fmt.Errorf("resolve recovery helper runtime: %w", err)
+ }
+ executablePath, err := os.Executable()
+ if err != nil {
+ return fmt.Errorf("resolve Arcane executable: %w", err)
+ }
+ helperExecutable, executableMount, err := recoveryHelperExecutableInternal(current.Mounts, executablePath)
+ if err != nil {
+ return fmt.Errorf("resolve recovery helper executable: %w", err)
+ }
+ mounts := append([]mount.Mount{}, runtimeMounts...)
+ mounts = append(mounts, *appDataMount)
+ if executableMount != nil {
+ mounts = append(mounts, *executableMount)
+ }
+ hostConfig := &containertypes.HostConfig{AutoRemove: true, Mounts: mounts, NetworkMode: networkMode}
+ if current.HostConfig != nil {
+ hostConfig.SecurityOpt = append([]string{}, current.HostConfig.SecurityOpt...)
+ hostConfig.Privileged = current.HostConfig.Privileged
+ }
+ created, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{
+ Config: &containertypes.Config{
+ Image: current.Config.Image, Cmd: []string{helperExecutable, "recovery-restore", "--request", requestFile}, User: "0:0",
+ Env: containerEnv,
+ Labels: map[string]string{"com.getarcaneapp.arcane.recovery": "true", "com.getarcaneapp.arcane": "true"},
+ },
+ HostConfig: hostConfig,
+ Name: fmt.Sprintf("%s-recovery-%d", strings.TrimPrefix(current.Name, "/"), time.Now().Unix()),
+ })
+ if err != nil {
+ return fmt.Errorf("create recovery helper container: %w", err)
+ }
+ if _, err := dockerClient.ContainerStart(ctx, created.ID, client.ContainerStartOptions{}); err != nil {
+ _, _ = dockerClient.ContainerRemove(ctx, created.ID, client.ContainerRemoveOptions{Force: true})
+ return fmt.Errorf("start recovery helper container: %w", err)
+ }
+ cleanupRequest = false
+ return nil
+}
+
+const systemRecoveryConfigID = "system-recovery"
+
+func (s *SystemBackupService) loadPoliciesInternal(ctx context.Context) ([]SystemBackupPolicy, error) {
+ var policies []SystemBackupPolicy
+ if err := s.db.WithContext(ctx).Order("created_at ASC").Find(&policies).Error; err != nil {
+ return nil, fmt.Errorf("failed to load system backup policies: %w", err)
+ }
+ return policies, nil
+}
+
+func (s *SystemBackupService) loadPolicyInternal(ctx context.Context, policyID string) (*SystemBackupPolicy, error) {
+ if strings.TrimSpace(policyID) == "" {
+ return nil, nil
+ }
+ var policy SystemBackupPolicy
+ err := s.db.WithContext(ctx).Where("id = ?", policyID).First(&policy).Error
+ if errors.Is(err, gorm.ErrRecordNotFound) {
+ return nil, nil
+ }
+ if err != nil {
+ return nil, fmt.Errorf("failed to load system backup policy: %w", err)
+ }
+ return &policy, nil
+}
+
+// rusticRestoreNetworkModeInternal picks a named network Arcane is attached to
+// so the detached restore's Rustic container can still resolve network-local
+// S3 endpoints once Arcane is stopped. The default bridge carries no DNS, so
+// it (and none) fall back to the default network.
+func rusticRestoreNetworkModeInternal(current *containertypes.InspectResponse) string {
+ if current == nil || current.NetworkSettings == nil {
+ return ""
+ }
+ for _, name := range slices.Sorted(maps.Keys(current.NetworkSettings.Networks)) {
+ if name != "bridge" && name != "none" {
+ return name
+ }
+ }
+ return ""
+}
+
+func (s *SystemBackupService) recoveryKeyConfiguredInternal(ctx context.Context) (bool, error) {
+ var count int64
+ if err := s.db.WithContext(ctx).Model(&SystemBackupRecoveryConfig{}).
+ Where("id = ? AND encrypted_recovery_key <> ''", systemRecoveryConfigID).Count(&count).Error; err != nil {
+ return false, fmt.Errorf("failed to load recovery key status: %w", err)
+ }
+ return count > 0, nil
+}
+
+func (s *SystemBackupService) GenerateRecoveryKey() (*backuptypes.SystemBackupRecoveryKey, error) {
+ raw := make([]byte, 30)
+ if _, err := rand.Read(raw); err != nil {
+ return nil, fmt.Errorf("failed to generate recovery key: %w", err)
+ }
+ encoded := base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(raw)
+ groups := make([]string, 0, len(encoded)/6)
+ for i := 0; i < len(encoded); i += 6 {
+ groups = append(groups, encoded[i:i+6])
+ }
+ return &backuptypes.SystemBackupRecoveryKey{RecoveryKey: strings.Join(groups, "-")}, nil
+}
+
+func (s *SystemBackupService) SetRecoveryKey(ctx context.Context, recoveryKey string) (*backuptypes.SystemBackupRecoveryKeyStatus, error) {
+ if err := validateRecoveryKeyInternal(recoveryKey); err != nil {
+ return nil, err
+ }
+ // The repositories are encrypted with the key that initialized them and a
+ // different key cannot open them, so a rotation would silently break every
+ // scheduled backup and keyless restore. Refuse it while backups exist.
+ configured, err := s.recoveryKeyConfiguredInternal(ctx)
+ if err != nil {
+ return nil, err
+ }
+ if configured {
+ if current, keyErr := s.recoveryKeyInternal(ctx, ""); keyErr == nil && current != recoveryKey {
+ var runs int64
+ if err := s.db.WithContext(ctx).Model(&SystemBackupRun{}).Count(&runs).Error; err != nil {
+ return nil, err
+ }
+ if runs > 0 {
+ return nil, errors.New("delete the existing system backups before replacing the recovery key; they can only be opened with the current key")
+ }
+ }
+ }
+ encrypted, err := crypto.Encrypt(recoveryKey)
+ if err != nil {
+ return nil, fmt.Errorf("failed to encrypt recovery key: %w", err)
+ }
+ config := SystemBackupRecoveryConfig{ID: systemRecoveryConfigID, EncryptedRecoveryKey: encrypted}
+ if err := s.db.WithContext(ctx).Save(&config).Error; err != nil {
+ return nil, fmt.Errorf("failed to save recovery key: %w", err)
+ }
+ return &backuptypes.SystemBackupRecoveryKeyStatus{Configured: true}, nil
+}
+
+func (s *SystemBackupService) GetPolicies(ctx context.Context) (*backuptypes.SystemBackupPolicyCollection, error) {
+ policies, err := s.loadPoliciesInternal(ctx)
+ if err != nil {
+ return nil, err
+ }
+ configured, err := s.recoveryKeyConfiguredInternal(ctx)
+ if err != nil {
+ return nil, err
+ }
+ result := &backuptypes.SystemBackupPolicyCollection{
+ Policies: make([]backuptypes.SystemBackupPolicy, 0, len(policies)), RecoveryKeyStored: configured,
+ }
+ destinations := make(map[string]string)
+ if s.s3Destinations != nil {
+ if available, listErr := s.s3Destinations.ListAllS3Destinations(ctx); listErr == nil {
+ for _, destination := range available {
+ destinations[destination.ID] = destination.Name
+ }
+ }
+ }
+ for i := range policies {
+ var lastRun *SystemBackupRun
+ var run SystemBackupRun
+ if runErr := s.db.WithContext(ctx).Where("policy_id = ?", policies[i].ID).Order("created_at DESC").First(&run).Error; runErr == nil {
+ run.S3DestinationName = destinations[run.S3DestinationID]
+ lastRun = &run
+ } else if !errors.Is(runErr, gorm.ErrRecordNotFound) {
+ return nil, fmt.Errorf("failed to load latest system backup: %w", runErr)
+ }
+ dto := policies[i].ToDTO(lastRun)
+ dto.S3DestinationName = destinations[policies[i].S3DestinationID]
+ result.Policies = append(result.Policies, dto)
+ }
+ return result, nil
+}
+
+func (s *SystemBackupService) UpdatePolicies(ctx context.Context, updates []backuptypes.UpdateSystemBackupPolicy) (*backuptypes.SystemBackupPolicyCollection, error) {
+ configured, err := s.recoveryKeyConfiguredInternal(ctx)
+ if err != nil {
+ return nil, err
+ }
+ existing, err := s.loadPoliciesInternal(ctx)
+ if err != nil {
+ return nil, err
+ }
+ reconcile := backup.PolicyReconciliation[SystemBackupPolicy]{
+ Domain: "system",
+ DB: s.db,
+ Existing: existing,
+ ID: func(policy *SystemBackupPolicy) string { return policy.ID },
+ New: func() SystemBackupPolicy { return SystemBackupPolicy{} },
+ Apply: func(policy *SystemBackupPolicy, update backuptypes.UpdateBackupPolicy) {
+ policy.Enabled, policy.Schedule, policy.RetentionCount = update.Enabled, update.Schedule, update.RetentionCount
+ policy.LocalEnabled, policy.S3Enabled, policy.S3DestinationID = update.LocalEnabled, update.S3Enabled, update.S3DestinationID
+ },
+ ValidateUpdate: func(update backuptypes.UpdateBackupPolicy) error {
+ if update.Enabled && !configured {
+ return errors.New("configure a recovery key before enabling scheduled system backups")
+ }
+ if update.Enabled && !s.SupportedDatabaseProvider() {
+ return errors.New("scheduled system backups require the SQLite database provider")
+ }
+ return nil
+ },
+ S3Configured: func(ctx context.Context, destinationID string) error {
+ if _, err := s.s3Destinations.Configuration(ctx, destinationID); err != nil {
+ return errors.New("select a valid S3 destination for system backups")
+ }
+ return nil
+ },
+ Unregister: s.jobs.Unregister,
+ Reschedule: s.rescheduleSystemBackupPolicyInternal,
+ }
+ if err := reconcile.Run(ctx, updates); err != nil {
+ return nil, err
+ }
+ return s.GetPolicies(ctx)
+}
+
+// SetScheduler injects the dynamic scheduler and admission gate for per-policy
+// system backup jobs. Agent mode leaves them unset.
+func (s *SystemBackupService) SetScheduler(ctx context.Context, scheduler schedulertypes.DynamicScheduler, admissionGate *actors.Gate[actors.AdmissionKey]) error {
+ return s.jobs.SetScheduler(ctx, scheduler, admissionGate)
+}
+
+func (s *SystemBackupService) runScheduledBackupInternal(ctx context.Context, policyID string) {
+ policy, loadErr := s.loadPolicyInternal(ctx, policyID)
+ if loadErr != nil || policy == nil || !policy.Enabled {
+ return
+ }
+ var run *SystemBackupRun
+ _, runErr := activitylib.RunHandlerActivity(ctx, s.activityService, activitylib.HandlerOptions{
+ EnvironmentID: "0", Type: activitytypes.TypeResourceAction, ResourceType: "system_backup",
+ ResourceID: policy.ID, ResourceName: "Arcane", User: &common.SystemUser,
+ Step: "Creating scheduled system backup", Message: "Creating scheduled Arcane system backup",
+ SuccessMessage: "Scheduled Arcane system backup created successfully",
+ Metadata: database.JSON{"action": "scheduled_system_backup", "policyId": policy.ID, "schedule": policy.Schedule,
+ "retentionCount": policy.RetentionCount, "localEnabled": policy.LocalEnabled,
+ "s3Enabled": policy.S3Enabled, "s3DestinationId": policy.S3DestinationID},
+ }, func(activityCtx context.Context) error {
+ var backupErr error
+ run, backupErr = s.CreateBackup(activityCtx, common.SystemUser, SystemBackupTriggerScheduled,
+ backuptypes.CreateSystemBackupRequest{PolicyID: policy.ID})
+ return backupErr
+ })
+ if errors.Is(runErr, ErrSystemBackupAlreadyRunning) {
+ slog.InfoContext(ctx, "Scheduled Arcane system backup skipped; another backup is running", "policyId", policy.ID)
+ return
+ }
+ if runErr != nil {
+ slog.ErrorContext(ctx, "Scheduled Arcane system backup failed", "policyId", policy.ID, "error", runErr)
+ return
+ }
+ if policy.RetentionCount > 0 {
+ if retentionErr := s.applyRetentionInternal(ctx, policy.ID, policy.RetentionCount); retentionErr != nil {
+ slog.ErrorContext(ctx, "System backup retention failed", "policyId", policy.ID, "error", retentionErr)
+ }
+ }
+ slog.InfoContext(ctx, "Scheduled Arcane system backup completed", "backupId", run.ID, "policyId", policy.ID)
+}
+
+func (s *SystemBackupService) rescheduleSystemBackupPolicyInternal(ctx context.Context, policy *SystemBackupPolicy) {
+ if policy == nil {
+ return
+ }
+ if !policy.Enabled {
+ s.jobs.Unregister(ctx, policy.ID)
+ return
+ }
+ policyID := policy.ID
+ s.jobs.Register(ctx, policyID,
+ func(ctx context.Context) string {
+ current, err := s.loadPolicyInternal(ctx, policyID)
+ if err != nil || current == nil {
+ return defaultSystemBackupSchedule
+ }
+ return current.Schedule
+ },
+ func(ctx context.Context) {
+ s.runScheduledBackupInternal(ctx, policyID)
+ },
+ )
+}
+
+func (s *SystemBackupService) RegisterBackupJobOnStartup(ctx context.Context) {
+ policies, err := s.loadPoliciesInternal(ctx)
+ if err != nil {
+ slog.ErrorContext(ctx, "Failed to load Arcane system backup policies", "error", err)
+ return
+ }
+ for i := range policies {
+ s.rescheduleSystemBackupPolicyInternal(ctx, &policies[i])
+ }
+ slog.InfoContext(ctx, "Registered scheduled Arcane system backup jobs", "count", len(policies))
+}
+
+func (s *SystemBackupService) applyRetentionInternal(ctx context.Context, policyID string, keep int) error {
+ expired, err := backup.ExpiredRunIDs(ctx, s.db, "system_backup_runs", policyID, keep)
+ if err != nil {
+ return err
+ }
+ for _, id := range expired {
+ if err := s.DeleteBackup(ctx, id, ""); err != nil {
+ return err
+ }
+ }
+ return nil
+}
diff --git a/backend/internal/systembackup/service_test.go b/backend/internal/systembackup/service_test.go
new file mode 100644
index 000000000..124ea7963
--- /dev/null
+++ b/backend/internal/systembackup/service_test.go
@@ -0,0 +1,157 @@
+package systembackup
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/backup"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/config"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/entityjobs"
+ backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"
+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
+ sqlite "github.com/libtnb/sqlite"
+ containertypes "github.com/moby/moby/api/types/container"
+ mounttypes "github.com/moby/moby/api/types/mount"
+ "github.com/stretchr/testify/require"
+ "go.getarcane.app/sys/crypto"
+ "go.uber.org/fx/fxtest"
+ "gorm.io/gorm"
+)
+
+func newSystemBackupAdmissionGateForTestInternal(t testing.TB) *actors.Gate[actors.AdmissionKey] {
+ t.Helper()
+ fxLifecycle := fxtest.NewLifecycle(t)
+ runtime, err := actors.NewRuntime(t.Context(), fxLifecycle)
+ require.NoError(t, err)
+ gate, err := actors.NewGate[actors.AdmissionKey](t.Context(), runtime, "system-backup-test-admission", t.Name())
+ require.NoError(t, err)
+ t.Cleanup(func() {
+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ require.NoError(t, gate.Stop(stopCtx))
+ require.NoError(t, fxLifecycle.Stop(stopCtx))
+ })
+ return gate
+}
+
+type systemBackupPolicySchedulerInternal struct {
+ jobs map[string]schedulertypes.Job
+}
+
+func (s *systemBackupPolicySchedulerInternal) AddJob(_ context.Context, job schedulertypes.Job) error {
+ s.jobs[job.Name()] = job
+ return nil
+}
+
+func (s *systemBackupPolicySchedulerInternal) RemoveJob(_ context.Context, name string) {
+ delete(s.jobs, name)
+}
+
+func (s *systemBackupPolicySchedulerInternal) HasJob(name string) bool {
+ _, ok := s.jobs[name]
+ return ok
+}
+
+func TestRecoveryHelperExecutableInternal(t *testing.T) {
+ path, executableMount, err := recoveryHelperExecutableInternal(nil, "/app/arcane")
+ require.NoError(t, err)
+ require.Equal(t, "/app/arcane", path)
+ require.Nil(t, executableMount)
+
+ path, executableMount, err = recoveryHelperExecutableInternal([]containertypes.MountPoint{{
+ Type: mounttypes.TypeBind, Source: "/workspace/backend", Destination: "/app/backend", RW: true,
+ }}, "/app/backend/.bin/arcane")
+ require.NoError(t, err)
+ require.Equal(t, systemRecoveryHelperPath, path)
+ require.Equal(t, mounttypes.TypeBind, executableMount.Type)
+ require.Equal(t, "/workspace/backend/.bin/arcane", executableMount.Source)
+ require.Equal(t, systemRecoveryHelperPath, executableMount.Target)
+ require.True(t, executableMount.ReadOnly)
+}
+
+func TestSystemBackupPoliciesRegisterIndependentJobs(t *testing.T) {
+ gormDB, err := gorm.Open(sqlite.Open("file:system-backup-schedules?mode=memory&cache=shared"), &gorm.Config{})
+ require.NoError(t, err)
+ require.NoError(t, gormDB.AutoMigrate(&SystemBackupPolicy{}, &SystemBackupRun{}, &SystemBackupRecoveryConfig{}))
+ crypto.InitEncryption(&crypto.Config{EncryptionKey: "system-backup-policy-test-key-32bytes", Environment: "test"})
+ service := &SystemBackupService{
+ db: &database.DB{DB: gormDB},
+ config: &config.Config{DatabaseURL: "file:system-backup-schedules-test.db"},
+ jobs: entityjobs.New("system-backup:", backup.SystemAdmissionScope),
+ }
+ scheduler := &systemBackupPolicySchedulerInternal{jobs: make(map[string]schedulertypes.Job)}
+ require.NoError(t, service.SetScheduler(context.Background(), scheduler, newSystemBackupAdmissionGateForTestInternal(t)))
+
+ status, err := service.SetRecoveryKey(context.Background(), "QWERTY-ABCDEF-234567-GHIJKL-MNOPQR-STUVWX-YZ2345-ZXCVBN")
+ require.NoError(t, err)
+ require.True(t, status.Configured)
+ storedKey, err := service.recoveryKeyInternal(context.Background(), "")
+ require.NoError(t, err)
+ require.Equal(t, "QWERTY-ABCDEF-234567-GHIJKL-MNOPQR-STUVWX-YZ2345-ZXCVBN", storedKey)
+
+ collection, err := service.UpdatePolicies(context.Background(), []backuptypes.UpdateSystemBackupPolicy{
+ {Enabled: true, Schedule: "0 0 2 * * *", RetentionCount: 5, LocalEnabled: true},
+ {Enabled: true, Schedule: "0 0 14 * * *", RetentionCount: 30, LocalEnabled: true},
+ })
+ require.NoError(t, err)
+ require.True(t, collection.RecoveryKeyStored)
+ require.Len(t, collection.Policies, 2)
+ require.Len(t, scheduler.jobs, 2)
+
+ firstID := collection.Policies[0].ID
+ collection, err = service.UpdatePolicies(context.Background(), []backuptypes.UpdateSystemBackupPolicy{{
+ ID: firstID, Enabled: true, Schedule: "0 */30 * * * *", RetentionCount: 9, LocalEnabled: true,
+ }})
+ require.NoError(t, err)
+ require.Len(t, collection.Policies, 1)
+ require.Equal(t, firstID, collection.Policies[0].ID)
+ require.Equal(t, 9, collection.Policies[0].RetentionCount)
+ require.Len(t, scheduler.jobs, 1)
+}
+
+func TestSystemBackupPolicyRequiresConfiguredRecoveryKeyWhenEnabled(t *testing.T) {
+ gormDB, err := gorm.Open(sqlite.Open("file:system-backup-key-required?mode=memory&cache=shared"), &gorm.Config{})
+ require.NoError(t, err)
+ require.NoError(t, gormDB.AutoMigrate(&SystemBackupPolicy{}, &SystemBackupRecoveryConfig{}))
+ service := &SystemBackupService{
+ db: &database.DB{DB: gormDB},
+ config: &config.Config{DatabaseURL: "file:system-backup-key-required-test.db"},
+ }
+
+ _, err = service.UpdatePolicies(context.Background(), []backuptypes.UpdateSystemBackupPolicy{{
+ Enabled: true, Schedule: "0 0 2 * * *", RetentionCount: 7, LocalEnabled: true,
+ }})
+ require.ErrorContains(t, err, "configure a recovery key")
+}
+
+func TestRecoveryEnvironmentInternalIncludesRuntimeSecrets(t *testing.T) {
+ cfg := &config.Config{
+ JWTSecret: "jwt-secret",
+ EncryptionKey: "encryption-secret",
+ AdminStaticAPIKey: "admin-key",
+ OidcClientSecret: "oidc-secret",
+ FilePerm: 0o640,
+ }
+ environment := (&SystemBackupService{config: cfg}).recoveryEnvironmentInternal()
+ require.Equal(t, "jwt-secret", environment["JWT_SECRET"])
+ require.Equal(t, "encryption-secret", environment["ENCRYPTION_KEY"])
+ require.Equal(t, "admin-key", environment["ADMIN_STATIC_API_KEY"])
+ require.Equal(t, "oidc-secret", environment["OIDC_CLIENT_SECRET"])
+ require.Equal(t, "0640", environment["FILE_PERM"])
+}
+
+func TestSystemSnapshotPathFromFilesInternal(t *testing.T) {
+ path, err := systemSnapshotPathFromFilesInternal([]string{"/arcane.db", "/.arcane-recovery.json"})
+ require.NoError(t, err)
+ require.Equal(t, "/", path)
+
+ path, err = systemSnapshotPathFromFilesInternal([]string{"/app/data/arcane.db", "/app/data/.arcane-recovery.json"})
+ require.NoError(t, err)
+ require.Equal(t, "/app/data", path)
+
+ _, err = systemSnapshotPathFromFilesInternal([]string{"/arcane.db"})
+ require.ErrorContains(t, err, "does not contain an Arcane recovery manifest")
+}
diff --git a/backend/internal/volume/backup.go b/backend/internal/volume/backup.go
index 161e1b1c4..61479710b 100644
--- a/backend/internal/volume/backup.go
+++ b/backend/internal/volume/backup.go
@@ -1,30 +1,34 @@
package volume
import (
- "github.com/getarcaneapp/arcane/backend/v2/internal/event"
-
- "github.com/getarcaneapp/arcane/backend/v2/internal/common"
-
- "github.com/getarcaneapp/arcane/backend/v2/internal/database"
-
"archive/tar"
"compress/gzip"
"context"
+ "crypto/sha256"
+ "encoding/hex"
"fmt"
"io"
"log/slog"
"path"
- "strconv"
"strings"
"time"
"uuid"
"emperror.dev/errors"
+ "gorm.io/gorm"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/backup"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/common"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/event"
+ s3domain "github.com/getarcaneapp/arcane/backend/v2/internal/s3"
docker "github.com/getarcaneapp/arcane/backend/v2/pkg/dockerutil"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"
+ activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/volumehelper"
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
+ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
+ volumetypes "github.com/getarcaneapp/arcane/types/v2/volume"
"github.com/moby/moby/api/types/container"
"github.com/moby/moby/api/types/mount"
"github.com/moby/moby/client"
@@ -45,6 +49,11 @@ const (
const backupMountMissingWarning = "No volume is mounted at /backups in the Arcane container. Backups will only live inside Docker unless you mount a host path."
+const (
+ volumeBackupContainerRecoveryTimeout = 30 * time.Second
+ volumeBackupContainerRecoveryInterval = 500 * time.Millisecond
+)
+
type backupStorageMountInternal struct {
mode backupStorageMode
mount mount.Mount
@@ -140,67 +149,13 @@ func (s *VolumeService) BackupMountWarning(ctx context.Context) string {
return backupMountWarningFromArcaneMountsInternal(inspect.Mounts)
}
-func (s *VolumeService) createBackupTempContainerWithMountInternal(ctx context.Context, dockerClient *client.Client, helperImage string, backupMount mount.Mount) (string, func(), error) {
- var err error
- if dockerClient == nil {
- dockerClient, err = s.dockerService.GetClient(ctx)
- if err != nil {
- return "", nil, err
- }
- }
-
- if strings.TrimSpace(helperImage) == "" {
- helperImage, err = getVolumeHelperImageInternal(ctx, s.dockerService, s.imageService, dockerClient)
- if err != nil {
- return "", nil, err
- }
- }
-
- config := &container.Config{
- Image: helperImage,
- Cmd: []string{"sleep", "infinity"},
- NetworkDisabled: true,
- Labels: volumehelper.Labels(),
- }
-
- hostConfig := volumehelper.HostConfig(helperImage, nil, []mount.Mount{backupMount})
-
- resp, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{
- Config: config,
- HostConfig: hostConfig,
- })
- if err != nil {
- return "", nil, errors.WrapIf(err, "failed to create backup temp container")
- }
-
- if _, err := dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); err != nil {
- _, _ = dockerClient.ContainerRemove(ctx, resp.ID, volumehelper.RemoveOptions())
- return "", nil, errors.WrapIf(err, "failed to start backup temp container")
- }
-
- cleanup := func() {
- _, _ = dockerClient.ContainerRemove(ctx, resp.ID, volumehelper.RemoveOptions())
- }
-
- return resp.ID, cleanup, nil
-}
-
-func (s *VolumeService) createBackupTempContainerInternal(ctx context.Context, dockerClient *client.Client, target string, readOnly bool) (string, func(), error) {
- slog.DebugContext(ctx, "volume service: create backup temp container", "target", target, "read_only", readOnly)
- var err error
- if dockerClient == nil {
- dockerClient, err = s.dockerService.GetClient(ctx)
- if err != nil {
- return "", nil, err
- }
- }
-
- backupStorage, err := s.resolveUsableBackupStorageMountInternal(ctx, dockerClient, target, readOnly)
+// BackupStorageMount resolves the repository mount shared with system-backup operations.
+func (s *VolumeService) BackupStorageMount(ctx context.Context, dockerClient *client.Client, target string, readOnly bool) (mount.Mount, error) {
+ storage, err := s.resolveUsableBackupStorageMountInternal(ctx, dockerClient, target, readOnly)
if err != nil {
- return "", nil, err
+ return mount.Mount{}, err
}
-
- return s.createBackupTempContainerWithMountInternal(ctx, dockerClient, "", backupStorage.mount)
+ return storage.mount, nil
}
func (s *VolumeService) ensureBackupVolumeInternal(ctx context.Context) error {
@@ -222,110 +177,145 @@ func (s *VolumeService) ensureBackupVolumeInternal(ctx context.Context) error {
return nil
}
-func (s *VolumeService) CreateBackup(ctx context.Context, volumeName string, user common.User) (*VolumeBackup, error) {
- slog.DebugContext(ctx, "volume service: create backup", "volume", volumeName, "user", user.ID)
- workspaceLock, _ := ctx.Value(volumeWorkspaceLockContextKeyInternal{}).(volumeWorkspaceLockContextInternal)
- if workspaceLock.service != s || workspaceLock.volumeName != volumeName {
- defer s.workspaceLocks.Lock(volumeName)()
- }
-
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return nil, err
- }
-
- backupID := fmt.Sprintf("%s-%d-%s", volumeName, time.Now().UnixNano(), uuid.New().String()[:8])
- filename, err := s.backupArchiveFilenameInternal(backupID)
- if err != nil {
- return nil, err
- }
-
- helperImage, err := getVolumeHelperImageInternal(ctx, s.dockerService, s.imageService, dockerClient)
- if err != nil {
- return nil, err
- }
-
- backupStorage, err := s.resolveUsableBackupStorageMountInternal(ctx, dockerClient, "/backups", false)
- if err != nil {
- return nil, err
- }
-
- config := &container.Config{
- Image: helperImage,
- Cmd: []string{"sh", "-c", fmt.Sprintf("tar -czf /backups/%s -C /volume .", filename)},
- Labels: volumehelper.Labels(),
+func (s *VolumeService) stopRunningContainersForBackupInternal(ctx context.Context, dockerClient *client.Client, volumeName string, user common.User, refuseArcaneWriters bool) ([]container.Summary, error) {
+ if s.containerService == nil {
+ return nil, errors.New("container service is unavailable")
}
-
- hostConfig := volumehelper.HostConfig(helperImage, []string{
- volumeName + ":/volume:ro",
- }, []mount.Mount{backupStorage.mount})
-
- resp, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{
- Config: config,
- HostConfig: hostConfig,
- })
+ containers, err := dockerClient.ContainerList(ctx, client.ContainerListOptions{})
if err != nil {
- return nil, errors.WrapIf(err, "failed to create backup container")
- }
-
- if _, err := dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); err != nil {
- _, _ = dockerClient.ContainerRemove(ctx, resp.ID, volumehelper.RemoveOptions())
- return nil, errors.WrapIf(err, "failed to start backup container")
+ return nil, fmt.Errorf("failed to list running containers before volume backup: %w", err)
}
- waitResult := dockerClient.ContainerWait(ctx, resp.ID, client.ContainerWaitOptions{Condition: container.WaitConditionNotRunning})
- select {
- case err := <-waitResult.Error:
- if err != nil {
- return nil, err
+ eligible := make([]container.Summary, 0, len(containers.Items))
+ arcaneOwned := make([]container.Summary, 0, 2)
+ for _, candidate := range containers.Items {
+ if strings.EqualFold(candidate.Labels["com.getarcaneapp.arcane"], "true") || strings.EqualFold(candidate.Labels["com.getarcaneapp.arcane.agent"], "true") {
+ arcaneOwned = append(arcaneOwned, candidate)
+ continue
}
- case status := <-waitResult.Result:
- if status.StatusCode != 0 {
- return nil, errors.Errorf("backup container exited with status %d", status.StatusCode)
+ // Arcane's own helper containers mount the volume as well, but they are
+ // auto-removed when stopped and recreated on demand. Stopping one here
+ // would leave nothing for the restart pass to find, failing an
+ // otherwise successful backup or restore.
+ if strings.EqualFold(candidate.Labels[libarcane.InternalResourceLabel], "true") {
+ continue
}
+ eligible = append(eligible, candidate)
}
-
- sizeCheckMount := backupStorage.mount
- sizeCheckMount.Target = "/volume"
- sizeCheckMount.ReadOnly = true
-
- tempContainerID, cleanup, err := s.createBackupTempContainerWithMountInternal(ctx, dockerClient, "", sizeCheckMount)
- if err != nil {
- return nil, err
- }
- defer cleanup()
-
- sizeStr, _, err := s.execInContainerInternal(ctx, tempContainerID, []string{"stat", "-c", "%s", path.Join("/volume", filename)})
- if err != nil {
- return nil, err
- }
- size, err := strconv.ParseInt(strings.TrimSpace(sizeStr), 10, 64)
- if err != nil {
- return nil, err
+ // Restores rewrite the volume in place; refusing beats corrupting the data
+ // under a labeled container this pass deliberately never stops.
+ if refuseArcaneWriters {
+ if writers := docker.FilterContainersUsingVolume(arcaneOwned, volumeName); len(writers) > 0 {
+ return nil, fmt.Errorf("volume %s is mounted by a running Arcane-managed container; restoring under it would corrupt its data", volumeName)
+ }
}
-
- backup := &VolumeBackup{
- VolumeName: volumeName,
- Size: size,
- CreatedAt: time.Now(),
+ containerIDs := docker.FilterContainersUsingVolume(eligible, volumeName)
+ containersByID := make(map[string]container.Summary, len(eligible))
+ for _, candidate := range eligible {
+ containersByID[candidate.ID] = candidate
+ }
+ stopped := make([]container.Summary, 0, len(containerIDs))
+ for _, containerID := range containerIDs {
+ candidate := containersByID[containerID]
+ if err := s.containerService.StopContainer(ctx, containerID, user); err != nil {
+ stillStopped, restartErr := s.startContainersAfterBackupInternal(context.WithoutCancel(ctx), dockerClient, stopped, user)
+ return stillStopped, errors.Combine(fmt.Errorf("failed to stop container %s before volume backup: %w", containerID, err), restartErr)
+ }
+ stopped = append(stopped, candidate)
}
- backup.ID = backupID
+ return stopped, nil
+}
- if err := s.db.WithContext(ctx).Create(backup).Error; err != nil {
- return nil, err
- }
+//nolint:gocognit // recovery retries must reconcile IDs, names, and Compose identities in one bounded loop
+func (s *VolumeService) startContainersAfterBackupInternal(ctx context.Context, dockerClient *client.Client, stoppedContainers []container.Summary, user common.User) ([]container.Summary, error) {
+ recoveryCtx, cancel := context.WithTimeout(ctx, volumeBackupContainerRecoveryTimeout)
+ defer cancel()
+
+ remaining := append([]container.Summary(nil), stoppedContainers...)
+ lastErrors := make(map[string]error, len(stoppedContainers))
+ for len(remaining) > 0 {
+ currentContainers, listErr := dockerClient.ContainerList(recoveryCtx, client.ContainerListOptions{All: true})
+ if listErr == nil {
+ currentByID := make(map[string]container.Summary, len(currentContainers.Items))
+ currentByName := make(map[string]container.Summary, len(currentContainers.Items))
+ for _, current := range currentContainers.Items {
+ currentByID[current.ID] = current
+ if name := docker.ContainerNameFromNames(current.Names); name != "" {
+ currentByName[name] = current
+ }
+ }
+
+ nextRemaining := make([]container.Summary, 0, len(remaining))
+ for _, stopped := range remaining {
+ current, found := currentByID[stopped.ID]
+ if !found {
+ name := docker.ContainerNameFromNames(stopped.Names)
+ current, found = currentByName[name]
+ }
+ if !found {
+ projectName := docker.ComposeProjectLabel(stopped.Labels)
+ serviceName := docker.ComposeServiceLabel(stopped.Labels)
+ containerNumber := strings.TrimSpace(stopped.Labels["com.docker.compose.container-number"])
+ if projectName != "" && serviceName != "" {
+ for _, candidate := range currentContainers.Items {
+ if docker.ComposeProjectLabel(candidate.Labels) != projectName || docker.ComposeServiceLabel(candidate.Labels) != serviceName {
+ continue
+ }
+ if containerNumber != "" && strings.TrimSpace(candidate.Labels["com.docker.compose.container-number"]) != containerNumber {
+ continue
+ }
+ current = candidate
+ found = true
+ break
+ }
+ }
+ }
+ if !found {
+ nextRemaining = append(nextRemaining, stopped)
+ continue
+ }
+ if current.State == container.StateRunning || current.State == container.StateRestarting {
+ if current.ID != stopped.ID {
+ slog.InfoContext(ctx, "volume service: container was replaced during backup and is already running", "previous_container", stopped.ID, "current_container", current.ID)
+ }
+ continue
+ }
+ if startErr := s.containerService.StartContainer(recoveryCtx, current.ID, user); startErr != nil {
+ lastErrors[stopped.ID] = startErr
+ nextRemaining = append(nextRemaining, stopped)
+ continue
+ }
+ if current.ID != stopped.ID {
+ slog.InfoContext(ctx, "volume service: restarted replacement container after backup", "previous_container", stopped.ID, "current_container", current.ID)
+ }
+ }
+ remaining = nextRemaining
+ } else {
+ for _, stopped := range remaining {
+ lastErrors[stopped.ID] = listErr
+ }
+ }
- metadata := database.JSON{
- "action": "backup_create",
- "backup_id": backup.ID,
- "filename": filename,
- "size": size,
- }
- if logErr := s.eventService.LogVolumeEvent(ctx, event.EventTypeVolumeBackupCreate, volumeName, volumeName, user.ID, user.Username, "0", metadata); logErr != nil {
- slog.WarnContext(ctx, "could not log volume backup create event", "volume", volumeName, "error", logErr.Error())
+ if len(remaining) == 0 {
+ return nil, nil
+ }
+ timer := time.NewTimer(volumeBackupContainerRecoveryInterval)
+ select {
+ case <-recoveryCtx.Done():
+ timer.Stop()
+ var restartErr error
+ for _, stopped := range remaining {
+ if lastErr := lastErrors[stopped.ID]; lastErr != nil {
+ restartErr = errors.Combine(restartErr, fmt.Errorf("failed to restart container %s after volume backup: %w", stopped.ID, lastErr))
+ } else {
+ restartErr = errors.Combine(restartErr, fmt.Errorf("failed to restart container %s after volume backup: replacement did not appear within %s", stopped.ID, volumeBackupContainerRecoveryTimeout))
+ }
+ }
+ return remaining, restartErr
+ case <-timer.C:
+ }
}
-
- return backup, nil
+ return nil, nil
}
func (s *VolumeService) ListBackupsPaginated(ctx context.Context, volumeName string, params pagination.QueryParams) ([]VolumeBackup, pagination.Response, error) {
@@ -334,7 +324,8 @@ func (s *VolumeService) ListBackupsPaginated(ctx context.Context, volumeName str
query := s.db.WithContext(ctx).Model(&VolumeBackup{}).Where("volume_name = ?", volumeName)
if params.Search != "" {
- query = query.Where("id LIKE ?", "%"+params.Search+"%")
+ pattern := "%" + params.Search + "%"
+ query = query.Where("id LIKE ? OR status LIKE ? OR trigger LIKE ? OR destination LIKE ? OR COALESCE(local_snapshot_id, '') LIKE ? OR COALESCE(remote_snapshot_id, '') LIKE ? OR COALESCE(error, '') LIKE ?", pattern, pattern, pattern, pattern, pattern, pattern, pattern)
}
var totalItems int64
@@ -352,6 +343,14 @@ func (s *VolumeService) ListBackupsPaginated(ctx context.Context, volumeName str
sortCol = "id"
case "size":
sortCol = "size"
+ case "status":
+ sortCol = "status"
+ case "trigger":
+ sortCol = "trigger"
+ case "destination":
+ sortCol = "destination"
+ case "remoteSnapshotId", "remote_snapshot_id":
+ sortCol = "remote_snapshot_id"
default:
sortCol = "created_at"
}
@@ -371,9 +370,30 @@ func (s *VolumeService) ListBackupsPaginated(ctx context.Context, volumeName str
if err := query.Find(&backups).Error; err != nil {
return nil, pagination.Response{}, err
}
+ hasS3Destination := false
+ for i := range backups {
+ if strings.TrimSpace(backups[i].S3DestinationID) != "" {
+ hasS3Destination = true
+ break
+ }
+ }
+ if s.s3Destinations != nil && hasS3Destination {
+ // Destination names are decoration; never fail the listing over them.
+ destinations, destinationErr := s.s3Destinations.ListAllS3Destinations(ctx)
+ if destinationErr != nil {
+ slog.WarnContext(ctx, "could not resolve volume backup S3 destination names", "volume", volumeName, "error", destinationErr)
+ } else {
+ destinationNames := make(map[string]string, len(destinations))
+ for _, destination := range destinations {
+ destinationNames[destination.ID] = destination.Name
+ }
+ for i := range backups {
+ backups[i].S3DestinationName = destinationNames[backups[i].S3DestinationID]
+ }
+ }
+ }
- paginationResp := pagination.BuildResponse(totalItems, totalItems, params)
- return backups, paginationResp, nil
+ return backups, pagination.BuildResponse(totalItems, totalItems, params), nil
}
func (s *VolumeService) ListBackups(ctx context.Context, volumeName string) ([]VolumeBackup, error) {
@@ -383,313 +403,856 @@ func (s *VolumeService) ListBackups(ctx context.Context, volumeName string) ([]V
return backups, err
}
-func (s *VolumeService) DeleteBackup(ctx context.Context, backupID string, user *common.User) error {
- slog.DebugContext(ctx, "volume service: delete backup", "backup_id", backupID)
- var backup VolumeBackup
- if err := s.db.WithContext(ctx).Where("id = ?", backupID).First(&backup).Error; err != nil {
- return err
- }
-
- // Delete from DB first - if this fails, no changes are made.
- // If file deletion fails afterward, we just have an orphan file (easier to clean up)
- // rather than an orphan DB record pointing to a non-existent file.
- volumeName := backup.VolumeName // Save before deletion
- if err := s.db.WithContext(ctx).Delete(&backup).Error; err != nil {
- return err
- }
-
- // Now delete the actual file - best effort since DB record is already gone
- containerID, cleanup, err := s.createBackupTempContainerInternal(ctx, nil, "/volume", false)
- if err != nil {
- slog.WarnContext(ctx, "failed to create container for backup file cleanup", "backup_id", backupID, "error", err.Error())
- } else {
- defer cleanup()
- filename, filenameErr := s.backupArchiveFilenameInternal(backupID)
- if filenameErr != nil {
- slog.WarnContext(ctx, "failed to sanitize backup id for file cleanup", "backup_id", backupID, "error", filenameErr.Error())
- } else if _, _, err = s.execInContainerInternal(ctx, containerID, []string{"rm", "-f", path.Join("/volume", filename)}); err != nil {
- slog.WarnContext(ctx, "failed to delete backup file (orphan file may remain)", "backup_id", backupID, "error", err.Error())
- }
+func (s *VolumeService) sanitizeBackupPathInternal(input string) (string, error) {
+ trimmed := strings.TrimSpace(input)
+ if trimmed == "" {
+ return "", errors.New("invalid path: empty")
}
-
- actingUser := user
- if actingUser == nil {
- actingUser = &common.SystemUser
+ cleaned := path.Clean(trimmed)
+ if cleaned == "." || cleaned == "/" {
+ return "", errors.Errorf("invalid path: %s", input)
}
- metadata := database.JSON{
- "action": "backup_delete",
- "backup_id": backupID,
+ if path.IsAbs(cleaned) {
+ cleaned = strings.TrimPrefix(cleaned, "/")
}
- if logErr := s.eventService.LogVolumeEvent(ctx, event.EventTypeVolumeBackupDelete, volumeName, volumeName, actingUser.ID, actingUser.Username, "0", metadata); logErr != nil {
- slog.WarnContext(ctx, "could not log volume backup delete event", "volume", volumeName, "error", logErr.Error())
+ if cleaned == "" || cleaned == "." || cleaned == "/" || strings.HasPrefix(cleaned, "..") || strings.Contains(cleaned, "/../") {
+ return "", errors.Errorf("invalid path: %s", input)
}
-
- return nil
+ return cleaned, nil
}
-func (s *VolumeService) RestoreBackup(ctx context.Context, volumeName, backupID string, user common.User) error {
- slog.DebugContext(ctx, "volume service: restore backup", "volume", volumeName, "backup_id", backupID, "user", user.ID)
- var backup VolumeBackup
- if err := s.db.WithContext(ctx).Where("id = ?", backupID).First(&backup).Error; err != nil {
- return err
+func (s *VolumeService) sanitizeBackupIDInternal(backupID string) (string, error) {
+ cleaned, err := s.sanitizeBackupPathInternal(backupID)
+ if err != nil {
+ return "", errors.WrapIf(err, "invalid backup id")
}
-
- // Validate backup belongs to volume
- if backup.VolumeName != volumeName {
- return errors.Errorf("backup does not belong to volume %s", volumeName)
+ if strings.Contains(cleaned, "/") {
+ return "", errors.New("invalid backup id: path separators not allowed")
}
- unlock := s.workspaceLocks.Lock(volumeName)
- defer unlock()
- ctx = context.WithValue(ctx, volumeWorkspaceLockContextKeyInternal{}, volumeWorkspaceLockContextInternal{
- service: s,
- volumeName: volumeName,
- })
+ return cleaned, nil
+}
- // Check if volume is in use by running containers
- inUse, containerIDs, err := s.GetVolumeUsage(ctx, volumeName)
+func (s *VolumeService) backupArchiveFilenameInternal(backupID string) (string, error) {
+ sanitizedBackupID, err := s.sanitizeBackupIDInternal(backupID)
if err != nil {
- slog.WarnContext(ctx, "could not check volume usage", "volume", volumeName, "error", err.Error())
- } else if inUse {
- return errors.Errorf("volume is in use by %d container(s): restoring while containers are running may cause data corruption. Stop the containers first or use selective file restore", len(containerIDs))
+ return "", err
}
+ return sanitizedBackupID + ".tar.gz", nil
+}
- preBackup, err := s.CreateBackup(ctx, volumeName, user)
- if err != nil {
- return errors.WrapIf(err, "failed to create pre-restore backup")
- }
+const (
+ volumeRusticRepositoryPath = "/repository/volumes"
+ localVolumeRepositoryID = "volumes:local"
+)
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return err
- }
+func (s *VolumeService) rusticPasswordInternal() string {
+ sum := sha256.Sum256([]byte("arcane-volume-backups:" + s.encryptionKey))
+ return hex.EncodeToString(sum[:])
+}
- filename, err := s.backupArchiveFilenameInternal(backupID)
+func (s *VolumeService) localRusticRepositoryInternal(ctx context.Context, dockerClient *client.Client, readOnly bool) (backup.Repository, error) {
+ storage, err := s.resolveUsableBackupStorageMountInternal(ctx, dockerClient, "/repository", readOnly)
if err != nil {
- return err
+ return backup.Repository{}, err
}
+ return backup.Repository{
+ ID: localVolumeRepositoryID,
+ Environment: []string{"RUSTIC_REPOSITORY=" + volumeRusticRepositoryPath},
+ Mounts: []mount.Mount{storage.mount},
+ }, nil
+}
- helperImage, err := getVolumeHelperImageInternal(ctx, s.dockerService, s.imageService, dockerClient)
- if err != nil {
- return err
+func (s *VolumeService) remoteRusticRepositoryInternal(ctx context.Context, destinationID string) (backup.Repository, error) {
+ if s.s3Destinations == nil {
+ return backup.Repository{}, errors.New("S3 backup service is unavailable")
}
-
- backupStorage, err := s.resolveUsableBackupStorageMountInternal(ctx, dockerClient, "/backups", true)
+ configuration, err := s.s3Destinations.Configuration(ctx, destinationID)
if err != nil {
- return err
+ return backup.Repository{}, errors.New("the selected S3 backup destination is not configured")
+ }
+ instanceID := strings.TrimSpace(s.settingsService.GetSettingsConfig().InstanceID.Value)
+ if instanceID == "" {
+ return backup.Repository{}, errors.New("arcane instance ID is unavailable")
}
+ return backup.Repository{
+ ID: "volumes:s3:" + destinationID,
+ Environment: configuration.RusticEnvironment("arcane-volume-backups", instanceID),
+ }, nil
+}
- config := &container.Config{
- Image: helperImage,
- Cmd: []string{
- "sh",
- "-c",
- fmt.Sprintf("set -e; tmp=$(mktemp -d /volume/.restore_tmp.XXXXXX); tar -tzf /backups/%s >/dev/null; tar -xzf /backups/%s -C \"$tmp\"; find /volume -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +; find \"$tmp\" -mindepth 1 -maxdepth 1 -exec mv -- {} /volume/ \\;; rmdir \"$tmp\"", filename, filename),
- },
- Labels: volumehelper.Labels(),
+func (s *VolumeService) rusticRepositoryForBackupInternal(ctx context.Context, dockerClient *client.Client, entry *VolumeBackup) (backup.Repository, string, error) {
+ if entry.LocalSnapshotID != "" {
+ repository, err := s.localRusticRepositoryInternal(ctx, dockerClient, true)
+ return repository, entry.LocalSnapshotID, err
}
+ if entry.RemoteSnapshotID != "" {
+ repository, err := s.remoteRusticRepositoryInternal(ctx, entry.S3DestinationID)
+ return repository, entry.RemoteSnapshotID, err
+ }
+ return backup.Repository{}, "", errors.New("volume backup has no Rustic snapshot")
+}
- hostConfig := volumehelper.HostConfig(helperImage, []string{
- volumeName + ":/volume",
- }, []mount.Mount{backupStorage.mount})
+func volumeSourceMountInternal(volumeName string) mount.Mount {
+ return mount.Mount{Type: mount.TypeVolume, Source: volumeName, Target: "/volume", ReadOnly: true}
+}
- resp, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{
- Config: config,
- HostConfig: hostConfig,
- })
- if err != nil {
- return errors.WrapIf(err, "failed to create restore container")
- }
+var ErrVolumeBackupAlreadyRunning = errors.New("a backup is already running for this volume")
- if _, err := dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); err != nil {
- _, _ = dockerClient.ContainerRemove(ctx, resp.ID, volumehelper.RemoveOptions())
- return errors.WrapIf(err, "failed to start restore container")
- }
+// backupPlanInternal is the resolved destination selection for one backup run.
+type backupPlanInternal struct {
+ policy *VolumeBackupPolicy
+ localEnabled bool
+ s3Enabled bool
+ s3DestinationID string
+ destination volumetypes.BackupDestination
+}
- waitResult := dockerClient.ContainerWait(ctx, resp.ID, client.ContainerWaitOptions{Condition: container.WaitConditionNotRunning})
- var waitBody container.WaitResponse
- select {
- case err := <-waitResult.Error:
+func (s *VolumeService) resolveBackupPlanInternal(ctx context.Context, volumeName string, trigger VolumeBackupTrigger, request volumetypes.CreateBackupRequest) (backupPlanInternal, error) {
+ destination := request.Destination
+ if destination != "" && destination != volumetypes.BackupDestinationLocal && destination != volumetypes.BackupDestinationS3 && destination != volumetypes.BackupDestinationLocalS3 {
+ return backupPlanInternal{}, errors.New("invalid volume backup destination")
+ }
+ var policy *VolumeBackupPolicy
+ if trigger != VolumeBackupTriggerSafety {
+ var err error
+ policy, err = s.loadVolumeBackupPolicyInternal(ctx, volumeName, request.PolicyID)
if err != nil {
- return err
+ return backupPlanInternal{}, err
}
- case waitBody = <-waitResult.Result:
+ if request.PolicyID != "" && policy == nil {
+ return backupPlanInternal{}, errors.New("volume backup policy not found")
+ }
+ }
+ localEnabled, s3Enabled, s3DestinationID := true, false, strings.TrimSpace(request.S3DestinationID)
+ if policy != nil {
+ localEnabled, s3Enabled, s3DestinationID = policy.LocalEnabled, policy.S3Enabled, policy.S3DestinationID
+ }
+ if trigger == VolumeBackupTriggerSafety {
+ localEnabled, s3Enabled = true, false
+ } else if destination != "" {
+ localEnabled = destination != volumetypes.BackupDestinationS3
+ s3Enabled = destination != volumetypes.BackupDestinationLocal
+ }
+ if !localEnabled && !s3Enabled {
+ return backupPlanInternal{}, errors.New("select at least one volume backup destination")
+ }
+ if s3Enabled && strings.TrimSpace(s3DestinationID) == "" {
+ return backupPlanInternal{}, errors.New("select an S3 destination for the volume backup")
+ }
+ if s3Enabled {
+ if s.s3Destinations == nil {
+ return backupPlanInternal{}, errors.New("S3 backup destinations are unavailable")
+ }
+ if _, err := s.s3Destinations.Configuration(ctx, s3DestinationID); err != nil {
+ return backupPlanInternal{}, errors.New("select a valid S3 destination for the volume backup")
+ }
+ } else {
+ s3DestinationID = ""
+ }
+ switch {
+ case localEnabled && s3Enabled:
+ destination = volumetypes.BackupDestinationLocalS3
+ case s3Enabled:
+ destination = volumetypes.BackupDestinationS3
+ default:
+ destination = volumetypes.BackupDestinationLocal
}
+ return backupPlanInternal{policy: policy, localEnabled: localEnabled, s3Enabled: s3Enabled, s3DestinationID: s3DestinationID, destination: destination}, nil
+}
- if waitBody.StatusCode != 0 {
- return errors.Errorf("restore container exited with code %d (volume may be partially wiped)", waitBody.StatusCode)
+//nolint:gocognit,gocyclo // backup orchestration keeps cleanup and persistence transitions in one transaction-like flow
+func (s *VolumeService) CreateBackup(ctx context.Context, volumeName string, user common.User, trigger VolumeBackupTrigger, request volumetypes.CreateBackupRequest) (_ *VolumeBackup, err error) {
+ workspaceLock, _ := ctx.Value(volumeWorkspaceLockContextKeyInternal{}).(volumeWorkspaceLockContextInternal)
+ if workspaceLock.service != s || workspaceLock.volumeName != volumeName {
+ defer s.workspaceLocks.Lock(volumeName)()
+ }
+ if trigger == "" {
+ trigger = VolumeBackupTriggerManual
+ }
+ plan, err := s.resolveBackupPlanInternal(ctx, volumeName, trigger, request)
+ if err != nil {
+ return nil, err
+ }
+ lease, admitted, leaseErr := s.engine.TryAcquireRun(ctx, backup.VolumeAdmissionScope, volumeName)
+ if leaseErr != nil {
+ return nil, leaseErr
}
+ if !admitted {
+ return nil, ErrVolumeBackupAlreadyRunning
+ }
+ defer lease.Release()
+ entry := &VolumeBackup{
+ VolumeName: volumeName, CreatedAt: time.Now(), Status: VolumeBackupStatusRunning,
+ Trigger: trigger, Destination: plan.destination, Format: VolumeBackupFormatRustic,
+ S3DestinationID: plan.s3DestinationID, PolicyID: request.PolicyID,
+ }
+ entry.ID = fmt.Sprintf("%s-%d-%s", volumeName, time.Now().UnixNano(), uuid.New().String()[:8])
+ if err := s.db.WithContext(ctx).Create(entry).Error; err != nil {
+ return nil, err
+ }
+ defer func() {
+ if err != nil {
+ entry.Status, entry.Error = VolumeBackupStatusFailed, err.Error()
+ } else {
+ entry.Status, entry.Error = VolumeBackupStatusSucceeded, ""
+ }
+ if saveErr := s.db.WithContext(context.WithoutCancel(ctx)).Save(entry).Error; saveErr != nil {
+ err = errors.Combine(err, fmt.Errorf("failed to save volume backup result: %w", saveErr))
+ }
+ }()
+ dockerClient, err := s.dockerService.GetClient(ctx)
+ if err != nil {
+ return entry, err
+ }
+ var stopped []container.Summary
+ containersStopped := false
+ if plan.policy != nil && plan.policy.StopContainers {
+ stopped, err = s.stopRunningContainersForBackupInternal(ctx, dockerClient, volumeName, user, false)
+ containersStopped = len(stopped) > 0
+ defer func() {
+ if containersStopped {
+ _, restartErr := s.startContainersAfterBackupInternal(context.WithoutCancel(ctx), dockerClient, stopped, user)
+ err = errors.Combine(err, restartErr)
+ }
+ }()
+ if err != nil {
+ return entry, err
+ }
+ }
+ // The live volume is read once. A local+S3 backup replicates the local
+ // snapshot to S3 instead of scanning the source a second time.
+ var localSnapshot backup.Snapshot
+ if plan.localEnabled {
+ repository, repoErr := s.localRusticRepositoryInternal(ctx, dockerClient, false)
+ if repoErr != nil {
+ return entry, repoErr
+ }
+ localSnapshot, err = s.engine.CreateSnapshot(ctx, dockerClient, repository, s.rusticPasswordInternal(), volumeName, volumeSourceMountInternal(volumeName))
+ if err != nil {
+ return entry, fmt.Errorf("failed to create local Rustic snapshot: %w", err)
+ }
+ entry.LocalSnapshotID = localSnapshot.ID
+ entry.Size = localSnapshot.Size
+ }
+ if plan.s3Enabled {
+ remoteRepository, repoErr := s.remoteRusticRepositoryInternal(ctx, plan.s3DestinationID)
+ if repoErr != nil {
+ return entry, repoErr
+ }
+ var remoteSnapshot backup.Snapshot
+ if plan.localEnabled {
+ localRepository, localErr := s.localRusticRepositoryInternal(ctx, dockerClient, true)
+ if localErr != nil {
+ return entry, localErr
+ }
+ remoteSnapshot, err = s.engine.Replicate(ctx, dockerClient, localRepository, localSnapshot.ID, remoteRepository, s.rusticPasswordInternal(), volumeName)
+ } else {
+ remoteSnapshot, err = s.engine.CreateSnapshot(ctx, dockerClient, remoteRepository, s.rusticPasswordInternal(), volumeName, volumeSourceMountInternal(volumeName))
+ }
+ if err != nil {
+ return entry, fmt.Errorf("failed to create S3 Rustic snapshot: %w", err)
+ }
+ entry.RemoteSnapshotID = remoteSnapshot.ID
+ if entry.Size == 0 {
+ entry.Size = remoteSnapshot.Size
+ }
+ }
+ if containersStopped {
+ stopped, err = s.startContainersAfterBackupInternal(context.WithoutCancel(ctx), dockerClient, stopped, user)
+ containersStopped = len(stopped) > 0
+ if err != nil {
+ return entry, err
+ }
+ }
+ // Retention failures must not fail the run: the backup itself succeeded,
+ // and the deferred status-save would otherwise mark it failed and hide it
+ // from ExpiredRunIDs forever.
+ if trigger != VolumeBackupTriggerSafety && plan.policy != nil && plan.policy.RetentionCount > 0 {
+ if retentionErr := s.applyVolumeBackupRetentionInternal(ctx, plan.policy.ID, plan.policy.RetentionCount); retentionErr != nil {
+ slog.ErrorContext(ctx, "Volume backup retention failed", "volume", volumeName, "policy", plan.policy.ID, "error", retentionErr)
+ }
+ }
+ metadata := database.JSON{"action": "backup_create", "backup_id": entry.ID, "size": entry.Size, "destination": entry.Destination}
+ if logErr := s.eventService.LogVolumeEvent(ctx, event.EventTypeVolumeBackupCreate, volumeName, volumeName, user.ID, user.Username, "0", metadata); logErr != nil {
+ slog.WarnContext(ctx, "could not log volume backup create event", "volume", volumeName, "error", logErr)
+ }
+ return entry, nil
+}
- metadata := database.JSON{
- "action": "backup_restore",
- "backup_id": backupID,
- "pre_restore_backupId": preBackup.ID,
+func (s *VolumeService) UploadBackup(ctx context.Context, backupID, s3DestinationID string) (*VolumeBackup, error) {
+ var entry VolumeBackup
+ if err := s.db.WithContext(ctx).Where("id = ?", backupID).First(&entry).Error; err != nil {
+ return nil, err
}
- if logErr := s.eventService.LogVolumeEvent(ctx, event.EventTypeVolumeBackupRestore, volumeName, volumeName, user.ID, user.Username, "0", metadata); logErr != nil {
- slog.WarnContext(ctx, "could not log volume backup restore event", "volume", volumeName, "error", logErr.Error())
+ lease, admitted, err := s.engine.TryAcquireRun(ctx, backup.VolumeAdmissionScope, entry.VolumeName)
+ if err != nil {
+ return nil, err
+ }
+ if !admitted {
+ return nil, ErrVolumeBackupAlreadyRunning
+ }
+ defer lease.Release()
+ // Reload under the lease: a delete may have raced the first read.
+ if err := s.db.WithContext(ctx).Where("id = ?", backupID).First(&entry).Error; err != nil {
+ return nil, err
+ }
+ if entry.Status != VolumeBackupStatusSucceeded || entry.LocalSnapshotID == "" {
+ return nil, errors.New("only successful local volume backups can be uploaded")
+ }
+ if entry.RemoteSnapshotID != "" {
+ return nil, errors.New("volume backup has already been uploaded")
+ }
+ if strings.TrimSpace(s3DestinationID) == "" {
+ return nil, errors.New("select an S3 destination for the upload")
+ }
+ if s.s3Destinations == nil {
+ return nil, errors.New("S3 backup destinations are unavailable")
+ }
+ if _, err := s.s3Destinations.Configuration(ctx, s3DestinationID); err != nil {
+ return nil, errors.New("select a valid S3 destination for the upload")
+ }
+ dockerClient, err := s.dockerService.GetClient(ctx)
+ if err != nil {
+ return nil, err
+ }
+ localRepository, err := s.localRusticRepositoryInternal(ctx, dockerClient, true)
+ if err != nil {
+ return nil, err
+ }
+ remoteRepository, err := s.remoteRusticRepositoryInternal(ctx, s3DestinationID)
+ if err != nil {
+ return nil, err
+ }
+ snapshot, err := s.engine.Replicate(ctx, dockerClient, localRepository, entry.LocalSnapshotID, remoteRepository, s.rusticPasswordInternal(), entry.VolumeName)
+ if err != nil {
+ return nil, fmt.Errorf("failed to upload Rustic snapshot to S3: %w", err)
+ }
+ entry.RemoteSnapshotID = snapshot.ID
+ entry.S3DestinationID = s3DestinationID
+ entry.Destination = volumetypes.BackupDestinationLocalS3
+ if err := s.db.WithContext(ctx).Save(&entry).Error; err != nil {
+ return nil, fmt.Errorf("failed to save uploaded volume backup: %w", err)
+ }
+ return &entry, nil
+}
+
+func (s *VolumeService) DeleteBackup(ctx context.Context, backupID string, user *common.User) error {
+ var entry VolumeBackup
+ if err := s.db.WithContext(ctx).Where("id = ?", backupID).First(&entry).Error; err != nil {
+ return err
+ }
+ // Deletes contend with create/restore/upload on the volume's run lease so
+ // a slow operation cannot resurrect or double-forget snapshots. Retention
+ // deletes run under CreateBackup's lease and call the internal path.
+ lease, admitted, err := s.engine.TryAcquireRun(ctx, backup.VolumeAdmissionScope, entry.VolumeName)
+ if err != nil {
+ return err
+ }
+ if !admitted {
+ return ErrVolumeBackupAlreadyRunning
}
+ defer lease.Release()
+ return s.deleteBackupInternal(ctx, backupID, user)
+}
+func (s *VolumeService) deleteBackupInternal(ctx context.Context, backupID string, user *common.User) error {
+ var entry VolumeBackup
+ if err := s.db.WithContext(ctx).Where("id = ?", backupID).First(&entry).Error; err != nil {
+ return err
+ }
+ if entry.Format == VolumeBackupFormatArchive {
+ return s.deleteArchiveBackupInternal(ctx, &entry, user)
+ }
+ dockerClient, err := s.dockerService.GetClient(ctx)
+ if err != nil {
+ return err
+ }
+ var deleteErr error
+ if entry.LocalSnapshotID != "" {
+ repository, repoErr := s.localRusticRepositoryInternal(ctx, dockerClient, false)
+ if repoErr == nil {
+ repoErr = s.engine.ForgetSnapshot(ctx, dockerClient, repository, s.rusticPasswordInternal(), entry.LocalSnapshotID)
+ }
+ if repoErr != nil {
+ deleteErr = errors.Combine(deleteErr, fmt.Errorf("failed to delete local Rustic snapshot: %w", repoErr))
+ } else {
+ entry.LocalSnapshotID = ""
+ }
+ }
+ if entry.RemoteSnapshotID != "" {
+ repository, repoErr := s.remoteRusticRepositoryInternal(ctx, entry.S3DestinationID)
+ if repoErr == nil {
+ repoErr = s.engine.ForgetSnapshot(ctx, dockerClient, repository, s.rusticPasswordInternal(), entry.RemoteSnapshotID)
+ }
+ if repoErr != nil {
+ deleteErr = errors.Combine(deleteErr, fmt.Errorf("failed to delete S3 Rustic snapshot: %w", repoErr))
+ } else {
+ entry.RemoteSnapshotID = ""
+ entry.S3DestinationID = ""
+ }
+ }
+ if deleteErr != nil {
+ switch {
+ case entry.LocalSnapshotID != "" && entry.RemoteSnapshotID != "":
+ entry.Destination = volumetypes.BackupDestinationLocalS3
+ case entry.LocalSnapshotID != "":
+ entry.Destination = volumetypes.BackupDestinationLocal
+ case entry.RemoteSnapshotID != "":
+ entry.Destination = volumetypes.BackupDestinationS3
+ }
+ entry.Error = deleteErr.Error()
+ if saveErr := s.db.WithContext(ctx).Save(&entry).Error; saveErr != nil {
+ deleteErr = errors.Combine(deleteErr, saveErr)
+ }
+ return deleteErr
+ }
+ if err := s.db.WithContext(ctx).Delete(&entry).Error; err != nil {
+ return fmt.Errorf("failed to delete volume backup record: %w", err)
+ }
+ s.logBackupDeleteEventInternal(ctx, entry.VolumeName, backupID, user)
return nil
}
-func (s *VolumeService) sanitizeBackupPathInternal(input string) (string, error) {
- trimmed := strings.TrimSpace(input)
- if trimmed == "" {
- return "", errors.New("invalid path: empty")
+func (s *VolumeService) logBackupDeleteEventInternal(ctx context.Context, volumeName, backupID string, user *common.User) {
+ actingUser := user
+ if actingUser == nil {
+ actingUser = &common.SystemUser
}
- cleaned := path.Clean(trimmed)
- if cleaned == "." || cleaned == "/" {
- return "", errors.Errorf("invalid path: %s", input)
+ if logErr := s.eventService.LogVolumeEvent(ctx, event.EventTypeVolumeBackupDelete, volumeName, volumeName, actingUser.ID, actingUser.Username, "0", database.JSON{"action": "backup_delete", "backup_id": backupID}); logErr != nil {
+ slog.WarnContext(ctx, "could not log volume backup delete event", "volume", volumeName, "error", logErr)
}
- if path.IsAbs(cleaned) {
- cleaned = strings.TrimPrefix(cleaned, "/")
+}
+
+func (s *VolumeService) RestoreBackup(ctx context.Context, volumeName, backupID string, user common.User) (err error) {
+ var entry VolumeBackup
+ if err := s.db.WithContext(ctx).Where("id = ?", backupID).First(&entry).Error; err != nil {
+ return err
}
- if cleaned == "" || cleaned == "." || cleaned == "/" || strings.HasPrefix(cleaned, "..") || strings.Contains(cleaned, "/../") {
- return "", errors.Errorf("invalid path: %s", input)
+ if entry.VolumeName != volumeName {
+ return errors.New("backup does not belong to volume")
}
- return cleaned, nil
+ unlock := s.workspaceLocks.Lock(volumeName)
+ defer unlock()
+ ctx = context.WithValue(ctx, volumeWorkspaceLockContextKeyInternal{}, volumeWorkspaceLockContextInternal{
+ service: s,
+ volumeName: volumeName,
+ })
+ dockerClient, err := s.dockerService.GetClient(ctx)
+ if err != nil {
+ return err
+ }
+ var repository backup.Repository
+ var snapshotID string
+ if entry.Format != VolumeBackupFormatArchive {
+ repository, snapshotID, err = s.rusticRepositoryForBackupInternal(ctx, dockerClient, &entry)
+ if err != nil {
+ return err
+ }
+ }
+ stopped, err := s.stopRunningContainersForBackupInternal(ctx, dockerClient, volumeName, user, true)
+ containersStopped := len(stopped) > 0
+ defer func() {
+ if containersStopped {
+ _, restartErr := s.startContainersAfterBackupInternal(context.WithoutCancel(ctx), dockerClient, stopped, user)
+ err = errors.Combine(err, restartErr)
+ }
+ }()
+ if err != nil {
+ return err
+ }
+ preBackup, err := s.CreateBackup(ctx, volumeName, user, VolumeBackupTriggerSafety, volumetypes.CreateBackupRequest{Destination: volumetypes.BackupDestinationLocal})
+ if err != nil {
+ return fmt.Errorf("failed to create pre-restore backup: %w", err)
+ }
+ if entry.Format == VolumeBackupFormatArchive {
+ if err := s.restoreArchiveBackupInternal(ctx, dockerClient, volumeName, backupID); err != nil {
+ return err
+ }
+ } else if err := s.engine.RestoreSnapshot(ctx, dockerClient, repository, s.rusticPasswordInternal(), snapshotID, mount.Mount{Type: mount.TypeVolume, Source: volumeName, Target: "/volume"}, backup.RestoreOptions{DeleteExtra: true}); err != nil {
+ return fmt.Errorf("failed to restore Rustic snapshot: %w", err)
+ }
+ if containersStopped {
+ stopped, err = s.startContainersAfterBackupInternal(context.WithoutCancel(ctx), dockerClient, stopped, user)
+ containersStopped = len(stopped) > 0
+ if err != nil {
+ return err
+ }
+ }
+ metadata := database.JSON{"action": "backup_restore", "backup_id": backupID, "pre_restore_backupId": preBackup.ID}
+ if logErr := s.eventService.LogVolumeEvent(ctx, event.EventTypeVolumeBackupRestore, volumeName, volumeName, user.ID, user.Username, "0", metadata); logErr != nil {
+ slog.WarnContext(ctx, "could not log volume backup restore event", "volume", volumeName, "error", logErr)
+ }
+ return nil
}
-func (s *VolumeService) sanitizeBackupIDInternal(backupID string) (string, error) {
- cleaned, err := s.sanitizeBackupPathInternal(backupID)
+func (s *VolumeService) ListBackupFiles(ctx context.Context, backupID string) ([]string, error) {
+ var entry VolumeBackup
+ if err := s.db.WithContext(ctx).Where("id = ?", backupID).First(&entry).Error; err != nil {
+ return nil, err
+ }
+ if entry.Format == VolumeBackupFormatArchive {
+ return s.listArchiveBackupFilesInternal(ctx, backupID)
+ }
+ dockerClient, err := s.dockerService.GetClient(ctx)
if err != nil {
- return "", errors.WrapIf(err, "invalid backup id")
+ return nil, err
}
- if strings.Contains(cleaned, "/") {
- return "", errors.New("invalid backup id: path separators not allowed")
+ repository, snapshotID, err := s.rusticRepositoryForBackupInternal(ctx, dockerClient, &entry)
+ if err != nil {
+ return nil, err
}
- return cleaned, nil
+ return s.engine.ListSnapshotFiles(ctx, dockerClient, repository, s.rusticPasswordInternal(), snapshotID)
}
-func (s *VolumeService) backupArchiveFilenameInternal(backupID string) (string, error) {
- sanitizedBackupID, err := s.sanitizeBackupIDInternal(backupID)
+func (s *VolumeService) BackupHasPath(ctx context.Context, backupID, filePath string) (bool, error) {
+ cleaned, err := s.sanitizeBackupPathInternal(filePath)
if err != nil {
- return "", err
+ return false, err
}
+ files, err := s.ListBackupFiles(ctx, backupID)
+ if err != nil {
+ return false, err
+ }
+ for _, candidate := range files {
+ if strings.TrimPrefix(candidate, "/") == cleaned {
+ return true, nil
+ }
+ }
+ return false, nil
+}
- return sanitizedBackupID + ".tar.gz", nil
+func (s *VolumeService) RestoreBackupFiles(ctx context.Context, volumeName, backupID string, paths []string, user common.User) (err error) {
+ if len(paths) == 0 {
+ return errors.New("no paths provided")
+ }
+ var entry VolumeBackup
+ if err := s.db.WithContext(ctx).Where("id = ?", backupID).First(&entry).Error; err != nil {
+ return err
+ }
+ if entry.VolumeName != volumeName {
+ return errors.New("backup does not belong to volume")
+ }
+ unlock := s.workspaceLocks.Lock(volumeName)
+ defer unlock()
+ ctx = context.WithValue(ctx, volumeWorkspaceLockContextKeyInternal{}, volumeWorkspaceLockContextInternal{
+ service: s,
+ volumeName: volumeName,
+ })
+ cleanedPaths := make([]string, 0, len(paths))
+ for _, requestedPath := range paths {
+ cleaned, cleanErr := s.sanitizeBackupPathInternal(requestedPath)
+ if cleanErr != nil {
+ return cleanErr
+ }
+ cleanedPaths = append(cleanedPaths, cleaned)
+ }
+ dockerClient, err := s.dockerService.GetClient(ctx)
+ if err != nil {
+ return err
+ }
+ var repository backup.Repository
+ var snapshotID string
+ if entry.Format != VolumeBackupFormatArchive {
+ repository, snapshotID, err = s.rusticRepositoryForBackupInternal(ctx, dockerClient, &entry)
+ if err != nil {
+ return err
+ }
+ }
+ stopped, err := s.stopRunningContainersForBackupInternal(ctx, dockerClient, volumeName, user, true)
+ containersStopped := len(stopped) > 0
+ defer func() {
+ if containersStopped {
+ _, restartErr := s.startContainersAfterBackupInternal(context.WithoutCancel(ctx), dockerClient, stopped, user)
+ err = errors.Combine(err, restartErr)
+ }
+ }()
+ if err != nil {
+ return err
+ }
+ preBackup, err := s.CreateBackup(ctx, volumeName, user, VolumeBackupTriggerSafety, volumetypes.CreateBackupRequest{Destination: volumetypes.BackupDestinationLocal})
+ if err != nil {
+ return fmt.Errorf("failed to create pre-restore backup: %w", err)
+ }
+ if entry.Format == VolumeBackupFormatArchive {
+ if err := s.restoreArchiveBackupFilesInternal(ctx, dockerClient, volumeName, backupID, cleanedPaths); err != nil {
+ return err
+ }
+ } else {
+ for _, cleaned := range cleanedPaths {
+ if err := s.engine.RestoreSnapshot(ctx, dockerClient, repository, s.rusticPasswordInternal(), snapshotID, mount.Mount{Type: mount.TypeVolume, Source: volumeName, Target: "/volume"}, backup.RestoreOptions{SourcePath: cleaned, DestinationPath: path.Join("/volume", cleaned)}); err != nil {
+ return fmt.Errorf("failed to restore %s from Rustic snapshot: %w", cleaned, err)
+ }
+ }
+ }
+ if containersStopped {
+ stopped, err = s.startContainersAfterBackupInternal(context.WithoutCancel(ctx), dockerClient, stopped, user)
+ containersStopped = len(stopped) > 0
+ if err != nil {
+ return err
+ }
+ }
+ metadata := database.JSON{"action": "backup_restore_files", "backup_id": backupID, "pre_restore_backupId": preBackup.ID, "paths_count": len(cleanedPaths)}
+ if logErr := s.eventService.LogVolumeEvent(ctx, event.EventTypeVolumeBackupRestoreFiles, volumeName, volumeName, user.ID, user.Username, "0", metadata); logErr != nil {
+ slog.WarnContext(ctx, "could not log volume backup restore files event", "volume", volumeName, "error", logErr)
+ }
+ return nil
}
-func (s *VolumeService) restoreBackupFilesInContainerInternal(ctx context.Context, containerID, filename string, cleanedPaths []string) (string, error) {
- args := make([]string, 0, len(cleanedPaths)+5)
- args = append(args, "sh", "-c", restoreBackupFilesScriptInternal, "sh", path.Join("/backups", filename))
- for _, cleaned := range cleanedPaths {
- args = append(args, "./"+cleaned)
+// DownloadBackup streams a backup as a tar.gz archive. Legacy archive rows
+// stream their stored file; local Rustic rows are restored into a scratch
+// volume and packaged on the fly.
+func (s *VolumeService) DownloadBackup(ctx context.Context, backupID string, user *common.User) (io.ReadCloser, int64, error) {
+ var entry VolumeBackup
+ if err := s.db.WithContext(ctx).Where("id = ?", backupID).First(&entry).Error; err != nil {
+ return nil, 0, err
}
- _, stderr, err := s.execInContainerInternal(ctx, containerID, args)
- return stderr, errors.WrapIf(err, "failed to restore files")
+ var reader io.ReadCloser
+ var size int64
+ var err error
+ if entry.Format == VolumeBackupFormatArchive {
+ reader, size, err = s.downloadArchiveBackupInternal(ctx, backupID)
+ } else {
+ reader, size, err = s.downloadRusticBackupInternal(ctx, &entry)
+ }
+ if err != nil {
+ return nil, 0, err
+ }
+ actingUser := user
+ if actingUser == nil {
+ actingUser = &common.SystemUser
+ }
+ metadata := database.JSON{"action": "backup_download", "backup_id": backupID, "size": size}
+ if logErr := s.eventService.LogVolumeEvent(ctx, event.EventTypeVolumeBackupDownload, entry.VolumeName, entry.VolumeName, actingUser.ID, actingUser.Username, "0", metadata); logErr != nil {
+ slog.WarnContext(ctx, "could not log volume backup download event", "volume", entry.VolumeName, "error", logErr)
+ }
+ return reader, size, nil
}
-const restoreBackupFilesScriptInternal = `set -e
-archive="$1"
-shift
-archive_mode=$(stat -c '%A' -- "$archive" 2>/dev/null) || { echo ARCANE_NOT_FOUND >&2; exit 44; }
-case "$archive_mode" in -*) ;; *) echo ARCANE_NOT_FOUND >&2; exit 44 ;; esac
-for member do
- if ! tar -tzf "$archive" -- "$member" >/dev/null 2>&1; then echo ARCANE_NOT_FOUND >&2; exit 44; fi
-done
-tar -xzf "$archive" -C /volume -- "$@"`
+func (s *VolumeService) downloadRusticBackupInternal(ctx context.Context, entry *VolumeBackup) (io.ReadCloser, int64, error) {
+ if entry.LocalSnapshotID == "" {
+ return nil, 0, errors.New("only local volume backups can be downloaded")
+ }
+ dockerClient, err := s.dockerService.GetClient(ctx)
+ if err != nil {
+ return nil, 0, err
+ }
+ repository, err := s.localRusticRepositoryInternal(ctx, dockerClient, true)
+ if err != nil {
+ return nil, 0, err
+ }
+ scratchVolume := "arcane-rustic-download-" + uuid.New().String()
+ if _, err := dockerClient.VolumeCreate(ctx, client.VolumeCreateOptions{Name: scratchVolume, Labels: volumehelper.Labels()}); err != nil {
+ return nil, 0, fmt.Errorf("failed to create download scratch volume: %w", err)
+ }
+ removeScratch := func() {
+ _, _ = dockerClient.VolumeRemove(context.WithoutCancel(ctx), scratchVolume, client.VolumeRemoveOptions{Force: true})
+ }
+ if err := s.engine.RestoreSnapshot(ctx, dockerClient, repository, s.rusticPasswordInternal(), entry.LocalSnapshotID, mount.Mount{Type: mount.TypeVolume, Source: scratchVolume, Target: "/volume"}, backup.RestoreOptions{DeleteExtra: true}); err != nil {
+ removeScratch()
+ return nil, 0, fmt.Errorf("failed to restore Rustic snapshot for download: %w", err)
+ }
+ helperImage, err := getVolumeHelperImageInternal(ctx, s.dockerService, s.imageService, dockerClient)
+ if err != nil {
+ removeScratch()
+ return nil, 0, err
+ }
+ archiveMount := mount.Mount{Type: mount.TypeVolume, Source: scratchVolume, Target: "/volume", ReadOnly: false}
+ containerID, removeContainer, err := s.createBackupTempContainerWithMountInternal(ctx, dockerClient, helperImage, archiveMount)
+ if err != nil {
+ removeScratch()
+ return nil, 0, err
+ }
+ cleanup := func() {
+ removeContainer()
+ removeScratch()
+ }
+ archivePath := "/tmp/" + entry.ID + ".tar.gz"
+ if _, _, err := s.execInContainerInternal(ctx, containerID, []string{"tar", "-czf", archivePath, "-C", "/volume", "."}); err != nil {
+ cleanup()
+ return nil, 0, fmt.Errorf("failed to package Rustic snapshot for download: %w", err)
+ }
+ return volumehelper.DownloadFileFromContainer(ctx, dockerClient, containerID, archivePath, cleanup)
+}
-func (s *VolumeService) BackupHasPath(ctx context.Context, backupID string, filePath string) (bool, error) {
- slog.DebugContext(ctx, "volume service: backup has path", "backup_id", backupID, "path", filePath)
- cleaned, err := s.sanitizeBackupPathInternal(filePath)
+func (s *VolumeService) createBackupTempContainerWithMountInternal(ctx context.Context, dockerClient *client.Client, helperImage string, backupMount mount.Mount) (string, func(), error) {
+ var err error
+ if dockerClient == nil {
+ dockerClient, err = s.dockerService.GetClient(ctx)
+ if err != nil {
+ return "", nil, err
+ }
+ }
+
+ if strings.TrimSpace(helperImage) == "" {
+ helperImage, err = getVolumeHelperImageInternal(ctx, s.dockerService, s.imageService, dockerClient)
+ if err != nil {
+ return "", nil, err
+ }
+ }
+
+ config := &container.Config{
+ Image: helperImage,
+ Cmd: []string{"sleep", "infinity"},
+ NetworkDisabled: true,
+ Labels: volumehelper.Labels(),
+ }
+
+ hostConfig := volumehelper.HostConfig(helperImage, nil, []mount.Mount{backupMount})
+
+ resp, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{
+ Config: config,
+ HostConfig: hostConfig,
+ })
+ if err != nil {
+ return "", nil, errors.WrapIf(err, "failed to create backup temp container")
+ }
+
+ if _, err := dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); err != nil {
+ _, _ = dockerClient.ContainerRemove(ctx, resp.ID, volumehelper.RemoveOptions())
+ return "", nil, errors.WrapIf(err, "failed to start backup temp container")
+ }
+
+ cleanup := func() {
+ _, _ = dockerClient.ContainerRemove(context.WithoutCancel(ctx), resp.ID, volumehelper.RemoveOptions())
+ }
+
+ return resp.ID, cleanup, nil
+}
+
+func (s *VolumeService) createBackupTempContainerInternal(ctx context.Context, dockerClient *client.Client, target string, readOnly bool) (string, func(), error) {
+ slog.DebugContext(ctx, "volume service: create backup temp container", "target", target, "read_only", readOnly)
+ var err error
+ if dockerClient == nil {
+ dockerClient, err = s.dockerService.GetClient(ctx)
+ if err != nil {
+ return "", nil, err
+ }
+ }
+
+ backupStorage, err := s.resolveUsableBackupStorageMountInternal(ctx, dockerClient, target, readOnly)
+ if err != nil {
+ return "", nil, err
+ }
+
+ return s.createBackupTempContainerWithMountInternal(ctx, dockerClient, "", backupStorage.mount)
+}
+
+func (s *VolumeService) downloadArchiveBackupInternal(ctx context.Context, backupID string) (io.ReadCloser, int64, error) {
+ filename, err := s.backupArchiveFilenameInternal(backupID)
+ if err != nil {
+ return nil, 0, err
+ }
+ dockerClient, err := s.dockerService.GetClient(ctx)
if err != nil {
- return false, err
+ return nil, 0, err
}
- filename, err := s.backupArchiveFilenameInternal(backupID)
+ containerID, cleanup, err := s.createBackupTempContainerInternal(ctx, dockerClient, "/volume", true)
if err != nil {
- return false, err
+ return nil, 0, err
}
+ return volumehelper.DownloadFileFromContainer(ctx, dockerClient, containerID, path.Join("/volume", filename), cleanup)
+}
- var backup VolumeBackup
- if err := s.db.WithContext(ctx).Where("id = ?", backupID).First(&backup).Error; err != nil {
- return false, err
+func (s *VolumeService) deleteArchiveBackupInternal(ctx context.Context, entry *VolumeBackup, user *common.User) error {
+ // Delete from DB first - if this fails, no changes are made. If file
+ // deletion fails afterward we just have an orphan file, easier to clean up
+ // than an orphan DB record pointing at a missing file.
+ volumeName := entry.VolumeName
+ backupID := entry.ID
+ if err := s.db.WithContext(ctx).Delete(entry).Error; err != nil {
+ return err
}
-
- dockerClient, err := s.dockerService.GetClient(ctx)
+ containerID, cleanup, err := s.createBackupTempContainerInternal(ctx, nil, "/volume", false)
if err != nil {
- return false, err
+ slog.WarnContext(ctx, "failed to create container for backup file cleanup", "backup_id", backupID, "error", err.Error())
+ } else {
+ defer cleanup()
+ filename, filenameErr := s.backupArchiveFilenameInternal(backupID)
+ if filenameErr != nil {
+ slog.WarnContext(ctx, "failed to sanitize backup id for file cleanup", "backup_id", backupID, "error", filenameErr.Error())
+ } else if _, _, err = s.execInContainerInternal(ctx, containerID, []string{"rm", "-f", path.Join("/volume", filename)}); err != nil {
+ slog.WarnContext(ctx, "failed to delete backup file (orphan file may remain)", "backup_id", backupID, "error", err.Error())
+ }
}
+ s.logBackupDeleteEventInternal(ctx, volumeName, backupID, user)
+ return nil
+}
- helperImage, err := getVolumeHelperImageInternal(ctx, s.dockerService, s.imageService, dockerClient)
+func (s *VolumeService) restoreArchiveBackupInternal(ctx context.Context, dockerClient *client.Client, volumeName, backupID string) error {
+ filename, err := s.backupArchiveFilenameInternal(backupID)
if err != nil {
- return false, err
+ return err
}
-
- backupStorage, err := s.resolveUsableBackupStorageMountInternal(ctx, dockerClient, "/volume", true)
+ helperImage, err := getVolumeHelperImageInternal(ctx, s.dockerService, s.imageService, dockerClient)
if err != nil {
- return false, err
+ return err
}
-
- containerID, cleanup, err := s.createBackupTempContainerWithMountInternal(ctx, dockerClient, helperImage, backupStorage.mount)
+ backupStorage, err := s.resolveUsableBackupStorageMountInternal(ctx, dockerClient, "/backups", true)
if err != nil {
- return false, err
+ return err
}
- defer cleanup()
-
- archivePath := path.Join("/volume", filename)
- cmd := []string{"tar", "-tzf", archivePath}
- stdout, stderr, err := s.execInContainerInternal(ctx, containerID, cmd)
+ config := &container.Config{
+ Image: helperImage,
+ Cmd: []string{
+ "sh",
+ "-c",
+ fmt.Sprintf("set -e; tmp=$(mktemp -d /volume/.restore_tmp.XXXXXX); tar -tzf /backups/%s >/dev/null; tar -xzf /backups/%s -C \"$tmp\"; find /volume -mindepth 1 -maxdepth 1 -not -path \"$tmp\" -exec rm -rf -- {} +; find \"$tmp\" -mindepth 1 -maxdepth 1 -exec mv -- {} /volume/ \\;; rmdir \"$tmp\"", filename, filename),
+ },
+ Labels: volumehelper.Labels(),
+ }
+ hostConfig := volumehelper.HostConfig(helperImage, []string{
+ volumeName + ":/volume",
+ }, []mount.Mount{backupStorage.mount})
+ resp, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{
+ Config: config,
+ HostConfig: hostConfig,
+ })
if err != nil {
- return false, err
+ return errors.WrapIf(err, "failed to create restore container")
}
- if strings.TrimSpace(stderr) != "" {
- return false, errors.Errorf("failed to list backup contents: %s", strings.TrimSpace(stderr))
+ defer func() {
+ _, _ = dockerClient.ContainerRemove(context.WithoutCancel(ctx), resp.ID, volumehelper.RemoveOptions())
+ }()
+ if _, err := dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); err != nil {
+ return errors.WrapIf(err, "failed to start restore container")
}
-
- for line := range strings.SplitSeq(stdout, "\n") {
- entry := strings.TrimSpace(line)
- if entry == "" {
- continue
- }
- entry = strings.TrimPrefix(entry, "./")
- if entry == cleaned || strings.TrimSuffix(entry, "/") == cleaned {
- return true, nil
+ waitResult := dockerClient.ContainerWait(ctx, resp.ID, client.ContainerWaitOptions{Condition: container.WaitConditionNotRunning})
+ var waitBody container.WaitResponse
+ select {
+ case err := <-waitResult.Error:
+ if err != nil {
+ return err
}
+ case waitBody = <-waitResult.Result:
}
-
- return false, nil
+ if waitBody.StatusCode != 0 {
+ return errors.Errorf("restore container exited with code %d (volume may be partially wiped)", waitBody.StatusCode)
+ }
+ return nil
}
-func (s *VolumeService) ListBackupFiles(ctx context.Context, backupID string) ([]string, error) {
- slog.DebugContext(ctx, "volume service: list backup files", "backup_id", backupID)
+func (s *VolumeService) listArchiveBackupFilesInternal(ctx context.Context, backupID string) ([]string, error) {
filename, err := s.backupArchiveFilenameInternal(backupID)
if err != nil {
return nil, err
}
-
- var backup VolumeBackup
- if err := s.db.WithContext(ctx).Where("id = ?", backupID).First(&backup).Error; err != nil {
- return nil, err
- }
-
dockerClient, err := s.dockerService.GetClient(ctx)
if err != nil {
return nil, err
}
-
- helperImage, err := getVolumeHelperImageInternal(ctx, s.dockerService, s.imageService, dockerClient)
- if err != nil {
- return nil, err
- }
-
backupStorage, err := s.resolveUsableBackupStorageMountInternal(ctx, dockerClient, "/volume", true)
if err != nil {
return nil, err
}
-
- containerID, cleanup, err := s.createBackupTempContainerWithMountInternal(ctx, dockerClient, helperImage, backupStorage.mount)
+ containerID, cleanup, err := s.createBackupTempContainerWithMountInternal(ctx, dockerClient, "", backupStorage.mount)
if err != nil {
return nil, err
}
defer cleanup()
-
- archivePath := path.Join("/volume", filename)
- cmd := []string{"tar", "-tzf", archivePath}
- stdout, _, err := s.execInContainerInternal(ctx, containerID, cmd)
+ stdout, _, err := s.execInContainerInternal(ctx, containerID, []string{"tar", "-tzf", path.Join("/volume", filename)})
if err != nil {
return nil, err
}
-
lines := strings.Split(strings.TrimSpace(stdout), "\n")
files := make([]string, 0, len(lines))
seen := make(map[string]struct{})
@@ -708,79 +1271,51 @@ func (s *VolumeService) ListBackupFiles(ctx context.Context, backupID string) ([
seen[clean] = struct{}{}
files = append(files, clean)
}
-
return files, nil
}
-func (s *VolumeService) RestoreBackupFiles(ctx context.Context, volumeName, backupID string, paths []string, user common.User) error {
- slog.DebugContext(ctx, "volume service: restore backup files", "volume", volumeName, "backup_id", backupID, "paths_count", len(paths), "user", user.ID)
- if len(paths) == 0 {
- return errors.New("no paths provided")
- }
- filename, err := s.backupArchiveFilenameInternal(backupID)
- if err != nil {
- return err
- }
-
- var backup VolumeBackup
- if err := s.db.WithContext(ctx).Where("id = ?", backupID).First(&backup).Error; err != nil {
- return err
- }
- if backup.VolumeName != volumeName {
- return errors.New("backup does not belong to volume")
- }
- unlock := s.workspaceLocks.Lock(volumeName)
- defer unlock()
- ctx = context.WithValue(ctx, volumeWorkspaceLockContextKeyInternal{}, volumeWorkspaceLockContextInternal{
- service: s,
- volumeName: volumeName,
- })
-
- // Create pre-restore backup for safety (consistent with RestoreBackup behavior)
- preBackup, err := s.CreateBackup(ctx, volumeName, user)
- if err != nil {
- return errors.WrapIf(err, "failed to create pre-restore backup")
+func (s *VolumeService) restoreBackupFilesInContainerInternal(ctx context.Context, containerID, filename string, cleanedPaths []string) (string, error) {
+ args := make([]string, 0, len(cleanedPaths)+5)
+ args = append(args, "sh", "-c", restoreBackupFilesScriptInternal, "sh", path.Join("/backups", filename))
+ for _, cleaned := range cleanedPaths {
+ args = append(args, "./"+cleaned)
}
- slog.DebugContext(ctx, "created pre-restore backup", "volume", volumeName, "pre_backup_id", preBackup.ID)
+ _, stderr, err := s.execInContainerInternal(ctx, containerID, args)
+ return stderr, errors.WrapIf(err, "failed to restore files")
+}
- cleanedPaths := make([]string, 0, len(paths))
- for _, p := range paths {
- cleaned, err := s.sanitizeBackupPathInternal(p)
- if err != nil {
- return err
- }
- cleanedPaths = append(cleanedPaths, cleaned)
- }
- if len(cleanedPaths) == 0 {
- return errors.New("no valid paths provided")
- }
+const restoreBackupFilesScriptInternal = `set -e
+archive="$1"
+shift
+archive_mode=$(stat -c '%A' -- "$archive" 2>/dev/null) || { echo ARCANE_NOT_FOUND >&2; exit 44; }
+case "$archive_mode" in -*) ;; *) echo ARCANE_NOT_FOUND >&2; exit 44 ;; esac
+for member do
+ if ! tar -tzf "$archive" -- "$member" >/dev/null 2>&1; then echo ARCANE_NOT_FOUND >&2; exit 44; fi
+done
+tar -xzf "$archive" -C /volume -- "$@"`
- dockerClient, err := s.dockerService.GetClient(ctx)
+func (s *VolumeService) restoreArchiveBackupFilesInternal(ctx context.Context, dockerClient *client.Client, volumeName, backupID string, cleanedPaths []string) error {
+ filename, err := s.backupArchiveFilenameInternal(backupID)
if err != nil {
return err
}
-
helperImage, err := getVolumeHelperImageInternal(ctx, s.dockerService, s.imageService, dockerClient)
if err != nil {
return err
}
-
backupStorage, err := s.resolveUsableBackupStorageMountInternal(ctx, dockerClient, "/backups", true)
if err != nil {
return err
}
-
config := &container.Config{
Image: helperImage,
Cmd: []string{"sleep", "infinity"},
NetworkDisabled: true,
Labels: volumehelper.Labels(),
}
-
hostConfig := volumehelper.HostConfig(helperImage, []string{
volumeName + ":/volume",
}, []mount.Mount{backupStorage.mount})
-
resp, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{
Config: config,
HostConfig: hostConfig,
@@ -788,17 +1323,12 @@ func (s *VolumeService) RestoreBackupFiles(ctx context.Context, volumeName, back
if err != nil {
return errors.WrapIf(err, "failed to create restore container")
}
-
+ defer func() {
+ _, _ = dockerClient.ContainerRemove(context.WithoutCancel(ctx), resp.ID, volumehelper.RemoveOptions())
+ }()
if _, err := dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); err != nil {
- _, _ = dockerClient.ContainerRemove(ctx, resp.ID, volumehelper.RemoveOptions())
return errors.WrapIf(err, "failed to start restore container")
}
-
- cleanup := func() {
- _, _ = dockerClient.ContainerRemove(ctx, resp.ID, volumehelper.RemoveOptions())
- }
- defer cleanup()
-
stderr, err := s.restoreBackupFilesInContainerInternal(ctx, resp.ID, filename, cleanedPaths)
if err != nil {
return errors.WrapIf(err, "failed to restore files")
@@ -806,68 +1336,9 @@ func (s *VolumeService) RestoreBackupFiles(ctx context.Context, volumeName, back
if strings.TrimSpace(stderr) != "" {
slog.DebugContext(ctx, "volume service: restore files stderr", "backup_id", backupID, "stderr", strings.TrimSpace(stderr))
}
-
- metadata := database.JSON{
- "action": "backup_restore_files",
- "backup_id": backupID,
- "pre_restore_backupId": preBackup.ID,
- "paths_count": len(cleanedPaths),
- }
- if len(cleanedPaths) > 0 {
- limit := min(len(cleanedPaths), 5)
- metadata["paths_sample"] = cleanedPaths[:limit]
- }
- if logErr := s.eventService.LogVolumeEvent(ctx, event.EventTypeVolumeBackupRestoreFiles, volumeName, volumeName, user.ID, user.Username, "0", metadata); logErr != nil {
- slog.WarnContext(ctx, "could not log volume backup restore files event", "volume", volumeName, "error", logErr.Error())
- }
-
return nil
}
-func (s *VolumeService) DownloadBackup(ctx context.Context, backupID string, user *common.User) (io.ReadCloser, int64, error) {
- slog.DebugContext(ctx, "volume service: download backup", "backup_id", backupID)
- filename, err := s.backupArchiveFilenameInternal(backupID)
- if err != nil {
- return nil, 0, err
- }
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return nil, 0, err
- }
-
- containerID, cleanup, err := s.createBackupTempContainerInternal(ctx, dockerClient, "/volume", true)
- if err != nil {
- return nil, 0, err
- }
-
- reader, size, err := volumehelper.DownloadFileFromContainer(ctx, dockerClient, containerID, path.Join("/volume", filename), cleanup)
- if err != nil {
- return nil, 0, err
- }
-
- actingUser := user
- if actingUser == nil {
- actingUser = &common.SystemUser
- }
- volumeName := ""
- var backup VolumeBackup
- if err := s.db.WithContext(ctx).Where("id = ?", backupID).First(&backup).Error; err == nil {
- volumeName = backup.VolumeName
- }
- if volumeName != "" {
- metadata := database.JSON{
- "action": "backup_download",
- "backup_id": backupID,
- "size": size,
- }
- if logErr := s.eventService.LogVolumeEvent(ctx, event.EventTypeVolumeBackupDownload, volumeName, volumeName, actingUser.ID, actingUser.Username, "0", metadata); logErr != nil {
- slog.WarnContext(ctx, "could not log volume backup download event", "volume", volumeName, "error", logErr.Error())
- }
- }
-
- return reader, size, nil
-}
-
func (s *VolumeService) UploadAndRestore(ctx context.Context, volumeName string, archive io.ReadSeeker, filename string, user common.User) error {
slog.DebugContext(ctx, "volume service: upload and restore", "volume", volumeName, "filename", filename, "user", user.ID)
@@ -880,14 +1351,14 @@ func (s *VolumeService) UploadAndRestore(ctx context.Context, volumeName string,
return errors.WrapIf(err, "invalid archive")
}
_ = gzr.Close()
+
unlock := s.workspaceLocks.Lock(volumeName)
defer unlock()
ctx = context.WithValue(ctx, volumeWorkspaceLockContextKeyInternal{}, volumeWorkspaceLockContextInternal{
service: s,
volumeName: volumeName,
})
-
- preBackup, err := s.CreateBackup(ctx, volumeName, user)
+ preBackup, err := s.CreateBackup(ctx, volumeName, user, VolumeBackupTriggerSafety, volumetypes.CreateBackupRequest{Destination: volumetypes.BackupDestinationLocal})
if err != nil {
return errors.WrapIf(err, "failed to create pre-restore backup")
}
@@ -959,3 +1430,207 @@ func (s *VolumeService) UploadAndRestore(ctx context.Context, volumeName string,
return nil
}
+
+const defaultVolumeBackupSchedule = "0 0 2 * * *"
+
+func (s *VolumeService) loadVolumeBackupPoliciesInternal(ctx context.Context, volumeName string) ([]VolumeBackupPolicy, error) {
+ var policies []VolumeBackupPolicy
+ if err := s.db.WithContext(ctx).Where("volume_name = ?", volumeName).Order("created_at ASC").Find(&policies).Error; err != nil {
+ return nil, fmt.Errorf("failed to load volume backup policies: %w", err)
+ }
+ return policies, nil
+}
+
+func (s *VolumeService) loadVolumeBackupPolicyInternal(ctx context.Context, volumeName, policyID string) (*VolumeBackupPolicy, error) {
+ if strings.TrimSpace(policyID) == "" {
+ return nil, nil
+ }
+ var policy VolumeBackupPolicy
+ err := s.db.WithContext(ctx).Where("id = ? AND volume_name = ?", policyID, volumeName).First(&policy).Error
+ if errors.Is(err, gorm.ErrRecordNotFound) {
+ return nil, nil
+ }
+ if err != nil {
+ return nil, fmt.Errorf("failed to load volume backup policy: %w", err)
+ }
+ return &policy, nil
+}
+
+func (s *VolumeService) GetBackupPolicies(ctx context.Context, volumeName string) (*volumetypes.BackupPolicyCollection, error) {
+ policies, err := s.loadVolumeBackupPoliciesInternal(ctx, volumeName)
+ if err != nil {
+ return nil, err
+ }
+ result := &volumetypes.BackupPolicyCollection{Policies: make([]volumetypes.BackupPolicy, 0, len(policies))}
+ destinations := make(map[string]s3domain.S3Destination)
+ if s.s3Destinations != nil {
+ available, listErr := s.s3Destinations.ListAllS3Destinations(ctx)
+ if listErr == nil {
+ result.S3Available = len(available) > 0
+ for _, destination := range available {
+ destinations[destination.ID] = s3domain.S3Destination{ID: destination.ID, Name: destination.Name, Bucket: destination.Bucket}
+ }
+ }
+ }
+ for i := range policies {
+ var lastRun *VolumeBackup
+ var entry VolumeBackup
+ if runErr := s.db.WithContext(ctx).Where("policy_id = ?", policies[i].ID).Order("created_at DESC").First(&entry).Error; runErr == nil {
+ if destination, ok := destinations[entry.S3DestinationID]; ok {
+ entry.S3DestinationName = destination.Name
+ }
+ lastRun = &entry
+ } else if !errors.Is(runErr, gorm.ErrRecordNotFound) {
+ return nil, fmt.Errorf("failed to load latest volume backup: %w", runErr)
+ }
+ dto := policies[i].ToDTO(lastRun)
+ dto.S3Available = result.S3Available
+ if destination, ok := destinations[policies[i].S3DestinationID]; ok {
+ dto.S3Bucket = destination.Bucket
+ dto.S3DestinationName = destination.Name
+ }
+ result.Policies = append(result.Policies, dto)
+ }
+ return result, nil
+}
+
+func (s *VolumeService) UpdateBackupPolicies(ctx context.Context, volumeName string, updates []volumetypes.UpdateBackupPolicy) (*volumetypes.BackupPolicyCollection, error) {
+ existing, err := s.loadVolumeBackupPoliciesInternal(ctx, volumeName)
+ if err != nil {
+ return nil, err
+ }
+ reconcile := backup.PolicyReconciliation[VolumeBackupPolicy]{
+ Domain: "volume",
+ DB: s.db,
+ Existing: existing,
+ ID: func(policy *VolumeBackupPolicy) string { return policy.ID },
+ New: func() VolumeBackupPolicy { return VolumeBackupPolicy{VolumeName: volumeName} },
+ Apply: func(policy *VolumeBackupPolicy, update volumetypes.UpdateBackupPolicy) {
+ policy.Enabled, policy.Schedule, policy.RetentionCount = update.Enabled, update.Schedule, update.RetentionCount
+ policy.StopContainers, policy.LocalEnabled, policy.S3Enabled = update.StopContainers, update.LocalEnabled, update.S3Enabled
+ policy.S3DestinationID = update.S3DestinationID
+ },
+ S3Configured: func(ctx context.Context, destinationID string) error {
+ if s.s3Destinations == nil {
+ return errors.New("S3 backup destinations are unavailable")
+ }
+ if _, err := s.s3Destinations.Configuration(ctx, destinationID); err != nil {
+ return errors.New("select a valid S3 destination for volume backups")
+ }
+ return nil
+ },
+ Unregister: s.jobs.Unregister,
+ Reschedule: s.rescheduleVolumeBackupPolicyInternal,
+ }
+ if err := reconcile.Run(ctx, updates); err != nil {
+ return nil, err
+ }
+ return s.GetBackupPolicies(ctx, volumeName)
+}
+
+func (s *VolumeService) applyVolumeBackupRetentionInternal(ctx context.Context, policyID string, retentionCount int) error {
+ expired, err := backup.ExpiredRunIDs(ctx, s.db, "volume_backups", policyID, retentionCount)
+ if err != nil {
+ return err
+ }
+ for _, id := range expired {
+ if err := s.deleteBackupInternal(ctx, id, nil); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (s *VolumeService) runScheduledBackupInternal(ctx context.Context, policyID string) {
+ var policy VolumeBackupPolicy
+ if err := s.db.WithContext(ctx).Where("id = ? AND enabled = ?", policyID, true).First(&policy).Error; err != nil {
+ return
+ }
+ var entry *VolumeBackup
+ _, err := activitylib.RunHandlerActivity(ctx, s.activityService, activitylib.HandlerOptions{
+ EnvironmentID: "0",
+ Type: activitytypes.TypeResourceAction,
+ ResourceType: "volume_backup",
+ ResourceID: policy.VolumeName,
+ ResourceName: policy.VolumeName,
+ User: &common.SystemUser,
+ Step: "Creating scheduled backup",
+ Message: "Creating scheduled volume backup",
+ SuccessMessage: "Scheduled volume backup created successfully",
+ Metadata: database.JSON{
+ "action": "scheduled_volume_backup",
+ "policyId": policy.ID,
+ "schedule": policy.Schedule,
+ "volumeName": policy.VolumeName,
+ "retentionCount": policy.RetentionCount,
+ "stopContainers": policy.StopContainers,
+ "localEnabled": policy.LocalEnabled,
+ "s3Enabled": policy.S3Enabled,
+ "s3DestinationId": policy.S3DestinationID,
+ },
+ }, func(activityCtx context.Context) error {
+ var backupErr error
+ entry, backupErr = s.CreateBackup(activityCtx, policy.VolumeName, common.SystemUser, VolumeBackupTriggerScheduled, volumetypes.CreateBackupRequest{PolicyID: policy.ID})
+ return backupErr
+ })
+ if errors.Is(err, ErrVolumeBackupAlreadyRunning) {
+ slog.InfoContext(ctx, "Scheduled volume backup skipped; another backup is running", "volume", policy.VolumeName)
+ return
+ }
+ if err != nil {
+ slog.ErrorContext(ctx, "Scheduled volume backup failed", "volume", policy.VolumeName, "error", err)
+ return
+ }
+ slog.InfoContext(ctx, "Scheduled volume backup completed", "volume", policy.VolumeName, "backup_id", entry.ID, "remote_snapshot_id", entry.RemoteSnapshotID)
+}
+
+func (s *VolumeService) rescheduleVolumeBackupPolicyInternal(ctx context.Context, policy *VolumeBackupPolicy) {
+ if policy == nil {
+ return
+ }
+ if !policy.Enabled {
+ s.jobs.Unregister(ctx, policy.ID)
+ return
+ }
+ policyID := policy.ID
+ s.jobs.Register(ctx, policyID,
+ func(ctx context.Context) string {
+ var current VolumeBackupPolicy
+ if err := s.db.WithContext(ctx).Where("id = ?", policyID).First(&current).Error; err != nil {
+ return defaultVolumeBackupSchedule
+ }
+ return current.Schedule
+ },
+ func(ctx context.Context) {
+ s.runScheduledBackupInternal(ctx, policyID)
+ },
+ )
+}
+
+func (s *VolumeService) RegisterBackupJobsOnStartup(ctx context.Context) {
+ if !s.jobs.Enabled() {
+ return
+ }
+ var policies []VolumeBackupPolicy
+ if err := s.db.WithContext(ctx).Where("enabled = ?", true).Find(&policies).Error; err != nil {
+ slog.ErrorContext(ctx, "Failed to load scheduled volume backups", "error", err)
+ return
+ }
+ for i := range policies {
+ s.rescheduleVolumeBackupPolicyInternal(ctx, &policies[i])
+ }
+ slog.InfoContext(ctx, "Registered scheduled volume backup jobs", "count", len(policies))
+}
+
+func (s *VolumeService) removeVolumeBackupPolicyInternal(ctx context.Context, volumeName string) {
+ policies, err := s.loadVolumeBackupPoliciesInternal(ctx, volumeName)
+ if err != nil {
+ return
+ }
+ for i := range policies {
+ s.jobs.Unregister(ctx, policies[i].ID)
+ }
+ if err := s.db.WithContext(ctx).Where("volume_name = ?", volumeName).Delete(&VolumeBackupPolicy{}).Error; err != nil {
+ slog.WarnContext(ctx, "Failed to delete volume backup policy", "volume", volumeName, "error", err)
+ }
+}
diff --git a/backend/internal/volume/handler.go b/backend/internal/volume/handler.go
index 5099f5306..0dbb539f3 100644
--- a/backend/internal/volume/handler.go
+++ b/backend/internal/volume/handler.go
@@ -1,19 +1,19 @@
package volume
import (
- "github.com/getarcaneapp/arcane/backend/v2/internal/common"
-
- "github.com/getarcaneapp/arcane/backend/v2/internal/database"
-
"context"
+ "fmt"
"io"
+ "log/slog"
"net/http"
+ "net/url"
"strconv"
"strings"
- activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
-
+ "github.com/getarcaneapp/arcane/backend/v2/internal/common"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/docker"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/environment"
"emperror.dev/errors"
"github.com/danielgtaylor/huma/v2"
@@ -25,6 +25,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"
+ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
"github.com/getarcaneapp/arcane/types/v2/base"
uploadtypes "github.com/getarcaneapp/arcane/types/v2/upload"
volumetypes "github.com/getarcaneapp/arcane/types/v2/volume"
@@ -34,11 +35,12 @@ import (
// VolumeHandler provides Huma-based volume management endpoints.
type VolumeHandler struct {
- volumeService *VolumeService
- dockerService *docker.DockerClientService
- activityService *activity.ActivityService
- uploadService *upload.UploadService
- appCtx context.Context
+ volumeService *VolumeService
+ dockerService *docker.DockerClientService
+ activityService *activity.ActivityService
+ environmentService *environment.EnvironmentService
+ uploadService *upload.UploadService
+ appCtx context.Context
}
// --- Huma Input/Output Wrappers ---
@@ -173,14 +175,34 @@ type ListBackupsOutput struct {
}
type CreateBackupInput struct {
- EnvironmentID string `path:"id" doc:"Environment ID"`
- VolumeName string `path:"volumeName" doc:"Volume name"`
+ EnvironmentID string `path:"id" doc:"Environment ID"`
+ VolumeName string `path:"volumeName" doc:"Volume name"`
+ Body *volumetypes.CreateBackupRequest `json:"body,omitempty"`
}
type CreateBackupOutput struct {
Body base.ApiResponse[*VolumeBackup]
}
+type GetVolumeBackupPolicyInput struct {
+ EnvironmentID string `path:"id" doc:"Environment ID"`
+ VolumeName string `path:"volumeName" doc:"Volume name"`
+}
+
+type GetVolumeBackupPolicyOutput struct {
+ Body base.ApiResponse[volumetypes.BackupPolicyCollection]
+}
+
+type UpdateVolumeBackupPolicyInput struct {
+ EnvironmentID string `path:"id" doc:"Environment ID"`
+ VolumeName string `path:"volumeName" doc:"Volume name"`
+ Body volumetypes.UpdateBackupPolicies `doc:"Scheduled volume backup policies"`
+}
+
+type UpdateVolumeBackupPolicyOutput struct {
+ Body base.ApiResponse[volumetypes.BackupPolicyCollection]
+}
+
type RestoreBackupInput struct {
EnvironmentID string `path:"id" doc:"Environment ID"`
VolumeName string `path:"volumeName" doc:"Volume name"`
@@ -232,15 +254,15 @@ type DeleteBackupInput struct {
BackupID string `path:"backupId" doc:"Backup ID"`
}
-type DeleteBackupOutput struct {
- Body base.ApiResponse[base.MessageResponse]
-}
-
type DownloadBackupInput struct {
EnvironmentID string `path:"id" doc:"Environment ID"`
BackupID string `path:"backupId" doc:"Backup ID"`
}
+type DeleteBackupOutput struct {
+ Body base.ApiResponse[base.MessageResponse]
+}
+
type UploadAndRestoreInput struct {
EnvironmentID string `path:"id" doc:"Environment ID"`
VolumeName string `path:"volumeName" doc:"Volume name"`
@@ -251,14 +273,25 @@ type UploadAndRestoreOutput struct {
Body base.ApiResponse[base.MessageResponse]
}
+type UploadBackupInput struct {
+ EnvironmentID string `path:"id" doc:"Environment ID"`
+ BackupID string `path:"backupId" doc:"Backup ID"`
+ Body volumetypes.UploadBackupRequest
+}
+
+type UploadBackupOutput struct {
+ Body base.ApiResponse[*VolumeBackup]
+}
+
// RegisterVolumes registers volume management routes using Huma.
-func RegisterVolumes(api huma.API, dockerService *docker.DockerClientService, volumeService *VolumeService, activityService *activity.ActivityService, uploadService *upload.UploadService, appCtx handlerutil.ActivityAppContext) {
+func RegisterVolumes(api huma.API, dockerService *docker.DockerClientService, volumeService *VolumeService, activityService *activity.ActivityService, environmentService *environment.EnvironmentService, uploadService *upload.UploadService, appCtx handlerutil.ActivityAppContext) {
h := &VolumeHandler{
- volumeService: volumeService,
- dockerService: dockerService,
- activityService: activityService,
- uploadService: uploadService,
- appCtx: appCtx.Context(),
+ volumeService: volumeService,
+ dockerService: dockerService,
+ activityService: activityService,
+ environmentService: environmentService,
+ uploadService: uploadService,
+ appCtx: appCtx.Context(),
}
middleware.RegisterWithPermission(api, huma.Operation{
@@ -347,6 +380,24 @@ func RegisterVolumes(api huma.API, dockerService *docker.DockerClientService, vo
// --- Volume Backup Endpoints ---
+ middleware.RegisterWithPermission(api, huma.Operation{
+ OperationID: "get-volume-backup-policy",
+ Method: http.MethodGet,
+ Path: "/environments/{id}/volumes/{volumeName}/backup-policy",
+ Summary: "Get volume backup policies",
+ Tags: []string{"Volume Backup"},
+ Security: handlerutil.DefaultOperationSecurity(),
+ }, authz.PermVolumesRead, h.GetBackupPolicy)
+
+ middleware.RegisterWithPermission(api, huma.Operation{
+ OperationID: "update-volume-backup-policy",
+ Method: http.MethodPut,
+ Path: "/environments/{id}/volumes/{volumeName}/backup-policy",
+ Summary: "Update volume backup policies",
+ Tags: []string{"Volume Backup"},
+ Security: handlerutil.DefaultOperationSecurity(),
+ }, authz.PermVolumesBackup, h.UpdateBackupPolicy)
+
middleware.RegisterWithPermission(api, huma.Operation{
OperationID: "list-volume-backups",
Method: http.MethodGet,
@@ -392,6 +443,16 @@ func RegisterVolumes(api huma.API, dockerService *docker.DockerClientService, vo
Security: handlerutil.DefaultOperationSecurity(),
}, authz.PermVolumesBackup, h.DeleteBackup)
+ middleware.RegisterWithPermission(api, huma.Operation{
+ OperationID: "upload-retained-volume-backup-to-s3",
+ Method: http.MethodPost,
+ Path: "/environments/{id}/volumes/backups/{backupId}/upload",
+ Summary: "Upload volume backup",
+ Description: "Upload an existing local volume backup to the selected S3 destination",
+ Tags: []string{"Volume Backup"},
+ Security: handlerutil.DefaultOperationSecurity(),
+ }, authz.PermVolumesBackup, h.UploadBackup)
+
middleware.RegisterWithPermission(api, huma.Operation{
OperationID: "download-volume-backup",
Method: http.MethodGet,
@@ -431,6 +492,117 @@ func RegisterVolumes(api huma.API, dockerService *docker.DockerClientService, vo
}, authz.PermVolumesUpload, h.UploadAndRestore)
}
+func (h *VolumeHandler) DownloadBackup(ctx context.Context, input *DownloadBackupInput) (*huma.StreamResponse, error) {
+ user, _ := common.CurrentUserFromContext(ctx)
+ reader, size, err := h.volumeService.DownloadBackup(ctx, input.BackupID, user)
+ if err != nil {
+ return nil, huma.Error500InternalServerError(err.Error())
+ }
+
+ return &huma.StreamResponse{
+ Body: func(humaCtx huma.Context) {
+ defer func() { _ = reader.Close() }()
+
+ humaCtx.SetHeader("Content-Type", "application/x-gzip")
+ humaCtx.SetHeader("Content-Disposition", "attachment; filename="+input.BackupID+".tar.gz")
+ humaCtx.SetHeader("Content-Length", strconv.FormatInt(size, 10))
+
+ writer := humaCtx.BodyWriter()
+ _, _ = io.Copy(writer, reader)
+ },
+ }, nil
+}
+
+func (h *VolumeHandler) GetBackupPolicy(ctx context.Context, input *GetVolumeBackupPolicyInput) (*GetVolumeBackupPolicyOutput, error) {
+ if input.EnvironmentID != "0" {
+ if h.environmentService == nil {
+ return nil, huma.Error500InternalServerError("environment service not available")
+ }
+ remotePath := fmt.Sprintf("/api/environments/0/volumes/%s/backup-policy", url.PathEscape(input.VolumeName))
+ response, err := handlerutil.ProxyRemoteJSON[base.ApiResponse[volumetypes.BackupPolicyCollection]](ctx, h.environmentService.ProxyJSONRequest, input.EnvironmentID, http.MethodGet, remotePath, nil)
+ if err != nil {
+ return nil, err
+ }
+ return &GetVolumeBackupPolicyOutput{Body: *response}, nil
+ }
+
+ policies, err := h.volumeService.GetBackupPolicies(ctx, input.VolumeName)
+ if err != nil {
+ return nil, huma.Error500InternalServerError(err.Error())
+ }
+ return &GetVolumeBackupPolicyOutput{Body: base.ApiResponse[volumetypes.BackupPolicyCollection]{Success: true, Data: *policies}}, nil
+}
+
+func (h *VolumeHandler) UpdateBackupPolicy(ctx context.Context, input *UpdateVolumeBackupPolicyInput) (*UpdateVolumeBackupPolicyOutput, error) {
+ user, _ := common.CurrentUserFromContext(ctx)
+ var policies *volumetypes.BackupPolicyCollection
+ var remoteResponse *base.ApiResponse[volumetypes.BackupPolicyCollection]
+ errorStatus := http.StatusBadRequest
+ runtimeCtx := utils.ActivityRuntimeContext(ctx, h.appCtx)
+ _, err := activitylib.RunHandlerActivity(runtimeCtx, h.activityService, activitylib.HandlerOptions{
+ EnvironmentID: input.EnvironmentID,
+ Type: activitytypes.TypeResourceAction,
+ ResourceType: "volume_backup_policy",
+ ResourceID: input.VolumeName,
+ ResourceName: input.VolumeName,
+ User: user,
+ Step: "Saving backup policies",
+ Message: "Saving volume backup policies",
+ SuccessMessage: "Volume backup policies saved successfully",
+ Metadata: database.JSON{
+ "action": "update_volume_backup_policy",
+ "policyCount": len(input.Body.Policies),
+ },
+ }, func(activityCtx context.Context) error {
+ if input.EnvironmentID != "0" {
+ if h.environmentService == nil {
+ errorStatus = http.StatusInternalServerError
+ return errors.New("environment service not available")
+ }
+ hasS3 := false
+ for _, policy := range input.Body.Policies {
+ if policy.S3Enabled {
+ hasS3 = true
+ break
+ }
+ }
+ // Destinations are reconciled before every policy write so the agent
+ // can validate S3 references. Only batches that actually need S3
+ // fail on a sync error; local-only edits proceed.
+ if syncErr := h.environmentService.SyncS3DestinationsToEnvironment(activityCtx, input.EnvironmentID); syncErr != nil {
+ if hasS3 {
+ errorStatus = http.StatusBadGateway
+ return fmt.Errorf("failed to synchronize S3 destinations to environment: %w", syncErr)
+ }
+ slog.WarnContext(activityCtx, "S3 destination sync failed before local-only volume backup policy write", "environment_id", input.EnvironmentID, "error", syncErr)
+ }
+ remotePath := fmt.Sprintf("/api/environments/0/volumes/%s/backup-policy", url.PathEscape(input.VolumeName))
+ var proxyErr error
+ remoteResponse, proxyErr = handlerutil.ProxyRemoteJSON[base.ApiResponse[volumetypes.BackupPolicyCollection]](activityCtx, h.environmentService.ProxyJSONRequest, input.EnvironmentID, http.MethodPut, remotePath, input.Body)
+ if proxyErr != nil {
+ errorStatus = http.StatusBadGateway
+ }
+ return proxyErr
+ }
+ var updateErr error
+ policies, updateErr = h.volumeService.UpdateBackupPolicies(activityCtx, input.VolumeName, input.Body.Policies)
+ return updateErr
+ })
+ if err != nil {
+ if errorStatus == http.StatusInternalServerError {
+ return nil, huma.Error500InternalServerError(err.Error())
+ }
+ if errorStatus == http.StatusBadGateway {
+ return nil, huma.NewError(http.StatusBadGateway, err.Error())
+ }
+ return nil, huma.Error400BadRequest(err.Error())
+ }
+ if remoteResponse != nil {
+ return &UpdateVolumeBackupPolicyOutput{Body: *remoteResponse}, nil
+ }
+ return &UpdateVolumeBackupPolicyOutput{Body: base.ApiResponse[volumetypes.BackupPolicyCollection]{Success: true, Data: *policies}}, nil
+}
+
// ListVolumes returns a paginated list of volumes.
func (h *VolumeHandler) ListVolumes(ctx context.Context, input *ListVolumesInput) (*ListVolumesOutput, error) {
params := handlerutil.PaginationParams(input.Start, input.Limit, input.Sort, input.Order, input.Search)
@@ -707,6 +879,14 @@ func (h *VolumeHandler) CreateBackup(ctx context.Context, input *CreateBackupInp
}
var backup *VolumeBackup
+ destination := volumetypes.BackupDestination("")
+ policyID := ""
+ s3DestinationID := ""
+ if input.Body != nil {
+ destination = input.Body.Destination
+ policyID = input.Body.PolicyID
+ s3DestinationID = input.Body.S3DestinationID
+ }
runtimeCtx := utils.ActivityRuntimeContext(ctx, h.appCtx)
activityID, err := activitylib.RunHandlerActivity(runtimeCtx, h.activityService, activitylib.HandlerOptions{
EnvironmentID: input.EnvironmentID,
@@ -718,12 +898,17 @@ func (h *VolumeHandler) CreateBackup(ctx context.Context, input *CreateBackupInp
Step: "Creating backup",
Message: "Creating volume backup",
SuccessMessage: "Volume backup created successfully",
- Metadata: database.JSON{"action": "create_volume_backup"},
+ Metadata: database.JSON{"action": "create_volume_backup", "destination": destination, "policyId": policyID, "s3DestinationId": s3DestinationID},
}, func(runtimeCtx context.Context) error {
var backupErr error
- backup, backupErr = h.volumeService.CreateBackup(runtimeCtx, input.VolumeName, *user)
+ backup, backupErr = h.volumeService.CreateBackup(runtimeCtx, input.VolumeName, *user, VolumeBackupTriggerManual, volumetypes.CreateBackupRequest{
+ Destination: destination, PolicyID: policyID, S3DestinationID: s3DestinationID,
+ })
return backupErr
})
+ if errors.Is(err, ErrVolumeBackupAlreadyRunning) {
+ return nil, huma.Error409Conflict(err.Error())
+ }
if err != nil {
return nil, huma.Error500InternalServerError(err.Error())
}
@@ -875,25 +1060,38 @@ func (h *VolumeHandler) DeleteBackup(ctx context.Context, input *DeleteBackupInp
}, nil
}
-func (h *VolumeHandler) DownloadBackup(ctx context.Context, input *DownloadBackupInput) (*huma.StreamResponse, error) {
+func (h *VolumeHandler) UploadBackup(ctx context.Context, input *UploadBackupInput) (*UploadBackupOutput, error) {
+ if h.volumeService == nil {
+ return nil, huma.Error500InternalServerError("service not available")
+ }
user, _ := common.CurrentUserFromContext(ctx)
- reader, size, err := h.volumeService.DownloadBackup(ctx, input.BackupID, user)
+ var backup *VolumeBackup
+ runtimeCtx := utils.ActivityRuntimeContext(ctx, h.appCtx)
+ activityID, err := activitylib.RunHandlerActivity(runtimeCtx, h.activityService, activitylib.HandlerOptions{
+ EnvironmentID: input.EnvironmentID,
+ Type: activitytypes.TypeResourceAction,
+ ResourceType: "volume_backup",
+ ResourceID: input.BackupID,
+ ResourceName: input.BackupID,
+ User: user,
+ Step: "Uploading backup",
+ Message: "Uploading volume backup to S3",
+ SuccessMessage: "Volume backup uploaded successfully",
+ Metadata: database.JSON{
+ "action": "upload_volume_backup",
+ "backupId": input.BackupID,
+ "s3DestinationId": input.Body.S3DestinationID,
+ },
+ }, func(runtimeCtx context.Context) error {
+ var uploadErr error
+ backup, uploadErr = h.volumeService.UploadBackup(runtimeCtx, input.BackupID, input.Body.S3DestinationID)
+ return uploadErr
+ })
if err != nil {
return nil, huma.Error500InternalServerError(err.Error())
}
-
- return &huma.StreamResponse{
- Body: func(humaCtx huma.Context) {
- defer func() { _ = reader.Close() }()
-
- humaCtx.SetHeader("Content-Type", "application/x-gzip")
- humaCtx.SetHeader("Content-Disposition", "attachment; filename="+input.BackupID+".tar.gz")
- humaCtx.SetHeader("Content-Length", strconv.FormatInt(size, 10))
-
- writer := humaCtx.BodyWriter()
- _, _ = io.Copy(writer, reader)
- },
- }, nil
+ backup.ActivityID = mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()
+ return &UploadBackupOutput{Body: base.ApiResponse[*VolumeBackup]{Success: true, Data: backup}}, nil
}
func (h *VolumeHandler) UploadAndRestore(ctx context.Context, input *UploadAndRestoreInput) (*UploadAndRestoreOutput, error) {
diff --git a/backend/internal/volume/model.go b/backend/internal/volume/model.go
index 91ecb356c..e8ab26ed8 100644
--- a/backend/internal/volume/model.go
+++ b/backend/internal/volume/model.go
@@ -8,13 +8,46 @@ import (
"github.com/getarcaneapp/arcane/types/v2/volume"
)
+type VolumeBackupStatus string
+
+const (
+ VolumeBackupStatusRunning VolumeBackupStatus = "running"
+ VolumeBackupStatusSucceeded VolumeBackupStatus = "succeeded"
+ VolumeBackupStatusFailed VolumeBackupStatus = "failed"
+)
+
+type VolumeBackupTrigger string
+
+const (
+ VolumeBackupTriggerManual VolumeBackupTrigger = "manual"
+ VolumeBackupTriggerScheduled VolumeBackupTrigger = "scheduled"
+ VolumeBackupTriggerSafety VolumeBackupTrigger = "safety"
+)
+
+type VolumeBackupFormat string
+
+const (
+ VolumeBackupFormatArchive VolumeBackupFormat = "archive"
+ VolumeBackupFormatRustic VolumeBackupFormat = "rustic"
+)
+
type VolumeBackup struct {
database.BaseModel
- VolumeName string `json:"volumeName" gorm:"column:volume_name;index"`
- Size int64 `json:"size" gorm:"column:size"`
- CreatedAt time.Time `json:"createdAt" gorm:"column:created_at"`
- ActivityID *string `json:"activityId,omitempty" gorm:"-"`
+ VolumeName string `json:"volumeName" gorm:"column:volume_name;index"`
+ Size int64 `json:"size" gorm:"column:size"`
+ CreatedAt time.Time `json:"createdAt" gorm:"column:created_at"`
+ Status VolumeBackupStatus `json:"status" gorm:"column:status;type:text;not null;default:succeeded"`
+ Trigger VolumeBackupTrigger `json:"trigger" gorm:"column:trigger;type:text;not null;default:manual"`
+ Destination volume.BackupDestination `json:"destination" gorm:"column:destination;type:text;not null;default:local"`
+ Format VolumeBackupFormat `json:"format" gorm:"column:format;type:text;not null;default:archive"`
+ LocalSnapshotID string `json:"localSnapshotId,omitempty" gorm:"column:local_snapshot_id;type:text"`
+ RemoteSnapshotID string `json:"remoteSnapshotId,omitempty" gorm:"column:remote_snapshot_id;type:text"`
+ S3DestinationID string `json:"s3DestinationId,omitempty" gorm:"column:s3_destination_id;type:text;index"`
+ S3DestinationName string `json:"s3DestinationName,omitempty" gorm:"-"`
+ PolicyID string `json:"policyId,omitempty" gorm:"column:policy_id;type:text;index"`
+ Error string `json:"error,omitempty" gorm:"column:error;type:text"`
+ ActivityID *string `json:"activityId,omitempty" gorm:"-"`
}
func (*VolumeBackup) TableName() string {
@@ -23,9 +56,55 @@ func (*VolumeBackup) TableName() string {
func (b *VolumeBackup) ToDTO() volume.BackupEntry {
return volume.BackupEntry{
- ID: b.ID,
- VolumeName: b.VolumeName,
- Size: b.Size,
- CreatedAt: b.CreatedAt.Format(time.RFC3339),
+ ID: b.ID,
+ VolumeName: b.VolumeName,
+ Size: b.Size,
+ CreatedAt: b.CreatedAt.Format(time.RFC3339),
+ Status: string(b.Status),
+ Trigger: string(b.Trigger),
+ Destination: b.Destination,
+ Format: volume.BackupFormat(b.Format),
+ LocalSnapshotID: b.LocalSnapshotID,
+ RemoteSnapshotID: b.RemoteSnapshotID,
+ S3DestinationID: b.S3DestinationID,
+ S3DestinationName: b.S3DestinationName,
+ PolicyID: b.PolicyID,
+ Error: b.Error,
+ }
+}
+
+type VolumeBackupPolicy struct {
+ database.BaseModel
+
+ VolumeName string `json:"volumeName" gorm:"column:volume_name;not null;index"`
+ Enabled bool `json:"enabled" gorm:"column:enabled;not null;default:false"`
+ Schedule string `json:"schedule" gorm:"column:schedule;type:text;not null"`
+ RetentionCount int `json:"retentionCount" gorm:"column:retention_count;not null;default:7"`
+ StopContainers bool `json:"stopContainers" gorm:"column:stop_containers;not null;default:false"`
+ LocalEnabled bool `json:"localEnabled" gorm:"column:local_enabled;not null"`
+ S3Enabled bool `json:"s3Enabled" gorm:"column:s3_enabled;not null;default:false"`
+ S3DestinationID string `json:"s3DestinationId,omitempty" gorm:"column:s3_destination_id;type:text;index"`
+}
+
+func (VolumeBackupPolicy) TableName() string {
+ return "volume_backup_policies"
+}
+
+func (p VolumeBackupPolicy) ToDTO(lastRun *VolumeBackup) volume.BackupPolicy {
+ dto := volume.BackupPolicy{
+ ID: p.ID,
+ VolumeName: p.VolumeName,
+ Enabled: p.Enabled,
+ Schedule: p.Schedule,
+ RetentionCount: p.RetentionCount,
+ StopContainers: p.StopContainers,
+ LocalEnabled: p.LocalEnabled,
+ S3Enabled: p.S3Enabled,
+ S3DestinationID: p.S3DestinationID,
+ }
+ if lastRun != nil {
+ lastRunDTO := lastRun.ToDTO()
+ dto.LastRun = &lastRunDTO
}
+ return dto
}
diff --git a/backend/internal/volume/module.go b/backend/internal/volume/module.go
index 8cbec5492..2f0955301 100644
--- a/backend/internal/volume/module.go
+++ b/backend/internal/volume/module.go
@@ -4,8 +4,12 @@ package volume
import (
"github.com/danielgtaylor/huma/v2"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/backup"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/container"
"github.com/getarcaneapp/arcane/backend/v2/internal/docker"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/environment"
"github.com/getarcaneapp/arcane/backend/v2/internal/image"
+ s3domain "github.com/getarcaneapp/arcane/backend/v2/internal/s3"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
@@ -18,14 +22,18 @@ import (
// Dependencies are the collaborators the volume domain needs.
type Dependencies struct {
- DB *database.DB
- Docker *docker.DockerClientService
- Event *event.EventService
- Settings *settings.SettingsService
- Image *image.ImageService
- Activity *activity.ActivityService
- Config *config.Config
- Upload *upload.UploadService
+ DB *database.DB
+ Docker *docker.DockerClientService
+ Event *event.EventService
+ Settings *settings.SettingsService
+ Image *image.ImageService
+ Activity *activity.ActivityService
+ Environment *environment.EnvironmentService
+ Container *container.ContainerService
+ Engine *backup.Engine
+ S3 *s3domain.S3DestinationService
+ Config *config.Config
+ Upload *upload.UploadService
}
// Module wires the volume domain and mounts its routes.
@@ -37,7 +45,7 @@ type Module struct {
// New builds the volume domain from its dependencies.
func New(deps Dependencies) *Module {
return &Module{
- service: NewVolumeService(deps.DB, deps.Docker, deps.Event, deps.Settings, deps.Image, deps.Config),
+ service: NewVolumeService(deps.DB, deps.Docker, deps.Event, deps.Activity, deps.Settings, deps.Container, deps.Image, deps.Engine, deps.S3, deps.Config),
deps: deps,
}
}
@@ -55,8 +63,8 @@ func (m *Module) Service() *VolumeService {
// OpenAPI spec generation can discover the routes without a service graph.
func (m *Module) RegisterRoutes(api huma.API, appCtx handlerutil.ActivityAppContext) {
if m == nil {
- RegisterVolumes(api, nil, nil, nil, nil, appCtx)
+ RegisterVolumes(api, nil, nil, nil, nil, nil, appCtx)
return
}
- RegisterVolumes(api, m.deps.Docker, m.service, m.deps.Activity, m.deps.Upload, appCtx)
+ RegisterVolumes(api, m.deps.Docker, m.service, m.deps.Activity, m.deps.Environment, m.deps.Upload, appCtx)
}
diff --git a/backend/internal/volume/service.go b/backend/internal/volume/service.go
index 1ed238ed1..883767d19 100644
--- a/backend/internal/volume/service.go
+++ b/backend/internal/volume/service.go
@@ -8,20 +8,27 @@ import (
"strings"
"sync"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/activity"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/backup"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/container"
"github.com/getarcaneapp/arcane/backend/v2/internal/docker"
"github.com/getarcaneapp/arcane/backend/v2/internal/image"
+ s3domain "github.com/getarcaneapp/arcane/backend/v2/internal/s3"
"emperror.dev/errors"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/event"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
dockerutil "github.com/getarcaneapp/arcane/backend/v2/pkg/dockerutil"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/entityjobs"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
workspacepkg "github.com/getarcaneapp/arcane/backend/v2/pkg/workspace"
+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
volumetypes "github.com/getarcaneapp/arcane/types/v2/volume"
"github.com/moby/moby/client"
"golang.org/x/sync/singleflight"
@@ -31,9 +38,14 @@ type VolumeService struct {
db *database.DB
dockerService *docker.DockerClientService
eventService *event.EventService
+ activityService *activity.ActivityService
settingsService *settings.SettingsService
+ containerService *container.ContainerService
imageService *image.ImageService
+ engine *backup.Engine
+ s3Destinations *s3domain.S3DestinationService
backupVolumeName string
+ encryptionKey string
workspaceMaxDepth int
workspaceMaxEntries int
workspaceMaxFileSizeBytes int64
@@ -45,6 +57,13 @@ type VolumeService struct {
// second overwrites the first in helperByVolume, orphaning a `sleep infinity`
// container that pins the volume until restart.
helperGroup singleflight.Group
+ jobs *entityjobs.Registry
+}
+
+// SetScheduler injects the dynamic scheduler and admission gate for per-policy
+// backup jobs. Agent mode passes them too: agents run their own volume backups.
+func (s *VolumeService) SetScheduler(ctx context.Context, scheduler schedulertypes.DynamicScheduler, admissionGate *actors.Gate[actors.AdmissionKey]) error {
+ return s.jobs.SetScheduler(ctx, scheduler, admissionGate)
}
type volumeWorkspaceLockContextKeyInternal struct{}
@@ -56,14 +75,16 @@ type volumeWorkspaceLockContextInternal struct {
const trivyCacheVolumePruneFilterValue = libarcane.InternalResourceLabel + "=true"
-func NewVolumeService(db *database.DB, dockerService *docker.DockerClientService, eventService *event.EventService, settingsService *settings.SettingsService, imageService *image.ImageService, cfg *config.Config) *VolumeService {
+func NewVolumeService(db *database.DB, dockerService *docker.DockerClientService, eventService *event.EventService, activityService *activity.ActivityService, settingsService *settings.SettingsService, containerService *container.ContainerService, imageService *image.ImageService, engine *backup.Engine, s3Destinations *s3domain.S3DestinationService, cfg *config.Config) *VolumeService {
slog.Debug("volume service: new")
backupVolumeName := ""
+ encryptionKey := ""
workspaceMaxDepth := 50
workspaceMaxEntries := 10000
workspaceMaxFileSizeMB := workspacepkg.DefaultMaxFileSizeMB
if cfg != nil {
backupVolumeName = cfg.BackupVolumeName
+ encryptionKey = cfg.EncryptionKey
workspaceMaxDepth = cfg.VolumeWorkspaceMaxDepth
workspaceMaxEntries = cfg.VolumeWorkspaceMaxEntries
workspaceMaxFileSizeMB = cfg.VolumeWorkspaceMaxFileSizeMB
@@ -75,13 +96,19 @@ func NewVolumeService(db *database.DB, dockerService *docker.DockerClientService
db: db,
dockerService: dockerService,
eventService: eventService,
+ activityService: activityService,
settingsService: settingsService,
+ containerService: containerService,
imageService: imageService,
+ engine: engine,
+ s3Destinations: s3Destinations,
backupVolumeName: backupVolumeName,
+ encryptionKey: encryptionKey,
workspaceMaxDepth: workspaceMaxDepth,
workspaceMaxEntries: workspaceMaxEntries,
workspaceMaxFileSizeBytes: workspacepkg.MaxFileSizeBytes(workspaceMaxFileSizeMB),
helperByVolume: make(map[string]*volumeHelper),
+ jobs: entityjobs.New("volume-backup:", backup.VolumeAdmissionScope),
}
}
@@ -190,6 +217,7 @@ func (s *VolumeService) DeleteVolume(ctx context.Context, name string, force boo
}
s.removeHelperEntry(name)
+ s.removeVolumeBackupPolicyInternal(ctx, name)
dockerutil.InvalidateVolumeUsageCache(dockerClient)
return nil
}
@@ -230,6 +258,7 @@ func (s *VolumeService) PruneVolumesWithOptions(ctx context.Context, all bool) (
for _, volumeName := range volumePruneResult.Report.VolumesDeleted {
s.removeHelperEntry(volumeName)
+ s.removeVolumeBackupPolicyInternal(ctx, volumeName)
}
dockerutil.InvalidateVolumeUsageCache(dockerClient)
diff --git a/backend/internal/volume/service_test.go b/backend/internal/volume/service_test.go
index 251d1cfc7..dc45d211d 100644
--- a/backend/internal/volume/service_test.go
+++ b/backend/internal/volume/service_test.go
@@ -6,20 +6,56 @@ import (
"net/http"
"net/http/httptest"
"strings"
+ "sync"
"testing"
"time"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/activity"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/backup"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/common"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/config"
+ containerdomain "github.com/getarcaneapp/arcane/backend/v2/internal/container"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/docker"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/event"
+ s3domain "github.com/getarcaneapp/arcane/backend/v2/internal/s3"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/volumehelper"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/entityjobs"
+ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"
volumetypes "github.com/getarcaneapp/arcane/types/v2/volume"
+ "github.com/libtnb/sqlite"
"github.com/moby/moby/api/types/container"
"github.com/moby/moby/api/types/mount"
"github.com/moby/moby/api/types/volume"
"github.com/moby/moby/client"
"github.com/stretchr/testify/require"
+ "go.getarcane.app/sys/crypto"
+ "go.uber.org/fx/fxtest"
+ "gorm.io/gorm"
)
+func newVolumeBackupEngineForTestInternal(t testing.TB) *backup.Engine {
+ t.Helper()
+ fxLifecycle := fxtest.NewLifecycle(t)
+ runtime, err := actors.NewRuntime(t.Context(), fxLifecycle)
+ require.NoError(t, err)
+ gate, err := actors.NewGate[actors.AdmissionKey](t.Context(), runtime, "volume-backup-test-admission", t.Name())
+ require.NoError(t, err)
+ engine := backup.NewEngine(t.Context(), runtime, gate, nil)
+ t.Cleanup(func() {
+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ require.NoError(t, engine.Stop(stopCtx))
+ require.NoError(t, gate.Stop(stopCtx))
+ require.NoError(t, fxLifecycle.Stop(stopCtx))
+ })
+ return engine
+}
+
func newVolumeServiceTestDockerClientInternal(t *testing.T, server *httptest.Server) *client.Client {
t.Helper()
dockerClient, err := client.New(
@@ -477,51 +513,6 @@ func TestBackupMountWarningFromArcaneMountsInternal(t *testing.T) {
}
}
-func TestBackupArchiveFilenameInternal(t *testing.T) {
- svc := &VolumeService{}
-
- tests := []struct {
- name string
- backupID string
- want string
- wantErr bool
- }{
- {
- name: "valid backup id",
- backupID: "volume-123-abc",
- want: "volume-123-abc.tar.gz",
- },
- {
- name: "trims surrounding whitespace",
- backupID: " volume-123-abc ",
- want: "volume-123-abc.tar.gz",
- },
- {
- name: "rejects traversal attempts",
- backupID: "../../bin/busybox",
- wantErr: true,
- },
- {
- name: "rejects path separators",
- backupID: "nested/path",
- wantErr: true,
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := svc.backupArchiveFilenameInternal(tt.backupID)
- if tt.wantErr {
- require.Error(t, err)
- return
- }
-
- require.NoError(t, err)
- require.Equal(t, tt.want, got)
- })
- }
-}
-
func TestCollectStaleHelperIDsInternal(t *testing.T) {
now := time.Now()
s := &VolumeService{
@@ -605,3 +596,403 @@ func TestTouchHelperInternal(t *testing.T) {
s.touchHelperInternal("missing")
require.NotContains(t, s.helperByVolume, "missing")
}
+
+type volumeBackupPolicySchedulerInternal struct {
+ jobs map[string]schedulertypes.Job
+}
+
+func (s *volumeBackupPolicySchedulerInternal) AddJob(_ context.Context, job schedulertypes.Job) error {
+ s.jobs[job.Name()] = job
+ return nil
+}
+
+func (s *volumeBackupPolicySchedulerInternal) RemoveJob(_ context.Context, name string) {
+ delete(s.jobs, name)
+}
+
+func (s *volumeBackupPolicySchedulerInternal) HasJob(name string) bool {
+ _, ok := s.jobs[name]
+ return ok
+}
+
+func TestVolumeBackupPolicy_UpdateRegistersIndependentJobsAndSettings(t *testing.T) {
+ gormDB, err := gorm.Open(sqlite.Open("file:volume-backup-schedule?mode=memory&cache=shared"), &gorm.Config{})
+ require.NoError(t, err)
+ require.NoError(t, gormDB.AutoMigrate(&VolumeBackupPolicy{}, &VolumeBackup{}))
+ db := &database.DB{DB: gormDB}
+ scheduler := &volumeBackupPolicySchedulerInternal{jobs: make(map[string]schedulertypes.Job)}
+ service := &VolumeService{db: db, jobs: entityjobs.New("volume-backup:", backup.VolumeAdmissionScope)}
+ fxLifecycle := fxtest.NewLifecycle(t)
+ runtime, err := actors.NewRuntime(t.Context(), fxLifecycle)
+ require.NoError(t, err)
+ gate, err := actors.NewGate[actors.AdmissionKey](t.Context(), runtime, "volume-policy-test-admission", t.Name())
+ require.NoError(t, err)
+ t.Cleanup(func() {
+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ require.NoError(t, gate.Stop(stopCtx))
+ require.NoError(t, fxLifecycle.Stop(stopCtx))
+ })
+ require.NoError(t, service.SetScheduler(context.Background(), scheduler, gate))
+
+ collection, err := service.UpdateBackupPolicies(context.Background(), "app-data", []volumetypes.UpdateBackupPolicy{
+ {Enabled: true, Schedule: "0 */15 * * * *", RetentionCount: 5, StopContainers: true, LocalEnabled: true},
+ {Enabled: true, Schedule: "0 0 2 * * *", RetentionCount: 30, LocalEnabled: true},
+ })
+ require.NoError(t, err)
+ require.Len(t, collection.Policies, 2)
+ require.Equal(t, 5, collection.Policies[0].RetentionCount)
+ require.True(t, collection.Policies[0].StopContainers)
+ require.Equal(t, 30, collection.Policies[1].RetentionCount)
+ require.Len(t, scheduler.jobs, 2)
+
+ firstID := collection.Policies[0].ID
+ collection, err = service.UpdateBackupPolicies(context.Background(), "app-data", []volumetypes.UpdateBackupPolicy{{
+ ID: firstID, Schedule: "0 */30 * * * *", RetentionCount: 9, LocalEnabled: true,
+ }})
+ require.NoError(t, err)
+ require.Len(t, collection.Policies, 1)
+ require.Equal(t, firstID, collection.Policies[0].ID)
+ require.Equal(t, 9, collection.Policies[0].RetentionCount)
+ require.Empty(t, scheduler.jobs)
+}
+
+func TestVolumeBackupPolicy_GetReturnsLastRunForEachPolicy(t *testing.T) {
+ gormDB, err := gorm.Open(sqlite.Open("file:volume-backup-last-runs?mode=memory&cache=shared"), &gorm.Config{})
+ require.NoError(t, err)
+ require.NoError(t, gormDB.AutoMigrate(&VolumeBackupPolicy{}, &VolumeBackup{}))
+ first := &VolumeBackupPolicy{VolumeName: "app-data", Schedule: "0 0 2 * * *", LocalEnabled: true}
+ second := &VolumeBackupPolicy{VolumeName: "app-data", Schedule: "0 0 14 * * *", LocalEnabled: true}
+ require.NoError(t, gormDB.Create(first).Error)
+ require.NoError(t, gormDB.Create(second).Error)
+ require.NoError(t, gormDB.Create(&VolumeBackup{VolumeName: "app-data", PolicyID: first.ID, Status: VolumeBackupStatusSucceeded}).Error)
+ require.NoError(t, gormDB.Create(&VolumeBackup{VolumeName: "app-data", PolicyID: second.ID, Status: VolumeBackupStatusFailed}).Error)
+
+ service := &VolumeService{db: &database.DB{DB: gormDB}}
+ collection, err := service.GetBackupPolicies(context.Background(), "app-data")
+ require.NoError(t, err)
+ require.Len(t, collection.Policies, 2)
+ require.Equal(t, "succeeded", collection.Policies[0].LastRun.Status)
+ require.Equal(t, "failed", collection.Policies[1].LastRun.Status)
+}
+
+func TestVolumeBackupPolicy_RetentionIgnoresFailedRuns(t *testing.T) {
+ gormDB, err := gorm.Open(sqlite.Open("file:volume-backup-retention-failed?mode=memory&cache=shared"), &gorm.Config{})
+ require.NoError(t, err)
+ require.NoError(t, gormDB.AutoMigrate(&VolumeBackup{}))
+
+ policyID := "policy-1"
+ require.NoError(t, gormDB.Create(&VolumeBackup{
+ VolumeName: "app-data", PolicyID: policyID, Status: VolumeBackupStatusSucceeded,
+ LocalSnapshotID: "snapshot-1", CreatedAt: time.Now().Add(-time.Hour),
+ }).Error)
+ require.NoError(t, gormDB.Create(&VolumeBackup{
+ VolumeName: "app-data", PolicyID: policyID, Status: VolumeBackupStatusFailed,
+ CreatedAt: time.Now(),
+ }).Error)
+
+ service := &VolumeService{db: &database.DB{DB: gormDB}}
+ require.NoError(t, service.applyVolumeBackupRetentionInternal(context.Background(), policyID, 1))
+
+ var backups []VolumeBackup
+ require.NoError(t, gormDB.Order("created_at ASC").Find(&backups).Error)
+ require.Len(t, backups, 2)
+ require.Equal(t, "snapshot-1", backups[0].LocalSnapshotID)
+ require.Equal(t, VolumeBackupStatusFailed, backups[1].Status)
+}
+
+func TestVolumeBackup_ValidationErrors(t *testing.T) {
+ gormDB, err := gorm.Open(sqlite.Open("file:volume-backup-validation?mode=memory&cache=shared"), &gorm.Config{})
+ require.NoError(t, err)
+ require.NoError(t, gormDB.AutoMigrate(&VolumeBackupPolicy{}))
+ service := &VolumeService{db: &database.DB{DB: gormDB}}
+ ctx := context.Background()
+
+ _, err = service.UpdateBackupPolicies(ctx, "app-data", []volumetypes.UpdateBackupPolicy{{Schedule: "not a cron", LocalEnabled: true}})
+ require.ErrorContains(t, err, "invalid volume backup schedule")
+
+ _, err = service.UpdateBackupPolicies(ctx, "app-data", []volumetypes.UpdateBackupPolicy{{Schedule: "0 0 2 * * *", RetentionCount: 7}})
+ require.ErrorContains(t, err, "select at least one volume backup destination")
+
+ _, err = service.CreateBackup(ctx, "app-data", common.User{}, VolumeBackupTriggerManual, volumetypes.CreateBackupRequest{Destination: volumetypes.BackupDestination("invalid")})
+ require.EqualError(t, err, "invalid volume backup destination")
+
+ _, err = service.CreateBackup(ctx, "app-data", common.User{}, VolumeBackupTriggerManual, volumetypes.CreateBackupRequest{Destination: volumetypes.BackupDestinationS3})
+ require.EqualError(t, err, "select an S3 destination for the volume backup")
+}
+
+func TestVolumeBackupPolicy_ScheduledRunCreatesActivity(t *testing.T) {
+ gormDB, err := gorm.Open(sqlite.Open("file:volume-backup-scheduled-activity?mode=memory&cache=shared"), &gorm.Config{})
+ require.NoError(t, err)
+ require.NoError(t, gormDB.AutoMigrate(&VolumeBackupPolicy{}, &VolumeBackup{}, &activity.Activity{}))
+ db := &database.DB{DB: gormDB}
+ policy := &VolumeBackupPolicy{
+ VolumeName: "app-data",
+ Enabled: true,
+ Schedule: "0 0 2 * * *",
+ RetentionCount: 7,
+ LocalEnabled: true,
+ }
+ require.NoError(t, gormDB.Create(policy).Error)
+ engine := newVolumeBackupEngineForTestInternal(t)
+ service := &VolumeService{
+ db: db,
+ activityService: activity.NewActivityService(db, nil),
+ engine: engine,
+ jobs: entityjobs.New("volume-backup:", backup.VolumeAdmissionScope),
+ }
+ // Holding the volume's admission lease makes the scheduled run fail with
+ // "already running", which still must record a failed activity.
+ lease, admitted, err := engine.TryAcquireRun(context.Background(), backup.VolumeAdmissionScope, policy.VolumeName)
+ require.NoError(t, err)
+ require.True(t, admitted)
+ defer lease.Release()
+
+ service.runScheduledBackupInternal(context.Background(), policy.ID)
+
+ var activity activity.Activity
+ require.NoError(t, gormDB.Where("resource_type = ?", "volume_backup").First(&activity).Error)
+ require.Equal(t, activitytypes.StatusFailed, activity.Status)
+ require.Equal(t, "scheduled_volume_backup", activity.Metadata["action"])
+ require.Equal(t, policy.Schedule, activity.Metadata["schedule"])
+}
+
+func TestVolumeBackupPolicy_UpdateUsesSelectedS3Destination(t *testing.T) {
+ gormDB, err := gorm.Open(sqlite.Open("file:volume-backup-s3-secret?mode=memory&cache=shared"), &gorm.Config{})
+ require.NoError(t, err)
+ require.NoError(t, gormDB.AutoMigrate(&VolumeBackupPolicy{}, &VolumeBackup{}, &s3domain.S3Destination{}))
+ db := &database.DB{DB: gormDB}
+ crypto.InitEncryption(&crypto.Config{
+ EncryptionKey: "test-encryption-key-for-volume-backups-32bytes",
+ Environment: "test",
+ })
+ encryptedSecret, err := crypto.Encrypt("destination-s3-secret")
+ require.NoError(t, err)
+ destination := &s3domain.S3Destination{
+ Name: "Offsite",
+ Bucket: "volume-backups",
+ Region: "us-east-1",
+ AccessKeyID: "destination-access-key",
+ SecretAccessKey: encryptedSecret,
+ UseSSL: true,
+ }
+ require.NoError(t, gormDB.Create(destination).Error)
+ service := &VolumeService{
+ db: db,
+ s3Destinations: s3domain.NewS3DestinationService(db),
+ jobs: entityjobs.New("volume-backup:", backup.VolumeAdmissionScope),
+ }
+ collection, err := service.UpdateBackupPolicies(context.Background(), "app-data", []volumetypes.UpdateBackupPolicy{{
+ Schedule: "0 0 2 * * *",
+ RetentionCount: 7,
+ S3Enabled: true,
+ S3DestinationID: destination.ID,
+ }})
+ require.NoError(t, err)
+ require.Len(t, collection.Policies, 1)
+ policy := collection.Policies[0]
+ require.False(t, policy.LocalEnabled)
+ require.True(t, policy.S3Enabled)
+ require.True(t, policy.S3Available)
+ require.Equal(t, destination.ID, policy.S3DestinationID)
+ require.Equal(t, "Offsite", policy.S3DestinationName)
+ require.Equal(t, "volume-backups", policy.S3Bucket)
+
+ var stored VolumeBackupPolicy
+ require.NoError(t, gormDB.Where("volume_name = ?", "app-data").First(&stored).Error)
+ require.False(t, stored.LocalEnabled)
+ require.True(t, stored.S3Enabled)
+ require.Equal(t, destination.ID, stored.S3DestinationID)
+}
+
+func TestVolumeBackup_ListResolvesDestinationName(t *testing.T) {
+ gormDB, err := gorm.Open(sqlite.Open("file:volume-backup-destination-name?mode=memory&cache=shared"), &gorm.Config{})
+ require.NoError(t, err)
+ require.NoError(t, gormDB.AutoMigrate(&VolumeBackup{}, &s3domain.S3Destination{}))
+ db := &database.DB{DB: gormDB}
+ destination := &s3domain.S3Destination{
+ Name: "Offsite",
+ Bucket: "volume-backups",
+ Region: "us-east-1",
+ AccessKeyID: "access-key",
+ SecretAccessKey: "encrypted-secret",
+ }
+ require.NoError(t, gormDB.Create(destination).Error)
+ require.NoError(t, gormDB.Create(&VolumeBackup{
+ VolumeName: "app-data",
+ Status: VolumeBackupStatusSucceeded,
+ Trigger: VolumeBackupTriggerManual,
+ Destination: volumetypes.BackupDestinationLocalS3,
+ S3DestinationID: destination.ID,
+ }).Error)
+
+ service := &VolumeService{db: db, s3Destinations: s3domain.NewS3DestinationService(db)}
+ backups, _, err := service.ListBackupsPaginated(context.Background(), "app-data", pagination.QueryParams{})
+ require.NoError(t, err)
+ require.Len(t, backups, 1)
+ require.Equal(t, volumetypes.BackupDestinationLocalS3, backups[0].Destination)
+ require.Equal(t, "Offsite", backups[0].S3DestinationName)
+}
+
+func setupVolumeBackupLifecycleTestInternal(t *testing.T, handler http.Handler) (*VolumeService, *client.Client) {
+ t.Helper()
+
+ server := httptest.NewServer(handler)
+ t.Cleanup(server.Close)
+ dockerClient, err := client.New(client.WithHost(server.URL), client.WithAPIVersion("1.41"))
+ require.NoError(t, err)
+ t.Cleanup(func() { _ = dockerClient.Close() })
+
+ gormDB, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{})
+ require.NoError(t, err)
+ require.NoError(t, gormDB.AutoMigrate(&event.Event{}))
+ db := &database.DB{DB: gormDB}
+ dockerService := docker.NewDockerClientService(t.Context(), db, &config.Config{}, nil).WithClient(dockerClient)
+ eventService := event.NewEventService(db, &config.Config{}, nil)
+ containerService := containerdomain.NewContainerService(t.Context(), eventService, dockerService, nil, nil, nil)
+ return &VolumeService{containerService: containerService}, dockerClient
+}
+
+func TestVolumeBackupContainerLifecycleStopsAndRestartsOnlyRunningContainersUsingVolume(t *testing.T) {
+ var mu sync.Mutex
+ var operations []string
+ serverHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ switch {
+ case strings.HasSuffix(r.URL.Path, "/containers/json"):
+ require.NoError(t, json.NewEncoder(w).Encode([]container.Summary{
+ {ID: "uses-volume", Mounts: []container.MountPoint{{Type: mount.TypeVolume, Name: "app-data"}}},
+ {ID: "other-volume", Mounts: []container.MountPoint{{Type: mount.TypeVolume, Name: "other-data"}}},
+ {ID: "arcane", Labels: map[string]string{"com.getarcaneapp.arcane": "true"}, Mounts: []container.MountPoint{{Type: mount.TypeVolume, Name: "app-data"}}},
+ }))
+ case strings.HasSuffix(r.URL.Path, "/containers/uses-volume/stop"):
+ mu.Lock()
+ operations = append(operations, "stop:uses-volume")
+ mu.Unlock()
+ w.WriteHeader(http.StatusNoContent)
+ case strings.HasSuffix(r.URL.Path, "/containers/uses-volume/start"):
+ mu.Lock()
+ operations = append(operations, "start:uses-volume")
+ mu.Unlock()
+ w.WriteHeader(http.StatusNoContent)
+ default:
+ http.NotFound(w, r)
+ }
+ })
+
+ service, dockerClient := setupVolumeBackupLifecycleTestInternal(t, serverHandler)
+ actor := common.User{BaseModel: database.BaseModel{ID: "user-1"}, Username: "tester"}
+ stopped, err := service.stopRunningContainersForBackupInternal(context.Background(), dockerClient, "app-data", actor, false)
+ require.NoError(t, err)
+ require.Len(t, stopped, 1)
+ require.Equal(t, "uses-volume", stopped[0].ID)
+
+ remaining, err := service.startContainersAfterBackupInternal(context.Background(), dockerClient, stopped, actor)
+ require.NoError(t, err)
+ require.Empty(t, remaining)
+ require.Equal(t, []string{"stop:uses-volume", "start:uses-volume"}, operations)
+}
+
+func TestVolumeBackupContainerLifecycleRollsBackStoppedContainersOnStopFailure(t *testing.T) {
+ var mu sync.Mutex
+ var operations []string
+ serverHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ switch {
+ case strings.HasSuffix(r.URL.Path, "/containers/json"):
+ require.NoError(t, json.NewEncoder(w).Encode([]container.Summary{
+ {ID: "first", Mounts: []container.MountPoint{{Type: mount.TypeVolume, Name: "app-data"}}},
+ {ID: "second", Mounts: []container.MountPoint{{Type: mount.TypeVolume, Name: "app-data"}}},
+ }))
+ case strings.HasSuffix(r.URL.Path, "/containers/first/stop"):
+ mu.Lock()
+ operations = append(operations, "stop:first")
+ mu.Unlock()
+ w.WriteHeader(http.StatusNoContent)
+ case strings.HasSuffix(r.URL.Path, "/containers/second/stop"):
+ mu.Lock()
+ operations = append(operations, "stop:second")
+ mu.Unlock()
+ http.Error(w, "stop failed", http.StatusInternalServerError)
+ case strings.HasSuffix(r.URL.Path, "/containers/first/start"):
+ mu.Lock()
+ operations = append(operations, "start:first")
+ mu.Unlock()
+ w.WriteHeader(http.StatusNoContent)
+ default:
+ http.NotFound(w, r)
+ }
+ })
+
+ service, dockerClient := setupVolumeBackupLifecycleTestInternal(t, serverHandler)
+ actor := common.User{BaseModel: database.BaseModel{ID: "user-1"}, Username: "tester"}
+ stillStopped, err := service.stopRunningContainersForBackupInternal(context.Background(), dockerClient, "app-data", actor, false)
+ require.ErrorContains(t, err, "failed to stop container second")
+ require.Empty(t, stillStopped)
+ require.Equal(t, []string{"stop:first", "stop:second", "start:first"}, operations)
+}
+
+func TestVolumeBackupContainerLifecycleWaitsForRunningComposeReplacement(t *testing.T) {
+ var mu sync.Mutex
+ var operations []string
+ listCalls := 0
+ serverHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ switch {
+ case strings.HasSuffix(r.URL.Path, "/containers/json"):
+ mu.Lock()
+ listCalls++
+ call := listCalls
+ mu.Unlock()
+ if call == 1 {
+ require.NoError(t, json.NewEncoder(w).Encode([]container.Summary{
+ {
+ ID: "old-id",
+ Names: []string{"/old-name"},
+ State: container.StateRunning,
+ Labels: map[string]string{"com.docker.compose.project": "vault", "com.docker.compose.service": "vaultwarden", "com.docker.compose.container-number": "1"},
+ Mounts: []container.MountPoint{{Type: mount.TypeVolume, Name: "app-data"}},
+ },
+ }))
+ return
+ }
+ if call == 2 {
+ require.NoError(t, json.NewEncoder(w).Encode([]container.Summary{}))
+ return
+ }
+ require.NoError(t, json.NewEncoder(w).Encode([]container.Summary{
+ {
+ ID: "new-id",
+ Names: []string{"/new-name"},
+ State: container.StateRunning,
+ Labels: map[string]string{"com.docker.compose.project": "vault", "com.docker.compose.service": "vaultwarden", "com.docker.compose.container-number": "1"},
+ Mounts: []container.MountPoint{{Type: mount.TypeVolume, Name: "app-data"}},
+ },
+ }))
+ case strings.HasSuffix(r.URL.Path, "/containers/old-id/stop"):
+ mu.Lock()
+ operations = append(operations, "stop:old-id")
+ mu.Unlock()
+ w.WriteHeader(http.StatusNoContent)
+ case strings.Contains(r.URL.Path, "/start"):
+ mu.Lock()
+ operations = append(operations, "unexpected:"+r.URL.Path)
+ mu.Unlock()
+ w.WriteHeader(http.StatusNoContent)
+ default:
+ http.NotFound(w, r)
+ }
+ })
+
+ service, dockerClient := setupVolumeBackupLifecycleTestInternal(t, serverHandler)
+ actor := common.User{BaseModel: database.BaseModel{ID: "user-1"}, Username: "tester"}
+ stopped, err := service.stopRunningContainersForBackupInternal(context.Background(), dockerClient, "app-data", actor, false)
+ require.NoError(t, err)
+ require.Len(t, stopped, 1)
+ require.Equal(t, "old-id", stopped[0].ID)
+
+ remaining, err := service.startContainersAfterBackupInternal(context.Background(), dockerClient, stopped, actor)
+ require.NoError(t, err)
+ require.Empty(t, remaining)
+ require.Equal(t, []string{"stop:old-id"}, operations)
+}
diff --git a/backend/pkg/authz/access_policy.go b/backend/pkg/authz/access_policy.go
index 480d50280..c17951792 100644
--- a/backend/pkg/authz/access_policy.go
+++ b/backend/pkg/authz/access_policy.go
@@ -48,6 +48,8 @@ var accessSurfacesInternal = []AccessSurface{
"settings.category.jobschedule",
"settings.category.notifications",
"settings.category.roles",
+ "settings.category.s3destinations",
+ "settings.category.systembackups",
"settings.category.timeouts",
"settings.category.users",
"settings.category.webhooks",
@@ -99,6 +101,8 @@ var accessSurfacesInternal = []AccessSurface{
settingsCategorySurfaceInternal("jobschedule", "/settings/jobs", "Automations", AccessScopeModeSelectedEnvPlusGlobal, []string{PermJobsManage}),
settingsCategorySurfaceInternal("notifications", "/settings/notifications", "Notifications", AccessScopeModeGlobalOnly, []string{PermNotificationsManage}),
settingsCategorySurfaceInternal("roles", "/settings/roles", "Roles", AccessScopeModeGlobalOnly, []string{PermRolesList, PermRolesRead}),
+ settingsCategorySurfaceInternal("s3destinations", "/settings/backups/s3", "S3 Destinations", AccessScopeModeGlobalOnly, []string{PermS3DestinationsList, PermS3DestinationsRead}),
+ globalAdminSettingsCategorySurfaceInternal("systembackups", "/settings/backups", "Backups"),
routeSurfaceInternal("route.settings.roles.new", "/settings/roles/new", "Create role", AccessScopeModeGlobalOnly, []string{PermRolesList, PermRolesRead}, 0),
routeSurfaceInternal("route.settings.roles.detail", "/settings/roles/{id}", "Role", AccessScopeModeGlobalOnly, []string{PermRolesList, PermRolesRead}, 0),
settingsCategorySurfaceInternal("timeouts", "/settings/timeouts", "Timeouts", AccessScopeModeGlobalOnly, []string{PermSettingsRead}),
@@ -243,6 +247,14 @@ func globalAdminRouteSurfaceInternal(id, url, label string) AccessSurface {
return surface
}
+// globalAdminSettingsCategorySurfaceInternal approximates the global-admin
+// gate its routes enforce: only admins hold every permission.
+func globalAdminSettingsCategorySurfaceInternal(categoryID, url, label string) AccessSurface {
+ surface := settingsCategorySurfaceInternal(categoryID, url, label, AccessScopeModeGlobalOnly, AllPermissions())
+ surface.MatchMode = AccessMatchModeAllOf
+ return surface
+}
+
func settingsCategorySurfaceInternal(categoryID, url, label, scopeMode string, permissions []string) AccessSurface {
return AccessSurface{
ID: "settings.category." + categoryID,
diff --git a/backend/pkg/authz/catalog.go b/backend/pkg/authz/catalog.go
index cd3b69f34..761daaded 100644
--- a/backend/pkg/authz/catalog.go
+++ b/backend/pkg/authz/catalog.go
@@ -93,6 +93,21 @@ var permissionCatalog = []PermissionCatalogResource{
{"test", PermGitReposTest, "Test", ""},
{"sync", PermGitReposSync, "Sync", ""},
}},
+ {"s3-destinations", "S3 Destinations", PermissionScopeGlobal, []PermissionCatalogAction{
+ {"list", PermS3DestinationsList, "List", ""},
+ {"read", PermS3DestinationsRead, "Read", ""},
+ {"create", PermS3DestinationsCreate, "Create", ""},
+ {"update", PermS3DestinationsUpdate, "Update", ""},
+ {"delete", PermS3DestinationsDelete, "Delete", ""},
+ {"test", PermS3DestinationsTest, "Test", ""},
+ {"sync", PermS3DestinationsSync, "Sync", ""},
+ }},
+ {"system-backups", "System Backups", PermissionScopeGlobal, []PermissionCatalogAction{
+ {"read", PermSystemBackupsRead, "Read", ""},
+ {"manage", PermSystemBackupsManage, "Manage", ""},
+ {"restore", PermSystemBackupsRestore, "Restore", ""},
+ {"recovery-key", PermSystemBackupsRecoveryKey, "Recovery Key", ""},
+ }},
{"events", "Events", PermissionScopeGlobal, []PermissionCatalogAction{
{"read", PermEventsRead, "Read", ""},
{"delete", PermEventsDelete, "Delete", ""},
diff --git a/backend/pkg/authz/permission_set_test.go b/backend/pkg/authz/permission_set_test.go
index 19adff6b6..a0b879a93 100644
--- a/backend/pkg/authz/permission_set_test.go
+++ b/backend/pkg/authz/permission_set_test.go
@@ -283,6 +283,85 @@ func TestVariablePermissionsAreSeparateGlobalGrantsWithBuiltInAccess(t *testing.
}
}
+func TestSystemBackupPermissionsAreAdminOnlyGlobalGrants(t *testing.T) {
+ builtInRoles := map[string][]string{
+ "Editor": BuiltInEditorPermissions(),
+ "No-Shell Editor": BuiltInNoShellEditorPermissions(),
+ "Viewer": BuiltInViewerPermissions(),
+ "Monitor": BuiltInMonitorPermissions(),
+ "Deployer": BuiltInDeployerPermissions(),
+ }
+ for _, permission := range []string{PermSystemBackupsRead, PermSystemBackupsManage, PermSystemBackupsRestore, PermSystemBackupsRecoveryKey} {
+ require.True(t, IsKnownPermission(permission), "system backup permission %q must be known", permission)
+ require.True(t, IsOrgLevel(permission), "system backup permission %q must be global", permission)
+ require.Contains(t, AllPermissions(), permission, "Admin must receive %q", permission)
+ for name, permissions := range builtInRoles {
+ require.NotContains(t, permissions, permission, "%s must not receive %q", name, permission)
+ }
+ }
+}
+
+func TestS3DestinationPermissionsAreSeparateGlobalGrantsWithBuiltInAccess(t *testing.T) {
+ s3Permissions := []string{
+ PermS3DestinationsList,
+ PermS3DestinationsRead,
+ PermS3DestinationsCreate,
+ PermS3DestinationsUpdate,
+ PermS3DestinationsDelete,
+ PermS3DestinationsTest,
+ PermS3DestinationsSync,
+ }
+
+ for _, permission := range s3Permissions {
+
+ require.True(t, IsKnownPermission(permission),
+ "S3 destination permission %q must be known", permission)
+
+ require.True(t, IsOrgLevel(permission),
+ "S3 destination permission %q must be global", permission)
+
+ require.Contains(t, AllPermissions(), permission,
+ "Admin must receive %q", permission)
+
+ require.NotContains(t, BuiltInMonitorPermissions(), permission,
+ "Monitor must not receive %q", permission)
+
+ require.NotContains(t, BuiltInDeployerPermissions(), permission,
+ "Deployer must not receive %q", permission)
+
+ }
+
+ // Roles that can run backups pick a destination, so they read but never
+ // manage the stored credentials.
+ for name, permissions := range map[string][]string{
+ "Editor": BuiltInEditorPermissions(),
+ "No-Shell Editor": BuiltInNoShellEditorPermissions(),
+ "Viewer": BuiltInViewerPermissions(),
+ } {
+ require.Contains(t, permissions, PermS3DestinationsList,
+ "%s must receive %q", name, PermS3DestinationsList)
+
+ require.Contains(t, permissions, PermS3DestinationsRead,
+ "%s must receive %q", name, PermS3DestinationsRead)
+
+ for _, permission := range s3Permissions[2:] {
+
+ require.NotContains(t, permissions, permission,
+ "%s must not receive %q", name, permission)
+
+ }
+ }
+
+ settingsOnly := NewPermissionSet()
+ settingsOnly.AddGlobal(PermSettingsRead, PermSettingsWrite)
+ for _, permission := range s3Permissions {
+
+ require.False(t, settingsOnly.Allows(permission, ""),
+ "settings grants must not satisfy %q", permission)
+
+ }
+}
+
func TestPermissionCatalogDerivesKnownPermissionsAndScopes(t *testing.T) {
catalog := PermissionCatalog()
diff --git a/backend/pkg/authz/permissions.go b/backend/pkg/authz/permissions.go
index 30060afff..07bd18d35 100644
--- a/backend/pkg/authz/permissions.go
+++ b/backend/pkg/authz/permissions.go
@@ -83,6 +83,29 @@ const (
PermGitReposTest = "git-repositories:test"
PermGitReposSync = "git-repositories:sync"
+ // PermS3DestinationsList and its siblings gate S3 backup destinations,
+ // which hold credentials and are manager-owned, so they are granted
+ // separately from general settings, like container registries.
+ PermS3DestinationsList = "s3-destinations:list"
+ PermS3DestinationsRead = "s3-destinations:read"
+ PermS3DestinationsCreate = "s3-destinations:create"
+ PermS3DestinationsUpdate = "s3-destinations:update"
+ PermS3DestinationsDelete = "s3-destinations:delete"
+ PermS3DestinationsTest = "s3-destinations:test"
+ PermS3DestinationsSync = "s3-destinations:sync"
+
+ // PermSystemBackupsRead and siblings gate Arcane's own system backups.
+ // Every route additionally requires a global admin; the dedicated grants
+ // exist so admin-owned scoped API keys can be limited to this surface
+ // instead of carrying settings:read/write. Restore is separate because it
+ // replaces the whole instance.
+ PermSystemBackupsRead = "system-backups:read"
+ PermSystemBackupsManage = "system-backups:manage"
+ PermSystemBackupsRestore = "system-backups:restore"
+ // PermSystemBackupsRecoveryKey is separate from manage: regenerating the
+ // key invalidates access to the existing repositories.
+ PermSystemBackupsRecoveryKey = "system-backups:recovery-key"
+
PermEventsRead = "events:read"
PermEventsDelete = "events:delete"
PermCustomizeManage = "customize:manage"
@@ -293,6 +316,8 @@ func BuiltInEditorPermissions() []string {
PermTemplatesList, PermTemplatesRead, PermTemplatesCreate, PermTemplatesUpdate, PermTemplatesDelete,
PermVariablesRead, PermVariablesCreate, PermVariablesUpdate, PermVariablesDelete, PermVariablesSync,
PermGitReposList, PermGitReposRead,
+ // Editor can run volume backups, so it needs to pick a destination.
+ PermS3DestinationsList, PermS3DestinationsRead,
PermEventsRead,
// Full env-scoped Docker management
PermContainersList, PermContainersRead, PermContainersLogs, PermContainersCreate, PermContainersStart, PermContainersStop, PermContainersRestart, PermContainersRedeploy, PermContainersKill, PermContainersPause, PermContainersDelete, PermContainersExec, PermContainersAutoUpdate,
@@ -351,6 +376,7 @@ func BuiltInViewerPermissions() []string {
PermTemplatesList, PermTemplatesRead,
PermVariablesRead,
PermGitReposList, PermGitReposRead,
+ PermS3DestinationsList, PermS3DestinationsRead,
PermEventsRead,
PermContainersList, PermContainersRead, PermContainersLogs,
PermProjectsList, PermProjectsRead, PermProjectsLogs,
diff --git a/backend/pkg/libarcane/activity/handler.go b/backend/pkg/libarcane/activity/handler.go
index 2f7abd111..e2827f5a5 100644
--- a/backend/pkg/libarcane/activity/handler.go
+++ b/backend/pkg/libarcane/activity/handler.go
@@ -26,6 +26,16 @@ import (
// cancelled (rather than failed) terminal status.
const ErrCanceled = errors.Sentinel("activity cancelled by user")
+type handlerActivityIDContextKey struct{}
+
+func IDFromContext(ctx context.Context) string {
+ if ctx == nil {
+ return ""
+ }
+ activityID, _ := ctx.Value(handlerActivityIDContextKey{}).(string)
+ return strings.TrimSpace(activityID)
+}
+
// cancelledMessage is the latest-message recorded when work is cancelled.
const cancelledMessage = "Cancelled by user"
@@ -125,6 +135,7 @@ func StartHandlerActivity(
if tracker, ok := activityService.(Tracker); ok {
workCtx = tracker.Track(ctx, activity.ID)
}
+ workCtx = context.WithValue(workCtx, handlerActivityIDContextKey{}, activity.ID)
return activity.ID, workCtx
}
diff --git a/backend/pkg/libarcane/edge/commands.go b/backend/pkg/libarcane/edge/commands.go
index a7afc0c9d..04c3c9b84 100644
--- a/backend/pkg/libarcane/edge/commands.go
+++ b/backend/pkg/libarcane/edge/commands.go
@@ -14,6 +14,8 @@ var commandRoutes = []commandRoute{
{Method: http.MethodGet, PathPattern: "/api/swarm/node-identity", CommandName: "swarm.node_identity"},
{Method: http.MethodPost, PathPattern: "/api/container-registries/sync", CommandName: "container_registry.sync"},
{Method: http.MethodPost, PathPattern: "/api/git-repositories/sync", CommandName: "git_repository.sync"},
+ {Method: http.MethodPost, PathPattern: "/api/backups/s3/sync", CommandName: "s3_destination.sync"},
+ {Method: http.MethodGet, PathPattern: "/api/backups/s3/{id}/in-use", CommandName: "s3_destination.in_use"},
{Method: http.MethodGet, PathPattern: "/api/environments/{id}/containers", CommandName: "container.list"},
{Method: http.MethodGet, PathPattern: "/api/environments/{id}/containers/counts", CommandName: "container.counts"},
@@ -122,9 +124,11 @@ var commandRoutes = []commandRoute{
{Method: http.MethodPost, PathPattern: "/api/environments/{id}/volumes/{volumeName}/backups/{backupId}/restore-files", CommandName: "volume.backup.restore_files"},
{Method: http.MethodDelete, PathPattern: "/api/environments/{id}/volumes/backups/{backupId}", CommandName: "volume.backup.delete"},
{Method: http.MethodGet, PathPattern: "/api/environments/{id}/volumes/backups/{backupId}/download", CommandName: "volume.backup.download"},
+ {Method: http.MethodGet, PathPattern: "/api/environments/{id}/volumes/{volumeName}/backup-policy", CommandName: "volume.backup_policy.get"},
+ {Method: http.MethodPut, PathPattern: "/api/environments/{id}/volumes/{volumeName}/backup-policy", CommandName: "volume.backup_policy.update"},
{Method: http.MethodGet, PathPattern: "/api/environments/{id}/volumes/backups/{backupId}/has-path", CommandName: "volume.backup.has_path"},
{Method: http.MethodGet, PathPattern: "/api/environments/{id}/volumes/backups/{backupId}/files", CommandName: "volume.backup.files"},
- {Method: http.MethodPost, PathPattern: "/api/environments/{id}/volumes/{volumeName}/backups/upload", CommandName: "volume.backup.upload"},
+ {Method: http.MethodPost, PathPattern: "/api/environments/{id}/volumes/backups/{backupId}/upload", CommandName: "volume.backup.upload"},
{Method: http.MethodGet, PathPattern: "/api/environments/{id}/builds/browse", CommandName: "build_workspace.browse.list"},
{Method: http.MethodGet, PathPattern: "/api/environments/{id}/builds/browse/content", CommandName: "build_workspace.browse.read_file"},
diff --git a/backend/pkg/libarcane/rustic/image.go b/backend/pkg/libarcane/rustic/image.go
new file mode 100644
index 000000000..c52487117
--- /dev/null
+++ b/backend/pkg/libarcane/rustic/image.go
@@ -0,0 +1,5 @@
+// Package rustic provides shared Rustic runtime configuration.
+package rustic
+
+// DefaultImage is the official Rustic container image used by backup and recovery flows.
+const DefaultImage = "ghcr.io/rustic-rs/rustic:v0.11.2"
diff --git a/backend/pkg/libarcane/rustic/run.go b/backend/pkg/libarcane/rustic/run.go
new file mode 100644
index 000000000..a42917e20
--- /dev/null
+++ b/backend/pkg/libarcane/rustic/run.go
@@ -0,0 +1,77 @@
+package rustic
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "strings"
+
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/volumehelper"
+ "github.com/moby/moby/api/pkg/stdcopy"
+ "github.com/moby/moby/api/types/container"
+ "github.com/moby/moby/api/types/mount"
+ "github.com/moby/moby/client"
+)
+
+// Run executes one Rustic command in an ephemeral container built from
+// DefaultImage and returns its trimmed stdout. It is the single Rustic
+// invocation contract, shared by the backup engine and the detached recovery
+// helper; the image must already be present. The standard RUSTIC_* variables
+// are appended to environment, and networkMode is applied verbatim ("" keeps
+// the default network).
+func Run(ctx context.Context, dockerClient *client.Client, password string, command, environment []string, mounts []mount.Mount, networkMode container.NetworkMode) (string, error) {
+ env := append([]string{}, environment...)
+ env = append(env,
+ "RUSTIC_PASSWORD="+password,
+ "RUSTIC_NO_PROGRESS=true",
+ "RUSTIC_LOG_LEVEL=error",
+ )
+ hostConfig := volumehelper.HostConfig(DefaultImage, nil, mounts)
+ hostConfig.AutoRemove = false
+ hostConfig.NetworkMode = networkMode
+ created, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{
+ Config: &container.Config{
+ Image: DefaultImage,
+ Cmd: command,
+ Env: env,
+ Labels: volumehelper.Labels(),
+ },
+ HostConfig: hostConfig,
+ })
+ if err != nil {
+ return "", fmt.Errorf("failed to create Rustic container: %w", err)
+ }
+ defer func() {
+ _, _ = dockerClient.ContainerRemove(context.WithoutCancel(ctx), created.ID, volumehelper.RemoveOptions())
+ }()
+ if _, err := dockerClient.ContainerStart(ctx, created.ID, client.ContainerStartOptions{}); err != nil {
+ return "", fmt.Errorf("failed to start Rustic container: %w", err)
+ }
+ wait := dockerClient.ContainerWait(ctx, created.ID, client.ContainerWaitOptions{Condition: container.WaitConditionNotRunning})
+ var status container.WaitResponse
+ select {
+ case err := <-wait.Error:
+ if err != nil {
+ return "", fmt.Errorf("failed to wait for Rustic container: %w", err)
+ }
+ case status = <-wait.Result:
+ }
+ logs, err := dockerClient.ContainerLogs(ctx, created.ID, client.ContainerLogsOptions{ShowStdout: true, ShowStderr: true})
+ if err != nil {
+ return "", fmt.Errorf("failed to read Rustic output: %w", err)
+ }
+ defer func() { _ = logs.Close() }()
+ var stdout bytes.Buffer
+ var stderr bytes.Buffer
+ if _, err := stdcopy.StdCopy(&stdout, &stderr, logs); err != nil {
+ return "", fmt.Errorf("failed to decode Rustic output: %w", err)
+ }
+ if status.StatusCode != 0 {
+ message := strings.TrimSpace(stderr.String())
+ if message == "" {
+ message = strings.TrimSpace(stdout.String())
+ }
+ return "", fmt.Errorf("rustic exited with code %d: %s", status.StatusCode, message)
+ }
+ return strings.TrimSpace(stdout.String()), nil
+}
diff --git a/backend/pkg/utils/s3/s3.go b/backend/pkg/utils/s3/s3.go
new file mode 100644
index 000000000..3983212ae
--- /dev/null
+++ b/backend/pkg/utils/s3/s3.go
@@ -0,0 +1,254 @@
+// Package s3 provides reusable S3 configuration and connectivity helpers. It
+// deals in infrastructure configuration only; mapping from API DTOs belongs to
+// the backup destination domain.
+package s3
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "io"
+ "net/url"
+ "path"
+ "strings"
+
+ "emperror.dev/errors"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ awsconfig "github.com/aws/aws-sdk-go-v2/config"
+ "github.com/aws/aws-sdk-go-v2/credentials"
+ awss3 "github.com/aws/aws-sdk-go-v2/service/s3"
+ "github.com/google/uuid"
+)
+
+const connectionTestPayload = "arcane-s3-connection-test"
+
+// Configuration contains the credentials and addressing options for an S3 destination.
+type Configuration struct {
+ ID string
+ Name string
+ Endpoint string
+ Bucket string
+ Region string
+ AccessKeyID string
+ SecretAccessKey string
+ Prefix string
+ UseSSL bool
+ ForcePathStyle bool
+}
+
+// Normalized returns a copy with surrounding whitespace and prefix separators removed.
+func (c Configuration) Normalized() Configuration {
+ c.ID = strings.TrimSpace(c.ID)
+ c.Name = strings.TrimSpace(c.Name)
+ c.Endpoint = strings.TrimSpace(c.Endpoint)
+ c.Bucket = strings.TrimSpace(c.Bucket)
+ c.Region = strings.TrimSpace(c.Region)
+ c.AccessKeyID = strings.TrimSpace(c.AccessKeyID)
+ c.SecretAccessKey = strings.TrimSpace(c.SecretAccessKey)
+ c.Prefix = strings.Trim(strings.TrimSpace(c.Prefix), "/")
+ return c
+}
+
+// ValidateEndpoint checks a custom endpoint: only HTTP or HTTPS with a host,
+// no userinfo, query, or fragment. Private and self-hosted hosts are allowed.
+func ValidateEndpoint(endpoint string) error {
+ endpoint = strings.TrimSpace(endpoint)
+ if endpoint == "" {
+ return nil
+ }
+ candidate := endpoint
+ if !strings.Contains(candidate, "://") {
+ candidate = "https://" + candidate
+ }
+ parsed, err := url.Parse(candidate)
+ if err != nil {
+ return fmt.Errorf("invalid S3 endpoint: %w", err)
+ }
+ if parsed.Scheme != "http" && parsed.Scheme != "https" {
+ return errors.New("invalid S3 endpoint: only http and https are supported")
+ }
+ if parsed.Host == "" {
+ return errors.New("invalid S3 endpoint: host is required")
+ }
+ if parsed.User != nil {
+ return errors.New("invalid S3 endpoint: credentials in the URL are not allowed")
+ }
+ if parsed.RawQuery != "" || parsed.Fragment != "" || parsed.RawFragment != "" {
+ return errors.New("invalid S3 endpoint: query strings and fragments are not allowed")
+ }
+ return nil
+}
+
+// Validate verifies the fields required to connect to the configured destination.
+func (c Configuration) Validate(requireSecret bool) error {
+ c = c.Normalized()
+ if c.Name == "" {
+ return errors.New("name is required")
+ }
+ if c.Bucket == "" {
+ return errors.New("bucket is required")
+ }
+ if c.Endpoint == "" && c.Region == "" {
+ return errors.New("region is required for AWS S3")
+ }
+ if err := ValidateEndpoint(c.Endpoint); err != nil {
+ return err
+ }
+ if c.AccessKeyID == "" {
+ return errors.New("access key ID is required")
+ }
+ if requireSecret && c.SecretAccessKey == "" {
+ return errors.New("secret access key is required")
+ }
+ return nil
+}
+
+// ConnectionFieldsEqual reports whether two configurations address the same
+// service the same way. A change in any of these fields means a stored secret
+// may no longer be combined with the configuration.
+func ConnectionFieldsEqual(a, b Configuration) bool {
+ a, b = a.Normalized(), b.Normalized()
+ return a.Endpoint == b.Endpoint &&
+ a.Bucket == b.Bucket &&
+ a.Region == b.Region &&
+ a.AccessKeyID == b.AccessKeyID &&
+ a.UseSSL == b.UseSSL &&
+ a.ForcePathStyle == b.ForcePathStyle
+}
+
+// EndpointURL returns the normalized endpoint URL, applying UseSSL to custom endpoints.
+func (c Configuration) EndpointURL() string {
+ endpoint := strings.TrimSpace(c.Endpoint)
+ if endpoint == "" {
+ return ""
+ }
+ endpoint = strings.TrimPrefix(strings.TrimPrefix(endpoint, "https://"), "http://")
+ scheme := "http://"
+ if c.UseSSL {
+ scheme = "https://"
+ }
+ return scheme + strings.TrimRight(endpoint, "/")
+}
+
+// RusticEnvironment builds the environment used by Rustic's OpenDAL S3 backend.
+func (c Configuration) RusticEnvironment(rootParts ...string) []string {
+ c = c.Normalized()
+ repositoryParts := append([]string{"/", c.Prefix}, rootParts...)
+ region := c.Region
+ if region == "" {
+ region = "auto"
+ }
+ environment := []string{
+ "RUSTIC_REPOSITORY=opendal:s3",
+ "RUSTIC_REPO_OPT_BUCKET=" + c.Bucket,
+ "RUSTIC_REPO_OPT_ROOT=" + path.Join(repositoryParts...),
+ "AWS_ACCESS_KEY_ID=" + c.AccessKeyID,
+ "AWS_SECRET_ACCESS_KEY=" + c.SecretAccessKey,
+ "AWS_EC2_METADATA_DISABLED=true",
+ }
+ if endpoint := c.EndpointURL(); endpoint != "" {
+ environment = append(environment, "RUSTIC_REPO_OPT_ENDPOINT="+endpoint)
+ }
+ environment = append(environment,
+ "RUSTIC_REPO_OPT_REGION="+region,
+ "AWS_REGION="+region,
+ )
+ if !c.ForcePathStyle {
+ environment = append(environment, "RUSTIC_REPO_OPT_ENABLE_VIRTUAL_HOST_STYLE=true")
+ }
+ return environment
+}
+
+// TestConnection verifies upload, download, and delete access for a destination.
+func TestConnection(ctx context.Context, configuration Configuration) (err error) {
+ configuration = configuration.Normalized()
+ if err := configuration.Validate(true); err != nil {
+ return err
+ }
+ client, err := newClientInternal(ctx, configuration)
+ if err != nil {
+ return fmt.Errorf("failed to upload S3 connection test object: %w", err)
+ }
+ payload := []byte(connectionTestPayload)
+ remoteKey := path.Join(configuration.Prefix, ".arcane-connection-test-"+uuid.NewString())
+ if err := putObjectInternal(ctx, client, configuration.Bucket, remoteKey, payload); err != nil {
+ return fmt.Errorf("failed to upload S3 connection test object: %w", err)
+ }
+
+ deleted := false
+ defer func() {
+ if deleted {
+ return
+ }
+ if cleanupErr := deleteObjectInternal(context.WithoutCancel(ctx), client, configuration.Bucket, remoteKey); cleanupErr != nil {
+ err = errors.Combine(err, fmt.Errorf("failed to clean up S3 connection test object: %w", cleanupErr))
+ }
+ }()
+
+ downloaded, err := getObjectInternal(ctx, client, configuration.Bucket, remoteKey)
+ if err != nil {
+ return fmt.Errorf("failed to download S3 connection test object: %w", err)
+ }
+ if !bytes.Equal(downloaded, payload) {
+ return errors.New("S3 connection test object contents did not match")
+ }
+ if err := deleteObjectInternal(ctx, client, configuration.Bucket, remoteKey); err != nil {
+ return fmt.Errorf("failed to delete S3 connection test object: %w", err)
+ }
+ deleted = true
+ return nil
+}
+
+func newClientInternal(ctx context.Context, configuration Configuration) (*awss3.Client, error) {
+ region := configuration.Region
+ if region == "" {
+ region = "us-east-1"
+ }
+ awsConfiguration, err := awsconfig.LoadDefaultConfig(ctx,
+ awsconfig.WithRegion(region),
+ awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(configuration.AccessKeyID, configuration.SecretAccessKey, "")),
+ )
+ if err != nil {
+ return nil, err
+ }
+ endpoint := configuration.EndpointURL()
+ return awss3.NewFromConfig(awsConfiguration, func(options *awss3.Options) {
+ options.UsePathStyle = configuration.ForcePathStyle
+ if endpoint != "" {
+ options.BaseEndpoint = aws.String(endpoint)
+ }
+ }), nil
+}
+
+func putObjectInternal(ctx context.Context, client *awss3.Client, bucket, key string, payload []byte) error {
+ _, err := client.PutObject(ctx, &awss3.PutObjectInput{
+ Bucket: aws.String(bucket),
+ Key: aws.String(key),
+ Body: bytes.NewReader(payload),
+ ContentLength: new(int64(len(payload))),
+ ContentType: aws.String("application/gzip"),
+ })
+ return err
+}
+
+func getObjectInternal(ctx context.Context, client *awss3.Client, bucket, key string) ([]byte, error) {
+ output, err := client.GetObject(ctx, &awss3.GetObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)})
+ if err != nil {
+ return nil, err
+ }
+ payload, readErr := io.ReadAll(output.Body)
+ closeErr := output.Body.Close()
+ if readErr != nil {
+ return nil, errors.Combine(readErr, closeErr)
+ }
+ if closeErr != nil {
+ return nil, closeErr
+ }
+ return payload, nil
+}
+
+func deleteObjectInternal(ctx context.Context, client *awss3.Client, bucket, key string) error {
+ _, err := client.DeleteObject(ctx, &awss3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)})
+ return err
+}
diff --git a/backend/pkg/utils/s3/s3_test.go b/backend/pkg/utils/s3/s3_test.go
new file mode 100644
index 000000000..4b459c440
--- /dev/null
+++ b/backend/pkg/utils/s3/s3_test.go
@@ -0,0 +1,53 @@
+package s3
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestConfigurationValidateRequiresRegionOnlyForAWS(t *testing.T) {
+ configuration := Configuration{Name: "Offsite", Bucket: "backups", AccessKeyID: "access-key", SecretAccessKey: "secret-key"}
+ require.EqualError(t, configuration.Validate(true), "region is required for AWS S3")
+
+ configuration.Endpoint = "s3.example.com"
+ require.NoError(t, configuration.Validate(true))
+}
+
+func TestValidateEndpoint(t *testing.T) {
+ require.NoError(t, ValidateEndpoint(""))
+ require.NoError(t, ValidateEndpoint("minio.internal:9000"))
+ require.NoError(t, ValidateEndpoint("http://10.0.0.5:9000"))
+ require.NoError(t, ValidateEndpoint("https://s3.example.com"))
+
+ require.ErrorContains(t, ValidateEndpoint("ftp://s3.example.com"), "only http and https")
+ require.ErrorContains(t, ValidateEndpoint("https://user:[email protected]"), "credentials in the URL")
+ require.ErrorContains(t, ValidateEndpoint("https://s3.example.com?x=1"), "query strings and fragments")
+ require.ErrorContains(t, ValidateEndpoint("https://s3.example.com#frag"), "query strings and fragments")
+ require.ErrorContains(t, ValidateEndpoint("https://"), "host is required")
+}
+
+func TestConfigurationRusticEnvironment(t *testing.T) {
+ configuration := Configuration{
+ Endpoint: "http://s3.example.com/",
+ Bucket: "backups",
+ AccessKeyID: "access-key",
+ SecretAccessKey: "secret-key",
+ Prefix: "/production/",
+ UseSSL: true,
+ ForcePathStyle: false,
+ }
+
+ require.Equal(t, []string{
+ "RUSTIC_REPOSITORY=opendal:s3",
+ "RUSTIC_REPO_OPT_BUCKET=backups",
+ "RUSTIC_REPO_OPT_ROOT=/production/arcane-volume-backups/instance-1",
+ "AWS_ACCESS_KEY_ID=access-key",
+ "AWS_SECRET_ACCESS_KEY=secret-key",
+ "AWS_EC2_METADATA_DISABLED=true",
+ "RUSTIC_REPO_OPT_ENDPOINT=https://s3.example.com",
+ "RUSTIC_REPO_OPT_REGION=auto",
+ "AWS_REGION=auto",
+ "RUSTIC_REPO_OPT_ENABLE_VIRTUAL_HOST_STYLE=true",
+ }, configuration.RusticEnvironment("arcane-volume-backups", "instance-1"))
+}
diff --git a/backend/pkg/utils/schedule/cron.go b/backend/pkg/utils/schedule/cron.go
new file mode 100644
index 000000000..30ac6f37e
--- /dev/null
+++ b/backend/pkg/utils/schedule/cron.go
@@ -0,0 +1,26 @@
+// Package schedule provides reusable schedule validation.
+package schedule
+
+import (
+ "fmt"
+ "strings"
+
+ "github.com/robfig/cron/v3"
+)
+
+// NormalizeSixField validates and normalizes a six-field cron schedule.
+func NormalizeSixField(value, subject string) (string, error) {
+ fields := strings.Fields(value)
+ if len(fields) == 0 {
+ return "", fmt.Errorf("%s schedule is required", subject)
+ }
+ if len(fields) != 6 {
+ return "", fmt.Errorf("invalid %s schedule %q: expected six fields", subject, strings.TrimSpace(value))
+ }
+ normalized := strings.Join(fields, " ")
+ parser := cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor)
+ if _, err := parser.Parse(normalized); err != nil {
+ return "", fmt.Errorf("invalid %s schedule %q: %w", subject, normalized, err)
+ }
+ return normalized, nil
+}
diff --git a/backend/resources/migrations/postgres/073_add_backup_support.sql b/backend/resources/migrations/postgres/073_add_backup_support.sql
new file mode 100644
index 000000000..d19148cea
--- /dev/null
+++ b/backend/resources/migrations/postgres/073_add_backup_support.sql
@@ -0,0 +1,112 @@
+-- +goose Up
+CREATE TABLE s3_destinations (
+ id TEXT PRIMARY KEY,
+ name TEXT NOT NULL UNIQUE,
+ endpoint TEXT,
+ bucket TEXT NOT NULL,
+ region TEXT NOT NULL,
+ access_key_id TEXT NOT NULL,
+ secret_access_key TEXT NOT NULL,
+ prefix TEXT,
+ use_ssl BOOLEAN NOT NULL DEFAULT TRUE,
+ force_path_style BOOLEAN NOT NULL DEFAULT TRUE,
+ created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ updated_at TIMESTAMPTZ
+);
+
+ALTER TABLE volume_backups ADD COLUMN status TEXT NOT NULL DEFAULT 'succeeded';
+ALTER TABLE volume_backups ADD COLUMN trigger TEXT NOT NULL DEFAULT 'manual';
+ALTER TABLE volume_backups ADD COLUMN s3_destination_id TEXT;
+ALTER TABLE volume_backups ADD COLUMN error TEXT;
+ALTER TABLE volume_backups ADD COLUMN destination TEXT NOT NULL DEFAULT 'local';
+ALTER TABLE volume_backups ADD COLUMN local_snapshot_id TEXT;
+ALTER TABLE volume_backups ADD COLUMN remote_snapshot_id TEXT;
+ALTER TABLE volume_backups ADD COLUMN policy_id TEXT;
+-- Pre-existing rows are tar.gz archive backups; new Rustic-snapshot rows set 'rustic'.
+ALTER TABLE volume_backups ADD COLUMN format TEXT NOT NULL DEFAULT 'archive';
+CREATE INDEX idx_volume_backups_s3_destination_id ON volume_backups(s3_destination_id);
+CREATE INDEX idx_volume_backups_policy_id ON volume_backups(policy_id);
+
+CREATE TABLE volume_backup_policies (
+ id TEXT PRIMARY KEY,
+ volume_name TEXT NOT NULL,
+ enabled BOOLEAN NOT NULL DEFAULT FALSE,
+ schedule TEXT NOT NULL,
+ retention_count INTEGER NOT NULL DEFAULT 7,
+ stop_containers BOOLEAN NOT NULL DEFAULT FALSE,
+ local_enabled BOOLEAN NOT NULL DEFAULT TRUE,
+ s3_enabled BOOLEAN NOT NULL DEFAULT FALSE,
+ s3_destination_id TEXT,
+ created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ updated_at TIMESTAMPTZ
+);
+CREATE INDEX idx_volume_backup_policies_volume_name ON volume_backup_policies(volume_name);
+CREATE INDEX idx_volume_backup_policies_s3_destination_id ON volume_backup_policies(s3_destination_id);
+
+CREATE TABLE system_backup_runs (
+ id TEXT PRIMARY KEY,
+ size BIGINT NOT NULL DEFAULT 0,
+ created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ updated_at TIMESTAMPTZ,
+ status TEXT NOT NULL,
+ trigger TEXT NOT NULL,
+ destination TEXT NOT NULL,
+ local_snapshot_id TEXT,
+ remote_snapshot_id TEXT,
+ s3_destination_id TEXT,
+ policy_id TEXT,
+ error TEXT
+);
+CREATE INDEX idx_system_backup_runs_s3_destination_id ON system_backup_runs(s3_destination_id);
+CREATE INDEX idx_system_backup_runs_policy_id ON system_backup_runs(policy_id);
+
+CREATE TABLE system_backup_policies (
+ id TEXT PRIMARY KEY,
+ enabled BOOLEAN NOT NULL DEFAULT FALSE,
+ schedule TEXT NOT NULL,
+ retention_count INTEGER NOT NULL DEFAULT 7,
+ local_enabled BOOLEAN NOT NULL DEFAULT TRUE,
+ s3_enabled BOOLEAN NOT NULL DEFAULT FALSE,
+ s3_destination_id TEXT,
+ created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ updated_at TIMESTAMPTZ
+);
+CREATE INDEX idx_system_backup_policies_s3_destination_id ON system_backup_policies(s3_destination_id);
+
+CREATE TABLE system_backup_recovery_config (
+ id TEXT PRIMARY KEY,
+ encrypted_recovery_key TEXT NOT NULL,
+ created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ updated_at TIMESTAMPTZ
+);
+
+-- +goose Down
+-- Refuse to downgrade while Rustic-format backups exist: the pre-073 schema cannot
+-- represent them and their snapshots would become unreachable.
+-- +goose StatementBegin
+DO $$
+BEGIN
+ IF EXISTS (SELECT 1 FROM volume_backups WHERE format <> 'archive') THEN
+ RAISE EXCEPTION 'downgrade blocked: Rustic-format volume backups exist; delete them first or stay on this version';
+ END IF;
+END
+$$;
+-- +goose StatementEnd
+
+DROP TABLE system_backup_recovery_config;
+DROP TABLE system_backup_policies;
+DROP TABLE system_backup_runs;
+DROP TABLE volume_backup_policies;
+
+DROP INDEX idx_volume_backups_policy_id;
+DROP INDEX idx_volume_backups_s3_destination_id;
+ALTER TABLE volume_backups DROP COLUMN format;
+ALTER TABLE volume_backups DROP COLUMN policy_id;
+ALTER TABLE volume_backups DROP COLUMN remote_snapshot_id;
+ALTER TABLE volume_backups DROP COLUMN local_snapshot_id;
+ALTER TABLE volume_backups DROP COLUMN destination;
+ALTER TABLE volume_backups DROP COLUMN error;
+ALTER TABLE volume_backups DROP COLUMN s3_destination_id;
+ALTER TABLE volume_backups DROP COLUMN trigger;
+ALTER TABLE volume_backups DROP COLUMN status;
+DROP TABLE s3_destinations;
diff --git a/backend/resources/migrations/sqlite/073_add_backup_support.sql b/backend/resources/migrations/sqlite/073_add_backup_support.sql
new file mode 100644
index 000000000..cde9d6cb2
--- /dev/null
+++ b/backend/resources/migrations/sqlite/073_add_backup_support.sql
@@ -0,0 +1,108 @@
+-- +goose Up
+CREATE TABLE s3_destinations (
+ id TEXT PRIMARY KEY,
+ name TEXT NOT NULL UNIQUE,
+ endpoint TEXT,
+ bucket TEXT NOT NULL,
+ region TEXT NOT NULL,
+ access_key_id TEXT NOT NULL,
+ secret_access_key TEXT NOT NULL,
+ prefix TEXT,
+ use_ssl INTEGER NOT NULL DEFAULT 1,
+ force_path_style INTEGER NOT NULL DEFAULT 1,
+ created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ updated_at DATETIME
+);
+
+ALTER TABLE volume_backups ADD COLUMN status TEXT NOT NULL DEFAULT 'succeeded';
+ALTER TABLE volume_backups ADD COLUMN trigger TEXT NOT NULL DEFAULT 'manual';
+ALTER TABLE volume_backups ADD COLUMN s3_destination_id TEXT;
+ALTER TABLE volume_backups ADD COLUMN error TEXT;
+ALTER TABLE volume_backups ADD COLUMN destination TEXT NOT NULL DEFAULT 'local';
+ALTER TABLE volume_backups ADD COLUMN local_snapshot_id TEXT;
+ALTER TABLE volume_backups ADD COLUMN remote_snapshot_id TEXT;
+ALTER TABLE volume_backups ADD COLUMN policy_id TEXT;
+-- Pre-existing rows are tar.gz archive backups; new Rustic-snapshot rows set 'rustic'.
+ALTER TABLE volume_backups ADD COLUMN format TEXT NOT NULL DEFAULT 'archive';
+CREATE INDEX idx_volume_backups_s3_destination_id ON volume_backups(s3_destination_id);
+CREATE INDEX idx_volume_backups_policy_id ON volume_backups(policy_id);
+
+CREATE TABLE volume_backup_policies (
+ id TEXT PRIMARY KEY,
+ volume_name TEXT NOT NULL,
+ enabled INTEGER NOT NULL DEFAULT 0,
+ schedule TEXT NOT NULL,
+ retention_count INTEGER NOT NULL DEFAULT 7,
+ stop_containers INTEGER NOT NULL DEFAULT 0,
+ local_enabled INTEGER NOT NULL DEFAULT 1,
+ s3_enabled INTEGER NOT NULL DEFAULT 0,
+ s3_destination_id TEXT,
+ created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ updated_at DATETIME
+);
+CREATE INDEX idx_volume_backup_policies_volume_name ON volume_backup_policies(volume_name);
+CREATE INDEX idx_volume_backup_policies_s3_destination_id ON volume_backup_policies(s3_destination_id);
+
+CREATE TABLE system_backup_runs (
+ id TEXT PRIMARY KEY,
+ size INTEGER NOT NULL DEFAULT 0,
+ created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ updated_at DATETIME,
+ status TEXT NOT NULL,
+ trigger TEXT NOT NULL,
+ destination TEXT NOT NULL,
+ local_snapshot_id TEXT,
+ remote_snapshot_id TEXT,
+ s3_destination_id TEXT,
+ policy_id TEXT,
+ error TEXT
+);
+CREATE INDEX idx_system_backup_runs_s3_destination_id ON system_backup_runs(s3_destination_id);
+CREATE INDEX idx_system_backup_runs_policy_id ON system_backup_runs(policy_id);
+
+CREATE TABLE system_backup_policies (
+ id TEXT PRIMARY KEY,
+ enabled INTEGER NOT NULL DEFAULT 0,
+ schedule TEXT NOT NULL,
+ retention_count INTEGER NOT NULL DEFAULT 7,
+ local_enabled INTEGER NOT NULL DEFAULT 1,
+ s3_enabled INTEGER NOT NULL DEFAULT 0,
+ s3_destination_id TEXT,
+ created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ updated_at DATETIME
+);
+CREATE INDEX idx_system_backup_policies_s3_destination_id ON system_backup_policies(s3_destination_id);
+
+CREATE TABLE system_backup_recovery_config (
+ id TEXT PRIMARY KEY,
+ encrypted_recovery_key TEXT NOT NULL,
+ created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ updated_at DATETIME
+);
+
+-- +goose Down
+-- Refuse to downgrade while Rustic-format backups exist: the pre-073 schema cannot
+-- represent them and their snapshots would become unreachable. The INSERT below
+-- violates NOT NULL only when such a row exists, aborting the downgrade.
+CREATE TABLE _downgrade_blocked_rustic_backups_exist (reason TEXT NOT NULL);
+INSERT INTO _downgrade_blocked_rustic_backups_exist (reason)
+SELECT NULL FROM volume_backups WHERE format <> 'archive' LIMIT 1;
+DROP TABLE _downgrade_blocked_rustic_backups_exist;
+
+DROP TABLE system_backup_recovery_config;
+DROP TABLE system_backup_policies;
+DROP TABLE system_backup_runs;
+DROP TABLE volume_backup_policies;
+
+DROP INDEX idx_volume_backups_policy_id;
+DROP INDEX idx_volume_backups_s3_destination_id;
+ALTER TABLE volume_backups DROP COLUMN format;
+ALTER TABLE volume_backups DROP COLUMN policy_id;
+ALTER TABLE volume_backups DROP COLUMN remote_snapshot_id;
+ALTER TABLE volume_backups DROP COLUMN local_snapshot_id;
+ALTER TABLE volume_backups DROP COLUMN destination;
+ALTER TABLE volume_backups DROP COLUMN error;
+ALTER TABLE volume_backups DROP COLUMN s3_destination_id;
+ALTER TABLE volume_backups DROP COLUMN trigger;
+ALTER TABLE volume_backups DROP COLUMN status;
+DROP TABLE s3_destinations;
diff --git a/frontend/messages/en.json b/frontend/messages/en.json
index 1bb45a6c6..5aa551eb4 100644
--- a/frontend/messages/en.json
+++ b/frontend/messages/en.json
@@ -56,6 +56,8 @@
"common_unused": "Unused",
"common_total": "Total",
"common_running": "Running",
+ "common_configured": "Configured",
+ "common_not_configured": "Not configured",
"common_stopped": "Stopped",
"common_size": "Size",
"common_overview": "Overview",
@@ -922,6 +924,8 @@
"volumes_workspace_restore_staged": "The selected backup restore will be applied when you save.",
"volumes_workspace_save_success": "Volume workspace updated successfully.",
"volumes_backups_title": "Volume Backups",
+ "volume_backups_remove_selected_title": "Remove {count} Volume Backup(s)",
+ "volume_backups_remove_selected_message": "Are you sure you want to remove the selected volume backup(s)? Arcane will also attempt to remove their local and S3 copies. This action cannot be undone.",
"volumes_backups_restore": "Restore",
"volumes_remove_in_use_warning": "Warning: This volume is currently in use by containers. Forcing removal may cause data loss or container issues.",
"volumes_containers_using_title": "Containers Using This Volume",
@@ -942,10 +946,11 @@
"volumes_backup_delete_confirm": "Are you sure you want to delete this backup?",
"volumes_backup_restore_title": "Restore Volume",
"volumes_backup_restore_message": "This will replace ALL data in the volume \"{volumeName}\" with the contents of this backup.\n\nA safety backup will be created automatically before restoring, so you can undo this operation if needed.",
- "volumes_backup_restore_in_use_warning": "\n\n⚠️ Warning: This volume is currently in use by {count} container(s). Restoring while containers are running may cause data corruption. Stop the containers first or use \"Restore files\" to restore specific files.",
+ "volumes_backup_restore_in_use_warning": "\n\nThis volume is currently in use by {count} container(s). Arcane will stop them before creating the safety backup and restart them after the restore.",
"volumes_backup_restore_success": "Volume restored successfully. A safety backup was created before restoring.",
"volumes_backup_restore_files_success": "{count} file(s) restored successfully. A safety backup was created before restoring.",
"volumes_backup_safety_info": "A safety backup will be created automatically before restoring, so you can undo this operation if needed.",
+ "volume_backup_restore_files_lifecycle_info": "Arcane will stop containers using this volume, create a safety backup, restore the selected files, and then restart the containers.",
"volumes_backup_overwrite_warning": "Selected files will overwrite existing files at those paths.",
"volumes_backup_file_restore_success": "File restored successfully. A safety backup was created before restoring.",
"volumes_symlink_external_tooltip": "This symlink points to a location outside the volume",
@@ -1895,8 +1900,6 @@
"git_sync_pre_deploy_acknowledgement": "Arcane runs this script on every deploy. Anyone who can push to the repo can change what it does.",
"git_sync_pre_deploy_managed_hint": "Managed by administrators",
"git_sync_pre_deploy_swarm_unsupported": "Pre-deploy scripts are only available for project syncs.",
- "git_sync_pre_deploy_status_configured": "Configured",
- "git_sync_pre_deploy_status_none": "Not configured",
"lifecycle_indicator_tooltip": "Runs a pre-deploy script: {path}",
"lifecycle_inline_runner_summary": "(image: {image}, network: {network})",
"common_syncing": "Syncing...",
@@ -3122,6 +3125,7 @@
"environments_connect_error": "Failed to Connect to Environment",
"common_access_denied": "Access denied",
"volumes_backup_restore_desc": "Select files from this backup to restore.",
+ "volumes_backup_load_failed": "Failed to load backups",
"volumes_workspace_backup_restore_desc": "Select a backup to restore the chosen file.",
"editor_commands_desc": "Run editor commands",
"environments_swarm_stack_source_label": "Swarm Stack Sources Directory",
@@ -3135,5 +3139,132 @@
"containers_status_removing": "Removing",
"paused": "Paused",
"containers_status_exited": "Exited",
- "containers_status_dead": "Dead"
+ "containers_status_dead": "Dead",
+ "backups_schedule_description": "Six-field cron expression interpreted in Arcane's configured timezone.",
+ "backups_retention_label": "Backups to keep",
+ "backups_retention_description": "Applied separately to local and remote backups. Set to 0 to keep all backups.",
+ "backups_s3_endpoint_label": "Endpoint",
+ "backups_s3_endpoint_description": "Leave empty for AWS S3, or enter a custom provider endpoint.",
+ "backups_s3_bucket_label": "Bucket",
+ "backups_s3_region_label": "Region",
+ "backups_s3_region_description": "Required for AWS S3. Optional for custom S3-compatible endpoints; Arcane uses us-east-1 for request signing when left empty.",
+ "backups_s3_access_key_label": "Access key ID",
+ "backups_s3_secret_key_label": "Secret access key",
+ "backups_s3_prefix_label": "Path prefix",
+ "backups_s3_prefix_description": "Optional folder-like prefix inside the bucket.",
+ "backups_s3_ssl_label": "Use SSL",
+ "backups_s3_ssl_description": "Use HTTPS when connecting to the custom endpoint.",
+ "backups_s3_path_style_label": "Use path-style addressing",
+ "backups_s3_path_style_description": "Recommended for MinIO and many S3-compatible providers.",
+ "backups_s3_bucket_required": "Bucket is required when S3 upload is enabled.",
+ "backups_s3_region_required": "Region is required when using AWS S3.",
+ "backups_s3_access_key_required": "Access key ID is required when S3 upload is enabled.",
+ "backups_s3_secret_key_required": "Secret access key is required when S3 upload is enabled.",
+ "backups_trigger_manual": "Manual",
+ "backups_trigger_scheduled": "Scheduled",
+ "backups_trigger_safety": "Pre-restore safety",
+ "backups_destination_s3": "S3",
+ "backups_destination_local_s3": "Local + S3",
+ "s3_destinations_title": "S3 Destinations",
+ "s3_destinations_description": "Create and manage reusable S3-compatible storage destinations for Arcane and volume backups.",
+ "s3_destinations_load_failed": "Failed to load S3 destinations",
+ "s3_destination_add": "Add destination",
+ "s3_destination_add_title": "Add S3 destination",
+ "s3_destination_edit_title": "Edit S3 destination",
+ "s3_destination_dialog_description": "Configure an AWS S3, Backblaze B2, MinIO, Hetzner, or compatible storage destination.",
+ "s3_destination_secret_keep_description": "Leave empty to preserve the saved encrypted secret.",
+ "s3_destination_aws_default": "AWS S3 default",
+ "s3_destination_created": "Created {name}",
+ "s3_destination_updated": "Updated {name}",
+ "s3_destination_save_failed": "Failed to save S3 destination",
+ "s3_destination_test_success": "Connection to {name} succeeded",
+ "s3_destination_test_failed": "Failed to connect to {name}",
+ "s3_destination_test_required": "Test this connection successfully before saving.",
+ "s3_destination_test_verified": "Connection verified. You can now save this destination.",
+ "s3_destination_delete_message": "This destination can only be deleted when no backup configuration or retained remote backup uses it.",
+ "s3_destination_deleted": "Deleted {name}",
+ "s3_destination_delete_failed": "Failed to delete S3 destination",
+ "volume_backup_policy_title": "Automatic volume backups",
+ "volume_backup_policy_description": "Add independent schedules with their own destination, retention, and container shutdown settings.",
+ "volume_backup_encryption_note": "Volume backups are encrypted automatically with Arcane's internal encryption key; no separate recovery key is required. Keep the original encryption key to restore these backups on another Arcane instance.",
+ "volume_backup_policy_configure": "Configure schedules",
+ "volume_backup_policy_saved": "Volume backup schedules saved",
+ "volume_backup_policy_save_failed": "Failed to save volume backup policy",
+ "backups_enabled": "Enable scheduled backups",
+ "volume_backup_policy_enabled_description": "Create backups of this volume according to this cron schedule.",
+ "volume_backup_add_schedule": "Add schedule",
+ "backups_remove_schedule": "Remove schedule",
+ "volume_backup_schedule_removed": "Volume backup schedule removed",
+ "volume_backup_retention_summary": "Keep {count}",
+ "volume_backup_retention_all": "Keep all",
+ "volume_backup_containers_stopped": "Stops containers",
+ "volume_backup_containers_running": "Containers stay running",
+ "volume_backup_schedule_number": "Cron expression {number}",
+ "volume_backup_retention_invalid": "Backups to keep must be a whole number between 0 and 3650.",
+ "volume_backup_stop_containers": "Stop containers during backup",
+ "volume_backup_stop_containers_description": "Stop running containers that use this volume while Rustic creates the snapshot, then start them again.",
+ "backups_destination_label": "Backup destination",
+ "volume_backup_destination_description": "Keep backups locally, remotely in a saved S3 destination, or in both places.",
+ "volume_backup_destination_local": "Local only",
+ "volume_backup_destination_local_description": "Keep Rustic snapshots in Arcane's local backup storage.",
+ "volume_backup_destination_s3": "S3 only",
+ "volume_backup_destination_s3_description": "Store Rustic snapshots directly in the selected S3 destination. Pre-restore safety backups remain local.",
+ "volume_backup_destination_local_s3_description": "Keep Rustic snapshots both locally and in the selected S3 destination.",
+ "volume_backup_s3_destination_label": "S3 destination",
+ "volume_backup_s3_destination_description": "Select the saved destination used for scheduled and manual volume backup uploads.",
+ "volume_backup_choose_s3_destination": "Choose S3 destination",
+ "volume_backup_choose_s3_destination_description": "Select where this on-demand backup should be stored remotely.",
+ "volume_backup_no_s3_destinations": "No S3 destinations are configured. Add one in Settings before creating an S3 backup.",
+ "volume_backup_s3_destination_required": "Select an S3 destination.",
+ "volume_backup_destination_unavailable": "The selected S3 destination is no longer available.",
+ "volume_backup_status_succeeded": "Succeeded",
+ "volume_backup_trigger": "Trigger",
+ "volume_backup_remote_snapshot": "S3 snapshot",
+ "volume_backup_not_uploaded": "Not uploaded",
+ "backups_upload_s3": "Upload to S3",
+ "backups_upload_s3_success": "Backup uploaded to S3",
+ "backups_upload_s3_failed": "Failed to upload backup to S3",
+ "system_backups_title": "Backups",
+ "system_backups_description": "Create encrypted recovery snapshots of Arcane's database, files, settings, and runtime secrets.",
+ "system_backups_id": "Backup ID",
+ "system_backups_configure": "Configure schedule",
+ "system_backups_schedules": "Backup schedules",
+ "system_backups_add_schedule": "Create schedule",
+ "system_backups_schedule_removed": "System backup schedule removed.",
+ "system_backups_no_schedules": "No system backup schedules configured yet.",
+ "system_backups_schedule_description": "Each schedule has its own destination and retention policy.",
+ "system_backups_dialog_description": "The separate recovery key encrypts this Rustic repository and is required to restore it on a fresh Arcane instance.",
+ "system_backups_recovery_key": "Recovery key",
+ "system_backups_recovery_key_saved_description": "Leave empty to use the configured recovery key.",
+ "system_backups_recovery_key_required": "Enter the generated recovery key (8 groups of 6 characters).",
+ "system_backups_recovery_key_needed": "Set up a recovery key before creating system backups.",
+ "system_backups_reset_recovery_key": "Reset recovery key",
+ "system_backups_recovery_key_alert": "Save this key outside Arcane now. It will not be shown again, and without it your system backups cannot be restored.",
+ "system_backups_recovery_key_enter_description": "Enter the recovery key that protects this backup.",
+ "system_backups_recovery_key_schedule_required": "Configure a recovery key before enabling this schedule.",
+ "system_backups_setup_recovery_key": "Set up recovery key",
+ "system_backups_recovery_key_saved": "Recovery key configured.",
+ "system_backups_recovery_key_save_failed": "Failed to configure the recovery key.",
+ "system_backups_enabled_description": "Run an encrypted Arcane recovery backup on the configured cron schedule.",
+ "system_backups_created": "Arcane system backup created successfully.",
+ "system_backups_create_failed": "Failed to create Arcane system backup.",
+ "system_backups_policy_saved": "System backup schedule saved.",
+ "system_backups_policy_failed": "Failed to save the system backup schedule.",
+ "system_backups_restore_title": "Restore Arcane",
+ "system_backups_restore_description": "Arcane will create a safety backup, stop itself, restore all data and runtime secrets, recreate its container, and start the recovered instance.",
+ "system_backups_restore_started": "Arcane restore started. This page will disconnect while Arcane restarts.",
+ "system_backups_restore_failed": "Failed to start the Arcane restore.",
+ "system_backups_upload_title": "Upload system backup",
+ "system_backups_delete_title": "Delete system backup?",
+ "system_backups_delete_description": "The Rustic snapshot will be removed from every reachable storage destination.",
+ "system_backups_deleted": "System backup deleted.",
+ "system_backups_delete_failed": "Failed to delete system backup.",
+ "system_backups_backup_configuration": "Backup configuration",
+ "system_backups_custom_configuration": "Custom destination",
+ "system_backups_custom_configuration_description": "Choose a destination for this backup only.",
+ "system_backups_discover": "Find S3 backups",
+ "system_backups_discover_title": "Find existing S3 backups",
+ "system_backups_discover_description": "Open an Arcane system-recovery repository with its separate recovery key and add its snapshots to this table.",
+ "system_backups_discovered": "Found {count} new system backup(s).",
+ "system_backups_discover_failed": "Failed to discover system backups. Check the destination and recovery key."
}
diff --git a/frontend/src/lib/components/arcane-table/cells/backup-destination-cell.svelte b/frontend/src/lib/components/arcane-table/cells/backup-destination-cell.svelte
new file mode 100644
index 000000000..f6978986f
--- /dev/null
+++ b/frontend/src/lib/components/arcane-table/cells/backup-destination-cell.svelte
@@ -0,0 +1,14 @@
+<script lang="ts">
+ import { Badge } from '#lib/components/ui/badge';
+ import type { BackupRun } from '#lib/types/backup';
+ import { backupDestinationLabel, backupDestinationName } from '#lib/utils/backups';
+
+ let { item }: { item: Pick<BackupRun, 'destination' | 's3DestinationId' | 's3DestinationName'> } = $props();
+</script>
+
+<div class="flex items-center gap-2">
+ <Badge variant={item.destination === 'local' ? 'gray' : 'blue'}>{backupDestinationLabel(item.destination)}</Badge>
+ {#if item.destination !== 'local' && backupDestinationName(item)}
+ <span class="max-w-48 truncate text-xs text-muted-foreground">{backupDestinationName(item)}</span>
+ {/if}
+</div>
diff --git a/frontend/src/lib/components/arcane-table/cells/backup-size-cell.svelte b/frontend/src/lib/components/arcane-table/cells/backup-size-cell.svelte
new file mode 100644
index 000000000..c11941b3b
--- /dev/null
+++ b/frontend/src/lib/components/arcane-table/cells/backup-size-cell.svelte
@@ -0,0 +1,7 @@
+<script lang="ts">
+ import { bytes } from '#lib/utils/formatting';
+
+ let { size }: { size: number } = $props();
+</script>
+
+{bytes.format(size, { unitSeparator: ' ' }) ?? '-'}
diff --git a/frontend/src/lib/components/arcane-table/cells/backup-status-cell.svelte b/frontend/src/lib/components/arcane-table/cells/backup-status-cell.svelte
new file mode 100644
index 000000000..423b337b8
--- /dev/null
+++ b/frontend/src/lib/components/arcane-table/cells/backup-status-cell.svelte
@@ -0,0 +1,9 @@
+<script lang="ts">
+ import { Badge } from '#lib/components/ui/badge';
+ import type { BackupStatus } from '#lib/types/backup';
+ import { backupStatusLabel, backupStatusVariant } from '#lib/utils/backups';
+
+ let { status }: { status: BackupStatus } = $props();
+</script>
+
+<Badge variant={backupStatusVariant(status)}>{backupStatusLabel(status)}</Badge>
diff --git a/frontend/src/lib/components/arcane-table/cells/backup-trigger-cell.svelte b/frontend/src/lib/components/arcane-table/cells/backup-trigger-cell.svelte
new file mode 100644
index 000000000..d5b7de6da
--- /dev/null
+++ b/frontend/src/lib/components/arcane-table/cells/backup-trigger-cell.svelte
@@ -0,0 +1,8 @@
+<script lang="ts">
+ import type { BackupTrigger } from '#lib/types/backup';
+ import { backupTriggerLabel } from '#lib/utils/backups';
+
+ let { trigger }: { trigger: BackupTrigger } = $props();
+</script>
+
+{backupTriggerLabel(trigger)}
diff --git a/frontend/src/lib/components/backup-policy-card.svelte b/frontend/src/lib/components/backup-policy-card.svelte
new file mode 100644
index 000000000..88605ee58
--- /dev/null
+++ b/frontend/src/lib/components/backup-policy-card.svelte
@@ -0,0 +1,67 @@
+<script lang="ts">
+ import { ArcaneButton } from '#lib/components/arcane-button';
+ import { Badge } from '#lib/components/ui/badge';
+ import { EditIcon } from '#lib/icons';
+ import type { BackupPolicy, BackupStatus } from '#lib/types/backup';
+ import { backupDestinationFromFlags, backupDestinationLabel, backupStatusLabel } from '#lib/utils/backups';
+ import { formatDateTimeShort } from '#lib/utils/formatting';
+ import * as m from '#lib/paraglide/messages.js';
+
+ type CardPolicy = BackupPolicy & {
+ s3DestinationName?: string;
+ s3Bucket?: string;
+ lastRun?: { status: BackupStatus; createdAt: string };
+ };
+
+ let {
+ policy,
+ showStopContainers = false,
+ onEdit,
+ editDisabled = false
+ }: {
+ policy: CardPolicy;
+ showStopContainers?: boolean;
+ onEdit?: () => void;
+ editDisabled?: boolean;
+ } = $props();
+
+ const s3Name = $derived(policy.s3DestinationName || policy.s3Bucket || policy.s3DestinationId || '');
+</script>
+
+<div class="grid min-w-0 grid-cols-[auto_minmax(0,1fr)_auto] items-center gap-x-1.5 gap-y-0.5 rounded-md border px-2 py-1">
+ <Badge variant={policy.enabled ? 'green' : 'gray'}>{policy.enabled ? m.common_enabled() : m.common_disabled()}</Badge>
+ <code class="truncate">{policy.schedule}</code>
+ <div class="flex items-center gap-1.5 justify-self-end">
+ <Badge variant="blue">{backupDestinationLabel(backupDestinationFromFlags(policy.localEnabled, policy.s3Enabled))}</Badge>
+ {#if onEdit}
+ <ArcaneButton
+ action="edit"
+ size="icon"
+ icon={EditIcon}
+ showLabel={false}
+ customLabel={m.jobs_edit_schedule()}
+ onclick={onEdit}
+ class="size-7"
+ disabled={editDisabled}
+ />
+ {/if}
+ </div>
+ <div class="col-span-2 flex min-w-0 items-center gap-1.5">
+ <span>
+ {policy.retentionCount === 0
+ ? m.volume_backup_retention_all()
+ : m.volume_backup_retention_summary({ count: policy.retentionCount })}
+ </span>
+ {#if showStopContainers}
+ <span>·</span>
+ <span>{policy.stopContainers ? m.volume_backup_containers_stopped() : m.volume_backup_containers_running()}</span>
+ {/if}
+ {#if policy.lastRun}
+ <span>·</span>
+ <span class="truncate">{backupStatusLabel(policy.lastRun.status)} · {formatDateTimeShort(policy.lastRun.createdAt)}</span>
+ {/if}
+ </div>
+ {#if policy.s3Enabled && s3Name}
+ <span class="max-w-28 justify-self-end truncate" title={s3Name}>{s3Name}</span>
+ {/if}
+</div>
diff --git a/frontend/src/lib/components/backup-policy-dialog.svelte b/frontend/src/lib/components/backup-policy-dialog.svelte
new file mode 100644
index 000000000..29a0cc378
--- /dev/null
+++ b/frontend/src/lib/components/backup-policy-dialog.svelte
@@ -0,0 +1,206 @@
+<script lang="ts" generics="TPolicy extends BackupPolicy">
+ import { ResponsiveDialog } from '#lib/components/ui/responsive-dialog';
+ import { ArcaneButton } from '#lib/components/arcane-button';
+ import BackupPolicyFields from '#lib/components/backup-policy-fields.svelte';
+ import type { BackupPolicy, BackupPolicyForm, BackupPolicyUpdate } from '#lib/types/backup';
+ import type { S3Destination } from '#lib/types/s3-destination';
+ import { s3DestinationService } from '#lib/services/s3-destination-service';
+ import { backupDestinationFromFlags, backupPolicyDestinationValues } from '#lib/utils/backups';
+ import { toast } from 'svelte-sonner';
+ import * as m from '#lib/paraglide/messages.js';
+
+ type PolicyForm = BackupPolicyForm & { id: string; serverError?: string };
+
+ let {
+ open = $bindable(),
+ idPrefix,
+ policies,
+ policyId,
+ addTitle,
+ description,
+ enabledDescription,
+ enabledError = null,
+ defaultSchedule,
+ defaultEnabled = true,
+ showStopContainers = false,
+ destinations,
+ updatePolicies,
+ messages,
+ onSaved
+ }: {
+ open: boolean;
+ idPrefix: string;
+ policies: TPolicy[];
+ policyId?: string;
+ addTitle: string;
+ description: string;
+ enabledDescription: string;
+ enabledError?: string | null;
+ defaultSchedule: string;
+ defaultEnabled?: boolean;
+ showStopContainers?: boolean;
+ destinations?: S3Destination[];
+ updatePolicies: (policies: BackupPolicyUpdate[]) => Promise<TPolicy[]>;
+ messages: { saved: string; saveFailed: string; removed: string };
+ onSaved: (policies: TPolicy[]) => void;
+ } = $props();
+
+ let saving = $state(false);
+ let deleting = $state(false);
+ let loadedDestinations = $state<S3Destination[]>([]);
+ let destinationsLoading = $state(false);
+ let form = $state<PolicyForm>(newPolicy());
+ const destinationList = $derived(destinations ?? loadedDestinations);
+ const scheduleError = $derived(
+ !form.schedule.trim()
+ ? m.jobs_cron_required()
+ : form.schedule.trim().split(/\s+/).length !== 6
+ ? m.jobs_cron_invalid()
+ : (form.serverError ?? null)
+ );
+ const retentionError = $derived(
+ Number.isInteger(Number(form.retentionCount)) && form.retentionCount >= 0 && form.retentionCount <= 3650
+ ? null
+ : m.volume_backup_retention_invalid()
+ );
+ const destinationError = $derived.by(() => {
+ if (form.destination === 'local' || destinationsLoading) return null;
+ if (!form.s3DestinationId) return m.volume_backup_s3_destination_required();
+ if (!destinations && !loadedDestinations.some((item) => item.id === form.s3DestinationId))
+ return m.volume_backup_destination_unavailable();
+ return null;
+ });
+ const activeEnabledError = $derived(form.enabled ? enabledError : null);
+ const formInvalid = $derived(Boolean(scheduleError || retentionError || destinationError || activeEnabledError));
+
+ function newPolicy(): PolicyForm {
+ return {
+ id: '',
+ enabled: defaultEnabled,
+ schedule: defaultSchedule,
+ retentionCount: 7,
+ stopContainers: false,
+ destination: 'local',
+ s3DestinationId: ''
+ };
+ }
+
+ async function loadDestinations() {
+ destinationsLoading = true;
+ try {
+ loadedDestinations = await s3DestinationService.listAll();
+ } catch (error) {
+ toast.error(error instanceof Error ? error.message : m.s3_destinations_load_failed());
+ } finally {
+ destinationsLoading = false;
+ }
+ }
+
+ $effect(() => {
+ if (!open) return;
+ const policy = policies.find((item) => item.id === policyId);
+ form = policy
+ ? {
+ id: policy.id,
+ enabled: policy.enabled,
+ schedule: policy.schedule,
+ retentionCount: policy.retentionCount,
+ stopContainers: policy.stopContainers ?? false,
+ s3DestinationId: policy.s3DestinationId || '',
+ destination: backupDestinationFromFlags(policy.localEnabled, policy.s3Enabled)
+ }
+ : newPolicy();
+ if (!destinations) void loadDestinations();
+ });
+
+ function updateForm(values: Partial<BackupPolicyForm>) {
+ form = { ...form, ...values, serverError: undefined };
+ }
+
+ function policyPayload(policy: BackupPolicy): BackupPolicyUpdate {
+ return {
+ id: policy.id,
+ enabled: policy.enabled,
+ schedule: policy.schedule,
+ retentionCount: policy.retentionCount,
+ localEnabled: policy.localEnabled,
+ s3Enabled: policy.s3Enabled,
+ s3DestinationId: policy.s3DestinationId ?? '',
+ ...(showStopContainers ? { stopContainers: policy.stopContainers ?? false } : {})
+ };
+ }
+
+ async function savePolicies() {
+ if (formInvalid) return;
+ saving = true;
+ try {
+ const current: BackupPolicyUpdate = {
+ id: form.id,
+ enabled: form.enabled,
+ schedule: form.schedule,
+ retentionCount: Number(form.retentionCount),
+ ...backupPolicyDestinationValues(form.destination, form.s3DestinationId),
+ ...(showStopContainers ? { stopContainers: form.stopContainers ?? false } : {})
+ };
+ const existing = policies.map(policyPayload);
+ const next = policyId ? existing.map((policy) => (policy.id === policyId ? current : policy)) : [...existing, current];
+ onSaved(await updatePolicies(next));
+ open = false;
+ toast.success(messages.saved);
+ } catch (error) {
+ const message = error instanceof Error ? error.message : messages.saveFailed;
+ if (/cron|schedule/i.test(message)) form = { ...form, serverError: message };
+ else toast.error(message);
+ } finally {
+ saving = false;
+ }
+ }
+
+ async function deletePolicy() {
+ if (!policyId) return;
+ deleting = true;
+ try {
+ onSaved(await updatePolicies(policies.filter((policy) => policy.id !== policyId).map(policyPayload)));
+ open = false;
+ toast.success(messages.removed);
+ } catch (error) {
+ toast.error(error instanceof Error ? error.message : messages.saveFailed);
+ } finally {
+ deleting = false;
+ }
+ }
+</script>
+
+<ResponsiveDialog bind:open title={policyId ? m.jobs_edit_schedule() : addTitle} {description} contentClass="sm:max-w-[720px]">
+ {#snippet children()}
+ <div class="space-y-4 py-2">
+ <BackupPolicyFields
+ {idPrefix}
+ {form}
+ destinations={destinationList}
+ {scheduleError}
+ {retentionError}
+ {destinationError}
+ enabledError={activeEnabledError}
+ {enabledDescription}
+ schedulePlaceholder={defaultSchedule}
+ {showStopContainers}
+ {destinationsLoading}
+ onChange={updateForm}
+ />
+ </div>
+ {/snippet}
+ {#snippet footer()}
+ {#if policyId}
+ <ArcaneButton
+ action="remove"
+ customLabel={m.backups_remove_schedule()}
+ onclick={deletePolicy}
+ loading={deleting}
+ disabled={saving || deleting}
+ />
+ {/if}
+ <ArcaneButton action="cancel" onclick={() => (open = false)} />
+ <ArcaneButton action="save" onclick={savePolicies} loading={saving} disabled={saving || deleting || formInvalid} />
+ {/snippet}
+</ResponsiveDialog>
diff --git a/frontend/src/lib/components/backup-policy-fields.svelte b/frontend/src/lib/components/backup-policy-fields.svelte
new file mode 100644
index 000000000..67cbd0ce7
--- /dev/null
+++ b/frontend/src/lib/components/backup-policy-fields.svelte
@@ -0,0 +1,99 @@
+<script lang="ts">
+ import LabeledSwitch from '#lib/components/form/labeled-switch.svelte';
+ import SelectWithLabel from '#lib/components/form/select-with-label.svelte';
+ import TextInputWithLabel from '#lib/components/form/text-input-with-label.svelte';
+ import type { BackupDestination, BackupPolicyForm } from '#lib/types/backup';
+ import type { S3Destination } from '#lib/types/s3-destination';
+ import { backupDestinationOptions, s3DestinationOptions } from '#lib/utils/backups';
+ import * as m from '#lib/paraglide/messages.js';
+
+ let {
+ idPrefix,
+ form,
+ destinations,
+ scheduleError,
+ retentionError,
+ destinationError,
+ enabledError,
+ enabledDescription,
+ schedulePlaceholder,
+ showStopContainers = false,
+ destinationsLoading = false,
+ onChange
+ }: {
+ idPrefix: string;
+ form: BackupPolicyForm;
+ destinations: S3Destination[];
+ scheduleError?: string | null;
+ retentionError?: string | null;
+ destinationError?: string | null;
+ enabledError?: string | null;
+ enabledDescription: string;
+ schedulePlaceholder: string;
+ showStopContainers?: boolean;
+ destinationsLoading?: boolean;
+ onChange: (values: Partial<BackupPolicyForm>) => void;
+ } = $props();
+
+ const destinationOptions = $derived(backupDestinationOptions(destinations.length > 0, true));
+ const s3Options = $derived(s3DestinationOptions(destinations));
+</script>
+
+<div class="space-y-5">
+ <LabeledSwitch
+ id={`${idPrefix}-enabled`}
+ checked={form.enabled}
+ onCheckedChange={(enabled) => onChange({ enabled })}
+ label={m.backups_enabled()}
+ description={enabledDescription}
+ />
+ {#if enabledError}<p class="text-sm text-destructive">{enabledError}</p>{/if}
+ <TextInputWithLabel
+ value={form.schedule}
+ onChange={(schedule) => onChange({ schedule })}
+ error={scheduleError}
+ label={m.jobs_schedule()}
+ description={m.backups_schedule_description()}
+ helpText={scheduleError ? undefined : m.jobs_cron_expression_help()}
+ reserveHelpTextSpace
+ placeholder={schedulePlaceholder}
+ />
+ <TextInputWithLabel
+ value={form.retentionCount}
+ onChange={(retentionCount) => onChange({ retentionCount: Number(retentionCount) })}
+ error={retentionError}
+ label={m.backups_retention_label()}
+ description={m.backups_retention_description()}
+ reserveHelpTextSpace
+ type="number"
+ />
+ {#if showStopContainers}
+ <LabeledSwitch
+ id={`${idPrefix}-stop-containers`}
+ checked={form.stopContainers ?? false}
+ onCheckedChange={(stopContainers) => onChange({ stopContainers })}
+ label={m.volume_backup_stop_containers()}
+ description={m.volume_backup_stop_containers_description()}
+ />
+ {/if}
+ <SelectWithLabel
+ id={`${idPrefix}-destination`}
+ value={form.destination}
+ onValueChange={(destination) => onChange({ destination: destination as BackupDestination })}
+ label={m.backups_destination_label()}
+ description={m.volume_backup_destination_description()}
+ options={destinationOptions}
+ />
+ {#if form.destination !== 'local'}
+ <SelectWithLabel
+ id={`${idPrefix}-s3-destination`}
+ value={form.s3DestinationId}
+ onValueChange={(s3DestinationId) => onChange({ s3DestinationId })}
+ label={m.volume_backup_s3_destination_label()}
+ description={m.volume_backup_s3_destination_description()}
+ error={destinationError}
+ disabled={destinationsLoading}
+ options={s3Options}
+ />
+ {/if}
+</div>
diff --git a/frontend/src/lib/components/dialogs/gitops-sync-dialog.svelte b/frontend/src/lib/components/dialogs/gitops-sync-dialog.svelte
index 05d866ee5..6b06efcc6 100644
--- a/frontend/src/lib/components/dialogs/gitops-sync-dialog.svelte
+++ b/frontend/src/lib/components/dialogs/gitops-sync-dialog.svelte
@@ -600,9 +600,9 @@
<span class="mt-0.5 inline-flex shrink-0 items-center gap-1.5 text-xs">
{#if hookConfigured}
<CodeIcon class="size-3.5" />
- {m.git_sync_pre_deploy_status_configured()}
+ {m.common_configured()}
{:else}
- {m.git_sync_pre_deploy_status_none()}
+ {m.common_not_configured()}
{/if}
</span>
</div>
diff --git a/frontend/src/lib/config/navigation-config.ts b/frontend/src/lib/config/navigation-config.ts
index 3cc750878..f306cc4a7 100644
--- a/frontend/src/lib/config/navigation-config.ts
+++ b/frontend/src/lib/config/navigation-config.ts
@@ -25,7 +25,8 @@ import {
GlobeIcon,
UpdateIcon,
VariableIcon,
- ActivityIcon
+ ActivityIcon,
+ BackupIcon
} from '#lib/icons';
import { m } from '#lib/paraglide/messages';
import type { ShortcutKey } from '#lib/utils/navigation';
@@ -43,6 +44,7 @@ export type NavigationItem = {
* item visible to every authenticated user.
*/
accessSurfaceId?: string;
+ adminOnly?: boolean;
};
export type NavigationSections = {
@@ -221,6 +223,13 @@ export const navigationItems: NavigationSections = {
icon: ActivityIcon,
accessSurfaceId: 'settings.category.activity'
},
+ {
+ title: m.system_backups_title(),
+ url: '/settings/backups',
+ icon: BackupIcon,
+ accessSurfaceId: 'settings.category.systembackups',
+ adminOnly: true
+ },
{
title: m.builds(),
url: '/settings/builds',
@@ -313,6 +322,7 @@ function canSeeItem(
currentEnvId: string | undefined,
accessManifest: PermissionsManifest | null | undefined
): boolean {
+ if (item.adminOnly && !user.isGlobalAdmin) return false;
if (!item.accessSurfaceId) return true;
if (!accessManifest?.accessSurfaces?.length) return true;
return canReachAccessSurface(accessManifest, item.accessSurfaceId, user, currentEnvId);
diff --git a/frontend/src/lib/icons/index.ts b/frontend/src/lib/icons/index.ts
index f0932c472..2b58d1a85 100644
--- a/frontend/src/lib/icons/index.ts
+++ b/frontend/src/lib/icons/index.ts
@@ -53,6 +53,7 @@ export { default as UserIcon } from 'virtual:icons/solar/user-circle-bold';
export { default as ClockIcon } from 'virtual:icons/solar/clock-circle-linear';
export { default as RegistryIcon } from 'virtual:icons/pajamas/infrastructure-registry';
export { default as DownloadIcon } from 'virtual:icons/solar/download-linear';
+export { default as BackupIcon } from 'virtual:icons/solar/archive-down-minimlistic-linear';
export { default as SuccessIcon } from 'virtual:icons/qlementine-icons/success-12';
export { default as ExternalLinkIcon } from 'virtual:icons/majesticons/external-link-line';
export { default as GithubIcon } from 'virtual:icons/garden/github-stroke-16';
@@ -118,3 +119,4 @@ export { default as ShieldXIcon } from 'virtual:icons/solar/shield-cross-linear'
export { default as ScanIcon } from 'virtual:icons/solar/magnifer-linear';
export { default as AlertTriangleIcon } from 'virtual:icons/solar/danger-triangle-linear';
export { default as HammerIcon } from 'virtual:icons/gravity-ui/hammer';
+export { default as CloudStorageIcon } from 'virtual:icons/solar/cloud-storage-linear';
diff --git a/frontend/src/lib/layouts/settings-page-layout.svelte b/frontend/src/lib/layouts/settings-page-layout.svelte
index 407958e2e..3f0f4c7cc 100644
--- a/frontend/src/lib/layouts/settings-page-layout.svelte
+++ b/frontend/src/lib/layouts/settings-page-layout.svelte
@@ -6,7 +6,7 @@
import * as DropdownMenu from '#lib/components/ui/dropdown-menu/index.js';
import { getContext } from 'svelte';
import { m } from '#lib/paraglide/messages';
- import { EllipsisIcon, ResetIcon, type IconType } from '#lib/icons';
+ import { EllipsisIcon, ResetIcon, type IconType, ArrowDownIcon } from '#lib/icons';
import { cn } from '#lib/utils';
import type { SettingsActionButton, SettingsPageType, SettingsStatCard } from './types.js';
@@ -49,15 +49,51 @@
{#snippet ActionButtonList(buttons: SettingsActionButton[])}
{#each buttons as button}
- <ArcaneButton
- action={button.action}
- customLabel={button.label}
- loadingLabel={button.loadingLabel}
- loading={button.loading}
- disabled={button.disabled}
- onclick={button.onclick}
- size="sm"
- />
+ {#if button.options?.length}
+ <DropdownMenu.Root>
+ <DropdownMenu.Trigger>
+ {#snippet child({ props })}
+ <ArcaneButton
+ {...props}
+ action={button.action}
+ icon={button.icon}
+ customLabel={button.label}
+ disabled={button.disabled}
+ size={button.size ?? 'default'}
+ >
+ <ArrowDownIcon class="size-3.5 opacity-60" />
+ </ArcaneButton>
+ {/snippet}
+ </DropdownMenu.Trigger>
+ <DropdownMenu.Content
+ align="end"
+ class="z-[var(--arcane-z-surface)] min-w-[160px] rounded-xl border bg-popover/90 p-1 shadow-lg backdrop-blur-md"
+ >
+ <DropdownMenu.Group>
+ {#each button.options as option}
+ <DropdownMenu.Item onclick={option.onclick} disabled={option.disabled}>
+ {#if option.icon}
+ {@const OptionIcon = option.icon}
+ <OptionIcon class="size-4" />
+ {/if}
+ {option.label}
+ </DropdownMenu.Item>
+ {/each}
+ </DropdownMenu.Group>
+ </DropdownMenu.Content>
+ </DropdownMenu.Root>
+ {:else}
+ <ArcaneButton
+ action={button.action}
+ icon={button.icon}
+ customLabel={button.label}
+ loadingLabel={button.loadingLabel}
+ loading={button.loading}
+ disabled={button.disabled}
+ onclick={button.onclick}
+ size={button.size ?? 'default'}
+ />
+ {/if}
{/each}
{/snippet}
@@ -163,9 +199,18 @@
>
<DropdownMenu.Group>
{#each mobileDropdownButtons as button}
- <DropdownMenu.Item onclick={button.onclick} disabled={button.disabled || button.loading}>
- {button.loading ? button.loadingLabel || button.label : button.label}
- </DropdownMenu.Item>
+ {#if button.options?.length}
+ <DropdownMenu.Label>{button.label}</DropdownMenu.Label>
+ {#each button.options as option}
+ <DropdownMenu.Item onclick={option.onclick} disabled={button.disabled || option.disabled}>
+ {option.label}
+ </DropdownMenu.Item>
+ {/each}
+ {:else}
+ <DropdownMenu.Item onclick={button.onclick} disabled={button.disabled || button.loading}>
+ {button.loading ? button.loadingLabel || button.label : button.label}
+ </DropdownMenu.Item>
+ {/if}
{/each}
</DropdownMenu.Group>
</DropdownMenu.Content>
diff --git a/frontend/src/lib/layouts/types.ts b/frontend/src/lib/layouts/types.ts
index 411bab82e..d86b2ce37 100644
--- a/frontend/src/lib/layouts/types.ts
+++ b/frontend/src/lib/layouts/types.ts
@@ -1,14 +1,25 @@
-import type { Action } from '#lib/components/arcane-button/index.js';
+import type { Action, ArcaneButtonSize } from '#lib/components/arcane-button/index.js';
import type { IconType } from '#lib/icons';
+export interface SettingsActionOption {
+ label: string;
+ icon?: IconType;
+ onclick: () => void;
+ disabled?: boolean;
+}
+
export interface SettingsActionButton {
id: string;
action: Action;
+ icon?: IconType;
label: string;
loadingLabel?: string;
loading?: boolean;
disabled?: boolean;
- onclick: () => void;
+ size?: ArcaneButtonSize;
+ onclick?: () => void;
+ // When set, the button renders as a dropdown trigger and onclick is unused.
+ options?: SettingsActionOption[];
showOnMobile?: boolean;
}
diff --git a/frontend/src/lib/services/s3-destination-service.ts b/frontend/src/lib/services/s3-destination-service.ts
new file mode 100644
index 000000000..64de09bd2
--- /dev/null
+++ b/frontend/src/lib/services/s3-destination-service.ts
@@ -0,0 +1,38 @@
+import BaseAPIService from './api-service';
+import type { CreateS3Destination, S3Destination, UpdateS3Destination } from '#lib/types/s3-destination';
+import type { Paginated, SearchPaginationSortRequest } from '#lib/types/shared';
+import { transformPaginationParams } from '#lib/utils/tables';
+
+class S3DestinationService extends BaseAPIService {
+ async list(options?: SearchPaginationSortRequest): Promise<Paginated<S3Destination>> {
+ const params = transformPaginationParams(options);
+ const response = await this.api.get('/backups/s3', { params });
+ return response.data;
+ }
+
+ async listAll(): Promise<S3Destination[]> {
+ return this.handleResponse(this.api.get('/backups/s3/options'));
+ }
+
+ async create(destination: CreateS3Destination): Promise<S3Destination> {
+ return this.handleResponse(this.api.post('/backups/s3', destination));
+ }
+
+ async update(id: string, destination: UpdateS3Destination): Promise<S3Destination> {
+ return this.handleResponse(this.api.put(`/backups/s3/${id}`, destination));
+ }
+
+ async test(id: string, destination?: UpdateS3Destination): Promise<void> {
+ await this.handleResponse(this.api.post(`/backups/s3/${id}/test`, destination));
+ }
+
+ async testConfiguration(destination: CreateS3Destination): Promise<void> {
+ await this.handleResponse(this.api.post('/backups/s3/test', destination));
+ }
+
+ async delete(id: string): Promise<void> {
+ await this.handleResponse(this.api.delete(`/backups/s3/${id}`));
+ }
+}
+
+export const s3DestinationService = new S3DestinationService();
diff --git a/frontend/src/lib/services/system-backup-service.ts b/frontend/src/lib/services/system-backup-service.ts
new file mode 100644
index 000000000..5359b03a2
--- /dev/null
+++ b/frontend/src/lib/services/system-backup-service.ts
@@ -0,0 +1,55 @@
+import BaseAPIService from './api-service';
+import type { Paginated, SearchPaginationSortRequest } from '#lib/types/shared';
+import type {
+ CreateSystemBackup,
+ SystemBackupPolicyCollection,
+ SystemBackupRecoveryKey,
+ SystemBackupRun,
+ UpdateSystemBackupPolicy
+} from '#lib/types/system-backup';
+import { transformPaginationParams } from '#lib/utils/tables';
+
+class SystemBackupService extends BaseAPIService {
+ async list(options?: SearchPaginationSortRequest): Promise<Paginated<SystemBackupRun>> {
+ const response = await this.api.get('/backups', { params: transformPaginationParams(options) });
+ return response.data;
+ }
+
+ async getPolicies(): Promise<SystemBackupPolicyCollection> {
+ return this.handleResponse(this.api.get('/backups/policies'));
+ }
+
+ async updatePolicies(policies: UpdateSystemBackupPolicy[]): Promise<SystemBackupPolicyCollection> {
+ return this.handleResponse(this.api.put('/backups/policies', { policies }));
+ }
+
+ async generateRecoveryKey(): Promise<SystemBackupRecoveryKey> {
+ return this.handleResponse(this.api.post('/backups/recovery-key/generate'));
+ }
+
+ async setRecoveryKey(recoveryKey: string): Promise<{ configured: boolean }> {
+ return this.handleResponse(this.api.put('/backups/recovery-key', { recoveryKey }));
+ }
+
+ async create(input: CreateSystemBackup): Promise<SystemBackupRun> {
+ return this.handleResponse(this.api.post('/backups', input));
+ }
+
+ async restore(id: string, recoveryKey: string): Promise<void> {
+ await this.handleResponse(this.api.post(`/backups/${id}/restore`, { recoveryKey }));
+ }
+
+ async upload(id: string, s3DestinationId: string, recoveryKey: string): Promise<SystemBackupRun> {
+ return this.handleResponse(this.api.post(`/backups/${id}/upload`, { s3DestinationId, recoveryKey }));
+ }
+
+ async delete(id: string, recoveryKey: string): Promise<void> {
+ await this.handleResponse(this.api.delete(`/backups/${id}`, { data: { recoveryKey } }));
+ }
+
+ async discover(s3DestinationId: string, recoveryKey: string): Promise<number> {
+ return this.handleResponse(this.api.post('/backups/discover', { s3DestinationId, recoveryKey }));
+ }
+}
+
+export const systemBackupService = new SystemBackupService();
diff --git a/frontend/src/lib/services/volume-backup-service.ts b/frontend/src/lib/services/volume-backup-service.ts
index 9413e9c11..3607a65cd 100644
--- a/frontend/src/lib/services/volume-backup-service.ts
+++ b/frontend/src/lib/services/volume-backup-service.ts
@@ -1,16 +1,31 @@
import BaseAPIService from './api-service';
import { uploadService, type UploadProgressCallback } from './upload-service';
import { environmentStore } from '#lib/stores/environment.store.svelte';
-import type { BackupEntry } from '#lib/types/shared';
+import type {
+ BackupEntry,
+ CreateVolumeBackupRequest,
+ UpdateVolumeBackupPolicy,
+ VolumeBackupPolicyCollection
+} from '#lib/types/shared';
import type { SearchPaginationSortRequest, Paginated } from '#lib/types/shared';
import { transformPaginationParams } from '#lib/utils/tables';
export type VolumeBackupListResponse = Paginated<BackupEntry> & { warnings?: string[] };
class VolumeBackupService extends BaseAPIService {
- async createBackup(volumeName: string): Promise<BackupEntry> {
+ async getPolicies(volumeName: string): Promise<VolumeBackupPolicyCollection> {
const envId = await environmentStore.getCurrentEnvironmentId();
- const res = await this.api.post(`/environments/${envId}/volumes/${volumeName}/backups`);
+ return this.handleResponse(this.api.get(`/environments/${envId}/volumes/${volumeName}/backup-policy`));
+ }
+
+ async updatePolicies(volumeName: string, policies: UpdateVolumeBackupPolicy[]): Promise<VolumeBackupPolicyCollection> {
+ const envId = await environmentStore.getCurrentEnvironmentId();
+ return this.handleResponse(this.api.put(`/environments/${envId}/volumes/${volumeName}/backup-policy`, { policies }));
+ }
+
+ async createBackup(volumeName: string, request?: CreateVolumeBackupRequest): Promise<BackupEntry> {
+ const envId = await environmentStore.getCurrentEnvironmentId();
+ const res = await this.api.post(`/environments/${envId}/volumes/${volumeName}/backups`, request);
return res.data.data;
}
@@ -49,11 +64,6 @@ class VolumeBackupService extends BaseAPIService {
return res.data.data ?? [];
}
- async deleteBackup(backupId: string): Promise<any> {
- const envId = await environmentStore.getCurrentEnvironmentId();
- return this.handleResponse(this.api.delete(`/environments/${envId}/volumes/backups/${backupId}`));
- }
-
async downloadBackup(backupId: string): Promise<void> {
const envId = await environmentStore.getCurrentEnvironmentId();
const res = await this.api.get(`/environments/${envId}/volumes/backups/${backupId}/download`, {
@@ -69,6 +79,16 @@ class VolumeBackupService extends BaseAPIService {
link.remove();
}
+ async deleteBackup(backupId: string): Promise<any> {
+ const envId = await environmentStore.getCurrentEnvironmentId();
+ return this.handleResponse(this.api.delete(`/environments/${envId}/volumes/backups/${backupId}`));
+ }
+
+ async uploadBackup(backupId: string, s3DestinationId: string): Promise<BackupEntry> {
+ const envId = await environmentStore.getCurrentEnvironmentId();
+ return this.handleResponse(this.api.post(`/environments/${envId}/volumes/backups/${backupId}/upload`, { s3DestinationId }));
+ }
+
async uploadAndRestore(volumeName: string, file: File, onProgress?: UploadProgressCallback): Promise<any> {
const envId = await environmentStore.getCurrentEnvironmentId();
const uploadId = await uploadService.uploadFile(envId, 'volume-backup', file, onProgress);
diff --git a/frontend/src/lib/types/backup.ts b/frontend/src/lib/types/backup.ts
new file mode 100644
index 000000000..d99d22f02
--- /dev/null
+++ b/frontend/src/lib/types/backup.ts
@@ -0,0 +1,50 @@
+export type BackupDestination = 'local' | 's3' | 'local_s3';
+export type BackupStatus = 'running' | 'succeeded' | 'failed';
+export type BackupTrigger = 'manual' | 'scheduled' | 'safety';
+
+export type BackupRun = {
+ id: string;
+ size: number;
+ createdAt: string;
+ status: BackupStatus;
+ trigger: BackupTrigger;
+ destination: BackupDestination;
+ format?: 'archive' | 'rustic';
+ localSnapshotId?: string;
+ remoteSnapshotId?: string;
+ s3DestinationId?: string;
+ s3DestinationName?: string;
+ policyId?: string;
+ error?: string;
+};
+
+export type BackupPolicyForm = {
+ enabled: boolean;
+ schedule: string;
+ retentionCount: number;
+ destination: BackupDestination;
+ s3DestinationId: string;
+ stopContainers?: boolean;
+};
+
+export type BackupPolicy = {
+ id: string;
+ enabled: boolean;
+ schedule: string;
+ retentionCount: number;
+ stopContainers?: boolean;
+ localEnabled: boolean;
+ s3Enabled: boolean;
+ s3DestinationId?: string;
+};
+
+export type BackupPolicyUpdate = {
+ id: string;
+ enabled: boolean;
+ schedule: string;
+ retentionCount: number;
+ localEnabled: boolean;
+ s3Enabled: boolean;
+ s3DestinationId: string;
+ stopContainers?: boolean;
+};
diff --git a/frontend/src/lib/types/s3-destination.ts b/frontend/src/lib/types/s3-destination.ts
new file mode 100644
index 000000000..4da0f7c4b
--- /dev/null
+++ b/frontend/src/lib/types/s3-destination.ts
@@ -0,0 +1,28 @@
+export type S3Destination = {
+ id: string;
+ name: string;
+ endpoint?: string;
+ bucket: string;
+ region: string;
+ accessKeyId: string;
+ prefix?: string;
+ useSsl: boolean;
+ forcePathStyle: boolean;
+ secretConfigured: boolean;
+ createdAt: string;
+ updatedAt?: string;
+};
+
+export type CreateS3Destination = {
+ name: string;
+ endpoint: string;
+ bucket: string;
+ region: string;
+ accessKeyId: string;
+ secretAccessKey: string;
+ prefix: string;
+ useSsl: boolean;
+ forcePathStyle: boolean;
+};
+
+export type UpdateS3Destination = CreateS3Destination;
diff --git a/frontend/src/lib/types/shared.ts b/frontend/src/lib/types/shared.ts
index a880f3045..08a773210 100644
--- a/frontend/src/lib/types/shared.ts
+++ b/frontend/src/lib/types/shared.ts
@@ -105,11 +105,41 @@ export interface FileContentResponse {
mimeType: string;
}
-export interface BackupEntry {
+import type { BackupDestination, BackupPolicyUpdate, BackupRun } from './backup';
+
+export type VolumeBackupDestination = BackupDestination;
+
+export interface CreateVolumeBackupRequest {
+ destination?: VolumeBackupDestination;
+ policyId?: string;
+ s3DestinationId?: string;
+}
+
+export interface BackupEntry extends BackupRun {
+ volumeName: string;
+}
+
+export interface VolumeBackupPolicy {
id: string;
volumeName: string;
- size: number;
- createdAt: string;
+ enabled: boolean;
+ schedule: string;
+ retentionCount: number;
+ stopContainers: boolean;
+ localEnabled: boolean;
+ s3Enabled: boolean;
+ s3DestinationId: string;
+ s3DestinationName?: string;
+ s3Available: boolean;
+ s3Bucket: string;
+ lastRun?: BackupEntry;
+}
+
+export type UpdateVolumeBackupPolicy = BackupPolicyUpdate;
+
+export interface VolumeBackupPolicyCollection {
+ policies: VolumeBackupPolicy[];
+ s3Available: boolean;
}
// --- Customize search ---
diff --git a/frontend/src/lib/types/system-backup.ts b/frontend/src/lib/types/system-backup.ts
new file mode 100644
index 000000000..c105888eb
--- /dev/null
+++ b/frontend/src/lib/types/system-backup.ts
@@ -0,0 +1,35 @@
+import type { BackupDestination, BackupRun } from './backup';
+
+export type SystemBackupDestination = BackupDestination;
+
+export type SystemBackupRun = BackupRun;
+
+export type SystemBackupPolicy = {
+ id: string;
+ enabled: boolean;
+ schedule: string;
+ retentionCount: number;
+ localEnabled: boolean;
+ s3Enabled: boolean;
+ s3DestinationId?: string;
+ s3DestinationName?: string;
+ lastRun?: SystemBackupRun;
+};
+
+export type UpdateSystemBackupPolicy = Omit<SystemBackupPolicy, 's3DestinationName' | 'lastRun'>;
+
+export type SystemBackupPolicyCollection = {
+ policies: SystemBackupPolicy[];
+ recoveryKeyStored: boolean;
+};
+
+export type SystemBackupRecoveryKey = {
+ recoveryKey: string;
+};
+
+export type CreateSystemBackup = {
+ destination?: SystemBackupDestination;
+ s3DestinationId?: string;
+ recoveryKey?: string;
+ policyId?: string;
+};
diff --git a/frontend/src/lib/utils/backups.ts b/frontend/src/lib/utils/backups.ts
new file mode 100644
index 000000000..07d2f6500
--- /dev/null
+++ b/frontend/src/lib/utils/backups.ts
@@ -0,0 +1,87 @@
+import { m } from '#lib/paraglide/messages';
+import type { BackupDestination, BackupRun, BackupStatus, BackupTrigger } from '#lib/types/backup';
+import type { S3Destination } from '#lib/types/s3-destination';
+
+export function backupStatusLabel(status: BackupStatus): string {
+ if (status === 'succeeded') return m.volume_backup_status_succeeded();
+ if (status === 'failed') return m.common_failed();
+ return m.common_running();
+}
+
+export function backupStatusVariant(status: BackupStatus): 'green' | 'red' | 'blue' {
+ if (status === 'succeeded') return 'green';
+ if (status === 'failed') return 'red';
+ return 'blue';
+}
+
+export function backupTriggerLabel(trigger: BackupTrigger): string {
+ if (trigger === 'scheduled') return m.backups_trigger_scheduled();
+ if (trigger === 'safety') return m.backups_trigger_safety();
+ return m.backups_trigger_manual();
+}
+
+export function backupDestinationLabel(destination: BackupDestination): string {
+ if (destination === 'local_s3') return m.backups_destination_local_s3();
+ if (destination === 's3') return m.backups_destination_s3();
+ return m.local();
+}
+
+export function backupDestinationFromFlags(localEnabled: boolean, s3Enabled: boolean): BackupDestination {
+ if (!s3Enabled) return 'local';
+ return localEnabled ? 'local_s3' : 's3';
+}
+
+export function backupDestinationName(item: Pick<BackupRun, 's3DestinationId' | 's3DestinationName'>): string {
+ return item.s3DestinationName || item.s3DestinationId || '';
+}
+
+export function backupDestinationDisplay(item: Pick<BackupRun, 'destination' | 's3DestinationId' | 's3DestinationName'>): string {
+ const label = backupDestinationLabel(item.destination);
+ const name = backupDestinationName(item);
+ return name && item.destination !== 'local' ? `${label} · ${name}` : label;
+}
+
+export function backupPolicyDestinationDisplay(policy: {
+ localEnabled: boolean;
+ s3Enabled: boolean;
+ s3DestinationName?: string;
+}): string {
+ const label = backupDestinationLabel(backupDestinationFromFlags(policy.localEnabled, policy.s3Enabled));
+ return policy.s3DestinationName ? `${label} · ${policy.s3DestinationName}` : label;
+}
+
+export function backupDestinationOptions(hasS3Destinations: boolean, descriptions = false) {
+ return [
+ {
+ label: m.volume_backup_destination_local(),
+ value: 'local',
+ description: descriptions ? m.volume_backup_destination_local_description() : undefined
+ },
+ ...(hasS3Destinations
+ ? [
+ {
+ label: m.volume_backup_destination_s3(),
+ value: 's3',
+ description: descriptions ? m.volume_backup_destination_s3_description() : undefined
+ },
+ {
+ label: m.backups_destination_local_s3(),
+ value: 'local_s3',
+ description: descriptions ? m.volume_backup_destination_local_s3_description() : undefined
+ }
+ ]
+ : [])
+ ];
+}
+
+export function s3DestinationOptions(destinations: S3Destination[]) {
+ return destinations.map((item) => ({ label: item.name, value: item.id, description: item.bucket }));
+}
+
+export function backupPolicyDestinationValues(destination: BackupDestination, s3DestinationId: string) {
+ return {
+ localEnabled: destination !== 's3',
+ s3Enabled: destination !== 'local',
+ s3DestinationId: destination === 'local' ? '' : s3DestinationId
+ };
+}
diff --git a/frontend/src/lib/utils/bulk-actions.ts b/frontend/src/lib/utils/bulk-actions.ts
index 2e4c4e974..e39e2ea43 100644
--- a/frontend/src/lib/utils/bulk-actions.ts
+++ b/frontend/src/lib/utils/bulk-actions.ts
@@ -40,7 +40,7 @@ export interface RunBulkOperationOptions<T> {
/** Clears the table selection after completion. Always called when there were ids. */
clearSelection?: () => void;
/** Optional per-item failure side effect, such as an item-specific toast. */
- onItemFailure?: (id: string) => void;
+ onItemFailure?: (id: string, error: Error) => void;
/** Run operations one at a time instead of concurrently. Defaults to concurrent. */
sequential?: boolean;
}
@@ -69,7 +69,7 @@ async function runBulkOperation<T>({
const tally = (id: string, outcome: Result<T>) => {
if (outcome.error) {
result.failed += 1;
- onItemFailure?.(id);
+ onItemFailure?.(id, outcome.error);
} else {
result.success += 1;
const activityId = extractActivityId(outcome.data);
diff --git a/frontend/src/routes/(app)/settings/+page.svelte b/frontend/src/routes/(app)/settings/+page.svelte
index 3263d63d1..f18388d45 100644
--- a/frontend/src/routes/(app)/settings/+page.svelte
+++ b/frontend/src/routes/(app)/settings/+page.svelte
@@ -12,7 +12,9 @@
JobsIcon,
CodeIcon,
GlobeIcon,
- ActivityIcon
+ ActivityIcon,
+ RemoteEnvironmentIcon,
+ BackupIcon
} from '#lib/icons';
import { m } from '#lib/paraglide/messages';
import { settingsSearchService } from '#lib/services/settings-search';
@@ -47,7 +49,9 @@
jobs: JobsIcon,
code: CodeIcon,
globe: GlobeIcon,
- activity: ActivityIcon
+ activity: ActivityIcon,
+ remoteenvironment: RemoteEnvironmentIcon,
+ backup: BackupIcon
};
const categoryMessages = {
@@ -109,6 +113,7 @@
}
function isAccessibleCategory(category: SettingsCategory) {
+ if (category.id === 'systembackups' && !user?.isGlobalAdmin) return false;
if (!permissionsManifest?.accessSurfaces?.length) return true;
return canReachAccessSurfaceUrl(permissionsManifest, category.url, user, environmentStore.selected?.id);
}
diff --git a/frontend/src/routes/(app)/settings/backups/+page.svelte b/frontend/src/routes/(app)/settings/backups/+page.svelte
new file mode 100644
index 000000000..c989aa489
--- /dev/null
+++ b/frontend/src/routes/(app)/settings/backups/+page.svelte
@@ -0,0 +1,434 @@
+<script lang="ts">
+ import { untrack } from 'svelte';
+ import { goto } from '$app/navigation';
+ import { toast } from 'svelte-sonner';
+ import settingsStore from '#lib/stores/config-store';
+ import { SettingsPageLayout, type SettingsActionButton } from '#lib/layouts';
+ import { AlertIcon, BackupIcon, CloudStorageIcon, LockIcon, ResetIcon } from '#lib/icons';
+ import * as Alert from '#lib/components/ui/alert';
+ import { CopyButton } from '#lib/components/ui/copy-button';
+ import { Input } from '#lib/components/ui/input';
+ import { ResponsiveDialog } from '#lib/components/ui/responsive-dialog';
+ import { ArcaneButton } from '#lib/components/arcane-button';
+ import SelectWithLabel from '#lib/components/form/select-with-label.svelte';
+ import TextInputWithLabel from '#lib/components/form/text-input-with-label.svelte';
+ import { systemBackupService } from '#lib/services/system-backup-service';
+ import { hasPermission } from '#lib/utils/auth';
+ import { backupDestinationOptions, backupPolicyDestinationDisplay, s3DestinationOptions } from '#lib/utils/backups';
+ import type { SearchPaginationSortRequest } from '#lib/types/shared';
+ import type { SystemBackupDestination, SystemBackupRun } from '#lib/types/system-backup';
+ import * as m from '#lib/paraglide/messages.js';
+ import SystemBackupTable from './system-backup-table.svelte';
+ import BackupPolicyDialog from '#lib/components/backup-policy-dialog.svelte';
+ import BackupPolicyCard from '#lib/components/backup-policy-card.svelte';
+
+ let { data } = $props();
+ let backups = $state(untrack(() => data.backups));
+ let policyCollection = $state(untrack(() => data.policyCollection));
+ let requestOptions = $state<SearchPaginationSortRequest>(untrack(() => data.requestOptions));
+ let policyOpen = $state(false);
+ let editingPolicyId = $state<string | undefined>();
+ let keyOpen = $state(false);
+ let actionOpen = $state(false);
+ let action = $state<'create' | 'restore' | 'upload' | 'delete' | 'discover'>('create');
+ let selected = $state<SystemBackupRun | null>(null);
+ let backupConfiguration = $state('custom');
+ let destination = $state<SystemBackupDestination>('local');
+ let s3DestinationId = $state('');
+ let recoveryKey = $state('');
+ let newRecoveryKey = $state('');
+ let loading = $state(false);
+ let generatingKey = $state(false);
+ const isReadOnly = $derived.by(() => $settingsStore.uiConfigDisabled);
+ const canManageRecoveryKey = $derived(hasPermission('system-backups:recovery-key'));
+ const destinationOptions = $derived(backupDestinationOptions(data.destinations.length > 0));
+ const s3Options = $derived(s3DestinationOptions(data.destinations));
+ const configurationOptions = $derived([
+ {
+ label: m.system_backups_custom_configuration(),
+ value: 'custom',
+ description: m.system_backups_custom_configuration_description()
+ },
+ ...policyCollection.policies.map((item) => ({
+ label: item.schedule,
+ value: item.id,
+ description: backupPolicyDestinationDisplay(item)
+ }))
+ ]);
+ // Restore and discover may target backups keyed by another instance, so
+ // they accept a typed (previously generated) key. Everything else only
+ // ever uses this instance's stored key.
+ const actionNeedsTypedKey = $derived(action === 'restore' || action === 'discover');
+ const recoveryKeyPattern = /^[A-Z2-7]{6}(-[A-Z2-7]{6}){7}$/;
+ const keyError = $derived(
+ actionNeedsTypedKey && recoveryKey.length > 0 && !recoveryKeyPattern.test(recoveryKey.trim())
+ ? m.system_backups_recovery_key_required()
+ : ''
+ );
+ const actionKeyInvalid = $derived.by(() => {
+ if (actionNeedsTypedKey) return !policyCollection.recoveryKeyStored && !recoveryKeyPattern.test(recoveryKey.trim());
+ // Deleting is always allowed; the backend only asks for the key when
+ // the run still has snapshots to forget.
+ if (action === 'delete') return false;
+ return !policyCollection.recoveryKeyStored;
+ });
+ const destinationError = $derived(
+ backupConfiguration === 'custom' &&
+ ((action === 'create' && destination !== 'local') || action === 'upload' || action === 'discover') &&
+ !s3DestinationId
+ ? m.volume_backup_s3_destination_required()
+ : ''
+ );
+ const invalid = $derived(Boolean(actionKeyInvalid || keyError || destinationError));
+
+ function openPolicy(id?: string) {
+ editingPolicyId = id;
+ policyOpen = true;
+ }
+
+ // Generating persists the key in the same step; the dialog only ever shows
+ // an already-saved key, with one job: get the user to store it elsewhere.
+ async function openRecoveryKey() {
+ newRecoveryKey = '';
+ generatingKey = true;
+ try {
+ const generated = await systemBackupService.generateRecoveryKey();
+ await systemBackupService.setRecoveryKey(generated.recoveryKey);
+ newRecoveryKey = generated.recoveryKey;
+ policyCollection = { ...policyCollection, recoveryKeyStored: true };
+ keyOpen = true;
+ } catch (error) {
+ toast.error(error instanceof Error ? error.message : m.system_backups_recovery_key_save_failed());
+ } finally {
+ generatingKey = false;
+ }
+ }
+
+ function openAction(next: typeof action, backup: SystemBackupRun | null = null) {
+ action = next;
+ selected = backup;
+ backupConfiguration = 'custom';
+ destination = 'local';
+ s3DestinationId =
+ backup?.s3DestinationId ||
+ policyCollection.policies.find((item) => item.s3DestinationId)?.s3DestinationId ||
+ data.destinations[0]?.id ||
+ '';
+ recoveryKey = '';
+ actionOpen = true;
+ }
+ function dialogTitle() {
+ if (action === 'restore') return m.system_backups_restore_title();
+ if (action === 'upload') return m.system_backups_upload_title();
+ if (action === 'delete') return m.system_backups_delete_title();
+ if (action === 'discover') return m.system_backups_discover_title();
+ return m.volumes_backup_create();
+ }
+ function dialogDescription() {
+ if (action === 'restore') return m.system_backups_restore_description();
+ if (action === 'delete') return m.system_backups_delete_description();
+ if (action === 'discover') return m.system_backups_discover_description();
+ return m.system_backups_dialog_description();
+ }
+ async function refresh() {
+ backups = await systemBackupService.list(requestOptions);
+ }
+
+ // With a stored key the S3 repositories are scanned automatically, so
+ // remote snapshots just appear in the table; the manual Find S3 backups
+ // flow only exists for fresh instances that must supply an old key.
+ let autoDiscovered = false;
+ $effect(() => {
+ if (autoDiscovered || !policyCollection.recoveryKeyStored || data.destinations.length === 0) return;
+ autoDiscovered = true;
+ void (async () => {
+ try {
+ const counts = await Promise.all(data.destinations.map((item) => systemBackupService.discover(item.id, '')));
+ if (counts.some((count) => count > 0)) await refresh();
+ } catch (error) {
+ console.warn('S3 backup discovery failed', error);
+ }
+ })();
+ });
+
+ async function submitAction() {
+ if (loading || invalid) return;
+ loading = true;
+ try {
+ if (action === 'create') {
+ await systemBackupService.create(
+ backupConfiguration === 'custom'
+ ? { destination, s3DestinationId: destination === 'local' ? '' : s3DestinationId, recoveryKey }
+ : { policyId: backupConfiguration, recoveryKey }
+ );
+ toast.success(m.system_backups_created());
+ await refresh();
+ } else if (action === 'restore' && selected) {
+ await systemBackupService.restore(selected.id, recoveryKey);
+ toast.success(m.system_backups_restore_started());
+ } else if (action === 'upload' && selected) {
+ await systemBackupService.upload(selected.id, s3DestinationId, recoveryKey);
+ toast.success(m.backups_upload_s3_success());
+ await refresh();
+ } else if (action === 'delete' && selected) {
+ await systemBackupService.delete(selected.id, recoveryKey);
+ toast.success(m.system_backups_deleted());
+ await refresh();
+ } else if (action === 'discover') {
+ const count = await systemBackupService.discover(s3DestinationId, recoveryKey);
+ toast.success(m.system_backups_discovered({ count }));
+ await refresh();
+ }
+ actionOpen = false;
+ } catch (error) {
+ const fallback =
+ action === 'restore'
+ ? m.system_backups_restore_failed()
+ : action === 'delete'
+ ? m.system_backups_delete_failed()
+ : action === 'upload'
+ ? m.backups_upload_s3_failed()
+ : action === 'discover'
+ ? m.system_backups_discover_failed()
+ : m.system_backups_create_failed();
+ toast.error(error instanceof Error ? error.message : fallback);
+ } finally {
+ loading = false;
+ }
+ }
+
+ const actionButtons: SettingsActionButton[] = $derived.by(() => [
+ ...(policyCollection.recoveryKeyStored && canManageRecoveryKey
+ ? [
+ {
+ id: 'reset-recovery-key',
+ action: 'refresh',
+ icon: ResetIcon,
+ label: m.system_backups_reset_recovery_key(),
+ loading: generatingKey,
+ onclick: openRecoveryKey,
+ disabled: isReadOnly
+ } satisfies SettingsActionButton
+ ]
+ : []),
+ {
+ id: 's3-destinations',
+ action: 'edit',
+ icon: CloudStorageIcon,
+ label: m.s3_destinations_title(),
+ onclick: () => goto('/settings/backups/s3')
+ },
+ ...(!policyCollection.recoveryKeyStored
+ ? [
+ {
+ id: 'discover',
+ action: 'inspect',
+ label: m.system_backups_discover(),
+ onclick: () => openAction('discover'),
+ disabled: isReadOnly || data.destinations.length === 0
+ } satisfies SettingsActionButton
+ ]
+ : []),
+ {
+ id: 'create',
+ action: 'create',
+ label: m.common_create(),
+ disabled: isReadOnly,
+ options: [
+ { label: m.jobs_schedule(), onclick: () => openPolicy() },
+ { label: m.volumes_workspace_backup(), onclick: () => openAction('create') }
+ ]
+ }
+ ]);
+</script>
+
+{#snippet recoveryKeySummary()}
+ {#if !policyCollection.recoveryKeyStored}
+ <div class="flex items-center justify-between gap-3 rounded-md border border-amber-500/40 bg-amber-500/5 px-3 py-2">
+ <div class="flex min-w-0 items-center gap-2">
+ <LockIcon class="size-4 shrink-0 text-amber-500" />
+ <p class="text-sm">{m.system_backups_recovery_key_needed()}</p>
+ </div>
+ <ArcaneButton
+ action="edit"
+ size="sm"
+ customLabel={m.system_backups_setup_recovery_key()}
+ loading={generatingKey}
+ onclick={openRecoveryKey}
+ disabled={isReadOnly || !canManageRecoveryKey}
+ />
+ </div>
+ {/if}
+{/snippet}
+
+{#snippet recoveryKeyDialog()}
+ <ResponsiveDialog
+ bind:open={keyOpen}
+ title={m.system_backups_recovery_key()}
+ description={m.system_backups_recovery_key_saved()}
+ contentClass="sm:max-w-[560px]"
+ >
+ {#snippet children()}
+ <div class="space-y-3 py-2">
+ <div class="flex items-center gap-2">
+ <Input
+ id="system-backup-recovery-key"
+ value={newRecoveryKey}
+ readonly
+ spellcheck={false}
+ class="font-mono tracking-wide uppercase"
+ />
+ <CopyButton text={newRecoveryKey} variant="outline" tabindex={0} class="shrink-0" />
+ </div>
+ <Alert.Root variant="destructive">
+ <AlertIcon class="size-4" />
+ <Alert.Description>{m.system_backups_recovery_key_alert()}</Alert.Description>
+ </Alert.Root>
+ </div>
+ {/snippet}
+ {#snippet footer()}
+ <ArcaneButton action="confirm" customLabel={m.common_done()} onclick={() => (keyOpen = false)} />
+ {/snippet}
+ </ResponsiveDialog>
+{/snippet}
+
+{#snippet actionDialog()}
+ <ResponsiveDialog
+ bind:open={actionOpen}
+ title={dialogTitle()}
+ description={dialogDescription()}
+ contentClass="sm:max-w-[560px]"
+ >
+ {#snippet children()}
+ <div class="space-y-5 py-2">
+ {#if action === 'create'}
+ <SelectWithLabel
+ id="manual-system-backup-configuration"
+ value={backupConfiguration}
+ onValueChange={(value) => (backupConfiguration = value)}
+ label={m.system_backups_backup_configuration()}
+ options={configurationOptions}
+ />
+ {#if backupConfiguration === 'custom'}
+ <SelectWithLabel
+ id="manual-system-backup-destination"
+ value={destination}
+ onValueChange={(value) => (destination = value as SystemBackupDestination)}
+ label={m.backups_destination_label()}
+ options={destinationOptions}
+ />
+ {/if}
+ {/if}
+ {#if action === 'upload' || action === 'discover' || (action === 'create' && backupConfiguration === 'custom' && destination !== 'local')}
+ <SelectWithLabel
+ id="manual-system-backup-s3"
+ value={s3DestinationId}
+ onValueChange={(value) => (s3DestinationId = value)}
+ label={m.volume_backup_s3_destination_label()}
+ error={destinationError || null}
+ options={s3Options}
+ />
+ {/if}
+ {#if actionNeedsTypedKey}
+ <TextInputWithLabel
+ value={recoveryKey}
+ onChange={(value) => (recoveryKey = value)}
+ error={keyError || null}
+ label={m.system_backups_recovery_key()}
+ description={policyCollection.recoveryKeyStored
+ ? m.system_backups_recovery_key_saved_description()
+ : m.system_backups_recovery_key_enter_description()}
+ type="password"
+ autocomplete="current-password"
+ />
+ {:else if action !== 'delete' && !policyCollection.recoveryKeyStored}
+ <div class="flex items-center justify-between gap-3 rounded-md border px-3 py-2">
+ <p class="text-sm text-muted-foreground">{m.system_backups_recovery_key_needed()}</p>
+ <ArcaneButton
+ action="edit"
+ size="sm"
+ customLabel={m.system_backups_setup_recovery_key()}
+ loading={generatingKey}
+ onclick={() => {
+ actionOpen = false;
+ openRecoveryKey();
+ }}
+ />
+ </div>
+ {/if}
+ </div>
+ {/snippet}
+ {#snippet footer()}
+ <ArcaneButton action="cancel" onclick={() => (actionOpen = false)} disabled={loading} />
+ <ArcaneButton
+ action={action === 'delete' ? 'remove' : action === 'restore' ? 'confirm' : action === 'create' ? 'create' : 'save'}
+ customLabel={dialogTitle()}
+ onclick={submitAction}
+ {loading}
+ disabled={loading || invalid}
+ />
+ {/snippet}
+ </ResponsiveDialog>
+{/snippet}
+
+<SettingsPageLayout
+ title={m.system_backups_title()}
+ description={m.system_backups_description()}
+ icon={BackupIcon}
+ pageType="management"
+ showReadOnlyTag={isReadOnly}
+ {actionButtons}
+>
+ {#snippet mainContent()}
+ <div class="space-y-4">
+ {@render recoveryKeySummary()}
+
+ <div class="space-y-2">
+ <h2 class="text-lg font-semibold">{m.system_backups_schedules()}</h2>
+ {#if policyCollection.policies.length}
+ <div class="grid grid-cols-1 gap-1.5 text-xs text-muted-foreground sm:grid-cols-2 xl:grid-cols-3">
+ {#each policyCollection.policies as policy (policy.id)}
+ <BackupPolicyCard {policy} onEdit={() => openPolicy(policy.id)} editDisabled={isReadOnly} />
+ {/each}
+ </div>
+ {:else}
+ <p class="text-sm text-muted-foreground">{m.system_backups_no_schedules()}</p>
+ {/if}
+ </div>
+
+ <SystemBackupTable
+ bind:backups
+ bind:requestOptions
+ onChanged={(options) => systemBackupService.list(options)}
+ onRestore={(item) => openAction('restore', item)}
+ onUpload={(item) => openAction('upload', item)}
+ onDelete={(item) => openAction('delete', item)}
+ />
+ </div>
+ {/snippet}
+ {#snippet additionalContent()}
+ <BackupPolicyDialog
+ bind:open={policyOpen}
+ idPrefix="system-backup-policy"
+ policies={policyCollection.policies}
+ policyId={editingPolicyId}
+ addTitle={m.system_backups_add_schedule()}
+ description={m.system_backups_schedule_description()}
+ enabledDescription={m.system_backups_enabled_description()}
+ enabledError={policyCollection.recoveryKeyStored ? null : m.system_backups_recovery_key_schedule_required()}
+ defaultSchedule="0 0 3 * * *"
+ defaultEnabled={policyCollection.recoveryKeyStored}
+ destinations={data.destinations}
+ updatePolicies={async (policies) => (await systemBackupService.updatePolicies(policies)).policies}
+ messages={{
+ saved: m.system_backups_policy_saved(),
+ saveFailed: m.system_backups_policy_failed(),
+ removed: m.system_backups_schedule_removed()
+ }}
+ onSaved={(policies) => (policyCollection = { ...policyCollection, policies })}
+ />
+
+ {@render recoveryKeyDialog()}
+ {@render actionDialog()}
+ {/snippet}
+</SettingsPageLayout>
diff --git a/frontend/src/routes/(app)/settings/backups/+page.ts b/frontend/src/routes/(app)/settings/backups/+page.ts
new file mode 100644
index 000000000..7cb2f0f63
--- /dev/null
+++ b/frontend/src/routes/(app)/settings/backups/+page.ts
@@ -0,0 +1,18 @@
+import type { PageLoad } from './$types';
+import { systemBackupService } from '#lib/services/system-backup-service';
+import { s3DestinationService } from '#lib/services/s3-destination-service';
+import { resolveInitialTableRequest } from '#lib/utils/tables';
+import type { SearchPaginationSortRequest } from '#lib/types/shared';
+
+export const load: PageLoad = async () => {
+ const requestOptions = resolveInitialTableRequest('arcane-system-backups-table', {
+ pagination: { page: 1, limit: 20 },
+ sort: { column: 'createdAt', direction: 'desc' }
+ } satisfies SearchPaginationSortRequest);
+ const [backups, policyCollection, destinations] = await Promise.all([
+ systemBackupService.list(requestOptions),
+ systemBackupService.getPolicies(),
+ s3DestinationService.listAll()
+ ]);
+ return { backups, policyCollection, destinations, requestOptions };
+};
diff --git a/frontend/src/routes/(app)/settings/backups/s3/+page.svelte b/frontend/src/routes/(app)/settings/backups/s3/+page.svelte
new file mode 100644
index 000000000..9c83591af
--- /dev/null
+++ b/frontend/src/routes/(app)/settings/backups/s3/+page.svelte
@@ -0,0 +1,104 @@
+<script lang="ts">
+ import { untrack } from 'svelte';
+ import { toast } from 'svelte-sonner';
+ import settingsStore from '#lib/stores/config-store';
+ import { SettingsPageLayout, type SettingsActionButton } from '#lib/layouts';
+ import { RemoteEnvironmentIcon } from '#lib/icons';
+ import { openConfirmDialog } from '#lib/components/confirm-dialog';
+ import { s3DestinationService } from '#lib/services/s3-destination-service';
+ import type { CreateS3Destination, S3Destination } from '#lib/types/s3-destination';
+ import type { SearchPaginationSortRequest } from '#lib/types/shared';
+ import * as m from '#lib/paraglide/messages.js';
+ import S3DestinationDialog from './s3-destination-dialog.svelte';
+ import S3DestinationTable from './s3-destination-table.svelte';
+
+ let { data } = $props();
+ let destinations = $state(untrack(() => data.destinations));
+ let requestOptions = $state<SearchPaginationSortRequest>(untrack(() => data.requestOptions));
+ let selected = $state<S3Destination | null>(null);
+ let dialogOpen = $state(false);
+ let saving = $state(false);
+ const isReadOnly = $derived.by(() => $settingsStore.uiConfigDisabled);
+
+ function openCreate() {
+ selected = null;
+ dialogOpen = true;
+ }
+
+ function openEdit(destination: S3Destination) {
+ selected = destination;
+ dialogOpen = true;
+ }
+
+ async function saveDestination(input: CreateS3Destination) {
+ saving = true;
+ try {
+ if (selected) {
+ await s3DestinationService.update(selected.id, input);
+ toast.success(m.s3_destination_updated({ name: input.name }));
+ } else {
+ await s3DestinationService.create(input);
+ toast.success(m.s3_destination_created({ name: input.name }));
+ }
+ destinations = await s3DestinationService.list(requestOptions);
+ dialogOpen = false;
+ selected = null;
+ } catch (error) {
+ toast.error(error instanceof Error ? error.message : m.s3_destination_save_failed());
+ } finally {
+ saving = false;
+ }
+ }
+
+ function deleteDestination(destination: S3Destination) {
+ openConfirmDialog({
+ title: m.delete_name({ name: destination.name }),
+ message: m.s3_destination_delete_message(),
+ confirm: {
+ label: m.common_delete(),
+ destructive: true,
+ action: async () => {
+ try {
+ await s3DestinationService.delete(destination.id);
+ destinations = await s3DestinationService.list(requestOptions);
+ toast.success(m.s3_destination_deleted({ name: destination.name }));
+ } catch (error) {
+ toast.error(error instanceof Error ? error.message : m.s3_destination_delete_failed());
+ }
+ }
+ }
+ });
+ }
+
+ const actionButtons: SettingsActionButton[] = $derived.by(() => [
+ {
+ id: 'create',
+ action: 'create',
+ label: m.s3_destination_add(),
+ onclick: openCreate,
+ disabled: isReadOnly
+ }
+ ]);
+</script>
+
+<SettingsPageLayout
+ title={m.s3_destinations_title()}
+ description={m.s3_destinations_description()}
+ icon={RemoteEnvironmentIcon}
+ pageType="management"
+ showReadOnlyTag={isReadOnly}
+ {actionButtons}
+>
+ {#snippet mainContent()}
+ <S3DestinationTable
+ bind:destinations
+ bind:requestOptions
+ onDestinationsChanged={(options) => s3DestinationService.list(options)}
+ onEdit={openEdit}
+ onDelete={deleteDestination}
+ />
+ {/snippet}
+ {#snippet additionalContent()}
+ <S3DestinationDialog bind:open={dialogOpen} destination={selected} {saving} onSubmit={saveDestination} />
+ {/snippet}
+</SettingsPageLayout>
diff --git a/frontend/src/routes/(app)/settings/backups/s3/+page.ts b/frontend/src/routes/(app)/settings/backups/s3/+page.ts
new file mode 100644
index 000000000..ecaf39977
--- /dev/null
+++ b/frontend/src/routes/(app)/settings/backups/s3/+page.ts
@@ -0,0 +1,13 @@
+import type { PageLoad } from './$types';
+import { s3DestinationService } from '#lib/services/s3-destination-service';
+import { resolveInitialTableRequest } from '#lib/utils/tables';
+import type { SearchPaginationSortRequest } from '#lib/types/shared';
+
+export const load: PageLoad = async () => {
+ const requestOptions = resolveInitialTableRequest('arcane-s3-destinations-table', {
+ pagination: { page: 1, limit: 20 },
+ sort: { column: 'name', direction: 'asc' }
+ } satisfies SearchPaginationSortRequest);
+ const destinations = await s3DestinationService.list(requestOptions);
+ return { destinations, requestOptions };
+};
diff --git a/frontend/src/routes/(app)/settings/backups/s3/s3-destination-dialog.svelte b/frontend/src/routes/(app)/settings/backups/s3/s3-destination-dialog.svelte
new file mode 100644
index 000000000..03a530666
--- /dev/null
+++ b/frontend/src/routes/(app)/settings/backups/s3/s3-destination-dialog.svelte
@@ -0,0 +1,195 @@
+<script lang="ts">
+ import { ResponsiveDialog } from '#lib/components/ui/responsive-dialog';
+ import SheetFooterActions from '#lib/components/sheets/sheet-footer-actions.svelte';
+ import FormInput from '#lib/components/form/form-input.svelte';
+ import LabeledSwitch from '#lib/components/form/labeled-switch.svelte';
+ import type { CreateS3Destination, S3Destination } from '#lib/types/s3-destination';
+ import { createForm, preventDefault } from '#lib/utils/settings';
+ import { s3DestinationService } from '#lib/services/s3-destination-service';
+ import { toast } from 'svelte-sonner';
+ import { z } from 'zod/v4';
+ import * as m from '#lib/paraglide/messages.js';
+
+ let {
+ open = $bindable(),
+ destination,
+ saving,
+ onSubmit
+ }: {
+ open: boolean;
+ destination: S3Destination | null;
+ saving: boolean;
+ onSubmit: (input: CreateS3Destination) => Promise<void>;
+ } = $props();
+
+ const formSchema = z
+ .object({
+ name: z.string().trim().min(1, m.common_name_required()),
+ endpoint: z.string(),
+ bucket: z.string().trim().min(1, m.backups_s3_bucket_required()),
+ region: z.string(),
+ accessKeyId: z.string().trim().min(1, m.backups_s3_access_key_required()),
+ secretAccessKey: z.string(),
+ prefix: z.string(),
+ useSsl: z.boolean(),
+ forcePathStyle: z.boolean()
+ })
+ .superRefine((data, ctx) => {
+ if (!data.endpoint.trim() && !data.region.trim()) {
+ ctx.addIssue({
+ code: z.ZodIssueCode.custom,
+ message: m.backups_s3_region_required(),
+ path: ['region']
+ });
+ }
+ if (!destination?.secretConfigured && !data.secretAccessKey.trim()) {
+ ctx.addIssue({
+ code: z.ZodIssueCode.custom,
+ message: m.backups_s3_secret_key_required(),
+ path: ['secretAccessKey']
+ });
+ }
+ });
+
+ let formData = $derived<CreateS3Destination>({
+ name: open && destination ? destination.name : '',
+ endpoint: open && destination ? (destination.endpoint ?? '') : '',
+ bucket: open && destination ? destination.bucket : '',
+ region: open && destination ? destination.region : 'us-east-1',
+ accessKeyId: open && destination ? destination.accessKeyId : '',
+ secretAccessKey: '',
+ prefix: open && destination ? (destination.prefix ?? '') : '',
+ useSsl: open && destination ? destination.useSsl : true,
+ forcePathStyle: open && destination ? destination.forcePathStyle : true
+ });
+
+ let { inputs, ...form } = $derived(createForm<typeof formSchema>(formSchema, formData));
+ let testing = $state(false);
+ let testedConfiguration = $state<string | null>(null);
+ const currentConfiguration = $derived(
+ JSON.stringify({
+ endpoint: $inputs.endpoint.value.trim(),
+ bucket: $inputs.bucket.value.trim(),
+ region: $inputs.region.value.trim(),
+ accessKeyId: $inputs.accessKeyId.value.trim(),
+ secretAccessKey: $inputs.secretAccessKey.value.trim(),
+ prefix: $inputs.prefix.value.trim().replace(/^\/+|\/+$/g, ''),
+ useSsl: $inputs.useSsl.value,
+ forcePathStyle: $inputs.forcePathStyle.value
+ })
+ );
+ const connectionVerified = $derived(testedConfiguration === currentConfiguration);
+
+ $effect(() => {
+ open;
+ destination?.id;
+ testedConfiguration = null;
+ });
+
+ function handleSubmit() {
+ const data = form.validate();
+ if (!data) return;
+ if (!connectionVerified) {
+ toast.error(m.s3_destination_test_required());
+ return;
+ }
+ onSubmit(data);
+ }
+
+ async function handleTest() {
+ const data = form.validate();
+ if (!data) return;
+ const testedCandidate = currentConfiguration;
+ testedConfiguration = null;
+ testing = true;
+ try {
+ if (destination) {
+ await s3DestinationService.test(destination.id, data);
+ } else {
+ await s3DestinationService.testConfiguration(data);
+ }
+ testedConfiguration = testedCandidate;
+ toast.success(m.s3_destination_test_success({ name: data.name }));
+ } catch (error) {
+ toast.error(error instanceof Error ? error.message : m.s3_destination_test_failed({ name: data.name }));
+ } finally {
+ testing = false;
+ }
+ }
+</script>
+
+{#snippet destinationFields()}
+ <FormInput bind:input={$inputs.name} label={m.common_name()} />
+ <FormInput
+ bind:input={$inputs.endpoint}
+ label={m.backups_s3_endpoint_label()}
+ description={m.backups_s3_endpoint_description()}
+ />
+ <FormInput bind:input={$inputs.bucket} label={m.backups_s3_bucket_label()} />
+ <FormInput bind:input={$inputs.region} label={m.backups_s3_region_label()} description={m.backups_s3_region_description()} />
+{/snippet}
+
+{#snippet credentialFields()}
+ <FormInput bind:input={$inputs.accessKeyId} label={m.backups_s3_access_key_label()} autocomplete="off" />
+ <FormInput
+ bind:input={$inputs.secretAccessKey}
+ label={m.backups_s3_secret_key_label()}
+ description={destination?.secretConfigured ? m.s3_destination_secret_keep_description() : undefined}
+ type="password"
+ autocomplete="new-password"
+ />
+{/snippet}
+
+{#snippet connectionOptions()}
+ <FormInput bind:input={$inputs.prefix} label={m.backups_s3_prefix_label()} description={m.backups_s3_prefix_description()} />
+ <div class="mt-1 grid gap-4 border-t pt-5 sm:grid-cols-2">
+ <LabeledSwitch
+ id="s3-destination-ssl"
+ bind:checked={$inputs.useSsl.value}
+ label={m.backups_s3_ssl_label()}
+ description={m.backups_s3_ssl_description()}
+ />
+ <LabeledSwitch
+ id="s3-destination-path-style"
+ bind:checked={$inputs.forcePathStyle.value}
+ label={m.backups_s3_path_style_label()}
+ description={m.backups_s3_path_style_description()}
+ />
+ </div>
+{/snippet}
+
+<ResponsiveDialog
+ bind:open
+ variant="sheet"
+ title={destination ? m.s3_destination_edit_title() : m.s3_destination_add_title()}
+ description={m.s3_destination_dialog_description()}
+ contentClass="sm:max-w-[640px]"
+>
+ {#snippet children()}
+ <form onsubmit={preventDefault(handleSubmit)} class="grid gap-4 py-6">
+ {@render destinationFields()}
+ {@render credentialFields()}
+ {@render connectionOptions()}
+ </form>
+ {/snippet}
+ {#snippet footer()}
+ <div class="flex w-full flex-col gap-2">
+ <p class={connectionVerified ? 'text-xs text-green-600' : 'text-xs text-muted-foreground'}>
+ {connectionVerified ? m.s3_destination_test_verified() : m.s3_destination_test_required()}
+ </p>
+ <SheetFooterActions
+ bind:open
+ cancelDisabled={saving || testing}
+ submitAction={connectionVerified ? (destination ? 'save' : 'create') : 'test'}
+ submitDisabled={saving || testing}
+ submitLoading={saving || testing}
+ onSubmit={() => (connectionVerified ? handleSubmit() : handleTest())}
+ submitLabel={connectionVerified
+ ? destination
+ ? m.common_save_changes()
+ : m.s3_destination_add_title()
+ : m.test_connection()}
+ />
+ </div>
+ {/snippet}
+</ResponsiveDialog>
diff --git a/frontend/src/routes/(app)/settings/backups/s3/s3-destination-table.svelte b/frontend/src/routes/(app)/settings/backups/s3/s3-destination-table.svelte
new file mode 100644
index 000000000..729588616
--- /dev/null
+++ b/frontend/src/routes/(app)/settings/backups/s3/s3-destination-table.svelte
@@ -0,0 +1,164 @@
+<script lang="ts">
+ import ArcaneTable from '#lib/components/arcane-table/arcane-table.svelte';
+ import RowActionsMenu from '#lib/components/arcane-table/row-actions-menu.svelte';
+ import { UniversalMobileCard } from '#lib/components/arcane-table';
+ import type { ColumnSpec, MobileFieldVisibility } from '#lib/components/arcane-table';
+ import * as DropdownMenu from '#lib/components/ui/dropdown-menu';
+ import type { S3Destination } from '#lib/types/s3-destination';
+ import type { Paginated, SearchPaginationSortRequest } from '#lib/types/shared';
+ import { EditIcon, RemoteEnvironmentIcon, TrashIcon, ClockIcon, GlobeIcon, TestIcon } from '#lib/icons';
+ import { formatOptionalDateTime } from '#lib/utils/formatting';
+ import * as m from '#lib/paraglide/messages.js';
+ import IfPermitted from '#lib/components/if-permitted.svelte';
+ import { Spinner } from '#lib/components/ui/spinner';
+ import { toast } from 'svelte-sonner';
+ import { s3DestinationService } from '#lib/services/s3-destination-service';
+
+ let {
+ destinations = $bindable(),
+ requestOptions = $bindable(),
+ onDestinationsChanged,
+ onEdit,
+ onDelete
+ }: {
+ destinations: Paginated<S3Destination>;
+ requestOptions: SearchPaginationSortRequest;
+ onDestinationsChanged: (options: SearchPaginationSortRequest) => Promise<Paginated<S3Destination>>;
+ onEdit: (destination: S3Destination) => void;
+ onDelete: (destination: S3Destination) => void;
+ } = $props();
+
+ const columns = [
+ { accessorKey: 'name', title: m.common_name(), sortable: true, cell: NameCell },
+ { accessorKey: 'bucket', title: m.backups_s3_bucket_label(), sortable: true },
+ { accessorKey: 'endpoint', title: m.backups_s3_endpoint_label(), sortable: true, cell: EndpointCell },
+ { accessorKey: 'region', title: m.backups_s3_region_label(), sortable: true },
+ { accessorKey: 'prefix', title: m.backups_s3_prefix_label(), sortable: true, cell: PrefixCell },
+ { accessorKey: 'updatedAt', title: m.common_updated(), sortable: true, cell: UpdatedCell }
+ ] satisfies ColumnSpec<S3Destination>[];
+
+ const mobileFields = [
+ { id: 'bucket', label: m.backups_s3_bucket_label(), defaultVisible: true },
+ { id: 'endpoint', label: m.backups_s3_endpoint_label(), defaultVisible: true },
+ { id: 'region', label: m.backups_s3_region_label(), defaultVisible: true },
+ { id: 'prefix', label: m.backups_s3_prefix_label(), defaultVisible: false }
+ ];
+ let mobileFieldVisibility = $state<Record<string, boolean>>({});
+ let testingId = $state<string | null>(null);
+
+ async function testDestination(destination: S3Destination) {
+ testingId = destination.id;
+ try {
+ await s3DestinationService.test(destination.id);
+ toast.success(m.s3_destination_test_success({ name: destination.name }));
+ } catch (error) {
+ toast.error(error instanceof Error ? error.message : m.s3_destination_test_failed({ name: destination.name }));
+ } finally {
+ testingId = null;
+ }
+ }
+</script>
+
+{#snippet NameCell({ item }: { item: S3Destination })}
+ <span class="font-medium">{item.name}</span>
+{/snippet}
+
+{#snippet EndpointCell({ item }: { item: S3Destination })}
+ <span class="text-muted-foreground">{item.endpoint || m.s3_destination_aws_default()}</span>
+{/snippet}
+
+{#snippet PrefixCell({ item }: { item: S3Destination })}
+ <code class="text-xs">{item.prefix || '-'}</code>
+{/snippet}
+
+{#snippet UpdatedCell({ item }: { item: S3Destination })}
+ {formatOptionalDateTime(item.updatedAt ?? item.createdAt)}
+{/snippet}
+
+{#snippet RowActions({ item }: { item: S3Destination })}
+ <RowActionsMenu>
+ <IfPermitted perm="s3-destinations:test">
+ <DropdownMenu.Item onclick={() => testDestination(item)} disabled={testingId === item.id}>
+ {#if testingId === item.id}
+ <Spinner class="size-4" />
+ {:else}
+ <TestIcon class="size-4" />
+ {/if}
+ {m.test_connection()}
+ </DropdownMenu.Item>
+ </IfPermitted>
+ <IfPermitted perm="s3-destinations:update">
+ <DropdownMenu.Item onclick={() => onEdit(item)}>
+ <EditIcon class="size-4" />
+ {m.common_edit()}
+ </DropdownMenu.Item>
+ </IfPermitted>
+ <IfPermitted perm="s3-destinations:delete">
+ <DropdownMenu.Separator />
+ <DropdownMenu.Item variant="destructive" onclick={() => onDelete(item)}>
+ <TrashIcon class="size-4" />
+ {m.common_delete()}
+ </DropdownMenu.Item>
+ </IfPermitted>
+ </RowActionsMenu>
+{/snippet}
+
+{#snippet MobileCard({ item, mobileFieldVisibility }: { item: S3Destination; mobileFieldVisibility: MobileFieldVisibility })}
+ <UniversalMobileCard
+ {item}
+ icon={{ component: RemoteEnvironmentIcon, variant: 'blue' }}
+ title={(item) => item.name}
+ fields={[
+ {
+ label: m.backups_s3_bucket_label(),
+ getValue: (item) => item.bucket,
+ icon: RemoteEnvironmentIcon,
+ iconVariant: 'gray',
+ show: mobileFieldVisibility['bucket'] ?? true
+ },
+ {
+ label: m.backups_s3_endpoint_label(),
+ getValue: (item) => item.endpoint || m.s3_destination_aws_default(),
+ icon: GlobeIcon,
+ iconVariant: 'gray',
+ show: mobileFieldVisibility['endpoint'] ?? true
+ },
+ {
+ label: m.backups_s3_region_label(),
+ getValue: (item) => item.region,
+ icon: GlobeIcon,
+ iconVariant: 'gray',
+ show: mobileFieldVisibility['region'] ?? true
+ },
+ {
+ label: m.backups_s3_prefix_label(),
+ getValue: (item) => item.prefix || '-',
+ icon: RemoteEnvironmentIcon,
+ iconVariant: 'gray',
+ show: mobileFieldVisibility['prefix'] ?? false
+ }
+ ]}
+ footer={{
+ label: m.common_updated(),
+ getValue: (item) => formatOptionalDateTime(item.updatedAt ?? item.createdAt),
+ icon: ClockIcon
+ }}
+ rowActions={RowActions}
+ />
+{/snippet}
+
+<ArcaneTable
+ persistKey="arcane-s3-destinations-table"
+ items={destinations}
+ bind:requestOptions
+ bind:mobileFieldVisibility
+ onRefresh={async (options) => {
+ requestOptions = options;
+ destinations = await onDestinationsChanged(options);
+ return destinations;
+ }}
+ {columns}
+ {mobileFields}
+ rowActions={RowActions}
+ mobileCard={MobileCard}
+/>
diff --git a/frontend/src/routes/(app)/settings/backups/system-backup-table.svelte b/frontend/src/routes/(app)/settings/backups/system-backup-table.svelte
new file mode 100644
index 000000000..df82d1578
--- /dev/null
+++ b/frontend/src/routes/(app)/settings/backups/system-backup-table.svelte
@@ -0,0 +1,122 @@
+<script lang="ts">
+ import ArcaneTable from '#lib/components/arcane-table/arcane-table.svelte';
+ import RowActionsMenu from '#lib/components/arcane-table/row-actions-menu.svelte';
+ import { UniversalMobileCard, type ColumnSpec, type MobileFieldVisibility } from '#lib/components/arcane-table';
+ import * as DropdownMenu from '#lib/components/ui/dropdown-menu';
+ import BackupStatusCell from '#lib/components/arcane-table/cells/backup-status-cell.svelte';
+ import BackupTriggerCell from '#lib/components/arcane-table/cells/backup-trigger-cell.svelte';
+ import BackupDestinationCell from '#lib/components/arcane-table/cells/backup-destination-cell.svelte';
+ import BackupSizeCell from '#lib/components/arcane-table/cells/backup-size-cell.svelte';
+ import CreatedAtCell from '#lib/components/arcane-table/cells/created-at-cell.svelte';
+ import type { Paginated, SearchPaginationSortRequest } from '#lib/types/shared';
+ import type { SystemBackupRun } from '#lib/types/system-backup';
+ import { BackupIcon, RestartIcon, TrashIcon, UploadIcon, ClockIcon } from '#lib/icons';
+ import { bytes, formatDateTimeShort } from '#lib/utils/formatting';
+ import { backupDestinationDisplay, backupStatusLabel, backupStatusVariant, backupTriggerLabel } from '#lib/utils/backups';
+ import * as m from '#lib/paraglide/messages.js';
+
+ let {
+ backups = $bindable(),
+ requestOptions = $bindable(),
+ onChanged,
+ onRestore,
+ onUpload,
+ onDelete
+ }: {
+ backups: Paginated<SystemBackupRun>;
+ requestOptions: SearchPaginationSortRequest;
+ onChanged: (options: SearchPaginationSortRequest) => Promise<Paginated<SystemBackupRun>>;
+ onRestore: (backup: SystemBackupRun) => void;
+ onUpload: (backup: SystemBackupRun) => void;
+ onDelete: (backup: SystemBackupRun) => void;
+ } = $props();
+
+ let mobileFieldVisibility = $state<Record<string, boolean>>({});
+ const columns = [
+ { accessorKey: 'id', title: m.system_backups_id(), sortable: false, cell: IdCell },
+ { accessorKey: 'status', title: m.common_status(), sortable: true, cell: StatusCell },
+ { accessorKey: 'trigger', title: m.volume_backup_trigger(), sortable: true, cell: TriggerCell },
+ { accessorKey: 'destination', title: m.backups_destination_label(), sortable: true, cell: DestinationCell },
+ { accessorKey: 'size', title: m.common_size(), sortable: true, cell: SizeCell },
+ { accessorKey: 'createdAt', title: m.common_created(), sortable: true, cell: CreatedCell },
+ { accessorKey: 'error', title: m.common_error(), sortable: false, cell: ErrorCell }
+ ] satisfies ColumnSpec<SystemBackupRun>[];
+ const mobileFields = [
+ { id: 'status', label: m.common_status(), defaultVisible: true },
+ { id: 'trigger', label: m.volume_backup_trigger(), defaultVisible: true },
+ { id: 'destination', label: m.backups_destination_label(), defaultVisible: true },
+ { id: 'size', label: m.common_size(), defaultVisible: true }
+ ];
+</script>
+
+{#snippet IdCell({ item }: { item: SystemBackupRun })}<code class="text-xs">{item.id.slice(0, 18)}…</code>{/snippet}
+{#snippet StatusCell({ item }: { item: SystemBackupRun })}<BackupStatusCell status={item.status} />{/snippet}
+{#snippet TriggerCell({ item }: { item: SystemBackupRun })}<BackupTriggerCell trigger={item.trigger} />{/snippet}
+{#snippet DestinationCell({ item }: { item: SystemBackupRun })}<BackupDestinationCell {item} />{/snippet}
+{#snippet SizeCell({ item }: { item: SystemBackupRun })}<BackupSizeCell size={item.size} />{/snippet}
+{#snippet CreatedCell({ item }: { item: SystemBackupRun })}<CreatedAtCell value={item.createdAt} />{/snippet}
+{#snippet ErrorCell({ item }: { item: SystemBackupRun })}<span class="max-w-72 truncate text-red-500">{item.error || '-'}</span
+ >{/snippet}
+
+{#snippet RowActions({ item }: { item: SystemBackupRun })}
+ <RowActionsMenu>
+ <DropdownMenu.Item onclick={() => onRestore(item)} disabled={item.status !== 'succeeded'}
+ ><RestartIcon class="size-4" />{m.volumes_backups_restore()}</DropdownMenu.Item
+ >
+ {#if item.localSnapshotId && !item.remoteSnapshotId}
+ <DropdownMenu.Item onclick={() => onUpload(item)}><UploadIcon class="size-4" />{m.backups_upload_s3()}</DropdownMenu.Item>
+ {/if}
+ <DropdownMenu.Separator />
+ <DropdownMenu.Item variant="destructive" onclick={() => onDelete(item)}
+ ><TrashIcon class="size-4" />{m.common_delete()}</DropdownMenu.Item
+ >
+ </RowActionsMenu>
+{/snippet}
+
+{#snippet MobileCard({ item, mobileFieldVisibility }: { item: SystemBackupRun; mobileFieldVisibility: MobileFieldVisibility })}
+ <UniversalMobileCard
+ {item}
+ icon={{ component: BackupIcon, variant: 'blue' }}
+ title={(item) => backupDestinationDisplay(item)}
+ badges={[
+ (item) => ({
+ variant: backupStatusVariant(item.status),
+ text: backupStatusLabel(item.status)
+ })
+ ]}
+ fields={[
+ {
+ label: m.volume_backup_trigger(),
+ getValue: (item) => backupTriggerLabel(item.trigger),
+ icon: BackupIcon,
+ iconVariant: 'gray',
+ show: mobileFieldVisibility['trigger'] ?? true
+ },
+ {
+ label: m.common_size(),
+ getValue: (item) => bytes(item.size),
+ icon: BackupIcon,
+ iconVariant: 'gray',
+ show: mobileFieldVisibility['size'] ?? true
+ }
+ ]}
+ footer={{ label: m.common_created(), getValue: (item) => formatDateTimeShort(item.createdAt), icon: ClockIcon }}
+ rowActions={RowActions}
+ />
+{/snippet}
+
+<ArcaneTable
+ persistKey="arcane-system-backups-table"
+ items={backups}
+ bind:requestOptions
+ bind:mobileFieldVisibility
+ onRefresh={async (options) => {
+ requestOptions = options;
+ backups = await onChanged(options);
+ return backups;
+ }}
+ {columns}
+ {mobileFields}
+ rowActions={RowActions}
+ mobileCard={MobileCard}
+/>
diff --git a/frontend/src/routes/(app)/volumes/components/volume-backup-table.svelte b/frontend/src/routes/(app)/volumes/components/volume-backup-table.svelte
index 8ed4bf8b1..7fee1b6f6 100644
--- a/frontend/src/routes/(app)/volumes/components/volume-backup-table.svelte
+++ b/frontend/src/routes/(app)/volumes/components/volume-backup-table.svelte
@@ -1,8 +1,10 @@
<script lang="ts">
import { m } from '#lib/paraglide/messages';
import { volumeBackupService, type VolumeBackupListResponse } from '#lib/services/volume-backup-service';
+ import { s3DestinationService } from '#lib/services/s3-destination-service';
import { volumeService } from '#lib/services/volume-service';
- import type { BackupEntry } from '#lib/types/shared';
+ import type { BackupEntry, CreateVolumeBackupRequest, VolumeBackupPolicy } from '#lib/types/shared';
+ import type { S3Destination } from '#lib/types/s3-destination';
import { onMount } from 'svelte';
import {
LoadingSpinnerIcon,
@@ -14,14 +16,22 @@
DownloadIcon,
RestartIcon,
FileTextIcon,
- AlertIcon
+ AlertIcon,
+ UploadIcon,
+ ArrowDownIcon
} from '#lib/icons';
- import { ArcaneButton } from '#lib/components/arcane-button';
+ import { ArcaneButton, arcaneButtonVariants } from '#lib/components/arcane-button';
+ import * as ButtonGroup from '#lib/components/ui/button-group';
import { toast } from 'svelte-sonner';
import { bytes, formatDateTimeShort } from '#lib/utils/formatting';
import ArcaneTable from '#lib/components/arcane-table/arcane-table.svelte';
import type { SearchPaginationSortRequest } from '#lib/types/shared';
- import { UniversalMobileCard, type ColumnSpec, type MobileFieldVisibility } from '#lib/components/arcane-table';
+ import {
+ UniversalMobileCard,
+ type BulkAction,
+ type ColumnSpec,
+ type MobileFieldVisibility
+ } from '#lib/components/arcane-table';
import * as DropdownMenu from '#lib/components/ui/dropdown-menu';
import RowActionsMenu from '#lib/components/file-browser/row-actions-menu.svelte';
import { openConfirmDialog } from '#lib/components/confirm-dialog';
@@ -34,11 +44,28 @@
import { hasPermission } from '#lib/utils/auth';
import IfPermitted from '#lib/components/if-permitted.svelte';
import { activityToastOptions, extractActivityId } from '#lib/utils/activity-toast';
+ import BackupPolicyDialog from '#lib/components/backup-policy-dialog.svelte';
+ import BackupPolicyCard from '#lib/components/backup-policy-card.svelte';
+ import BackupStatusCell from '#lib/components/arcane-table/cells/backup-status-cell.svelte';
+ import BackupTriggerCell from '#lib/components/arcane-table/cells/backup-trigger-cell.svelte';
+ import BackupDestinationCell from '#lib/components/arcane-table/cells/backup-destination-cell.svelte';
+ import BackupSizeCell from '#lib/components/arcane-table/cells/backup-size-cell.svelte';
+ import CreatedAtCell from '#lib/components/arcane-table/cells/created-at-cell.svelte';
+ import { cn } from '#lib/utils';
+ import { bulkConfirmAndRun } from '#lib/utils/bulk-actions';
+ import { extractApiErrorMessage } from '#lib/utils/api';
+ import {
+ backupDestinationDisplay,
+ backupTriggerLabel,
+ s3DestinationOptions as buildS3DestinationOptions
+ } from '#lib/utils/backups';
+ import SelectWithLabel from '#lib/components/form/select-with-label.svelte';
let { volumeName }: { volumeName: string } = $props();
const currentEnvId = $derived(environmentStore.selected?.id || '0');
const canBackupVolume = $derived(hasPermission('volumes:backup', currentEnvId));
+ const canDeleteBackup = $derived(hasPermission('volumes:backup', currentEnvId));
let backupsPaginated = $state<VolumeBackupListResponse>({
data: [],
@@ -50,6 +77,14 @@
}
});
let backupWarnings = $state<string[]>([]);
+ let backupPolicies = $state<VolumeBackupPolicy[]>([]);
+ let s3Destinations = $state<S3Destination[]>([]);
+ let showBackupPolicy = $state(false);
+ let editingBackupPolicyId = $state<string | undefined>();
+ let showS3DestinationDialog = $state(false);
+ let onDemandDestination = $state<'s3' | 'local_s3'>('s3');
+ let onDemandS3DestinationId = $state('');
+ const s3DestinationOptions = $derived(buildS3DestinationOptions(s3Destinations));
let requestOptions = $state<SearchPaginationSortRequest>({
pagination: { page: 1, limit: 10 },
@@ -57,6 +92,9 @@
});
let creating = $state(false);
+ let deletingSelected = $state(false);
+ let selectedIds = $state<string[]>([]);
+ let uploadingBackupId = $state<string | null>(null);
let restoringFiles = $state(false);
let showRestoreFiles = $state(false);
let restoreTarget = $state<BackupEntry | null>(null);
@@ -76,28 +114,48 @@
backupsPaginated = result;
backupWarnings = result.warnings ?? [];
return result;
- } catch (e: any) {
- toast.error(e.message || 'Failed to load backups');
+ } catch (error) {
+ toast.error(error instanceof Error ? error.message : m.volumes_backup_load_failed());
return backupsPaginated;
}
}
- async function handleCreate() {
+ async function handleCreate(request?: CreateVolumeBackupRequest) {
creating = true;
try {
- const result = await volumeBackupService.createBackup(volumeName);
+ const result = await volumeBackupService.createBackup(volumeName, request);
toast.success(m.common_success(), activityToastOptions(extractActivityId(result)));
await loadData(requestOptions);
- } catch (e: any) {
- toast.error(e.message || m.common_failed());
+ return true;
+ } catch (error) {
+ toast.error(error instanceof Error ? error.message : m.common_failed());
+ return false;
} finally {
creating = false;
}
}
+ function openS3DestinationDialog(destination: 's3' | 'local_s3') {
+ onDemandDestination = destination;
+ onDemandS3DestinationId = s3Destinations.length === 1 ? (s3Destinations[0]?.id ?? '') : '';
+ showS3DestinationDialog = true;
+ }
+
+ async function createS3Backup() {
+ if (!onDemandS3DestinationId) return;
+ const created = await handleCreate({
+ destination: onDemandDestination,
+ s3DestinationId: onDemandS3DestinationId
+ });
+ if (created) {
+ showS3DestinationDialog = false;
+ onDemandS3DestinationId = '';
+ }
+ }
+
async function handleDelete(backup: BackupEntry) {
openConfirmDialog({
- title: m.common_remove_title({ resource: 'Backup' }),
+ title: m.common_remove_title({ resource: m.volumes_workspace_backup() }),
message: m.volumes_backup_delete_confirm(),
confirm: {
label: m.common_remove(),
@@ -105,16 +163,54 @@
action: async () => {
try {
const result = await volumeBackupService.deleteBackup(backup.id);
- toast.success(m.common_delete_success({ resource: 'Backup' }), activityToastOptions(extractActivityId(result)));
+ toast.success(
+ m.common_delete_success({ resource: m.volumes_workspace_backup() }),
+ activityToastOptions(extractActivityId(result))
+ );
+ await loadData(requestOptions);
+ } catch (error) {
+ toast.error(extractApiErrorMessage(error));
await loadData(requestOptions);
- } catch (e: any) {
- toast.error(e.message || m.common_delete_failed({ resource: 'Backup' }));
}
}
}
});
}
+ function handleDeleteSelected(ids: string[]) {
+ bulkConfirmAndRun({
+ ids,
+ title: m.volume_backups_remove_selected_title({ count: ids.length }),
+ message: m.volume_backups_remove_selected_message({ count: ids.length }),
+ confirmLabel: m.common_remove(),
+ destructive: true,
+ run: (id) => volumeBackupService.deleteBackup(id),
+ messages: {
+ success: (count) => m.common_bulk_remove_success({ count, resource: m.volumes_backups_title() }),
+ partial: (success, total, failed) =>
+ m.common_bulk_remove_partial({ success, total, failed, resource: m.volumes_backups_title() }),
+ failure: () => m.common_bulk_remove_failed({ count: ids.length, resource: m.volumes_backups_title() })
+ },
+ setLoading: (loading) => (deletingSelected = loading),
+ onItemFailure: (_id, error) => toast.error(extractApiErrorMessage(error)),
+ onComplete: () => loadData(requestOptions),
+ clearSelection: () => (selectedIds = [])
+ });
+ }
+
+ async function handleUpload(backup: BackupEntry, s3DestinationId: string) {
+ uploadingBackupId = backup.id;
+ try {
+ const result = await volumeBackupService.uploadBackup(backup.id, s3DestinationId);
+ toast.success(m.backups_upload_s3_success(), activityToastOptions(extractActivityId(result)));
+ await loadData(requestOptions);
+ } catch (error) {
+ toast.error(error instanceof Error ? error.message : m.backups_upload_s3_failed());
+ } finally {
+ uploadingBackupId = null;
+ }
+ }
+
async function openRestoreFilesDialog(backup: BackupEntry) {
restoreTarget = backup;
selectedPaths = [];
@@ -124,8 +220,8 @@
backupFilesLoading = true;
try {
backupFiles = await volumeBackupService.listBackupFiles(backup.id);
- } catch (e: any) {
- toast.error(e.message || m.common_failed());
+ } catch (error) {
+ toast.error(error instanceof Error ? error.message : m.common_failed());
} finally {
backupFilesLoading = false;
}
@@ -176,8 +272,8 @@
const result = await volumeBackupService.restoreBackup(volumeName, backup.id);
toast.success(m.volumes_backup_restore_success(), activityToastOptions(extractActivityId(result)));
await loadData(requestOptions);
- } catch (e: any) {
- toast.error(e.message || m.common_failed());
+ } catch (error) {
+ toast.error(error instanceof Error ? error.message : m.common_failed());
}
}
}
@@ -196,8 +292,8 @@
activityToastOptions(extractActivityId(result))
);
showRestoreFiles = false;
- } catch (e: any) {
- toast.error(e.message || m.common_failed());
+ } catch (error) {
+ toast.error(error instanceof Error ? error.message : m.common_failed());
} finally {
restoringFiles = false;
}
@@ -208,30 +304,85 @@
}
onMount(async () => {
- await loadData(requestOptions);
+ const [collection, destinations] = await Promise.all([
+ volumeBackupService.getPolicies(volumeName),
+ canBackupVolume ? s3DestinationService.listAll().catch(() => []) : Promise.resolve([]),
+ loadData(requestOptions)
+ ]);
+ backupPolicies = collection.policies;
+ s3Destinations = destinations;
});
const columns = [
{ accessorKey: 'id', title: m.common_id(), sortable: true, cell: IdCell },
+ { accessorKey: 'status', title: m.common_status(), sortable: true, cell: StatusCell },
+ { accessorKey: 'trigger', title: m.volume_backup_trigger(), sortable: true, cell: TriggerCell },
+ { accessorKey: 'destination', title: m.backups_destination_label(), sortable: true, cell: DestinationCell },
{ accessorKey: 'size', title: m.common_size(), sortable: true, cell: SizeCell },
- { accessorKey: 'createdAt', title: m.common_created(), sortable: true, cell: CreatedCell }
+ { accessorKey: 'createdAt', title: m.common_created(), sortable: true, cell: CreatedCell },
+ {
+ accessorKey: 'remoteSnapshotId',
+ title: m.volume_backup_remote_snapshot(),
+ sortable: true,
+ cell: RemoteSnapshotCell,
+ hidden: true
+ },
+ { accessorKey: 'error', title: m.common_error(), sortable: false, cell: ErrorCell, hidden: true }
] satisfies ColumnSpec<BackupEntry>[];
- const mobileFields = [{ id: 'size', label: m.common_size(), defaultVisible: true }];
+ const mobileFields = [
+ { id: 'status', label: m.common_status(), defaultVisible: true },
+ { id: 'trigger', label: m.volume_backup_trigger(), defaultVisible: true },
+ { id: 'destination', label: m.backups_destination_label(), defaultVisible: true },
+ { id: 'size', label: m.common_size(), defaultVisible: true },
+ { id: 'remoteSnapshotId', label: m.volume_backup_remote_snapshot(), defaultVisible: false }
+ ];
+
+ const bulkActions = $derived.by<BulkAction[]>(() => [
+ {
+ id: 'remove',
+ label: m.common_remove_selected_count({ count: selectedIds.length }),
+ action: 'remove',
+ onClick: handleDeleteSelected,
+ loading: deletingSelected,
+ disabled: !canDeleteBackup || deletingSelected || selectedIds.length === 0,
+ icon: TrashIcon
+ }
+ ]);
let mobileFieldVisibility = $state<Record<string, boolean>>({});
</script>
-{#snippet IdCell({ value }: { value: any })}
- <code class="font-mono text-xs font-medium">{value}</code>
+{#snippet IdCell({ item }: { item: BackupEntry })}
+ <code class="font-mono text-xs font-medium">{item.id}</code>
+{/snippet}
+
+{#snippet StatusCell({ item }: { item: BackupEntry })}
+ <BackupStatusCell status={item.status} />
+{/snippet}
+
+{#snippet TriggerCell({ item }: { item: BackupEntry })}
+ <BackupTriggerCell trigger={item.trigger} />
+{/snippet}
+
+{#snippet DestinationCell({ item }: { item: BackupEntry })}
+ <BackupDestinationCell {item} />
{/snippet}
-{#snippet SizeCell({ value }: { value: any })}
- {formatBytes(Number(value))}
+{#snippet SizeCell({ item }: { item: BackupEntry })}
+ <BackupSizeCell size={item.size} />
{/snippet}
-{#snippet CreatedCell({ value }: { value: any })}
- {formatDateTimeShort(String(value))}
+{#snippet CreatedCell({ item }: { item: BackupEntry })}
+ <CreatedAtCell value={item.createdAt} />
+{/snippet}
+
+{#snippet RemoteSnapshotCell({ item }: { item: BackupEntry })}
+ <code class="text-xs">{item.remoteSnapshotId || m.volume_backup_not_uploaded()}</code>
+{/snippet}
+
+{#snippet ErrorCell({ item }: { item: BackupEntry })}
+ <span class="line-clamp-2 max-w-80 text-xs text-destructive">{item.error || '-'}</span>
{/snippet}
{#snippet RowActions({ item }: { item: BackupEntry })}
@@ -239,18 +390,31 @@
{#if canBackupVolume}
<DropdownMenu.Item onclick={() => handleRestore(item)}>
<RestartIcon class="size-4" />
- Restore
+ {m.volumes_backups_restore()}
</DropdownMenu.Item>
<DropdownMenu.Item onclick={() => openRestoreFilesDialog(item)}>
<FileTextIcon class="size-4" />
- Restore files
+ {m.volume_restore_files()}
</DropdownMenu.Item>
{/if}
- <DropdownMenu.Item onclick={() => volumeBackupService.downloadBackup(item.id)}>
- <DownloadIcon class="size-4" />
- {m.templates_download()}
- </DropdownMenu.Item>
- <IfPermitted perm="volumes:delete">
+ {#if item.format === 'archive' || item.localSnapshotId}
+ <DropdownMenu.Item onclick={() => volumeBackupService.downloadBackup(item.id)}>
+ <DownloadIcon class="size-4" />
+ {m.templates_download()}
+ </DropdownMenu.Item>
+ {/if}
+ {#if canBackupVolume && s3Destinations.length}
+ {#each s3Destinations as destination (destination.id)}
+ <DropdownMenu.Item
+ disabled={item.status !== 'succeeded' || Boolean(item.remoteSnapshotId) || uploadingBackupId === item.id}
+ onclick={() => handleUpload(item, destination.id)}
+ >
+ <UploadIcon class="size-4" />
+ {m.backups_upload_s3()} · {destination.name}
+ </DropdownMenu.Item>
+ {/each}
+ {/if}
+ <IfPermitted perm="volumes:backup">
<DropdownMenu.Separator />
<DropdownMenu.Item variant="destructive" onclick={() => handleDelete(item)}>
<TrashIcon class="size-4" />
@@ -262,15 +426,50 @@
{#snippet ToolbarActions()}
{#if canBackupVolume}
- <ArcaneButton
- action="create"
- customLabel={m.volumes_backup_create()}
- loading={creating}
- disabled={creating}
- onclick={handleCreate}
- size="sm"
- icon={AddIcon}
- />
+ <div class="flex items-center gap-2">
+ <ArcaneButton
+ action="create"
+ customLabel={m.volume_backup_add_schedule()}
+ onclick={() => {
+ editingBackupPolicyId = undefined;
+ showBackupPolicy = true;
+ }}
+ size="sm"
+ icon={AddIcon}
+ />
+ <ButtonGroup.Root>
+ <ArcaneButton
+ action="create"
+ customLabel={m.volumes_backup_create()}
+ loading={creating}
+ disabled={creating}
+ onclick={() => handleCreate()}
+ size="sm"
+ icon={AddIcon}
+ />
+ <DropdownMenu.Root>
+ <DropdownMenu.Trigger
+ class={cn(arcaneButtonVariants({ tone: 'outline-primary', size: 'icon' }), 'size-8 rounded-md')}
+ aria-label={m.common_open_menu()}
+ disabled={creating}
+ >
+ <ArrowDownIcon class="size-4" />
+ </DropdownMenu.Trigger>
+ <DropdownMenu.Content align="end" class="w-64">
+ <DropdownMenu.Label>{m.backups_destination_label()}</DropdownMenu.Label>
+ <DropdownMenu.Item onclick={() => handleCreate({ destination: 'local' })}>
+ {m.local()}
+ </DropdownMenu.Item>
+ <DropdownMenu.Item onclick={() => openS3DestinationDialog('s3')}>
+ {m.backups_destination_s3()}
+ </DropdownMenu.Item>
+ <DropdownMenu.Item onclick={() => openS3DestinationDialog('local_s3')}>
+ {m.backups_destination_local_s3()}
+ </DropdownMenu.Item>
+ </DropdownMenu.Content>
+ </DropdownMenu.Root>
+ </ButtonGroup.Root>
+ </div>
{/if}
{/snippet}
@@ -286,12 +485,33 @@
icon={{ component: VolumesIcon, variant: 'blue' }}
title={(item) => item.id}
fields={[
+ {
+ label: m.volume_backup_trigger(),
+ getValue: (item) => backupTriggerLabel(item.trigger),
+ icon: ClockIcon,
+ iconVariant: 'gray',
+ show: mobileFieldVisibility['trigger'] ?? true
+ },
{
label: m.common_size(),
getValue: (item) => formatBytes(item.size),
icon: InfoIcon,
iconVariant: 'gray',
show: mobileFieldVisibility['size'] ?? true
+ },
+ {
+ label: m.backups_destination_label(),
+ getValue: (item) => backupDestinationDisplay(item),
+ icon: DownloadIcon,
+ iconVariant: 'gray',
+ show: mobileFieldVisibility['destination'] ?? true
+ },
+ {
+ label: m.volume_backup_remote_snapshot(),
+ getValue: (item) => item.remoteSnapshotId || m.volume_backup_not_uploaded(),
+ icon: DownloadIcon,
+ iconVariant: 'gray',
+ show: mobileFieldVisibility['remoteSnapshotId'] ?? false
}
]}
footer={{
@@ -307,6 +527,27 @@
<div class="flex items-center justify-between">
<h2 class="text-lg font-semibold">{m.volumes_backups_title()}</h2>
</div>
+ <div class="grid grid-cols-1 gap-1.5 text-xs text-muted-foreground sm:grid-cols-2 xl:grid-cols-3">
+ {#each backupPolicies as policy (policy.id)}
+ <BackupPolicyCard
+ {policy}
+ showStopContainers
+ onEdit={canBackupVolume
+ ? () => {
+ editingBackupPolicyId = policy.id;
+ showBackupPolicy = true;
+ }
+ : undefined}
+ />
+ {/each}
+ </div>
+
+ <Alert.Root class="py-2 [&>svg]:top-2">
+ <InfoIcon class="size-4" />
+ <Alert.Description class="text-xs">
+ {m.volume_backup_encryption_note()}
+ </Alert.Description>
+ </Alert.Root>
{#if backupWarnings.length > 0}
<Alert.Root variant="warning" class="py-2 [&>svg]:top-2">
@@ -320,11 +561,13 @@
<ArcaneTable
persistKey="arcane-volume-backup-table"
items={backupsPaginated}
+ bind:selectedIds
bind:requestOptions
bind:mobileFieldVisibility
onRefresh={loadData}
{columns}
{mobileFields}
+ {bulkActions}
rowActions={RowActions}
mobileCard={BackupMobileCardSnippet}
customToolbarActions={ToolbarActions}
@@ -342,7 +585,7 @@
<Alert.Root class="py-2 [&>svg]:top-2">
<InfoIcon class="size-4" />
<Alert.Description class="text-xs">
- {m.volumes_backup_safety_info()}
+ {m.volume_backup_restore_files_lifecycle_info()}
</Alert.Description>
</Alert.Root>
@@ -405,3 +648,55 @@
{/if}
{/snippet}
</ResponsiveDialog>
+
+<ResponsiveDialog
+ bind:open={showS3DestinationDialog}
+ title={m.volume_backup_choose_s3_destination()}
+ description={m.volume_backup_choose_s3_destination_description()}
+ contentClass="sm:max-w-[520px]"
+>
+ {#snippet children()}
+ <div class="py-2">
+ <SelectWithLabel
+ id="on-demand-volume-backup-s3-destination"
+ value={onDemandS3DestinationId}
+ onValueChange={(value) => (onDemandS3DestinationId = value)}
+ label={m.volume_backup_s3_destination_label()}
+ description={m.volume_backup_s3_destination_description()}
+ options={s3DestinationOptions}
+ />
+ {#if s3Destinations.length === 0}
+ <p class="mt-2 text-xs text-muted-foreground">{m.volume_backup_no_s3_destinations()}</p>
+ {/if}
+ </div>
+ {/snippet}
+ {#snippet footer()}
+ <ArcaneButton action="cancel" onclick={() => (showS3DestinationDialog = false)} disabled={creating} />
+ <ArcaneButton
+ action="create"
+ customLabel={m.volumes_backup_create()}
+ onclick={createS3Backup}
+ loading={creating}
+ disabled={creating || !onDemandS3DestinationId}
+ />
+ {/snippet}
+</ResponsiveDialog>
+
+<BackupPolicyDialog
+ bind:open={showBackupPolicy}
+ idPrefix="volume-backup-policy"
+ policies={backupPolicies}
+ policyId={editingBackupPolicyId}
+ addTitle={m.volume_backup_add_schedule()}
+ description={m.volume_backup_policy_description()}
+ enabledDescription={m.volume_backup_policy_enabled_description()}
+ defaultSchedule="0 0 2 * * *"
+ showStopContainers
+ updatePolicies={async (policies) => (await volumeBackupService.updatePolicies(volumeName, policies)).policies}
+ messages={{
+ saved: m.volume_backup_policy_saved(),
+ saveFailed: m.volume_backup_policy_save_failed(),
+ removed: m.volume_backup_schedule_removed()
+ }}
+ onSaved={(policies) => (backupPolicies = policies)}
+/>
diff --git a/go.work.sum b/go.work.sum
index d6a9542e7..fbe20fb65 100644
--- a/go.work.sum
+++ b/go.work.sum
@@ -472,6 +472,7 @@ github.com/aws/aws-sdk-go-v2 v1.39.6/go.mod h1:c9pm7VwuW0UPxAEYGyTmyurVcNrbF6Rt/
github.com/aws/aws-sdk-go-v2 v1.41.1 h1:ABlyEARCDLN034NhxlRUSZr4l71mh+T5KAeGh6cerhU=
github.com/aws/aws-sdk-go-v2 v1.41.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
github.com/aws/aws-sdk-go-v2 v1.42.0/go.mod h1:27+ACypSLljLAEKsCYOmrjKh83vuTRkuAe9Uv/3A4bg=
+github.com/aws/aws-sdk-go-v2 v1.43.0/go.mod h1:5pKeft2eJj+gElQ38Jqg4ibCqh+/AK33/0X3hip7IjM=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0 h1:6GMWV6CNpA/6fbFHnoAjrv4+LGfyTqZz2LtCHnspgDg=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0/go.mod h1:/mXlTIVG9jbxkqDnr5UQNQxW1HRYxeGklkM9vAFeabg=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 h1:t9yYsydLYNBk9cJ73rgPhPWqOh/52fcWDQB5b1JsKSY=
@@ -480,10 +481,6 @@ github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5 h1:zWFmPmgw4sveAYi1mRqG+E/g0461cJ5M4bJ8/nc6d3Q=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5/go.mod h1:nVUlMLVV8ycXSb7mSkcNu9e3v/1TJq2RTlrPwhYWr5c=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.13 h1:p1BBrg/Hhp6uK7zpejeI8QFXHJeC/mynzi04Sl03k9g=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.13/go.mod h1:8cIfkE9MDhkRZGpQ22aV6/lkYeYSozpz16Smrs5x4Ls=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 h1:3IZY0XAJquT3aHzbkHfPzy4ACPcEjVG0x87KOwtpqGY=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14/go.mod h1:zwM6veDkhGgQFqkBy+uT28AAYpLu+uFMlPl+rCg/73E=
github.com/aws/aws-sdk-go-v2/config v1.31.3 h1:RIb3yr/+PZ18YYNe6MDiG/3jVoJrPmdoCARwNkMGvco=
github.com/aws/aws-sdk-go-v2/config v1.31.3/go.mod h1:jjgx1n7x0FAKl6TnakqrpkHWWKcX3xfWtdnIJs5K9CE=
github.com/aws/aws-sdk-go-v2/config v1.31.20 h1:/jWF4Wu90EhKCgjTdy1DGxcbcbNrjfBHvksEL79tfQc=
@@ -520,6 +517,7 @@ github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13/go.mod h1:oGnKwIYZ4X
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 h1:xOLELNKGp2vsiteLsvLPwxC+mYmO6OZ8PYgiuPJzF8U=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17/go.mod h1:5M5CI3D12dNOtH3/mk6minaRwI2/37ifCURZISxA/IQ=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.29/go.mod h1:MzoLFUArKGpGD+ukmPiTPG1X5x4o6M2kq4v2dr1FiEc=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.31/go.mod h1:aVyUoytEyOViR6jhq6jula0xkc5NfBE2hgeF6BvOrao=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.4 h1:j7vjtr1YIssWQOMeOWRbh3z8g2oY/xPjnZH2gLY4sGw=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.4/go.mod h1:yDmJgqOiH4EA8Hndnv4KwAo8jCGTSnM5ASG1nBI+toA=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13 h1:HBSI2kDkMdWz4ZM7FjwE7e/pWDEZ+nR95x8Ztet1ooY=
@@ -527,6 +525,7 @@ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13/go.mod h1:YE94ZoDArI7
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 h1:WWLqlh79iO48yLkj1v3ISRNiv+3KdQoZ6JWyfcsyQik=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17/go.mod h1:EhG22vHRrvF8oXSTYStZhJc1aUgKtnJe+aOiFEV90cM=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.29/go.mod h1:71wt8W2EgswdZy9Mf9KNnzxZ3TiZlv4caKghPktDOkA=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.31/go.mod h1:OERqI9k0draSLB8O8woxY3q25ZWTELRK4RRoLMuMZFo=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
@@ -538,6 +537,7 @@ github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.4/go.mod h1:SPBBhkJxjcrzJBc+qY85e
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12 h1:itu4KHu8JK/N6NcLIISlf3LL1LccMqruLUXZ9y7yBZw=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12/go.mod h1:i+6vTU3xziikTY3vcox23X8pPGW5X3wVgd1VZ7ha+x8=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.30/go.mod h1:AS0HycUvJRFvTt613AYDOgO2jzw+00cVSMny8XB3yMY=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.32/go.mod h1:2tNZkuWz54arj8mHVf+8Y7cKkcD8Wr/fBpENgEXpjLc=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.0 h1:6+lZi2JeGKtCraAj1rpoZfKqnQ9SptseRZioejfUOLM=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.0/go.mod h1:eb3gfbVIxIoGgJsi9pGne19dhCBpK6opTYpQqAmdy44=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 h1:x2Ibm/Af8Fi+BH+Hsn9TXGdT+hKbDd5XOTZxTMxDk7o=
@@ -549,10 +549,6 @@ github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.4 h1:Beh9oVgtQnBgR4s
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.4/go.mod h1:b17At0o8inygF+c6FOD3rNyYZufPw62o9XJbSfQPgbo=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3 h1:NEe7FaViguRQEm8zl8Ay/kC/QRsMtWUiCGZajQIsLdc=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3/go.mod h1:JLuCKu5VfiLBBBl/5IzZILU7rxS0koQpHzMOCzycOJU=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.22 h1:V51LGlOq/1VsDsHUdoklAQi7rMmx4qQubvFYAlP2254=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.22/go.mod h1:4Pzhyz8hJOm2bepgl+NjvRx8vlUFAIIvJnZ/MkcNPpU=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.24 h1:mdPwDQPqxlw9Sc62Nt15yjEcARaDbPXkjRYtXsUripo=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.24/go.mod h1:ls5ytnwLTcQaUu32fMYXFI3MjpKuTwL840PAm9iqyEg=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.4 h1:ueB2Te0NacDMnaC+68za9jLwkjzxGWm0KB5HTUHjLTI=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.4/go.mod h1:nLEfLnVMmLvyIG58/6gsSA03F1voKGaCfHV7+lR8S7s=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13 h1:kDqdFvMY4AtKoACfzIGD8A0+hbT41KTKF//gq7jITfM=
@@ -560,22 +556,15 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13/go.mod h1:l
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 h1:RuNSMoozM8oXlgLG/n6WLaFGoea7/CddrCfIiSA+xdY=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17/go.mod h1:F2xxQ9TZz5gDWsclCtPQscGpP0VUOc8RqgFM3vDENmU=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.29/go.mod h1:LfRkPCD8YHDM2E5eTkos2UpwYeZnBcVarTa8L59bJHA=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.31/go.mod h1:wAhpCQbkov+IcvjozJbd2xRCoZybUEHNkcFunssNACg=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.4 h1:HVSeukL40rHclNcUqVcBwE1YoZhOkoLeBfhUqR3tjIU=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.4/go.mod h1:DnbBOv4FlIXHj2/xmrUQYtawRFC9L9ZmQPz+DBc6X5I=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12 h1:R3uW0iKl8rgNEXNjVGliW/oMEh9fO/LlUEV8RvIFr1I=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12/go.mod h1:XEttbEr5yqsw8ebi7vlDoGJJjMXRez4/s9pibpJyL5s=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.29 h1:hiME6pBzC7OTl9LMtlyTWBuEl1f4QBcUmFDKC7MLXtc=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.29/go.mod h1:G7RP+uhagpKtKhd1BM9N6JQqjCcGEU47K5lBVZQyRQw=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.32 h1:jWXtZdCnhXa9sGFixRaU2AxT4DIVse9HS4E2f+/KwV0=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.32/go.mod h1:9JS1UpfVvyD/ZPX8GsKb/Pq8scEM+7GP5fqh9SwH7po=
github.com/aws/aws-sdk-go-v2/service/s3 v1.87.1 h1:2n6Pd67eJwAb/5KCX62/8RTU0aFAAW7V5XIGSghiHrw=
github.com/aws/aws-sdk-go-v2/service/s3 v1.87.1/go.mod h1:w5PC+6GHLkvMJKasYGVloB3TduOtROEMqm15HSuIbw4=
github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1 h1:Dq82AV+Qxpno/fG162eAhnD8d48t9S+GZCfz7yv1VeA=
github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1/go.mod h1:MbKLznDKpf7PnSonNRUVYZzfP0CeLkRIUexeblgKcU4=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.103.3 h1:JRseEu/vIDMaWis4bSw0QbXL+cvIGc1XnX076H5ZXLE=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.103.3/go.mod h1:77ZAgynvx1txMvDG8gGWoWkO1augYDxkp9JElWFgjQU=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.106.0 h1:7QZWVJZWzHivHWIa+5TELLaBBkbuoj0GPwQtMlJ0sqk=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.106.0/go.mod h1:fcvq5L7dK+5cQFicEJwpI6e6Wn8NY2i6yT5wRLYVc7s=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 h1:VrhDvQib/i0lxvr3zqlUwLwJP4fpmpyD9wYG1vfSu+Y=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.5/go.mod h1:k029+U8SY30/3/ras4G/Fnv/b88N4mAfliNn08Dem4M=
github.com/aws/aws-sdk-go-v2/service/signin v1.1.5/go.mod h1:LxYujSTLPRlp2vTtcUO/+1ilrew8ytt6SvQyOgejzFQ=
@@ -884,6 +873,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cu
github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
+github.com/docker/buildx v0.36.0 h1:5UYbvkIXhkjiRZG15ZoWuA1BIuFatexA9yeL+b72MpM=
+github.com/docker/buildx v0.36.0/go.mod h1:APOldMT9ejTIiYJ44OxhFX1zt+MM1cR9zz4DQLq8qLw=
github.com/docker/cli v29.0.3+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
github.com/docker/cli v29.1.4+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
github.com/docker/cli v29.1.5+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
@@ -894,6 +885,8 @@ github.com/docker/cli-docs-tool v0.10.0/go.mod h1:5EM5zPnT2E7yCLERZmrDA234Vwn09f
github.com/docker/cli-docs-tool v0.11.0 h1:7d8QARFb7QEobizqxmEM7fOteZEHwH/zWgHQtHZEcfE=
github.com/docker/cli-docs-tool v0.11.0/go.mod h1:ma8BKiisUo8D6W05XEYIh3oa1UbgrZhi1nowyKFJa8Q=
github.com/docker/compose/v5 v5.0.0/go.mod h1:BurapGv8zmYnsbSmlpCz5EU2Pi3YFV/PjeUnoFpcw64=
+github.com/docker/compose/v5 v5.4.0 h1:i0xFmVvWv9QILfRE2PD07GmQqJYjpYAGutafIrJLURE=
+github.com/docker/compose/v5 v5.4.0/go.mod h1:bNeEncwZhG0F4MN+0zgln4EA2ILqGZC7knRI8QYJz9U=
github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0=
github.com/docker/docker-credential-helpers v0.9.3/go.mod h1:x+4Gbw9aGmChi3qTLZj8Dfn0TD20M/fuWy0E5+WDeCo=
@@ -1525,6 +1518,8 @@ github.com/moby/buildkit v0.26.2/go.mod h1:ylDa7IqzVJgLdi/wO7H1qLREFQpmhFbw2fbn4
github.com/moby/buildkit v0.26.3/go.mod h1:4T4wJzQS4kYWIfFRjsbJry4QoxDBjK+UGOEOs1izL7w=
github.com/moby/buildkit v0.27.0/go.mod h1:4STUkNc5t1nf03HS+01UmI2X6FdfOI3XaKt9QNoTsms=
github.com/moby/go-archive v0.1.0/go.mod h1:G9B+YoujNohJmrIYFBpSd54GTUB4lt9S+xVQvsJyFuo=
+github.com/moby/go-archive v0.3.2 h1:x893kC3zRygv2C+k4Y9kMxYRPLCj4XEJB0srbAP06Hw=
+github.com/moby/go-archive v0.3.2/go.mod h1:Npdv43fFqlhZW7Xo8fbm3ZMYFvAGNviUPqX21VERbcE=
github.com/moby/moby/api v1.52.0/go.mod h1:8mb+ReTlisw4pS6BRzCMts5M49W5M7bKt1cJy/YbAqc=
github.com/moby/moby/api v1.54.2/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs=
github.com/moby/moby/client v0.2.1/go.mod h1:O+/tw5d4a1Ha/ZA/tPxIZJapJRUS6LNZ1wiVRxYHyUE=
diff --git a/tests/setup/compose-postgres.yaml b/tests/setup/compose-postgres.yaml
index 678a52872..7a6ae1d5a 100644
--- a/tests/setup/compose-postgres.yaml
+++ b/tests/setup/compose-postgres.yaml
@@ -1,4 +1,19 @@
services:
+ minio:
+ image: quay.io/minio/minio:RELEASE.2025-09-07T16-13-09Z
+ # A directory created under the data dir is served as a bucket, so backups
+ # have somewhere to write without a separate provisioning container.
+ entrypoint:
+ ['sh', '-c', 'mkdir -p /data/arcane-backups && exec minio server /data --address :9000']
+ environment:
+ - MINIO_ROOT_USER=arcane-test
+ - MINIO_ROOT_PASSWORD=arcane-test-secret
+ healthcheck:
+ test: ['CMD', 'curl', '-fs', 'http://localhost:9000/minio/health/live']
+ interval: 2s
+ timeout: 5s
+ retries: 15
+
postgres:
image: postgres:18-alpine
environment:
@@ -33,6 +48,8 @@ services:
depends_on:
postgres:
condition: service_healthy
+ minio:
+ condition: service_healthy
build:
args:
- BUILD_TAGS=playwright
diff --git a/tests/setup/compose-proxy.yaml b/tests/setup/compose-proxy.yaml
index ab944bcf5..b308392a7 100644
--- a/tests/setup/compose-proxy.yaml
+++ b/tests/setup/compose-proxy.yaml
@@ -1,4 +1,19 @@
services:
+ minio:
+ image: quay.io/minio/minio:RELEASE.2025-09-07T16-13-09Z
+ # A directory created under the data dir is served as a bucket, so backups
+ # have somewhere to write without a separate provisioning container.
+ entrypoint:
+ ['sh', '-c', 'mkdir -p /data/arcane-backups && exec minio server /data --address :9000']
+ environment:
+ - MINIO_ROOT_USER=arcane-test
+ - MINIO_ROOT_PASSWORD=arcane-test-secret
+ healthcheck:
+ test: ['CMD', 'curl', '-fs', 'http://localhost:9000/minio/health/live']
+ interval: 2s
+ timeout: 5s
+ retries: 15
+
docker-socket-proxy:
image: tecnativa/docker-socket-proxy:latest
container_name: arcane-docker-proxy-test
@@ -49,7 +64,10 @@ services:
- arcane_data:/app/data
- ./projects:/app/data/projects
depends_on:
- - docker-socket-proxy
+ docker-socket-proxy:
+ condition: service_started
+ minio:
+ condition: service_healthy
build:
args:
- BUILD_TAGS=playwright
diff --git a/tests/setup/compose.yaml b/tests/setup/compose.yaml
index 88c94d3bf..7f7ab14a7 100644
--- a/tests/setup/compose.yaml
+++ b/tests/setup/compose.yaml
@@ -1,4 +1,19 @@
services:
+ minio:
+ image: quay.io/minio/minio:RELEASE.2025-09-07T16-13-09Z
+ # A directory created under the data dir is served as a bucket, so backups
+ # have somewhere to write without a separate provisioning container.
+ entrypoint:
+ ['sh', '-c', 'mkdir -p /data/arcane-backups && exec minio server /data --address :9000']
+ environment:
+ - MINIO_ROOT_USER=arcane-test
+ - MINIO_ROOT_PASSWORD=arcane-test-secret
+ healthcheck:
+ test: ['CMD', 'curl', '-fs', 'http://localhost:9000/minio/health/live']
+ interval: 2s
+ timeout: 5s
+ retries: 15
+
arcane:
image: arcane:playwright-tests
ports:
@@ -15,6 +30,9 @@ services:
- /var/run/docker.sock:/var/run/docker.sock
- arcane_data:/app/data
- ./projects:/app/data/projects
+ depends_on:
+ minio:
+ condition: service_healthy
build:
args:
- BUILD_TAGS=playwright
diff --git a/tests/spec/s3-backups.spec.ts b/tests/spec/s3-backups.spec.ts
new file mode 100644
index 000000000..2af32afaa
--- /dev/null
+++ b/tests/spec/s3-backups.spec.ts
@@ -0,0 +1,325 @@
+import { test, expect, type Page } from '@playwright/test';
+
+// The MinIO sidecar in the compose stack. Arcane reaches it over the compose
+// network, and so must the Rustic helper container it starts for each backup.
+const S3_ENDPOINT = process.env.E2E_S3_ENDPOINT ?? 'http://minio:9000';
+const S3_BUCKET = process.env.E2E_S3_BUCKET ?? 'arcane-backups';
+const S3_ACCESS_KEY_ID = process.env.E2E_S3_ACCESS_KEY_ID ?? 'arcane-test';
+const S3_SECRET_ACCESS_KEY = process.env.E2E_S3_SECRET_ACCESS_KEY ?? 'arcane-test-secret';
+
+type S3Destination = { id: string; name: string; bucket: string; secretConfigured: boolean };
+type BackupEntry = {
+ id: string;
+ status: string;
+ destination: string;
+ format: string;
+ remoteSnapshotId?: string;
+ localSnapshotId?: string;
+ error?: string;
+};
+
+function destinationPayload(overrides: Record<string, unknown> = {}) {
+ return {
+ name: `e2e-minio-${Date.now()}`,
+ endpoint: S3_ENDPOINT,
+ bucket: S3_BUCKET,
+ region: '',
+ accessKeyId: S3_ACCESS_KEY_ID,
+ secretAccessKey: S3_SECRET_ACCESS_KEY,
+ prefix: `e2e/${Date.now()}`,
+ useSsl: false,
+ forcePathStyle: true,
+ ...overrides
+ };
+}
+
+async function createDestinationViaApi(
+ page: Page,
+ overrides: Record<string, unknown> = {}
+): Promise<S3Destination> {
+ const response = await page.request.post('/api/backups/s3', {
+ data: destinationPayload(overrides)
+ });
+ if (!response.ok()) {
+ throw new Error(
+ `Failed to create S3 destination: ${response.status()} ${await response.text()}`
+ );
+ }
+ return (await response.json()) as S3Destination;
+}
+
+async function deleteDestinationViaApi(page: Page, destinationId: string) {
+ await page.request.delete(`/api/backups/s3/${destinationId}`).catch(() => undefined);
+}
+
+async function createVolumeViaApi(page: Page, volumeName: string) {
+ const response = await page.request.post('/api/environments/0/volumes', {
+ data: { name: volumeName, driver: 'local' }
+ });
+ if (!response.ok()) {
+ throw new Error(`Failed to create volume ${volumeName}: ${response.status()}`);
+ }
+}
+
+async function removeVolumeViaApi(page: Page, volumeName: string) {
+ await page.request
+ .delete(`/api/environments/0/volumes/${encodeURIComponent(volumeName)}?force=true`)
+ .catch(() => undefined);
+}
+
+async function getVolumeWorkspaceRevision(page: Page, volumeName: string) {
+ const response = await page.request.get(
+ `/api/environments/0/volumes/${encodeURIComponent(volumeName)}/workspace`
+ );
+ if (!response.ok()) {
+ throw new Error(
+ `Failed to read ${volumeName} workspace: ${response.status()} ${await response.text()}`
+ );
+ }
+ const body = await response.json();
+ return body.data.fileTreeRevision as string;
+}
+
+async function writeVolumeFile(page: Page, volumeName: string, fileName: string, content: string) {
+ const fileTreeRevision = await getVolumeWorkspaceRevision(page, volumeName);
+ const response = await page.request.put(
+ `/api/environments/0/volumes/${encodeURIComponent(volumeName)}/workspace`,
+ {
+ multipart: {
+ manifest: JSON.stringify({
+ fileTreeRevision,
+ fileChanges: [{ operation: 'create_file', relativePath: fileName, uploadIndex: 0 }]
+ }),
+ files: { name: fileName, mimeType: 'text/plain', buffer: Buffer.from(content) }
+ }
+ }
+ );
+ if (!response.ok()) {
+ throw new Error(`Failed to upload ${fileName}: ${response.status()} ${await response.text()}`);
+ }
+}
+
+async function readVolumeFile(page: Page, volumeName: string, filePath: string) {
+ const relativePath = filePath.replace(/^\/+/, '');
+ const response = await page.request.get(
+ `/api/environments/0/volumes/${encodeURIComponent(volumeName)}/workspace/file?relativePath=${encodeURIComponent(relativePath)}`
+ );
+ if (!response.ok()) {
+ return null;
+ }
+ const body = await response.json();
+ const content = body?.data?.content;
+ return typeof content === 'string' ? content : null;
+}
+
+async function deleteVolumeFile(page: Page, volumeName: string, filePath: string) {
+ const relativePath = filePath.replace(/^\/+/, '');
+ const fileTreeRevision = await getVolumeWorkspaceRevision(page, volumeName);
+ const response = await page.request.put(
+ `/api/environments/0/volumes/${encodeURIComponent(volumeName)}/workspace`,
+ {
+ multipart: {
+ manifest: JSON.stringify({
+ fileTreeRevision,
+ fileChanges: [{ operation: 'delete', relativePath }]
+ })
+ }
+ }
+ );
+ if (!response.ok()) {
+ throw new Error(`Failed to delete ${filePath}: ${response.status()} ${await response.text()}`);
+ }
+}
+
+async function listBackups(page: Page, volumeName: string): Promise<BackupEntry[]> {
+ const response = await page.request.get(
+ `/api/environments/0/volumes/${encodeURIComponent(volumeName)}/backups`
+ );
+ if (!response.ok()) {
+ return [];
+ }
+ const body = await response.json();
+ return (body?.data ?? []) as BackupEntry[];
+}
+
+// Backups run Rustic in a helper container, so the request returns before the
+// snapshot finishes. Poll the record until it leaves the running state.
+async function waitForBackup(page: Page, volumeName: string, backupId: string) {
+ let last: BackupEntry | undefined;
+ await expect
+ .poll(
+ async () => {
+ last = (await listBackups(page, volumeName)).find((entry) => entry.id === backupId);
+ return last?.status ?? 'missing';
+ },
+ { timeout: 180000, intervals: [2000] }
+ )
+ .not.toBe('running');
+ expect(last?.status, `backup failed: ${last?.error ?? 'unknown error'}`).toBe('succeeded');
+ return last as BackupEntry;
+}
+
+// Deleting a destination also verifies that no managed environment still
+// references it, and is refused when one cannot be reached. Other specs create
+// environments, so the release path depends on what is registered right now.
+async function countRemoteEnvironments(page: Page) {
+ const response = await page.request.get('/api/environments');
+ if (!response.ok()) {
+ return 0;
+ }
+ const body = await response.json();
+ const environments = (body?.data ?? []) as { id: string }[];
+ return environments.filter((environment) => environment.id !== '0').length;
+}
+
+async function createBackupViaApi(
+ page: Page,
+ volumeName: string,
+ data: Record<string, unknown>
+): Promise<BackupEntry> {
+ const response = await page.request.post(
+ `/api/environments/0/volumes/${encodeURIComponent(volumeName)}/backups`,
+ { data, timeout: 180000 }
+ );
+ if (!response.ok()) {
+ throw new Error(`Failed to create backup: ${response.status()} ${await response.text()}`);
+ }
+ const body = await response.json();
+ return body.data as BackupEntry;
+}
+
+test.describe('S3 Backups', () => {
+ test('creates a destination and verifies connectivity against MinIO', async ({ page }) => {
+ const destination = await createDestinationViaApi(page);
+ try {
+ expect(destination.bucket).toBe(S3_BUCKET);
+ expect(destination.secretConfigured).toBe(true);
+
+ // Uploads, downloads, and deletes a probe object in the bucket.
+ const test = await page.request.post(`/api/backups/s3/${destination.id}/test`);
+ expect(test.status(), await test.text()).toBe(200);
+ } finally {
+ await deleteDestinationViaApi(page, destination.id);
+ }
+ });
+
+ test('refuses the stored secret once connection settings change', async ({ page }) => {
+ const destination = await createDestinationViaApi(page);
+ try {
+ const withoutSecret = await page.request.put(`/api/backups/s3/${destination.id}`, {
+ data: destinationPayload({
+ name: destination.name,
+ bucket: 'someone-elses-bucket',
+ secretAccessKey: ''
+ })
+ });
+ expect(withoutSecret.status()).toBe(400);
+ expect(await withoutSecret.text()).toContain('re-enter the secret access key');
+
+ // The same edit is accepted once the secret is supplied again.
+ const withSecret = await page.request.put(`/api/backups/s3/${destination.id}`, {
+ data: destinationPayload({ name: destination.name, prefix: `e2e/${Date.now()}-moved` })
+ });
+ expect(withSecret.status(), await withSecret.text()).toBe(200);
+ } finally {
+ await deleteDestinationViaApi(page, destination.id);
+ }
+ });
+
+ test('backs a volume up to S3 and restores it from the bucket', async ({ page }) => {
+ test.setTimeout(300000);
+
+ const volumeName = `e2e-s3-backup-${Date.now()}`;
+ const fileContent = `arcane-e2e-${Date.now()}`;
+ const destination = await createDestinationViaApi(page);
+ let backupId: string | undefined;
+
+ try {
+ await createVolumeViaApi(page, volumeName);
+ await writeVolumeFile(page, volumeName, 'payload.txt', fileContent);
+ expect(await readVolumeFile(page, volumeName, '/payload.txt')).toBe(fileContent);
+
+ const created = await createBackupViaApi(page, volumeName, {
+ destination: 's3',
+ s3DestinationId: destination.id
+ });
+ backupId = created.id;
+ const backup = await waitForBackup(page, volumeName, created.id);
+
+ expect(backup.destination).toBe('s3');
+ expect(backup.format).toBe('rustic');
+ expect(backup.remoteSnapshotId, 'no snapshot was recorded in S3').toBeTruthy();
+ expect(backup.localSnapshotId ?? '').toBe('');
+
+ // Rustic can only list these paths by reading the repository back out
+ // of the bucket, so this proves the snapshot really landed in MinIO.
+ const files = await page.request.get(
+ `/api/environments/0/volumes/backups/${created.id}/files`
+ );
+ expect(files.status(), await files.text()).toBe(200);
+ expect(JSON.stringify((await files.json())?.data ?? [])).toContain('payload.txt');
+
+ await deleteVolumeFile(page, volumeName, '/payload.txt');
+ expect(await readVolumeFile(page, volumeName, '/payload.txt')).toBeNull();
+
+ const restore = await page.request.post(
+ `/api/environments/0/volumes/${encodeURIComponent(volumeName)}/backups/${created.id}/restore`,
+ { data: {}, timeout: 180000 }
+ );
+ expect(restore.status(), await restore.text()).toBe(200);
+ expect(await readVolumeFile(page, volumeName, '/payload.txt')).toBe(fileContent);
+ } finally {
+ if (backupId) {
+ await page.request
+ .delete(`/api/environments/0/volumes/backups/${backupId}`)
+ .catch(() => undefined);
+ }
+ await removeVolumeViaApi(page, volumeName);
+ await deleteDestinationViaApi(page, destination.id);
+ }
+ });
+
+ test('blocks deleting a destination a backup still references', async ({ page }) => {
+ test.setTimeout(300000);
+
+ const volumeName = `e2e-s3-inuse-${Date.now()}`;
+ const destination = await createDestinationViaApi(page);
+ let backupId: string | undefined;
+
+ try {
+ await createVolumeViaApi(page, volumeName);
+ await writeVolumeFile(page, volumeName, 'payload.txt', 'in-use');
+
+ const created = await createBackupViaApi(page, volumeName, {
+ destination: 's3',
+ s3DestinationId: destination.id
+ });
+ backupId = created.id;
+ await waitForBackup(page, volumeName, created.id);
+
+ const blocked = await page.request.delete(`/api/backups/s3/${destination.id}`);
+ expect(blocked.status()).toBe(409);
+
+ await page.request.delete(`/api/environments/0/volumes/backups/${created.id}`);
+ backupId = undefined;
+
+ const remoteEnvironments = await countRemoteEnvironments(page);
+ const allowed = await page.request.delete(`/api/backups/s3/${destination.id}`);
+ if (remoteEnvironments === 0) {
+ expect(allowed.status(), await allowed.text()).toBe(200);
+ } else {
+ // A registered environment that cannot be checked keeps the
+ // destination, so the local reference is no longer the blocker.
+ expect([200, 409]).toContain(allowed.status());
+ }
+ } finally {
+ if (backupId) {
+ await page.request
+ .delete(`/api/environments/0/volumes/backups/${backupId}`)
+ .catch(() => undefined);
+ }
+ await removeVolumeViaApi(page, volumeName);
+ await deleteDestinationViaApi(page, destination.id);
+ }
+ });
+});
diff --git a/types/backup/s3.go b/types/backup/s3.go
new file mode 100644
index 000000000..e4f5dc5fa
--- /dev/null
+++ b/types/backup/s3.go
@@ -0,0 +1,46 @@
+package backup
+
+import "time"
+
+type S3Destination struct {
+ ID string `json:"id"`
+ Name string `json:"name"`
+ Endpoint string `json:"endpoint,omitempty"`
+ Bucket string `json:"bucket"`
+ Region string `json:"region"`
+ AccessKeyID string `json:"accessKeyId"`
+ Prefix string `json:"prefix,omitempty"`
+ UseSSL bool `json:"useSsl"`
+ ForcePathStyle bool `json:"forcePathStyle"`
+ SecretConfigured bool `json:"secretConfigured"`
+ CreatedAt time.Time `json:"createdAt"`
+ UpdatedAt *time.Time `json:"updatedAt,omitempty"`
+}
+
+type CreateS3Destination struct {
+ Name string `json:"name"`
+ Endpoint string `json:"endpoint,omitempty"`
+ Bucket string `json:"bucket"`
+ Region string `json:"region"`
+ AccessKeyID string `json:"accessKeyId"`
+ SecretAccessKey string `json:"secretAccessKey"`
+ Prefix string `json:"prefix,omitempty"`
+ UseSSL bool `json:"useSsl"`
+ ForcePathStyle bool `json:"forcePathStyle"`
+}
+
+// UpdateS3Destination carries the same connection fields as create; an empty
+// SecretAccessKey keeps the stored secret.
+type UpdateS3Destination = CreateS3Destination
+
+type S3DestinationSync struct {
+ CreateS3Destination
+
+ ID string `json:"id"`
+ CreatedAt time.Time `json:"createdAt"`
+ UpdatedAt *time.Time `json:"updatedAt,omitempty"`
+}
+
+type S3DestinationSyncRequest struct {
+ Destinations []S3DestinationSync `json:"destinations"`
+}
diff --git a/types/backup/system_backup.go b/types/backup/system_backup.go
new file mode 100644
index 000000000..2a563f7f0
--- /dev/null
+++ b/types/backup/system_backup.go
@@ -0,0 +1,95 @@
+package backup
+
+import "time"
+
+type SystemBackupDestination string
+
+const (
+ SystemBackupDestinationLocal SystemBackupDestination = "local"
+ SystemBackupDestinationS3 SystemBackupDestination = "s3"
+ SystemBackupDestinationLocalS3 SystemBackupDestination = "local_s3"
+)
+
+type SystemBackupRun struct {
+ ID string `json:"id"`
+ Size int64 `json:"size"`
+ CreatedAt time.Time `json:"createdAt"`
+ Status string `json:"status"`
+ Trigger string `json:"trigger"`
+ Destination SystemBackupDestination `json:"destination"`
+ LocalSnapshotID string `json:"localSnapshotId,omitempty"`
+ RemoteSnapshotID string `json:"remoteSnapshotId,omitempty"`
+ S3DestinationID string `json:"s3DestinationId,omitempty"`
+ S3DestinationName string `json:"s3DestinationName,omitempty"`
+ PolicyID string `json:"policyId,omitempty"`
+ Error string `json:"error,omitempty"`
+}
+
+type SystemBackupPolicy struct {
+ ID string `json:"id"`
+ Enabled bool `json:"enabled"`
+ Schedule string `json:"schedule"`
+ RetentionCount int `json:"retentionCount"`
+ LocalEnabled bool `json:"localEnabled"`
+ S3Enabled bool `json:"s3Enabled"`
+ S3DestinationID string `json:"s3DestinationId,omitempty"`
+ S3DestinationName string `json:"s3DestinationName,omitempty"`
+ LastRun *SystemBackupRun `json:"lastRun,omitempty"`
+}
+
+// UpdateBackupPolicy is the shared policy-update payload for every backup
+// domain; StopContainers only applies to volume backups.
+type UpdateBackupPolicy struct {
+ ID string `json:"id,omitempty"`
+ Enabled bool `json:"enabled"`
+ Schedule string `json:"schedule"`
+ RetentionCount int `json:"retentionCount"`
+ StopContainers bool `json:"stopContainers,omitempty"`
+ LocalEnabled bool `json:"localEnabled"`
+ S3Enabled bool `json:"s3Enabled"`
+ S3DestinationID string `json:"s3DestinationId,omitempty"`
+}
+
+type UpdateSystemBackupPolicy = UpdateBackupPolicy
+
+type SystemBackupPolicyCollection struct {
+ Policies []SystemBackupPolicy `json:"policies"`
+ RecoveryKeyStored bool `json:"recoveryKeyStored"`
+}
+
+type UpdateSystemBackupPolicies struct {
+ Policies []UpdateSystemBackupPolicy `json:"policies"`
+}
+
+type SystemBackupRecoveryKeyStatus struct {
+ Configured bool `json:"configured"`
+}
+
+type SystemBackupRecoveryKey struct {
+ RecoveryKey string `json:"recoveryKey"`
+}
+
+type CreateSystemBackupRequest struct {
+ Destination SystemBackupDestination `json:"destination,omitempty"`
+ S3DestinationID string `json:"s3DestinationId,omitempty"`
+ RecoveryKey string `json:"recoveryKey,omitempty"`
+ PolicyID string `json:"policyId,omitempty"`
+}
+
+type RestoreSystemBackupRequest struct {
+ RecoveryKey string `json:"recoveryKey"`
+}
+
+type UploadSystemBackupRequest struct {
+ S3DestinationID string `json:"s3DestinationId"`
+ RecoveryKey string `json:"recoveryKey"`
+}
+
+type DeleteSystemBackupRequest struct {
+ RecoveryKey string `json:"recoveryKey,omitempty"`
+}
+
+type DiscoverSystemBackupsRequest struct {
+ S3DestinationID string `json:"s3DestinationId"`
+ RecoveryKey string `json:"recoveryKey"`
+}
diff --git a/types/volume/backup.go b/types/volume/backup.go
index e9f9aca8f..90ec5688e 100644
--- a/types/volume/backup.go
+++ b/types/volume/backup.go
@@ -1,8 +1,72 @@
package volume
+import "github.com/getarcaneapp/arcane/types/v2/backup"
+
+type BackupDestination string
+
+const (
+ BackupDestinationLocal BackupDestination = "local"
+ BackupDestinationS3 BackupDestination = "s3"
+ BackupDestinationLocalS3 BackupDestination = "local_s3"
+)
+
+type CreateBackupRequest struct {
+ Destination BackupDestination `json:"destination,omitempty" doc:"Optional destination override for this manual backup"`
+ PolicyID string `json:"policyId,omitempty" doc:"Optional backup policy whose settings should be used"`
+ S3DestinationID string `json:"s3DestinationId,omitempty" doc:"Saved S3 destination for a manual remote backup"`
+}
+
+type UploadBackupRequest struct {
+ S3DestinationID string `json:"s3DestinationId" doc:"S3 destination for the uploaded backup"`
+}
+
+type BackupFormat string
+
+const (
+ BackupFormatArchive BackupFormat = "archive"
+ BackupFormatRustic BackupFormat = "rustic"
+)
+
type BackupEntry struct {
- ID string `json:"id" doc:"Unique identifier of the backup"`
- VolumeName string `json:"volumeName" doc:"Name of the volume"`
- Size int64 `json:"size" doc:"Size of the backup archive in bytes"`
- CreatedAt string `json:"createdAt" doc:"When the backup was created"`
+ ID string `json:"id" doc:"Unique identifier of the backup"`
+ VolumeName string `json:"volumeName" doc:"Name of the volume"`
+ Size int64 `json:"size" doc:"Total size of the backup contents"`
+ CreatedAt string `json:"createdAt" doc:"When the backup was created"`
+ Status string `json:"status" doc:"Backup result status"`
+ Trigger string `json:"trigger" doc:"How the backup was started"`
+ Destination BackupDestination `json:"destination" doc:"Requested backup storage target"`
+ Format BackupFormat `json:"format" doc:"Storage format of the backup: legacy tar.gz archive or Rustic snapshot"`
+ LocalSnapshotID string `json:"localSnapshotId,omitempty" doc:"Snapshot ID in the local Rustic repository"`
+ RemoteSnapshotID string `json:"remoteSnapshotId,omitempty" doc:"Snapshot ID in the S3 Rustic repository"`
+ S3DestinationID string `json:"s3DestinationId,omitempty" doc:"S3 destination used by the backup"`
+ S3DestinationName string `json:"s3DestinationName,omitempty" doc:"Name of the S3 destination used by the backup"`
+ PolicyID string `json:"policyId,omitempty" doc:"Backup policy that created the backup"`
+ Error string `json:"error,omitempty" doc:"Backup error when the run failed"`
+}
+
+type BackupPolicy struct {
+ ID string `json:"id"`
+ VolumeName string `json:"volumeName"`
+ Enabled bool `json:"enabled"`
+ Schedule string `json:"schedule"`
+ RetentionCount int `json:"retentionCount"`
+ StopContainers bool `json:"stopContainers"`
+ LocalEnabled bool `json:"localEnabled"`
+ S3Enabled bool `json:"s3Enabled"`
+ S3DestinationID string `json:"s3DestinationId,omitempty"`
+ S3DestinationName string `json:"s3DestinationName,omitempty"`
+ S3Available bool `json:"s3Available"`
+ S3Bucket string `json:"s3Bucket,omitempty"`
+ LastRun *BackupEntry `json:"lastRun,omitempty"`
+}
+
+type UpdateBackupPolicy = backup.UpdateBackupPolicy
+
+type BackupPolicyCollection struct {
+ Policies []BackupPolicy `json:"policies"`
+ S3Available bool `json:"s3Available"`
+}
+
+type UpdateBackupPolicies struct {
+ Policies []UpdateBackupPolicy `json:"policies"`
}