Skip to content

Modern Docker Management, Designed for Everyone

fix: run backups in an activity instead of locking the ui (#3847)

Kyle Mendelldd16fd942afa
diff --git a/backend/internal/activity/backup.go b/backend/internal/activity/backup.go
new file mode 100644
index 000000000..7b0faf959
--- /dev/null
+++ b/backend/internal/activity/backup.go
@@ -0,0 +1,30 @@
+package activity
+
+import (
+ "context"
+
+ "emperror.dev/errors"
+ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
+)
+
+// FailInterruptedBackups finalizes backup activities before startup admits work.
+func (s *ActivityService) FailInterruptedBackups(ctx context.Context) error {
+ var pending []Activity
+ if err := s.db.WithContext(ctx).Where("status IN ?", []activitytypes.Status{activitytypes.StatusQueued, activitytypes.StatusRunning}).Find(&pending).Error; err != nil {
+ return errors.WrapIf(err, "find interrupted backup activities")
+ }
+ const message = "Backup interrupted by Arcane restart"
+ var result error
+ for _, entry := range pending {
+ if s.isTrackedInternal(entry.ID) {
+ continue
+ }
+ switch entry.Metadata["action"] {
+ case "create_volume_backup", "create_system_backup", "run_system_volume_backups", "scheduled_volume_backup", "scheduled_system_backup":
+ errMessage := message
+ _, err := s.CompleteActivity(ctx, entry.ID, activitytypes.StatusFailed, message, &errMessage)
+ result = errors.Combine(result, err)
+ }
+ }
+ return result
+}
diff --git a/backend/internal/backup/engine.go b/backend/internal/backup/engine.go
index d6df12468..19d3654f9 100644
--- a/backend/internal/backup/engine.go
+++ b/backend/internal/backup/engine.go
@@ -75,6 +75,8 @@ type Engine struct {
admission *actors.Gate[actors.AdmissionKey]
lifecycleCtx context.Context
executors *actors.StateMap[string, *actors.Executor]
+ runs *actors.StateMap[string, *backupRunInternal]
+ stopping bool // Accessed inside runs.Apply.
}
// NewEngine creates the shared backup engine on the actor runtime. ctx is the
@@ -86,6 +88,7 @@ func NewEngine(ctx context.Context, runtime *actors.Runtime, admission *actors.G
admission: admission,
lifecycleCtx: ctx,
executors: actors.NewStateMap[string, *actors.Executor](),
+ runs: actors.NewStateMap[string, *backupRunInternal](),
}
}
@@ -98,16 +101,19 @@ func (e *Engine) TryAcquireRun(ctx context.Context, scope, id string) (*actors.L
return e.admission.TryAcquire(ctx, actors.AdmissionKey{Scope: scope, ID: id})
}
-// Stop joins every repository executor.
+// Stop cancels and joins backup runs before stopping their repository executors.
func (e *Engine) Stop(ctx context.Context) error {
if e == nil {
return nil
}
+ stopErr := e.stopRunsInternal(ctx)
+ if stopErr != nil {
+ return stopErr
+ }
executors, err := e.executors.Drain(ctx, "stop backup repository executors")
if err != nil {
return err
}
- var stopErr error
for _, executor := range executors {
stopErr = errors.Combine(stopErr, executor.Stop(ctx))
}
@@ -321,7 +327,7 @@ func (e *Engine) executorForInternal(repositoryID string) (*actors.Executor, err
if executor, ok := values[repositoryID]; ok {
return executor, false, nil
}
- executor, err := actors.NewExecutor(e.lifecycleCtx, e.runtime, "backup-repository", repositoryID, 3)
+ executor, err := actors.NewExecutor(context.WithoutCancel(e.lifecycleCtx), e.runtime, "backup-repository", repositoryID, 3)
if err != nil {
return nil, false, err
}
@@ -337,13 +343,18 @@ func (e *Engine) runInternal(ctx context.Context, dockerClient *client.Client, r
if strings.TrimSpace(repository.ID) == "" {
return "", errors.New("backup repository ID is required")
}
- executor, err := e.executorForInternal(repository.ID)
+ executor, err := e.executorForInternal(repository.ID) //nolint:contextcheck // Repository executors outlive requests so shutdown cleanup can finish.
if err != nil {
return "", err
}
- return executor.Execute(ctx, "rustic "+command[0], func(workCtx context.Context) (string, error) {
+ task, err := executor.Submit(ctx, "rustic "+command[0], func(workCtx context.Context) (string, error) {
return e.runContainerInternal(workCtx, dockerClient, repository, password, command, extraMounts...)
}, nil)
+ if err != nil {
+ return "", err
+ }
+ // Cancellation must finish helper cleanup before callers release source locks.
+ return task.Wait(context.WithoutCancel(ctx))
}
func (e *Engine) ensureImageInternal(ctx context.Context, dockerClient *client.Client) error {
diff --git a/backend/internal/backup/runs.go b/backend/internal/backup/runs.go
new file mode 100644
index 000000000..62ee554b1
--- /dev/null
+++ b/backend/internal/backup/runs.go
@@ -0,0 +1,84 @@
+package backup
+
+import (
+ "context"
+ "log/slog"
+
+ "emperror.dev/errors"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
+)
+
+type backupRunInternal struct {
+ executor *actors.Executor
+ cancel context.CancelFunc
+}
+
+// SubmitRun accepts one backup on an executor independent of repository work.
+// after owns finalization only when submission succeeds.
+func (e *Engine) SubmitRun(ctx context.Context, id string, work func(context.Context) error, after func(error)) error {
+ if e == nil || e.runs == nil {
+ return errors.New("backup engine is unavailable")
+ }
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ return e.runs.Apply(ctx, "submit backup run", func(runs map[string]*backupRunInternal) (bool, error) {
+ if e.stopping {
+ return false, errors.New("backup engine is stopping")
+ }
+ if _, exists := runs[id]; exists {
+ return false, errors.New("backup run already submitted")
+ }
+ runCtx, cancel := context.WithCancel(ctx)
+ // Keep the mailbox alive long enough to finalize cancelled work.
+ executor, err := actors.NewExecutor(context.WithoutCancel(runCtx), e.runtime, "backup-runs", id, 0)
+ if err != nil {
+ cancel()
+ return false, err
+ }
+ _, err = executor.Submit(runCtx, "run backup", func(workerCtx context.Context) (actors.NoPayload, error) {
+ return actors.NoPayload{}, work(workerCtx)
+ }, func(_ actors.NoPayload, runErr error) {
+ defer func() {
+ cancel()
+ // Queue termination without waiting on our own mailbox.
+ if stopErr := executor.Stop(runCtx); stopErr != nil && !errors.Is(stopErr, context.Canceled) {
+ slog.ErrorContext(context.WithoutCancel(ctx), "Failed to stop backup executor", "runId", id, "error", stopErr)
+ }
+ if _, _, removeErr := e.runs.Remove(context.WithoutCancel(ctx), "complete backup run", id); removeErr != nil {
+ slog.ErrorContext(context.WithoutCancel(ctx), "Failed to remove completed backup run", "runId", id, "error", removeErr)
+ }
+ }()
+ after(runErr)
+ })
+ if err != nil {
+ cancel()
+ stopErr := executor.Stop(runCtx)
+ if errors.Is(stopErr, context.Canceled) {
+ stopErr = nil // The stop was queued; only its wait was cancelled.
+ }
+ return false, errors.Combine(err, stopErr)
+ }
+ runs[id] = &backupRunInternal{executor: executor, cancel: cancel}
+ return true, nil
+ })
+}
+
+func (e *Engine) stopRunsInternal(ctx context.Context) error {
+ var pending []*backupRunInternal
+ if err := e.runs.Apply(ctx, "stop backup runs", func(runs map[string]*backupRunInternal) (bool, error) {
+ e.stopping = true
+ for _, run := range runs {
+ pending = append(pending, run)
+ run.cancel()
+ }
+ return false, nil
+ }); err != nil {
+ return err
+ }
+ var stopErr error
+ for _, run := range pending {
+ stopErr = errors.Combine(stopErr, run.executor.Stop(ctx))
+ }
+ return stopErr
+}
diff --git a/backend/internal/bootstrap/jobs_bootstrap.go b/backend/internal/bootstrap/jobs_bootstrap.go
index 8572d2c17..0283bc4bc 100644
--- a/backend/internal/bootstrap/jobs_bootstrap.go
+++ b/backend/internal/bootstrap/jobs_bootstrap.go
@@ -106,6 +106,9 @@ func registerJobs(params registerJobsParams) error {
// Bootstrap owns registration, agent-mode gating, and settings callbacks.
if params.Activity != nil {
+ if err := params.Activity.FailInterruptedBackups(params.AppCtx); err != nil {
+ return err
+ }
failed, err := params.Activity.FailStaleImageUpdateChecks(params.AppCtx)
if err != nil {
slog.WarnContext(params.AppCtx, "Failed to mark stale image update checks as failed", "count", failed, "error", err)
@@ -212,12 +215,18 @@ func registerDynamicJobs(params dynamicJobsParams) error {
// registered on managers and agents; environment proxying persists each policy
// in the correct Arcane database.
if params.Volume != nil {
+ if err := params.Volume.ReconcileInterruptedBackups(params.AppCtx); err != nil {
+ return err
+ }
if err := params.Volume.SetScheduler(params.AppCtx, params.Scheduler, params.Admission); err != nil {
return err
}
params.Volume.RegisterBackupJobsOnStartup(params.AppCtx)
}
if !params.Config.AgentMode && params.SystemBackup != nil {
+ if err := params.SystemBackup.ReconcileInterruptedBackups(params.AppCtx); err != nil {
+ return err
+ }
if err := params.SystemBackup.SetScheduler(params.AppCtx, params.Scheduler, params.Admission); err != nil {
return err
}
diff --git a/backend/internal/systembackup/background.go b/backend/internal/systembackup/background.go
new file mode 100644
index 000000000..7316fdee9
--- /dev/null
+++ b/backend/internal/systembackup/background.go
@@ -0,0 +1,125 @@
+package systembackup
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+
+ "emperror.dev/errors"
+
+ "github.com/getarcaneapp/arcane/backend/v2/internal/backup"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/common"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/volume"
+ activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"
+ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
+ backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"
+)
+
+// StartBackup prepares a manual backup before submitting its snapshot work.
+func (s *SystemBackupService) StartBackup(ctx context.Context, user common.User, request backuptypes.CreateSystemBackupRequest) (*backuptypes.SystemBackupRun, error) {
+ lease, admitted, err := s.engine.TryAcquireRun(ctx, backup.SystemAdmissionScope, systemAdmissionID)
+ if err != nil {
+ return nil, err
+ }
+ if !admitted {
+ return nil, ErrSystemBackupAlreadyRunning
+ }
+ prepared, err := s.prepareBackupInternal(ctx, SystemBackupTriggerManual, request)
+ if err != nil {
+ lease.Release()
+ return nil, err
+ }
+ activityID, workCtx := activitylib.StartHandlerActivity(ctx, s.activityService, "0", activitytypes.TypeResourceAction, "system_backup", "arcane", "Arcane", &user,
+ "Creating system backup", "Creating Arcane system backup", database.JSON{"action": "create_system_backup", "backupId": prepared.run.ID, "destination": prepared.run.Destination, "s3DestinationId": prepared.run.S3DestinationID, "policyId": prepared.run.PolicyID}, false)
+ finish := func(runErr error) {
+ defer lease.Release()
+ if runErr != nil {
+ if saveErr := s.db.WithContext(context.WithoutCancel(workCtx)).Model(&SystemBackupRun{}).Where("id = ?", prepared.run.ID).Updates(map[string]any{"status": SystemBackupStatusFailed, "error": runErr.Error()}).Error; saveErr != nil {
+ runErr = errors.Combine(runErr, errors.WrapIf(saveErr, "save backup failure"))
+ }
+ }
+ activitylib.CompleteHandlerActivity(workCtx, s.activityService, activityID, "Arcane system backup created successfully", runErr)
+ }
+ if activityID == "" {
+ err = errors.New("failed to create system backup activity")
+ finish(err)
+ return nil, err
+ }
+ dto := prepared.run.ToDTO()
+ dto.ActivityID = activityID
+ if err = s.engine.SubmitRun(workCtx, prepared.run.ID, func(runCtx context.Context) error {
+ _, runErr := s.executeBackupInternal(runCtx, prepared)
+ return runErr
+ }, finish); err != nil {
+ finish(err)
+ return nil, err
+ }
+ return &dto, nil
+}
+
+// ReconcileInterruptedBackups marks work interrupted by the previous process.
+func (s *SystemBackupService) ReconcileInterruptedBackups(ctx context.Context) error {
+ return s.db.WithContext(ctx).Model(&SystemBackupRun{}).Where("status = ?", SystemBackupStatusRunning).
+ Updates(map[string]any{"status": SystemBackupStatusFailed, "error": "Backup interrupted by application restart"}).Error
+}
+
+// StartSystemVolumeBackups freezes the selected policy and volumes before returning.
+func (s *SystemBackupService) StartSystemVolumeBackups(ctx context.Context, user common.User, request backuptypes.RunSystemVolumeBackupsRequest) (*backuptypes.BackupRunAccepted, error) {
+ prepared, err := s.prepareSystemVolumeBackupsInternal(ctx, request)
+ if err != nil {
+ return nil, err
+ }
+ policy, manualPolicy, candidates, lease := prepared.policy, prepared.manualPolicy, prepared.candidates, prepared.lease
+
+ names := make([]string, len(candidates))
+ for i, candidate := range candidates {
+ names[i] = candidate.Name
+ }
+ activityID, workCtx := activitylib.StartHandlerActivity(ctx, s.activityService, "0", activitytypes.TypeResourceAction, "system_backup", "volumes", "Volumes", &user,
+ "Backing up volumes", "Creating system-managed volume backups", database.JSON{"action": "run_system_volume_backups", "policyId": policy.ID, "volumeNames": names, "matched": len(candidates), "succeeded": 0, "failed": 0, "skipped": 0, "failures": []backuptypes.SystemVolumeBackupFailure{}}, false)
+ if activityID == "" {
+ lease.Release()
+ return nil, errors.New("failed to create system-managed volume backup activity")
+ }
+ finish := func(runErr error) {
+ defer lease.Release()
+ activitylib.CompleteHandlerActivity(workCtx, s.activityService, activityID, "System-managed volume backups completed", runErr)
+ }
+ err = s.engine.SubmitRun(workCtx, activityID, func(runCtx context.Context) error {
+ result, runErr := s.executeSystemVolumeBackupsInternal(runCtx, policy, manualPolicy, candidates, volume.VolumeBackupTriggerManual, activityID)
+ if runErr != nil {
+ return runErr
+ }
+ if result.Failed > 0 {
+ return fmt.Errorf("%d volume backups failed", result.Failed)
+ }
+ return nil
+ }, finish)
+ if err != nil {
+ finish(err)
+ return nil, err
+ }
+ return &backuptypes.BackupRunAccepted{ActivityID: activityID, Status: "running"}, nil
+}
+
+func (s *SystemBackupService) updateSystemVolumeProgressInternal(ctx context.Context, activityID string, policyID string, candidates []backuptypes.SystemVolumeBackupOption, result *backuptypes.SystemVolumeBackupRunResult) {
+ if activityID == "" {
+ return
+ }
+ names := make([]string, len(candidates))
+ for i, candidate := range candidates {
+ names[i] = candidate.Name
+ }
+ progress := 100
+ if result.Matched > 0 {
+ progress = 100 * (result.Succeeded + result.Failed + result.Skipped) / result.Matched
+ }
+ _, err := s.activityService.UpdateActivity(ctx, activityID, activitylib.UpdateRequest{Progress: &progress, Metadata: database.JSON{
+ "action": "run_system_volume_backups", "policyId": policyID, "volumeNames": names,
+ "matched": result.Matched, "succeeded": result.Succeeded, "failed": result.Failed, "skipped": result.Skipped, "failures": result.Failures,
+ }})
+ if err != nil {
+ slog.WarnContext(ctx, "Failed to report system-managed volume backup progress", "activityId", activityID, "policyId", policyID, "error", err)
+ }
+}
diff --git a/backend/internal/systembackup/handler.go b/backend/internal/systembackup/handler.go
index f9328a88d..be308b9d4 100644
--- a/backend/internal/systembackup/handler.go
+++ b/backend/internal/systembackup/handler.go
@@ -58,7 +58,7 @@ type SystemVolumeBackupOptionsOutput struct {
}
type SystemVolumeBackupRunOutput struct {
- Body backuptypes.SystemVolumeBackupRunResult
+ Body backuptypes.BackupRunAccepted
}
type RunSystemVolumeBackupsInput struct {
@@ -124,12 +124,12 @@ func RegisterSystemBackups(api huma.API, service *SystemBackupService, activityS
middleware.RegisterWithPermission(api, huma.Operation{OperationID: "get-system-volume-backup-config", Method: http.MethodGet, Path: "/backups/volumes/config", Summary: "Get system-managed volume backup policies", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsRead, h.GetSystemVolumeConfig)
middleware.RegisterWithPermission(api, huma.Operation{OperationID: "update-system-volume-backup-config", Method: http.MethodPut, Path: "/backups/volumes/config", Summary: "Update system-managed volume backup policies", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsManage, h.UpdateSystemVolumeConfig)
middleware.RegisterWithPermission(api, huma.Operation{OperationID: "list-system-volume-backup-options", Method: http.MethodGet, Path: "/backups/volumes/options", Summary: "List volumes available to system-managed backups", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsRead, h.ListSystemVolumeOptions)
- middleware.RegisterWithPermission(api, huma.Operation{OperationID: "run-system-volume-backups", Method: http.MethodPost, Path: "/backups/volumes/run", Summary: "Run system-managed volume backups", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsManage, h.RunSystemVolumeBackups)
+ middleware.RegisterWithPermission(api, huma.Operation{OperationID: "run-system-volume-backups", DefaultStatus: http.StatusAccepted, Method: http.MethodPost, Path: "/backups/volumes/run", Summary: "Run system-managed volume backups", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsManage, h.RunSystemVolumeBackups)
middleware.RegisterWithPermission(api, huma.Operation{OperationID: "get-system-backup-policies", Method: http.MethodGet, Path: "/backups/policies", Summary: "Get Arcane system backup policies", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsRead, h.GetPolicies)
middleware.RegisterWithPermission(api, huma.Operation{OperationID: "update-system-backup-policies", Method: http.MethodPut, Path: "/backups/policies", Summary: "Update Arcane system backup policies", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsManage, h.UpdatePolicies)
middleware.RegisterWithPermission(api, huma.Operation{OperationID: "generate-system-backup-recovery-key", Method: http.MethodPost, Path: "/backups/recovery-key/generate", Summary: "Generate an Arcane system backup recovery key", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsRecoveryKey, h.GenerateRecoveryKey)
middleware.RegisterWithPermission(api, huma.Operation{OperationID: "set-system-backup-recovery-key", Method: http.MethodPut, Path: "/backups/recovery-key", Summary: "Configure Arcane system backup recovery key", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsRecoveryKey, h.SetRecoveryKey)
- middleware.RegisterWithPermission(api, huma.Operation{OperationID: "create-system-backup", Method: http.MethodPost, Path: "/backups", Summary: "Create Arcane system backup", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsManage, h.Create)
+ middleware.RegisterWithPermission(api, huma.Operation{OperationID: "create-system-backup", DefaultStatus: http.StatusAccepted, Method: http.MethodPost, Path: "/backups", Summary: "Create Arcane system backup", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsManage, h.Create)
middleware.RegisterWithPermission(api, huma.Operation{OperationID: "discover-system-backups", Method: http.MethodPost, Path: "/backups/discover", Summary: "Discover Arcane system backups in S3", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsManage, h.Discover)
middleware.RegisterWithPermission(api, huma.Operation{OperationID: "restore-system-backup", Method: http.MethodPost, Path: "/backups/{id}/restore", Summary: "Restore Arcane system backup", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsRestore, h.Restore)
middleware.RegisterWithPermission(api, huma.Operation{OperationID: "browse-system-backup-files", Method: http.MethodPost, Path: "/backups/{id}/files/browse", Summary: "Browse project files in an Arcane system backup", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsRead, h.BrowseFiles)
@@ -183,7 +183,11 @@ func (h *SystemBackupHandler) RunSystemVolumeBackups(ctx context.Context, input
if input.Body != nil {
request = *input.Body
}
- result, err := h.service.RunSystemVolumeBackups(ctx, request)
+ user, err := handlerutil.RequireUser(ctx)
+ if err != nil {
+ return nil, err
+ }
+ result, err := h.service.StartSystemVolumeBackups(utils.ActivityRuntimeContext(ctx, h.appCtx), *user, request)
if errors.Is(err, ErrSystemBackupAlreadyRunning) {
return nil, huma.Error409Conflict(err.Error())
}
@@ -292,24 +296,14 @@ func (h *SystemBackupHandler) Create(ctx context.Context, input *CreateSystemBac
if err != nil {
return nil, err
}
- var run *SystemBackupRun
- _, err = activitylib.RunHandlerActivity(utils.ActivityRuntimeContext(ctx, h.appCtx), h.activity, activitylib.HandlerOptions{
- EnvironmentID: "0", Type: activitytypes.TypeResourceAction, ResourceType: "system_backup", ResourceID: "arcane", ResourceName: "Arcane", User: user,
- Step: "Creating system backup", Message: "Creating Arcane system backup", SuccessMessage: "Arcane system backup created successfully",
- Metadata: database.JSON{"action": "create_system_backup", "destination": input.Body.Destination, "s3DestinationId": input.Body.S3DestinationID, "policyId": input.Body.PolicyID},
- }, func(activityCtx context.Context) error {
- var e error
- run, e = h.service.CreateBackup(activityCtx, *user, SystemBackupTriggerManual, input.Body)
- return e
- })
+ run, err := h.service.StartBackup(utils.ActivityRuntimeContext(ctx, h.appCtx), *user, input.Body)
if errors.Is(err, ErrSystemBackupAlreadyRunning) {
return nil, huma.Error409Conflict(err.Error())
}
if err != nil {
return nil, huma.Error500InternalServerError(err.Error())
}
- dto := run.ToDTO()
- return &SystemBackupOutput{Body: dto}, nil
+ return &SystemBackupOutput{Body: *run}, nil
}
func (h *SystemBackupHandler) Restore(ctx context.Context, input *RestoreSystemBackupInput) (*handlerutil.Out[base.MessageResponse], error) {
diff --git a/backend/internal/systembackup/service.go b/backend/internal/systembackup/service.go
index b3339a673..35b73cc19 100644
--- a/backend/internal/systembackup/service.go
+++ b/backend/internal/systembackup/service.go
@@ -369,7 +369,22 @@ func (s *SystemBackupService) createStagedSystemRecoverySnapshotInternal(ctx con
return remoteSnapshot, remoteRepository, true, nil
}
-func (s *SystemBackupService) createBackupInternal(ctx context.Context, trigger SystemBackupTrigger, request backuptypes.CreateSystemBackupRequest) (_ *SystemBackupRun, err error) {
+type preparedSystemBackupInternal struct {
+ run *SystemBackupRun
+ recoveryKey string
+ localEnabled bool
+ s3Enabled bool
+}
+
+func (s *SystemBackupService) createBackupInternal(ctx context.Context, trigger SystemBackupTrigger, request backuptypes.CreateSystemBackupRequest) (*SystemBackupRun, error) {
+ prepared, err := s.prepareBackupInternal(ctx, trigger, request)
+ if err != nil {
+ return nil, err
+ }
+ return s.executeBackupInternal(ctx, prepared)
+}
+
+func (s *SystemBackupService) prepareBackupInternal(ctx context.Context, trigger SystemBackupTrigger, request backuptypes.CreateSystemBackupRequest) (*preparedSystemBackupInternal, error) {
if !s.SupportedDatabaseProvider() {
return nil, errors.New("arcane system recovery currently requires the SQLite database provider")
}
@@ -386,7 +401,17 @@ func (s *SystemBackupService) createBackupInternal(ctx context.Context, trigger
if err := s.db.WithContext(ctx).Create(run).Error; err != nil {
return nil, err
}
+ return &preparedSystemBackupInternal{run: run, recoveryKey: recoveryKey, localEnabled: localEnabled, s3Enabled: s3Enabled}, nil
+}
+
+func (s *SystemBackupService) executeBackupInternal(ctx context.Context, prepared *preparedSystemBackupInternal) (_ *SystemBackupRun, err error) {
+ run, recoveryKey := prepared.run, prepared.recoveryKey
+ localEnabled, s3Enabled, destinationID := prepared.localEnabled, prepared.s3Enabled, run.S3DestinationID
+
defer func() {
+ if err == nil {
+ err = ctx.Err()
+ }
if err != nil {
run.Status, run.Error = SystemBackupStatusFailed, err.Error()
} else {
@@ -396,6 +421,7 @@ func (s *SystemBackupService) createBackupInternal(ctx context.Context, trigger
err = errors.Combine(err, fmt.Errorf("failed to save system backup result: %w", saveErr))
}
}()
+ defer utils.RecoverToError(&err, "system backup")
dockerClient, err := s.dockerService.GetClient(ctx)
if err != nil {
return run, err
diff --git a/backend/internal/systembackup/system_volume.go b/backend/internal/systembackup/system_volume.go
index 67b0f6a26..821504e73 100644
--- a/backend/internal/systembackup/system_volume.go
+++ b/backend/internal/systembackup/system_volume.go
@@ -12,6 +12,7 @@ import (
"emperror.dev/errors"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/backup"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
s3domain "github.com/getarcaneapp/arcane/backend/v2/internal/s3"
@@ -268,11 +269,23 @@ func (s *SystemBackupService) resolveSystemVolumeRunPolicyInternal(ctx context.C
return policy, true, err
}
-func (s *SystemBackupService) RunSystemVolumeBackups(ctx context.Context, request backuptypes.RunSystemVolumeBackupsRequest) (*backuptypes.SystemVolumeBackupRunResult, error) {
- return s.runSystemVolumeBackupsInternal(ctx, request, volume.VolumeBackupTriggerManual)
+type preparedSystemVolumeBackupInternal struct {
+ policy backuptypes.SystemVolumeBackupPolicy
+ manualPolicy bool
+ candidates []backuptypes.SystemVolumeBackupOption
+ lease *actors.Lease[actors.AdmissionKey]
}
func (s *SystemBackupService) runSystemVolumeBackupsInternal(ctx context.Context, request backuptypes.RunSystemVolumeBackupsRequest, trigger volume.VolumeBackupTrigger) (*backuptypes.SystemVolumeBackupRunResult, error) {
+ prepared, err := s.prepareSystemVolumeBackupsInternal(ctx, request)
+ if err != nil {
+ return nil, err
+ }
+ defer prepared.lease.Release()
+ return s.executeSystemVolumeBackupsInternal(ctx, prepared.policy, prepared.manualPolicy, prepared.candidates, trigger, "")
+}
+
+func (s *SystemBackupService) prepareSystemVolumeBackupsInternal(ctx context.Context, request backuptypes.RunSystemVolumeBackupsRequest) (*preparedSystemVolumeBackupInternal, error) {
if s.volumeService == nil {
return nil, errors.New("volume service is unavailable")
}
@@ -287,21 +300,32 @@ func (s *SystemBackupService) runSystemVolumeBackupsInternal(ctx context.Context
if !admitted {
return nil, ErrSystemBackupAlreadyRunning
}
- defer lease.Release()
options, err := s.volumeService.ListBackupVolumeOptions(ctx)
if err != nil {
+ lease.Release()
return nil, err
}
candidates := selectSystemVolumeBackupCandidatesInternal(policyConfig, options)
- result := &backuptypes.SystemVolumeBackupRunResult{
+ return &preparedSystemVolumeBackupInternal{policy: policyConfig, manualPolicy: manualPolicy, candidates: candidates, lease: lease}, nil
+}
+
+func (s *SystemBackupService) executeSystemVolumeBackupsInternal(ctx context.Context, policyConfig backuptypes.SystemVolumeBackupPolicy, manualPolicy bool, candidates []backuptypes.SystemVolumeBackupOption, trigger volume.VolumeBackupTrigger, activityID string) (result *backuptypes.SystemVolumeBackupRunResult, err error) {
+ result = &backuptypes.SystemVolumeBackupRunResult{
Matched: len(candidates), Failures: make([]backuptypes.SystemVolumeBackupFailure, 0),
}
+ defer s.updateSystemVolumeProgressInternal(context.WithoutCancel(ctx), activityID, policyConfig.ID, candidates, result)
+ defer utils.RecoverToError(&err, "system-managed volume backup")
+
policy := backuptypes.UpdateBackupPolicy{
Enabled: true, Schedule: policyConfig.Schedule, RetentionCount: policyConfig.RetentionCount,
StopContainers: policyConfig.StopContainers, LocalEnabled: policyConfig.LocalEnabled, S3Enabled: policyConfig.S3Enabled,
S3DestinationID: policyConfig.S3DestinationID,
}
for _, candidate := range candidates {
+ if err := ctx.Err(); err != nil {
+ return result, err
+ }
+ s.updateSystemVolumeProgressInternal(ctx, activityID, policyConfig.ID, candidates, result)
overridden, policyErr := s.volumeService.HasEnabledBackupPolicy(ctx, candidate.Name)
if policyErr != nil {
result.Failed++
diff --git a/backend/internal/volume/backup.go b/backend/internal/volume/backup.go
index 41ce07218..6a15b9497 100644
--- a/backend/internal/volume/backup.go
+++ b/backend/internal/volume/backup.go
@@ -18,6 +18,7 @@ import (
"emperror.dev/errors"
"gorm.io/gorm"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/backup"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
@@ -27,6 +28,7 @@ import (
activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/volumehelper"
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/backupbrowser"
activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"
@@ -621,20 +623,25 @@ func (s *VolumeService) CreateSystemManagedBackup(ctx context.Context, volumeNam
return s.createBackupInternal(ctx, volumeName, user, trigger, policyID, plan)
}
-//nolint:gocognit // backup orchestration keeps cleanup and persistence transitions in one transaction-like flow
-func (s *VolumeService) createBackupInternal(ctx context.Context, volumeName string, user common.User, trigger VolumeBackupTrigger, policyID string, plan backupPlanInternal) (_ *VolumeBackup, err error) {
- workspaceLock, _ := ctx.Value(volumeWorkspaceLockContextKeyInternal{}).(volumeWorkspaceLockContextInternal)
- if workspaceLock.service != s || workspaceLock.volumeName != volumeName {
- defer s.workspaceLocks.Lock(volumeName)()
+func (s *VolumeService) createBackupInternal(ctx context.Context, volumeName string, user common.User, trigger VolumeBackupTrigger, policyID string, plan backupPlanInternal) (*VolumeBackup, error) {
+ entry, lease, err := s.prepareBackupInternal(ctx, volumeName, trigger, policyID, plan)
+ if err != nil {
+ return nil, err
}
- lease, admitted, leaseErr := s.engine.TryAcquireRun(ctx, backup.VolumeAdmissionScope, volumeName)
- if leaseErr != nil {
- return nil, leaseErr
+ defer lease.Release()
+ err = s.executeBackupInternal(ctx, entry, user, plan)
+ err = s.completeBackupInternal(ctx, entry, err)
+ return entry, err
+}
+
+func (s *VolumeService) prepareBackupInternal(ctx context.Context, volumeName string, trigger VolumeBackupTrigger, policyID string, plan backupPlanInternal) (*VolumeBackup, *actors.Lease[actors.AdmissionKey], error) {
+ lease, admitted, err := s.engine.TryAcquireRun(ctx, backup.VolumeAdmissionScope, volumeName)
+ if err != nil {
+ return nil, nil, err
}
if !admitted {
- return nil, ErrVolumeBackupAlreadyRunning
+ return nil, nil, ErrVolumeBackupAlreadyRunning
}
- defer lease.Release()
entry := &VolumeBackup{
VolumeName: volumeName, CreatedAt: time.Now(), Status: VolumeBackupStatusRunning,
Trigger: trigger, Destination: plan.destination, Format: VolumeBackupFormatRustic,
@@ -642,21 +649,41 @@ func (s *VolumeService) createBackupInternal(ctx context.Context, volumeName str
}
entry.ID = fmt.Sprintf("%s-%d-%s", volumeName, time.Now().UnixNano(), uuid.New().String()[:8])
if err := s.db.WithContext(ctx).Create(entry).Error; err != nil {
- return nil, err
+ lease.Release()
+ return nil, nil, err
+ }
+ return entry, lease, nil
+}
+
+func (s *VolumeService) completeBackupInternal(ctx context.Context, entry *VolumeBackup, err error) error {
+ if err == nil {
+ err = ctx.Err()
+ }
+ if err != nil {
+ entry.Status, entry.Error = VolumeBackupStatusFailed, err.Error()
+ } else {
+ entry.Status, entry.Error = VolumeBackupStatusSucceeded, ""
+ }
+ if saveErr := s.db.WithContext(context.WithoutCancel(ctx)).Save(entry).Error; saveErr != nil {
+ return errors.Combine(err, fmt.Errorf("failed to save volume backup result: %w", saveErr))
+ }
+ return err
+}
+
+//nolint:gocognit // backup execution keeps container recovery with snapshot work
+func (s *VolumeService) executeBackupInternal(ctx context.Context, entry *VolumeBackup, user common.User, plan backupPlanInternal) (err error) {
+ defer utils.RecoverToError(&err, "volume backup")
+ volumeName, trigger := entry.VolumeName, entry.Trigger
+ workspaceLock, _ := ctx.Value(volumeWorkspaceLockContextKeyInternal{}).(volumeWorkspaceLockContextInternal)
+ if workspaceLock.service != s || workspaceLock.volumeName != volumeName {
+ defer s.workspaceLocks.Lock(volumeName)()
+ }
+ if err := ctx.Err(); err != nil {
+ return err
}
- defer func() {
- if err != nil {
- entry.Status, entry.Error = VolumeBackupStatusFailed, err.Error()
- } else {
- entry.Status, entry.Error = VolumeBackupStatusSucceeded, ""
- }
- if saveErr := s.db.WithContext(context.WithoutCancel(ctx)).Save(entry).Error; saveErr != nil {
- err = errors.Combine(err, fmt.Errorf("failed to save volume backup result: %w", saveErr))
- }
- }()
dockerClient, err := s.dockerService.GetClient(ctx)
if err != nil {
- return entry, err
+ return err
}
var stopped []container.Summary
containersStopped := false
@@ -670,7 +697,7 @@ func (s *VolumeService) createBackupInternal(ctx context.Context, volumeName str
}
}()
if err != nil {
- return entry, err
+ return err
}
}
// The live volume is read once. A local+S3 backup replicates the local
@@ -679,11 +706,11 @@ func (s *VolumeService) createBackupInternal(ctx context.Context, volumeName str
if plan.localEnabled {
repository, repoErr := s.localRusticRepositoryInternal(ctx, dockerClient, false)
if repoErr != nil {
- return entry, repoErr
+ return repoErr
}
localSnapshot, err = s.engine.CreateSnapshot(ctx, dockerClient, repository, s.rusticPasswordInternal(), volumeName, volumeSourceMountInternal(volumeName))
if err != nil {
- return entry, fmt.Errorf("failed to create local Rustic snapshot: %w", err)
+ return fmt.Errorf("failed to create local Rustic snapshot: %w", err)
}
entry.LocalSnapshotID = localSnapshot.ID
entry.Size = localSnapshot.Size
@@ -691,20 +718,20 @@ func (s *VolumeService) createBackupInternal(ctx context.Context, volumeName str
if plan.s3Enabled {
remoteRepository, repoErr := s.remoteRusticRepositoryInternal(ctx, plan.s3DestinationID)
if repoErr != nil {
- return entry, repoErr
+ return repoErr
}
var remoteSnapshot backup.Snapshot
if plan.localEnabled {
localRepository, localErr := s.localRusticRepositoryInternal(ctx, dockerClient, true)
if localErr != nil {
- return entry, localErr
+ return localErr
}
remoteSnapshot, err = s.engine.Replicate(ctx, dockerClient, localRepository, localSnapshot.ID, remoteRepository, s.rusticPasswordInternal(), volumeName)
} else {
remoteSnapshot, err = s.engine.CreateSnapshot(ctx, dockerClient, remoteRepository, s.rusticPasswordInternal(), volumeName, volumeSourceMountInternal(volumeName))
}
if err != nil {
- return entry, fmt.Errorf("failed to create S3 Rustic snapshot: %w", err)
+ return fmt.Errorf("failed to create S3 Rustic snapshot: %w", err)
}
entry.RemoteSnapshotID = remoteSnapshot.ID
if entry.Size == 0 {
@@ -715,7 +742,7 @@ func (s *VolumeService) createBackupInternal(ctx context.Context, volumeName str
stopped, err = s.startContainersAfterBackupInternal(context.WithoutCancel(ctx), dockerClient, stopped, user)
containersStopped = len(stopped) > 0
if err != nil {
- return entry, err
+ return err
}
}
// Retention failures must not fail the run: the backup itself succeeded,
@@ -730,7 +757,7 @@ func (s *VolumeService) createBackupInternal(ctx context.Context, volumeName str
if logErr := s.eventService.LogVolumeEvent(ctx, event.EventTypeVolumeBackupCreate, volumeName, volumeName, user.ID, user.Username, "0", metadata); logErr != nil {
slog.WarnContext(ctx, "could not log volume backup create event", "volume", volumeName, "error", logErr)
}
- return entry, nil
+ return ctx.Err()
}
// HasEnabledBackupPolicy reports whether a volume-level schedule takes precedence over centralized backups.
diff --git a/backend/internal/volume/backup_run.go b/backend/internal/volume/backup_run.go
new file mode 100644
index 000000000..934289298
--- /dev/null
+++ b/backend/internal/volume/backup_run.go
@@ -0,0 +1,51 @@
+package volume
+
+import (
+ "context"
+
+ "emperror.dev/errors"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/common"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"
+ activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"
+ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
+ volumetypes "github.com/getarcaneapp/arcane/types/v2/volume"
+)
+
+// StartBackup persists a running backup and submits application-owned work.
+func (s *VolumeService) StartBackup(ctx context.Context, environmentID, volumeName string, user common.User, request volumetypes.CreateBackupRequest) (volumetypes.BackupEntry, error) {
+ plan, err := s.resolveBackupPlanInternal(ctx, volumeName, VolumeBackupTriggerManual, request, nil)
+ if err != nil {
+ return volumetypes.BackupEntry{}, err
+ }
+ entry, lease, err := s.prepareBackupInternal(ctx, volumeName, VolumeBackupTriggerManual, request.PolicyID, plan)
+ if err != nil {
+ return volumetypes.BackupEntry{}, err
+ }
+ activityID, workCtx := activitylib.StartHandlerActivity(ctx, s.activityService, environmentID, activitytypes.TypeResourceAction,
+ "volume", volumeName, volumeName, &user, "Creating backup", "Creating volume backup",
+ database.JSON{"action": "create_volume_backup", "backupId": entry.ID, "destination": entry.Destination, "policyId": request.PolicyID, "s3DestinationId": entry.S3DestinationID}, false)
+ if activityID == "" {
+ defer lease.Release()
+ return volumetypes.BackupEntry{}, s.completeBackupInternal(ctx, entry, errors.New("failed to start backup activity"))
+ }
+ entry.ActivityID = &activityID
+ accepted := entry.ToDTO()
+ complete := func(runErr error) {
+ defer lease.Release()
+ runErr = s.completeBackupInternal(workCtx, entry, runErr)
+ activitylib.CompleteHandlerActivity(workCtx, s.activityService, activityID, "Volume backup created successfully", runErr)
+ }
+ if err := s.engine.SubmitRun(workCtx, entry.ID, func(workerCtx context.Context) error {
+ return s.executeBackupInternal(workerCtx, entry, user, plan)
+ }, complete); err != nil {
+ complete(err)
+ return volumetypes.BackupEntry{}, err
+ }
+ return accepted, nil
+}
+
+// ReconcileInterruptedBackups runs before this process can accept backup work.
+func (s *VolumeService) ReconcileInterruptedBackups(ctx context.Context) error {
+ return s.db.WithContext(ctx).Model(&VolumeBackup{}).Where("status = ?", VolumeBackupStatusRunning).
+ Updates(map[string]any{"status": VolumeBackupStatusFailed, "error": "Backup interrupted by Arcane restart"}).Error
+}
diff --git a/backend/internal/volume/handler.go b/backend/internal/volume/handler.go
index 3792f2d71..327c611df 100644
--- a/backend/internal/volume/handler.go
+++ b/backend/internal/volume/handler.go
@@ -365,12 +365,13 @@ func RegisterVolumes(api huma.API, dockerService *docker.DockerClientService, vo
}, authz.PermVolumesRead, h.ListBackups)
middleware.RegisterWithPermission(api, huma.Operation{
- OperationID: "create-volume-backup",
- Method: http.MethodPost,
- Path: "/environments/{id}/volumes/{volumeName}/backups",
- Summary: "Create volume backup",
- Tags: []string{"Volume Backup"},
- Security: handlerutil.DefaultOperationSecurity(),
+ OperationID: "create-volume-backup",
+ DefaultStatus: http.StatusAccepted,
+ Method: http.MethodPost,
+ Path: "/environments/{id}/volumes/{volumeName}/backups",
+ Summary: "Create volume backup",
+ Tags: []string{"Volume Backup"},
+ Security: handlerutil.DefaultOperationSecurity(),
}, authz.PermVolumesBackup, h.CreateBackup)
middleware.RegisterWithPermission(api, huma.Operation{
@@ -904,52 +905,24 @@ func (h *VolumeHandler) ListBackups(ctx context.Context, input *ListBackupsInput
}, nil
}
-func (h *VolumeHandler) CreateBackup(ctx context.Context, input *CreateBackupInput) (*handlerutil.Out[*VolumeBackup], error) {
+func (h *VolumeHandler) CreateBackup(ctx context.Context, input *CreateBackupInput) (*handlerutil.Out[volumetypes.BackupEntry], error) {
user, err := handlerutil.RequireUser(ctx)
if err != nil {
return nil, err
}
-
- var backup *VolumeBackup
- destination := volumetypes.BackupDestination("")
- policyID := ""
- s3DestinationID := ""
+ request := volumetypes.CreateBackupRequest{}
if input.Body != nil {
- destination = input.Body.Destination
- policyID = input.Body.PolicyID
- s3DestinationID = input.Body.S3DestinationID
+ request = *input.Body
}
- runtimeCtx := utils.ActivityRuntimeContext(ctx, h.appCtx)
- activityID, err := activitylib.RunHandlerActivity(runtimeCtx, h.activityService, activitylib.HandlerOptions{
- EnvironmentID: input.EnvironmentID,
- Type: activitytypes.TypeResourceAction,
- ResourceType: "volume",
- ResourceID: input.VolumeName,
- ResourceName: input.VolumeName,
- User: user,
- Step: "Creating backup",
- Message: "Creating volume backup",
- SuccessMessage: "Volume backup created successfully",
- Metadata: database.JSON{"action": "create_volume_backup", "destination": destination, "policyId": policyID, "s3DestinationId": s3DestinationID},
- }, func(runtimeCtx context.Context) error {
- var backupErr error
- backup, backupErr = h.volumeService.CreateBackup(runtimeCtx, input.VolumeName, *user, VolumeBackupTriggerManual, volumetypes.CreateBackupRequest{
- Destination: destination, PolicyID: policyID, S3DestinationID: s3DestinationID,
- })
- return backupErr
- })
+ entry, err := h.volumeService.StartBackup(utils.ActivityRuntimeContext(ctx, h.appCtx), input.EnvironmentID, input.VolumeName, *user, request)
if errors.Is(err, ErrVolumeBackupAlreadyRunning) {
return nil, huma.Error409Conflict(err.Error())
}
if err != nil {
return nil, huma.Error500InternalServerError(err.Error())
}
- backup.ActivityID = mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()
- return &handlerutil.Out[*VolumeBackup]{
- Body: base.ApiResponse[*VolumeBackup]{
- Success: true,
- Data: backup,
- },
+ return &handlerutil.Out[volumetypes.BackupEntry]{
+ Body: base.ApiResponse[volumetypes.BackupEntry]{Success: true, Data: entry},
}, nil
}
diff --git a/backend/internal/volume/model.go b/backend/internal/volume/model.go
index 6a1638cb6..205d8cf57 100644
--- a/backend/internal/volume/model.go
+++ b/backend/internal/volume/model.go
@@ -57,6 +57,7 @@ func (*VolumeBackup) TableName() string {
func (b *VolumeBackup) ToDTO() volume.BackupEntry {
return volume.BackupEntry{
+ ActivityID: b.ActivityID,
ID: b.ID,
VolumeName: b.VolumeName,
Size: b.Size,
diff --git a/backend/internal/volume/service_test.go b/backend/internal/volume/service_test.go
index 32a8cc717..5f4d92224 100644
--- a/backend/internal/volume/service_test.go
+++ b/backend/internal/volume/service_test.go
@@ -740,10 +740,10 @@ func TestVolumeBackup_ValidationErrors(t *testing.T) {
_, err = service.UpdateBackupPolicies(ctx, "app-data", []volumetypes.UpdateBackupPolicy{{Schedule: "0 0 2 * * *", RetentionCount: 7}})
require.ErrorContains(t, err, "select at least one volume backup destination")
- _, err = service.CreateBackup(ctx, "app-data", common.User{}, VolumeBackupTriggerManual, volumetypes.CreateBackupRequest{Destination: volumetypes.BackupDestination("invalid")})
+ _, err = service.StartBackup(ctx, "0", "app-data", common.User{}, volumetypes.CreateBackupRequest{Destination: volumetypes.BackupDestination("invalid")})
require.EqualError(t, err, "invalid volume backup destination")
- _, err = service.CreateBackup(ctx, "app-data", common.User{}, VolumeBackupTriggerManual, volumetypes.CreateBackupRequest{Destination: volumetypes.BackupDestinationS3})
+ _, err = service.StartBackup(ctx, "0", "app-data", common.User{}, volumetypes.CreateBackupRequest{Destination: volumetypes.BackupDestinationS3})
require.EqualError(t, err, "select an S3 destination for the volume backup")
}
diff --git a/cli/pkg/volumes/backup.go b/cli/pkg/volumes/backup.go
index 4e2222997..715431e4c 100644
--- a/cli/pkg/volumes/backup.go
+++ b/cli/pkg/volumes/backup.go
@@ -380,7 +380,7 @@ var backupsCreateCmd = &cobra.Command{
return cmdutil.PrintJSON(result.Data)
}
- output.Success("Backup created for volume %s", resolved.Name)
+ output.Success("Backup started for volume %s", resolved.Name)
output.KeyValue("ID", result.Data.ID)
output.KeyValue("Status", result.Data.Status)
output.KeyValue("Destination", string(result.Data.Destination))
diff --git a/frontend/messages/en.json b/frontend/messages/en.json
index 048c56079..52148b50e 100644
--- a/frontend/messages/en.json
+++ b/frontend/messages/en.json
@@ -3347,7 +3347,9 @@
"system_backups_recovery_key_saved": "Recovery key configured.",
"system_backups_recovery_key_save_failed": "Failed to configure the recovery key.",
"system_backups_enabled_description": "Run an encrypted Arcane recovery backup on the configured cron schedule.",
- "system_backups_created": "Arcane system backup created successfully.",
+ "backups_started": "Backup started",
+ "backups_running": "Backup in progress",
+ "system_backups_created": "Arcane system backup created successfully.",
"system_backups_create_failed": "Failed to create Arcane system backup.",
"system_backups_policy_saved": "System backup schedule saved.",
"system_backups_policy_failed": "Failed to save the system backup schedule.",
diff --git a/frontend/src/lib/hooks/use-backup-activity.svelte.ts b/frontend/src/lib/hooks/use-backup-activity.svelte.ts
new file mode 100644
index 000000000..d291ec97b
--- /dev/null
+++ b/frontend/src/lib/hooks/use-backup-activity.svelte.ts
@@ -0,0 +1,86 @@
+import { untrack } from 'svelte';
+import { createQuery, useQueryClient } from '@tanstack/svelte-query';
+import { queryKeys } from '#lib/query/query-keys';
+import { activityService } from '#lib/services/activity-service';
+import { activityStore } from '#lib/stores/activity.store.svelte';
+import { environmentStore } from '#lib/stores/environment.store.svelte';
+import type { Activity } from '#lib/types/activity.type';
+
+export function useBackupActivity(
+ getEnvironmentId: () => string,
+ matches: (activity: Activity) => boolean,
+ onChange: () => Promise<unknown>,
+ getResourceKey: () => string = () => '',
+ getDiscovery: () => ((environmentId: string) => Promise<string[]>) | undefined = () => undefined
+) {
+ const queryClient = useQueryClient();
+ const queryKey = $derived(queryKeys.backupActivities.active(getEnvironmentId(), getResourceKey(), !!getDiscovery()));
+ const activeQuery = createQuery(() => {
+ const environmentId = getEnvironmentId();
+ const discover = getDiscovery();
+ return {
+ queryKey,
+ queryFn: async ({ signal }): Promise<string[]> => {
+ await environmentStore.ready;
+ signal.throwIfAborted();
+ if (discover) return discover(environmentId);
+ const activities: Activity[] = [];
+ for (const status of ['queued', 'running']) {
+ let page = 1;
+ let totalPages = 1;
+ do {
+ const result = await activityService.getActivities(
+ { pagination: { page, limit: 100 }, filters: { status } },
+ environmentId
+ );
+ signal.throwIfAborted();
+ activities.push(...result.data);
+ totalPages = result.pagination.totalPages;
+ page++;
+ } while (page <= totalPages);
+ }
+ return activities.filter(matches).map((activity) => activity.id);
+ },
+ refetchInterval: (query) => (discover || query.state.data?.length ? 5000 : false),
+ refetchIntervalInBackground: true,
+ staleTime: 0
+ };
+ });
+ const activeIds = $derived(activeQuery.data ?? []);
+
+ $effect(() => {
+ if (!activeQuery.dataUpdatedAt) return;
+ untrack(() => {
+ void onChange().catch((error) => console.warn('Failed to refresh backup history', error));
+ });
+ });
+
+ $effect(() => {
+ const environmentId = getEnvironmentId();
+ const key = queryKey;
+ activityStore.connected;
+ const events = activityStore.activities.filter(
+ (activity) => (activity.sourceEnvironmentId || activity.environmentId) === environmentId && matches(activity)
+ );
+ // Include progress changes as well as terminal transitions.
+ JSON.stringify(events);
+ untrack(() => {
+ void queryClient.cancelQueries({ queryKey: key, exact: true });
+ void queryClient.invalidateQueries({ queryKey: key, exact: true });
+ });
+ });
+
+ return {
+ get activeIds() {
+ return activeIds;
+ },
+ accepted(activityId?: string) {
+ // Cancel older discovery before adding the accepted operation to the cache.
+ void queryClient.cancelQueries({ queryKey, exact: true });
+ if (activityId) {
+ queryClient.setQueryData<string[]>(queryKey, (ids = []) => (ids.includes(activityId) ? ids : [...ids, activityId]));
+ }
+ void queryClient.invalidateQueries({ queryKey, exact: true });
+ }
+ };
+}
diff --git a/frontend/src/lib/query/query-keys.ts b/frontend/src/lib/query/query-keys.ts
index ef34dc1ab..19fbfd82f 100644
--- a/frontend/src/lib/query/query-keys.ts
+++ b/frontend/src/lib/query/query-keys.ts
@@ -13,6 +13,10 @@ function stableSerialize(value: unknown): string {
}
export const queryKeys = {
+ backupActivities: {
+ active: (environmentId: string, resourceKey: string, discoverFromHistory: boolean) =>
+ ['backup-activities', environmentId, resourceKey, discoverFromHistory] as const
+ },
auth: {
all: ['auth'] as const,
logout: () => ['auth', 'logout'] as const,
diff --git a/frontend/src/lib/services/volume-backup-service.ts b/frontend/src/lib/services/volume-backup-service.ts
index ec7afc84d..d1977b819 100644
--- a/frontend/src/lib/services/volume-backup-service.ts
+++ b/frontend/src/lib/services/volume-backup-service.ts
@@ -30,8 +30,12 @@ class VolumeBackupService extends BaseAPIService {
return res.data.data;
}
- async listBackups(volumeName: string, options?: SearchPaginationSortRequest): Promise<VolumeBackupListResponse> {
- const envId = await environmentStore.getCurrentEnvironmentId();
+ async listBackups(
+ volumeName: string,
+ options?: SearchPaginationSortRequest,
+ environmentId?: string
+ ): Promise<VolumeBackupListResponse> {
+ const envId = environmentId ?? (await environmentStore.getCurrentEnvironmentId());
const params = transformPaginationParams(options);
const res = await this.api.get(`/environments/${envId}/volumes/${volumeName}/backups`, { params });
return res.data;
diff --git a/frontend/src/lib/types/backup.ts b/frontend/src/lib/types/backup.ts
index a5970db2c..06be6ff3a 100644
--- a/frontend/src/lib/types/backup.ts
+++ b/frontend/src/lib/types/backup.ts
@@ -6,6 +6,7 @@ export type BackupTrigger = 'manual' | 'scheduled' | 'safety';
export type BackupManagementType = 'system' | 'volume';
export type BackupRun = {
+ activityId?: string;
id: string;
size: number;
createdAt: string;
diff --git a/frontend/src/lib/types/system-backup.ts b/frontend/src/lib/types/system-backup.ts
index 0e6f190ec..155fee8eb 100644
--- a/frontend/src/lib/types/system-backup.ts
+++ b/frontend/src/lib/types/system-backup.ts
@@ -35,11 +35,8 @@ export type SystemVolumeBackupOption = {
};
export type SystemVolumeBackupRunResult = {
- matched: number;
- succeeded: number;
- failed: number;
- skipped: number;
- failures: { volumeName: string; error: string }[];
+ activityId: string;
+ status: 'running';
};
export type SystemVolumeBackupCustomRun = {
diff --git a/frontend/src/lib/utils/activity-toast.ts b/frontend/src/lib/utils/activity-toast.ts
index 1ecb32808..8cbc01acc 100644
--- a/frontend/src/lib/utils/activity-toast.ts
+++ b/frontend/src/lib/utils/activity-toast.ts
@@ -2,14 +2,14 @@ import { activityStore } from '#lib/stores/activity.store.svelte';
import { markActivityToastShown } from '#lib/components/activity/activity-completion-toasts';
import { m } from '#lib/paraglide/messages';
-export function activityToastOptions(activityId?: string) {
+export function activityToastOptions(activityId?: string, suppressCompletion = true) {
if (!activityId) {
return undefined;
}
// The caller is about to show a response toast for this activity; keep the
// stream-driven completion toast from duplicating it.
- markActivityToastShown(activityId);
+ if (suppressCompletion) markActivityToastShown(activityId);
return {
action: {
diff --git a/frontend/src/routes/(app)/settings/backups/+page.svelte b/frontend/src/routes/(app)/settings/backups/+page.svelte
index cd497becc..8f137adb4 100644
--- a/frontend/src/routes/(app)/settings/backups/+page.svelte
+++ b/frontend/src/routes/(app)/settings/backups/+page.svelte
@@ -1,5 +1,7 @@
<script lang="ts">
import { goto } from '$app/navigation';
+ import { useBackupActivity } from '#lib/hooks/use-backup-activity.svelte';
+ import { activityStore } from '#lib/stores/activity.store.svelte';
import { toast } from 'svelte-sonner';
import settingsStore from '#lib/stores/config-store';
import { SettingsPageLayout, type SettingsActionButton } from '#lib/layouts';
@@ -300,18 +302,28 @@
})();
});
+ const backupActivity = useBackupActivity(
+ () => '0',
+ (activity) =>
+ activity.metadata?.['action'] === 'create_system_backup' ||
+ activity.metadata?.['action'] === 'scheduled_system_backup' ||
+ activity.metadata?.['action'] === 'run_system_volume_backups',
+ refresh
+ );
+
async function submitAction() {
- if (loading || invalid) return;
+ if (loading || invalid || (action === 'create' && backupActivity.activeIds.length)) return;
loading = true;
try {
if (action === 'create') {
if (backupType === 'system') {
- await systemBackupService.create(
+ const result = await systemBackupService.create(
backupConfiguration === 'custom'
? { destination, s3DestinationId: destination === 'local' ? '' : s3DestinationId, recoveryKey }
: { policyId: backupConfiguration, recoveryKey }
);
- toast.success(m.system_backups_created());
+ backupActivity.accepted(extractActivityId(result));
+ toast.success(m.backups_started(), activityToastOptions(extractActivityId(result), false));
} else {
const result = await systemBackupService.runSystemVolumeBackups(
backupConfiguration === 'custom'
@@ -327,16 +339,12 @@
}
: { policyId: backupConfiguration }
);
- toast.success(
- m.system_volume_backups_run_result({
- matched: result.matched,
- succeeded: result.succeeded,
- failed: result.failed,
- skipped: result.skipped
- })
- );
+ backupActivity.accepted(result.activityId);
+ toast.success(m.backups_started(), activityToastOptions(result.activityId, false));
}
- await refresh();
+ actionOpen = false;
+ loading = false;
+ void refresh().catch((error) => console.warn('Failed to refresh backup history', error));
} else if (action === 'restore' && selected) {
await systemBackupService.restore(selected.id, recoveryKey);
toast.success(m.system_backups_restore_started());
@@ -412,7 +420,11 @@
disabled: isReadOnly,
options: [
{ label: m.jobs_schedule(), onclick: () => openSchedule() },
- { label: m.volumes_workspace_backup(), onclick: () => openAction('create') }
+ {
+ label: m.volumes_workspace_backup(),
+ onclick: () => openAction('create'),
+ disabled: backupActivity.activeIds.length > 0
+ }
]
}
]);
@@ -573,7 +585,7 @@
customLabel={dialogTitle()}
onclick={submitAction}
{loading}
- disabled={loading || invalid}
+ disabled={loading || invalid || (action === 'create' && backupActivity.activeIds.length > 0)}
/>
{/snippet}
</ResponsiveDialog>
@@ -690,6 +702,20 @@
{/if}
</div>
+ {#each backupActivity.activeIds as activityId (activityId)}
+ <Alert.Root
+ ><Alert.Description>
+ {m.backups_running()}
+ <ArcaneButton
+ action="inspect"
+ size="sm"
+ customLabel={m.activity_view_activity()}
+ onclick={() => activityStore.openCenter(activityId)}
+ />
+ </Alert.Description></Alert.Root
+ >
+ {/each}
+
<SystemBackupTable
bind:backups
bind:requestOptions
diff --git a/frontend/src/routes/(app)/volumes/components/volume-backup-table.svelte b/frontend/src/routes/(app)/volumes/components/volume-backup-table.svelte
index b6e3421ed..a388538e1 100644
--- a/frontend/src/routes/(app)/volumes/components/volume-backup-table.svelte
+++ b/frontend/src/routes/(app)/volumes/components/volume-backup-table.svelte
@@ -6,6 +6,8 @@
import type { BackupEntry, CreateVolumeBackupRequest, VolumeBackupPolicy } from '#lib/types/shared';
import type { S3Destination } from '#lib/types/s3-destination';
import { onMount } from 'svelte';
+ import { useBackupActivity } from '#lib/hooks/use-backup-activity.svelte';
+ import { activityStore } from '#lib/stores/activity.store.svelte';
import {
TrashIcon,
AddIcon,
@@ -73,6 +75,7 @@
} = $props();
const currentEnvId = $derived(environmentStore.selected?.id || '0');
+ const canReadActivities = $derived(hasPermission('activities:read', currentEnvId));
const canBackupVolume = $derived(hasPermission('volumes:backup', currentEnvId));
const canDeleteBackup = $derived(hasPermission('volumes:backup', currentEnvId));
@@ -123,12 +126,55 @@
}
}
+ const backupActivity = useBackupActivity(
+ () => currentEnvId,
+ (activity) => {
+ const action = activity.metadata?.['action'];
+ return (
+ ((action === 'create_volume_backup' || action === 'scheduled_volume_backup') && activity.resourceName === volumeName) ||
+ (action === 'run_system_volume_backups' &&
+ Array.isArray(activity.metadata?.['volumeNames']) &&
+ activity.metadata['volumeNames'].includes(volumeName))
+ );
+ },
+ () => loadData(requestOptions),
+ () => volumeName,
+ () => {
+ const name = volumeName;
+ if (canReadActivities) return undefined;
+ return async (environmentId) => {
+ const ids: string[] = [];
+ let page = 1;
+ let totalPages = 1;
+ do {
+ const result = await volumeBackupService.listBackups(
+ name,
+ {
+ search: 'running',
+ pagination: { page, limit: 100 }
+ },
+ environmentId
+ );
+ ids.push(...result.data.filter((backup) => backup.status === 'running').map((backup) => backup.id));
+ totalPages = result.pagination.totalPages;
+ page++;
+ } while (page <= totalPages);
+ return ids;
+ };
+ }
+ );
+
async function handleCreate(request?: CreateVolumeBackupRequest) {
+ if (creating || backupActivity.activeIds.length) return false;
creating = true;
try {
const result = await volumeBackupService.createBackup(volumeName, request);
- toast.success(m.common_success(), activityToastOptions(extractActivityId(result)));
- await loadData(requestOptions);
+ showS3DestinationDialog = false;
+ onDemandS3DestinationId = '';
+ creating = false;
+ backupActivity.accepted(canReadActivities ? extractActivityId(result) : result.id);
+ toast.success(m.backups_started(), canReadActivities ? activityToastOptions(extractActivityId(result), false) : undefined);
+ void loadData(requestOptions);
return true;
} catch (error) {
toast.error(error instanceof Error ? error.message : m.common_failed());
@@ -434,7 +480,7 @@
action="create"
customLabel={m.volumes_backup_create()}
loading={creating}
- disabled={creating}
+ disabled={creating || backupActivity.activeIds.length > 0}
onclick={() => handleCreate()}
size="sm"
icon={AddIcon}
@@ -443,7 +489,7 @@
<DropdownMenu.Trigger
class={cn(arcaneButtonVariants({ tone: 'outline-primary', size: 'icon' }), 'size-8 rounded-md')}
aria-label={m.common_open_menu()}
- disabled={creating}
+ disabled={creating || backupActivity.activeIds.length > 0}
>
<ArrowDownIcon class="size-4" />
</DropdownMenu.Trigger>
@@ -522,6 +568,21 @@
{/snippet}
<div class="space-y-4">
+ {#each backupActivity.activeIds as activityId (activityId)}
+ <Alert.Root
+ ><Alert.Description>
+ {m.backups_running()}
+ {#if canReadActivities}
+ <ArcaneButton
+ action="inspect"
+ size="sm"
+ customLabel={m.activity_view_activity()}
+ onclick={() => activityStore.openCenter(activityId)}
+ />
+ {/if}
+ </Alert.Description></Alert.Root
+ >
+ {/each}
<div class="flex items-center justify-between">
<h2 class="text-lg font-semibold">{m.volumes_backups_title()}</h2>
</div>
@@ -665,7 +726,7 @@
customLabel={m.volumes_backup_create()}
onclick={createS3Backup}
loading={creating}
- disabled={creating || !onDemandS3DestinationId}
+ disabled={creating || backupActivity.activeIds.length > 0 || !onDemandS3DestinationId}
/>
{/snippet}
</ResponsiveDialog>
diff --git a/tests/spec/system-volume-backups.spec.ts b/tests/spec/system-volume-backups.spec.ts
index 5f12d7da0..28cf54e9f 100644
--- a/tests/spec/system-volume-backups.spec.ts
+++ b/tests/spec/system-volume-backups.spec.ts
@@ -1,4 +1,5 @@
import { expect, test, type Page } from '../fixtures/test.fixture';
+import type { Activity } from '../../frontend/src/lib/types/activity.type';
type ManagementType = 'system' | 'volume';
@@ -93,6 +94,13 @@ async function mockSystemBackupPage(
) {
let savedCollection = structuredClone(collection);
const runRequests: unknown[] = [];
+ const activities: Activity[] = [];
+ await page.route('**/api/environments/0/activities?**', (route) => {
+ const status = new URL(route.request().url()).searchParams.get('status');
+ return route.fulfill({
+ json: paginated(activities.filter((activity) => !status || activity.status === status))
+ });
+ });
await page.route('**/api/backups/volumes/config', async (route) => {
if (route.request().method() === 'PUT') {
@@ -128,9 +136,18 @@ async function mockSystemBackupPage(
await page.route('**/api/backups/volumes/options', (route) => route.fulfill({ json: options }));
await page.route('**/api/backups/volumes/run', async (route) => {
runRequests.push(await route.request().postDataJSON());
- return route.fulfill({
- json: { matched: 2, succeeded: 1, failed: 0, skipped: 1, failures: [] }
- });
+ const now = new Date().toISOString();
+ const activity: Activity = {
+ id: `system-volume-run-${runRequests.length}`,
+ environmentId: '0',
+ type: 'resource_action',
+ status: 'running',
+ metadata: { action: 'run_system_volume_backups' },
+ startedAt: now,
+ createdAt: now
+ };
+ activities.push(activity);
+ return route.fulfill({ status: 202, json: { activityId: activity.id, status: 'running' } });
});
await page.route('**/api/backups/history**', (route) => {
const type = new URL(route.request().url()).searchParams.get('type');
@@ -139,7 +156,29 @@ async function mockSystemBackupPage(
return route.fulfill({ json: paginated(rows) });
});
- return { savedCollection: () => savedCollection, runRequests: () => runRequests };
+ return {
+ savedCollection: () => savedCollection,
+ runRequests: () => runRequests,
+ completeRun: () => {
+ const activity = activities.at(-1);
+ if (!activity) throw new Error('No backup run was accepted');
+ activity.status = 'success';
+ activity.endedAt = new Date().toISOString();
+ activity.metadata = {
+ ...activity.metadata,
+ matched: 2,
+ succeeded: 1,
+ failed: 0,
+ skipped: 1,
+ failures: []
+ };
+ history.push({
+ ...historyEntry('completed-batch-volume', 'system'),
+ id: activity.id,
+ trigger: 'manual'
+ });
+ }
+ };
}
test.describe('System-managed volume backups', () => {
@@ -200,12 +239,25 @@ test.describe('System-managed volume backups', () => {
await page.getByRole('button', { name: 'Create', exact: true }).click();
await page.getByRole('menuitem', { name: 'Backup' }).click();
const createBackup = page.getByRole('dialog', { name: 'Create Backup' });
+ const runningBackup = page.getByRole('alert').filter({ hasText: 'Backup in progress' });
await createBackup.getByLabel('Backup type').click();
await page.getByRole('option', { name: 'Volume' }).click();
await createBackup.getByLabel('Backup configuration').click();
await page.getByRole('option', { name: '0 0 2 * * *' }).click();
await createBackup.getByRole('button', { name: 'Create Backup' }).click();
- await expect(page.getByText('Matched 2; 1 succeeded, 0 failed, and 1 skipped.')).toBeVisible();
+ await expect(createBackup).toBeHidden();
+ await expect(page.getByText('Backup started', { exact: true })).toBeVisible();
+ await expect(runningBackup).toBeVisible();
+ await page.getByRole('button', { name: 'Create', exact: true }).click();
+ await expect(page.getByRole('menuitem', { name: 'Backup', exact: true })).toBeDisabled();
+ await page.keyboard.press('Escape');
+ await page.reload();
+ await expect(runningBackup).toBeVisible();
+ mock.completeRun();
+ await expect(runningBackup).toBeHidden({
+ timeout: 15000
+ });
+ await expect(page.getByRole('table').getByText('completed-batch-volume')).toBeVisible();
expect(mock.runRequests()).toContainEqual({ policyId: 'volume-nightly' });
await page.getByRole('button', { name: 'Create', exact: true }).click();
@@ -217,6 +269,8 @@ test.describe('System-managed volume backups', () => {
await page.getByRole('option', { name: 'Allowlist' }).click();
await customBackup.locator('label').filter({ hasText: liveName }).getByRole('checkbox').click();
await customBackup.getByRole('button', { name: 'Create Backup' }).click();
+ await expect(customBackup).toBeHidden();
+ await expect(runningBackup).toBeVisible();
expect(mock.runRequests()).toContainEqual({
custom: {
destination: 'local',
diff --git a/types/backup/system_backup.go b/types/backup/system_backup.go
index c9a53a7b4..58bf2926d 100644
--- a/types/backup/system_backup.go
+++ b/types/backup/system_backup.go
@@ -11,6 +11,7 @@ const (
)
type SystemBackupRun struct {
+ ActivityID string `json:"activityId,omitempty"`
ID string `json:"id"`
Size int64 `json:"size"`
CreatedAt time.Time `json:"createdAt"`
diff --git a/types/backup/system_volume_backup.go b/types/backup/system_volume_backup.go
index acac23276..78ee13ad7 100644
--- a/types/backup/system_volume_backup.go
+++ b/types/backup/system_volume_backup.go
@@ -113,3 +113,9 @@ type HistoryEntry struct {
ResourceType string `json:"resourceType"`
ResourceName string `json:"resourceName" sortable:"true"`
}
+
+// BackupRunAccepted identifies a backup operation accepted for background execution.
+type BackupRunAccepted struct {
+ ActivityID string `json:"activityId"`
+ Status string `json:"status"`
+}
diff --git a/types/volume/backup.go b/types/volume/backup.go
index c923d6f7e..7b6feb799 100644
--- a/types/volume/backup.go
+++ b/types/volume/backup.go
@@ -28,6 +28,7 @@ const (
)
type BackupEntry struct {
+ ActivityID *string `json:"activityId,omitempty"`
ID string `json:"id" doc:"Unique identifier of the backup"`
VolumeName string `json:"volumeName" doc:"Name of the volume"`
Size int64 `json:"size" doc:"Total size of the backup contents"`