Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix: run backups in an activity instead of locking the ui (#3847)
Kyle Mendelldd16fd942afa
diff --git a/backend/internal/activity/backup.go b/backend/internal/activity/backup.gonew file mode 100644index 000000000..7b0faf959--- /dev/null+++ b/backend/internal/activity/backup.go@@ -0,0 +1,30 @@+package activity++import (+ "context"++ "emperror.dev/errors"+ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"+)++// FailInterruptedBackups finalizes backup activities before startup admits work.+func (s *ActivityService) FailInterruptedBackups(ctx context.Context) error {+ var pending []Activity+ if err := s.db.WithContext(ctx).Where("status IN ?", []activitytypes.Status{activitytypes.StatusQueued, activitytypes.StatusRunning}).Find(&pending).Error; err != nil {+ return errors.WrapIf(err, "find interrupted backup activities")+ }+ const message = "Backup interrupted by Arcane restart"+ var result error+ for _, entry := range pending {+ if s.isTrackedInternal(entry.ID) {+ continue+ }+ switch entry.Metadata["action"] {+ case "create_volume_backup", "create_system_backup", "run_system_volume_backups", "scheduled_volume_backup", "scheduled_system_backup":+ errMessage := message+ _, err := s.CompleteActivity(ctx, entry.ID, activitytypes.StatusFailed, message, &errMessage)+ result = errors.Combine(result, err)+ }+ }+ return result+}diff --git a/backend/internal/backup/engine.go b/backend/internal/backup/engine.goindex d6df12468..19d3654f9 100644--- a/backend/internal/backup/engine.go+++ b/backend/internal/backup/engine.go@@ -75,6 +75,8 @@ type Engine struct {admission *actors.Gate[actors.AdmissionKey]lifecycleCtx context.Contextexecutors *actors.StateMap[string, *actors.Executor]+ runs *actors.StateMap[string, *backupRunInternal]+ stopping bool // Accessed inside runs.Apply.}// NewEngine creates the shared backup engine on the actor runtime. ctx is the@@ -86,6 +88,7 @@ func NewEngine(ctx context.Context, runtime *actors.Runtime, admission *actors.Gadmission: admission,lifecycleCtx: ctx,executors: actors.NewStateMap[string, *actors.Executor](),+ runs: actors.NewStateMap[string, *backupRunInternal](),}}@@ -98,16 +101,19 @@ func (e *Engine) TryAcquireRun(ctx context.Context, scope, id string) (*actors.Lreturn e.admission.TryAcquire(ctx, actors.AdmissionKey{Scope: scope, ID: id})}-// Stop joins every repository executor.+// Stop cancels and joins backup runs before stopping their repository executors.func (e *Engine) Stop(ctx context.Context) error {if e == nil {return nil}+ stopErr := e.stopRunsInternal(ctx)+ if stopErr != nil {+ return stopErr+ }executors, err := e.executors.Drain(ctx, "stop backup repository executors")if err != nil {return err}- var stopErr errorfor _, executor := range executors {stopErr = errors.Combine(stopErr, executor.Stop(ctx))}@@ -321,7 +327,7 @@ func (e *Engine) executorForInternal(repositoryID string) (*actors.Executor, errif executor, ok := values[repositoryID]; ok {return executor, false, nil}- executor, err := actors.NewExecutor(e.lifecycleCtx, e.runtime, "backup-repository", repositoryID, 3)+ executor, err := actors.NewExecutor(context.WithoutCancel(e.lifecycleCtx), e.runtime, "backup-repository", repositoryID, 3)if err != nil {return nil, false, err}@@ -337,13 +343,18 @@ func (e *Engine) runInternal(ctx context.Context, dockerClient *client.Client, rif strings.TrimSpace(repository.ID) == "" {return "", errors.New("backup repository ID is required")}- executor, err := e.executorForInternal(repository.ID)+ executor, err := e.executorForInternal(repository.ID) //nolint:contextcheck // Repository executors outlive requests so shutdown cleanup can finish.if err != nil {return "", err}- return executor.Execute(ctx, "rustic "+command[0], func(workCtx context.Context) (string, error) {+ task, err := executor.Submit(ctx, "rustic "+command[0], func(workCtx context.Context) (string, error) {return e.runContainerInternal(workCtx, dockerClient, repository, password, command, extraMounts...)}, nil)+ if err != nil {+ return "", err+ }+ // Cancellation must finish helper cleanup before callers release source locks.+ return task.Wait(context.WithoutCancel(ctx))}func (e *Engine) ensureImageInternal(ctx context.Context, dockerClient *client.Client) error {diff --git a/backend/internal/backup/runs.go b/backend/internal/backup/runs.gonew file mode 100644index 000000000..62ee554b1--- /dev/null+++ b/backend/internal/backup/runs.go@@ -0,0 +1,84 @@+package backup++import (+ "context"+ "log/slog"++ "emperror.dev/errors"+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors"+)++type backupRunInternal struct {+ executor *actors.Executor+ cancel context.CancelFunc+}++// SubmitRun accepts one backup on an executor independent of repository work.+// after owns finalization only when submission succeeds.+func (e *Engine) SubmitRun(ctx context.Context, id string, work func(context.Context) error, after func(error)) error {+ if e == nil || e.runs == nil {+ return errors.New("backup engine is unavailable")+ }+ if err := ctx.Err(); err != nil {+ return err+ }+ return e.runs.Apply(ctx, "submit backup run", func(runs map[string]*backupRunInternal) (bool, error) {+ if e.stopping {+ return false, errors.New("backup engine is stopping")+ }+ if _, exists := runs[id]; exists {+ return false, errors.New("backup run already submitted")+ }+ runCtx, cancel := context.WithCancel(ctx)+ // Keep the mailbox alive long enough to finalize cancelled work.+ executor, err := actors.NewExecutor(context.WithoutCancel(runCtx), e.runtime, "backup-runs", id, 0)+ if err != nil {+ cancel()+ return false, err+ }+ _, err = executor.Submit(runCtx, "run backup", func(workerCtx context.Context) (actors.NoPayload, error) {+ return actors.NoPayload{}, work(workerCtx)+ }, func(_ actors.NoPayload, runErr error) {+ defer func() {+ cancel()+ // Queue termination without waiting on our own mailbox.+ if stopErr := executor.Stop(runCtx); stopErr != nil && !errors.Is(stopErr, context.Canceled) {+ slog.ErrorContext(context.WithoutCancel(ctx), "Failed to stop backup executor", "runId", id, "error", stopErr)+ }+ if _, _, removeErr := e.runs.Remove(context.WithoutCancel(ctx), "complete backup run", id); removeErr != nil {+ slog.ErrorContext(context.WithoutCancel(ctx), "Failed to remove completed backup run", "runId", id, "error", removeErr)+ }+ }()+ after(runErr)+ })+ if err != nil {+ cancel()+ stopErr := executor.Stop(runCtx)+ if errors.Is(stopErr, context.Canceled) {+ stopErr = nil // The stop was queued; only its wait was cancelled.+ }+ return false, errors.Combine(err, stopErr)+ }+ runs[id] = &backupRunInternal{executor: executor, cancel: cancel}+ return true, nil+ })+}++func (e *Engine) stopRunsInternal(ctx context.Context) error {+ var pending []*backupRunInternal+ if err := e.runs.Apply(ctx, "stop backup runs", func(runs map[string]*backupRunInternal) (bool, error) {+ e.stopping = true+ for _, run := range runs {+ pending = append(pending, run)+ run.cancel()+ }+ return false, nil+ }); err != nil {+ return err+ }+ var stopErr error+ for _, run := range pending {+ stopErr = errors.Combine(stopErr, run.executor.Stop(ctx))+ }+ return stopErr+}diff --git a/backend/internal/bootstrap/jobs_bootstrap.go b/backend/internal/bootstrap/jobs_bootstrap.goindex 8572d2c17..0283bc4bc 100644--- a/backend/internal/bootstrap/jobs_bootstrap.go+++ b/backend/internal/bootstrap/jobs_bootstrap.go@@ -106,6 +106,9 @@ func registerJobs(params registerJobsParams) error {// Bootstrap owns registration, agent-mode gating, and settings callbacks.if params.Activity != nil {+ if err := params.Activity.FailInterruptedBackups(params.AppCtx); err != nil {+ return err+ }failed, err := params.Activity.FailStaleImageUpdateChecks(params.AppCtx)if err != nil {slog.WarnContext(params.AppCtx, "Failed to mark stale image update checks as failed", "count", failed, "error", err)@@ -212,12 +215,18 @@ func registerDynamicJobs(params dynamicJobsParams) error {// registered on managers and agents; environment proxying persists each policy// in the correct Arcane database.if params.Volume != nil {+ if err := params.Volume.ReconcileInterruptedBackups(params.AppCtx); err != nil {+ return err+ }if err := params.Volume.SetScheduler(params.AppCtx, params.Scheduler, params.Admission); err != nil {return err}params.Volume.RegisterBackupJobsOnStartup(params.AppCtx)}if !params.Config.AgentMode && params.SystemBackup != nil {+ if err := params.SystemBackup.ReconcileInterruptedBackups(params.AppCtx); err != nil {+ return err+ }if err := params.SystemBackup.SetScheduler(params.AppCtx, params.Scheduler, params.Admission); err != nil {return err}diff --git a/backend/internal/systembackup/background.go b/backend/internal/systembackup/background.gonew file mode 100644index 000000000..7316fdee9--- /dev/null+++ b/backend/internal/systembackup/background.go@@ -0,0 +1,125 @@+package systembackup++import (+ "context"+ "fmt"+ "log/slog"++ "emperror.dev/errors"++ "github.com/getarcaneapp/arcane/backend/v2/internal/backup"+ "github.com/getarcaneapp/arcane/backend/v2/internal/common"+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"+ "github.com/getarcaneapp/arcane/backend/v2/internal/volume"+ activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"+ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"+ backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"+)++// StartBackup prepares a manual backup before submitting its snapshot work.+func (s *SystemBackupService) StartBackup(ctx context.Context, user common.User, request backuptypes.CreateSystemBackupRequest) (*backuptypes.SystemBackupRun, error) {+ lease, admitted, err := s.engine.TryAcquireRun(ctx, backup.SystemAdmissionScope, systemAdmissionID)+ if err != nil {+ return nil, err+ }+ if !admitted {+ return nil, ErrSystemBackupAlreadyRunning+ }+ prepared, err := s.prepareBackupInternal(ctx, SystemBackupTriggerManual, request)+ if err != nil {+ lease.Release()+ return nil, err+ }+ activityID, workCtx := activitylib.StartHandlerActivity(ctx, s.activityService, "0", activitytypes.TypeResourceAction, "system_backup", "arcane", "Arcane", &user,+ "Creating system backup", "Creating Arcane system backup", database.JSON{"action": "create_system_backup", "backupId": prepared.run.ID, "destination": prepared.run.Destination, "s3DestinationId": prepared.run.S3DestinationID, "policyId": prepared.run.PolicyID}, false)+ finish := func(runErr error) {+ defer lease.Release()+ if runErr != nil {+ if saveErr := s.db.WithContext(context.WithoutCancel(workCtx)).Model(&SystemBackupRun{}).Where("id = ?", prepared.run.ID).Updates(map[string]any{"status": SystemBackupStatusFailed, "error": runErr.Error()}).Error; saveErr != nil {+ runErr = errors.Combine(runErr, errors.WrapIf(saveErr, "save backup failure"))+ }+ }+ activitylib.CompleteHandlerActivity(workCtx, s.activityService, activityID, "Arcane system backup created successfully", runErr)+ }+ if activityID == "" {+ err = errors.New("failed to create system backup activity")+ finish(err)+ return nil, err+ }+ dto := prepared.run.ToDTO()+ dto.ActivityID = activityID+ if err = s.engine.SubmitRun(workCtx, prepared.run.ID, func(runCtx context.Context) error {+ _, runErr := s.executeBackupInternal(runCtx, prepared)+ return runErr+ }, finish); err != nil {+ finish(err)+ return nil, err+ }+ return &dto, nil+}++// ReconcileInterruptedBackups marks work interrupted by the previous process.+func (s *SystemBackupService) ReconcileInterruptedBackups(ctx context.Context) error {+ return s.db.WithContext(ctx).Model(&SystemBackupRun{}).Where("status = ?", SystemBackupStatusRunning).+ Updates(map[string]any{"status": SystemBackupStatusFailed, "error": "Backup interrupted by application restart"}).Error+}++// StartSystemVolumeBackups freezes the selected policy and volumes before returning.+func (s *SystemBackupService) StartSystemVolumeBackups(ctx context.Context, user common.User, request backuptypes.RunSystemVolumeBackupsRequest) (*backuptypes.BackupRunAccepted, error) {+ prepared, err := s.prepareSystemVolumeBackupsInternal(ctx, request)+ if err != nil {+ return nil, err+ }+ policy, manualPolicy, candidates, lease := prepared.policy, prepared.manualPolicy, prepared.candidates, prepared.lease++ names := make([]string, len(candidates))+ for i, candidate := range candidates {+ names[i] = candidate.Name+ }+ activityID, workCtx := activitylib.StartHandlerActivity(ctx, s.activityService, "0", activitytypes.TypeResourceAction, "system_backup", "volumes", "Volumes", &user,+ "Backing up volumes", "Creating system-managed volume backups", database.JSON{"action": "run_system_volume_backups", "policyId": policy.ID, "volumeNames": names, "matched": len(candidates), "succeeded": 0, "failed": 0, "skipped": 0, "failures": []backuptypes.SystemVolumeBackupFailure{}}, false)+ if activityID == "" {+ lease.Release()+ return nil, errors.New("failed to create system-managed volume backup activity")+ }+ finish := func(runErr error) {+ defer lease.Release()+ activitylib.CompleteHandlerActivity(workCtx, s.activityService, activityID, "System-managed volume backups completed", runErr)+ }+ err = s.engine.SubmitRun(workCtx, activityID, func(runCtx context.Context) error {+ result, runErr := s.executeSystemVolumeBackupsInternal(runCtx, policy, manualPolicy, candidates, volume.VolumeBackupTriggerManual, activityID)+ if runErr != nil {+ return runErr+ }+ if result.Failed > 0 {+ return fmt.Errorf("%d volume backups failed", result.Failed)+ }+ return nil+ }, finish)+ if err != nil {+ finish(err)+ return nil, err+ }+ return &backuptypes.BackupRunAccepted{ActivityID: activityID, Status: "running"}, nil+}++func (s *SystemBackupService) updateSystemVolumeProgressInternal(ctx context.Context, activityID string, policyID string, candidates []backuptypes.SystemVolumeBackupOption, result *backuptypes.SystemVolumeBackupRunResult) {+ if activityID == "" {+ return+ }+ names := make([]string, len(candidates))+ for i, candidate := range candidates {+ names[i] = candidate.Name+ }+ progress := 100+ if result.Matched > 0 {+ progress = 100 * (result.Succeeded + result.Failed + result.Skipped) / result.Matched+ }+ _, err := s.activityService.UpdateActivity(ctx, activityID, activitylib.UpdateRequest{Progress: &progress, Metadata: database.JSON{+ "action": "run_system_volume_backups", "policyId": policyID, "volumeNames": names,+ "matched": result.Matched, "succeeded": result.Succeeded, "failed": result.Failed, "skipped": result.Skipped, "failures": result.Failures,+ }})+ if err != nil {+ slog.WarnContext(ctx, "Failed to report system-managed volume backup progress", "activityId", activityID, "policyId", policyID, "error", err)+ }+}diff --git a/backend/internal/systembackup/handler.go b/backend/internal/systembackup/handler.goindex f9328a88d..be308b9d4 100644--- a/backend/internal/systembackup/handler.go+++ b/backend/internal/systembackup/handler.go@@ -58,7 +58,7 @@ type SystemVolumeBackupOptionsOutput struct {}type SystemVolumeBackupRunOutput struct {- Body backuptypes.SystemVolumeBackupRunResult+ Body backuptypes.BackupRunAccepted}type RunSystemVolumeBackupsInput struct {@@ -124,12 +124,12 @@ func RegisterSystemBackups(api huma.API, service *SystemBackupService, activitySmiddleware.RegisterWithPermission(api, huma.Operation{OperationID: "get-system-volume-backup-config", Method: http.MethodGet, Path: "/backups/volumes/config", Summary: "Get system-managed volume backup policies", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsRead, h.GetSystemVolumeConfig)middleware.RegisterWithPermission(api, huma.Operation{OperationID: "update-system-volume-backup-config", Method: http.MethodPut, Path: "/backups/volumes/config", Summary: "Update system-managed volume backup policies", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsManage, h.UpdateSystemVolumeConfig)middleware.RegisterWithPermission(api, huma.Operation{OperationID: "list-system-volume-backup-options", Method: http.MethodGet, Path: "/backups/volumes/options", Summary: "List volumes available to system-managed backups", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsRead, h.ListSystemVolumeOptions)- middleware.RegisterWithPermission(api, huma.Operation{OperationID: "run-system-volume-backups", Method: http.MethodPost, Path: "/backups/volumes/run", Summary: "Run system-managed volume backups", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsManage, h.RunSystemVolumeBackups)+ middleware.RegisterWithPermission(api, huma.Operation{OperationID: "run-system-volume-backups", DefaultStatus: http.StatusAccepted, Method: http.MethodPost, Path: "/backups/volumes/run", Summary: "Run system-managed volume backups", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsManage, h.RunSystemVolumeBackups)middleware.RegisterWithPermission(api, huma.Operation{OperationID: "get-system-backup-policies", Method: http.MethodGet, Path: "/backups/policies", Summary: "Get Arcane system backup policies", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsRead, h.GetPolicies)middleware.RegisterWithPermission(api, huma.Operation{OperationID: "update-system-backup-policies", Method: http.MethodPut, Path: "/backups/policies", Summary: "Update Arcane system backup policies", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsManage, h.UpdatePolicies)middleware.RegisterWithPermission(api, huma.Operation{OperationID: "generate-system-backup-recovery-key", Method: http.MethodPost, Path: "/backups/recovery-key/generate", Summary: "Generate an Arcane system backup recovery key", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsRecoveryKey, h.GenerateRecoveryKey)middleware.RegisterWithPermission(api, huma.Operation{OperationID: "set-system-backup-recovery-key", Method: http.MethodPut, Path: "/backups/recovery-key", Summary: "Configure Arcane system backup recovery key", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsRecoveryKey, h.SetRecoveryKey)- middleware.RegisterWithPermission(api, huma.Operation{OperationID: "create-system-backup", Method: http.MethodPost, Path: "/backups", Summary: "Create Arcane system backup", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsManage, h.Create)+ middleware.RegisterWithPermission(api, huma.Operation{OperationID: "create-system-backup", DefaultStatus: http.StatusAccepted, Method: http.MethodPost, Path: "/backups", Summary: "Create Arcane system backup", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsManage, h.Create)middleware.RegisterWithPermission(api, huma.Operation{OperationID: "discover-system-backups", Method: http.MethodPost, Path: "/backups/discover", Summary: "Discover Arcane system backups in S3", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsManage, h.Discover)middleware.RegisterWithPermission(api, huma.Operation{OperationID: "restore-system-backup", Method: http.MethodPost, Path: "/backups/{id}/restore", Summary: "Restore Arcane system backup", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsRestore, h.Restore)middleware.RegisterWithPermission(api, huma.Operation{OperationID: "browse-system-backup-files", Method: http.MethodPost, Path: "/backups/{id}/files/browse", Summary: "Browse project files in an Arcane system backup", Tags: []string{"System Backups"}, Middlewares: adminOnly}, authz.PermSystemBackupsRead, h.BrowseFiles)@@ -183,7 +183,11 @@ func (h *SystemBackupHandler) RunSystemVolumeBackups(ctx context.Context, inputif input.Body != nil {request = *input.Body}- result, err := h.service.RunSystemVolumeBackups(ctx, request)+ user, err := handlerutil.RequireUser(ctx)+ if err != nil {+ return nil, err+ }+ result, err := h.service.StartSystemVolumeBackups(utils.ActivityRuntimeContext(ctx, h.appCtx), *user, request)if errors.Is(err, ErrSystemBackupAlreadyRunning) {return nil, huma.Error409Conflict(err.Error())}@@ -292,24 +296,14 @@ func (h *SystemBackupHandler) Create(ctx context.Context, input *CreateSystemBacif err != nil {return nil, err}- var run *SystemBackupRun- _, err = activitylib.RunHandlerActivity(utils.ActivityRuntimeContext(ctx, h.appCtx), h.activity, activitylib.HandlerOptions{- EnvironmentID: "0", Type: activitytypes.TypeResourceAction, ResourceType: "system_backup", ResourceID: "arcane", ResourceName: "Arcane", User: user,- Step: "Creating system backup", Message: "Creating Arcane system backup", SuccessMessage: "Arcane system backup created successfully",- Metadata: database.JSON{"action": "create_system_backup", "destination": input.Body.Destination, "s3DestinationId": input.Body.S3DestinationID, "policyId": input.Body.PolicyID},- }, func(activityCtx context.Context) error {- var e error- run, e = h.service.CreateBackup(activityCtx, *user, SystemBackupTriggerManual, input.Body)- return e- })+ run, err := h.service.StartBackup(utils.ActivityRuntimeContext(ctx, h.appCtx), *user, input.Body)if errors.Is(err, ErrSystemBackupAlreadyRunning) {return nil, huma.Error409Conflict(err.Error())}if err != nil {return nil, huma.Error500InternalServerError(err.Error())}- dto := run.ToDTO()- return &SystemBackupOutput{Body: dto}, nil+ return &SystemBackupOutput{Body: *run}, nil}func (h *SystemBackupHandler) Restore(ctx context.Context, input *RestoreSystemBackupInput) (*handlerutil.Out[base.MessageResponse], error) {diff --git a/backend/internal/systembackup/service.go b/backend/internal/systembackup/service.goindex b3339a673..35b73cc19 100644--- a/backend/internal/systembackup/service.go+++ b/backend/internal/systembackup/service.go@@ -369,7 +369,22 @@ func (s *SystemBackupService) createStagedSystemRecoverySnapshotInternal(ctx conreturn remoteSnapshot, remoteRepository, true, nil}-func (s *SystemBackupService) createBackupInternal(ctx context.Context, trigger SystemBackupTrigger, request backuptypes.CreateSystemBackupRequest) (_ *SystemBackupRun, err error) {+type preparedSystemBackupInternal struct {+ run *SystemBackupRun+ recoveryKey string+ localEnabled bool+ s3Enabled bool+}++func (s *SystemBackupService) createBackupInternal(ctx context.Context, trigger SystemBackupTrigger, request backuptypes.CreateSystemBackupRequest) (*SystemBackupRun, error) {+ prepared, err := s.prepareBackupInternal(ctx, trigger, request)+ if err != nil {+ return nil, err+ }+ return s.executeBackupInternal(ctx, prepared)+}++func (s *SystemBackupService) prepareBackupInternal(ctx context.Context, trigger SystemBackupTrigger, request backuptypes.CreateSystemBackupRequest) (*preparedSystemBackupInternal, error) {if !s.SupportedDatabaseProvider() {return nil, errors.New("arcane system recovery currently requires the SQLite database provider")}@@ -386,7 +401,17 @@ func (s *SystemBackupService) createBackupInternal(ctx context.Context, triggerif err := s.db.WithContext(ctx).Create(run).Error; err != nil {return nil, err}+ return &preparedSystemBackupInternal{run: run, recoveryKey: recoveryKey, localEnabled: localEnabled, s3Enabled: s3Enabled}, nil+}++func (s *SystemBackupService) executeBackupInternal(ctx context.Context, prepared *preparedSystemBackupInternal) (_ *SystemBackupRun, err error) {+ run, recoveryKey := prepared.run, prepared.recoveryKey+ localEnabled, s3Enabled, destinationID := prepared.localEnabled, prepared.s3Enabled, run.S3DestinationID+defer func() {+ if err == nil {+ err = ctx.Err()+ }if err != nil {run.Status, run.Error = SystemBackupStatusFailed, err.Error()} else {@@ -396,6 +421,7 @@ func (s *SystemBackupService) createBackupInternal(ctx context.Context, triggererr = errors.Combine(err, fmt.Errorf("failed to save system backup result: %w", saveErr))}}()+ defer utils.RecoverToError(&err, "system backup")dockerClient, err := s.dockerService.GetClient(ctx)if err != nil {return run, errdiff --git a/backend/internal/systembackup/system_volume.go b/backend/internal/systembackup/system_volume.goindex 67b0f6a26..821504e73 100644--- a/backend/internal/systembackup/system_volume.go+++ b/backend/internal/systembackup/system_volume.go@@ -12,6 +12,7 @@ import ("emperror.dev/errors"+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors""github.com/getarcaneapp/arcane/backend/v2/internal/backup""github.com/getarcaneapp/arcane/backend/v2/internal/common"s3domain "github.com/getarcaneapp/arcane/backend/v2/internal/s3"@@ -268,11 +269,23 @@ func (s *SystemBackupService) resolveSystemVolumeRunPolicyInternal(ctx context.Creturn policy, true, err}-func (s *SystemBackupService) RunSystemVolumeBackups(ctx context.Context, request backuptypes.RunSystemVolumeBackupsRequest) (*backuptypes.SystemVolumeBackupRunResult, error) {- return s.runSystemVolumeBackupsInternal(ctx, request, volume.VolumeBackupTriggerManual)+type preparedSystemVolumeBackupInternal struct {+ policy backuptypes.SystemVolumeBackupPolicy+ manualPolicy bool+ candidates []backuptypes.SystemVolumeBackupOption+ lease *actors.Lease[actors.AdmissionKey]}func (s *SystemBackupService) runSystemVolumeBackupsInternal(ctx context.Context, request backuptypes.RunSystemVolumeBackupsRequest, trigger volume.VolumeBackupTrigger) (*backuptypes.SystemVolumeBackupRunResult, error) {+ prepared, err := s.prepareSystemVolumeBackupsInternal(ctx, request)+ if err != nil {+ return nil, err+ }+ defer prepared.lease.Release()+ return s.executeSystemVolumeBackupsInternal(ctx, prepared.policy, prepared.manualPolicy, prepared.candidates, trigger, "")+}++func (s *SystemBackupService) prepareSystemVolumeBackupsInternal(ctx context.Context, request backuptypes.RunSystemVolumeBackupsRequest) (*preparedSystemVolumeBackupInternal, error) {if s.volumeService == nil {return nil, errors.New("volume service is unavailable")}@@ -287,21 +300,32 @@ func (s *SystemBackupService) runSystemVolumeBackupsInternal(ctx context.Contextif !admitted {return nil, ErrSystemBackupAlreadyRunning}- defer lease.Release()options, err := s.volumeService.ListBackupVolumeOptions(ctx)if err != nil {+ lease.Release()return nil, err}candidates := selectSystemVolumeBackupCandidatesInternal(policyConfig, options)- result := &backuptypes.SystemVolumeBackupRunResult{+ return &preparedSystemVolumeBackupInternal{policy: policyConfig, manualPolicy: manualPolicy, candidates: candidates, lease: lease}, nil+}++func (s *SystemBackupService) executeSystemVolumeBackupsInternal(ctx context.Context, policyConfig backuptypes.SystemVolumeBackupPolicy, manualPolicy bool, candidates []backuptypes.SystemVolumeBackupOption, trigger volume.VolumeBackupTrigger, activityID string) (result *backuptypes.SystemVolumeBackupRunResult, err error) {+ result = &backuptypes.SystemVolumeBackupRunResult{Matched: len(candidates), Failures: make([]backuptypes.SystemVolumeBackupFailure, 0),}+ defer s.updateSystemVolumeProgressInternal(context.WithoutCancel(ctx), activityID, policyConfig.ID, candidates, result)+ defer utils.RecoverToError(&err, "system-managed volume backup")+policy := backuptypes.UpdateBackupPolicy{Enabled: true, Schedule: policyConfig.Schedule, RetentionCount: policyConfig.RetentionCount,StopContainers: policyConfig.StopContainers, LocalEnabled: policyConfig.LocalEnabled, S3Enabled: policyConfig.S3Enabled,S3DestinationID: policyConfig.S3DestinationID,}for _, candidate := range candidates {+ if err := ctx.Err(); err != nil {+ return result, err+ }+ s.updateSystemVolumeProgressInternal(ctx, activityID, policyConfig.ID, candidates, result)overridden, policyErr := s.volumeService.HasEnabledBackupPolicy(ctx, candidate.Name)if policyErr != nil {result.Failed++diff --git a/backend/internal/volume/backup.go b/backend/internal/volume/backup.goindex 41ce07218..6a15b9497 100644--- a/backend/internal/volume/backup.go+++ b/backend/internal/volume/backup.go@@ -18,6 +18,7 @@ import ("emperror.dev/errors""gorm.io/gorm"+ "github.com/getarcaneapp/arcane/backend/v2/internal/actors""github.com/getarcaneapp/arcane/backend/v2/internal/backup""github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/internal/database"@@ -27,6 +28,7 @@ import (activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/volumehelper""github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/backupbrowser"activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"@@ -621,20 +623,25 @@ func (s *VolumeService) CreateSystemManagedBackup(ctx context.Context, volumeNamreturn s.createBackupInternal(ctx, volumeName, user, trigger, policyID, plan)}-//nolint:gocognit // backup orchestration keeps cleanup and persistence transitions in one transaction-like flow-func (s *VolumeService) createBackupInternal(ctx context.Context, volumeName string, user common.User, trigger VolumeBackupTrigger, policyID string, plan backupPlanInternal) (_ *VolumeBackup, err error) {- workspaceLock, _ := ctx.Value(volumeWorkspaceLockContextKeyInternal{}).(volumeWorkspaceLockContextInternal)- if workspaceLock.service != s || workspaceLock.volumeName != volumeName {- defer s.workspaceLocks.Lock(volumeName)()+func (s *VolumeService) createBackupInternal(ctx context.Context, volumeName string, user common.User, trigger VolumeBackupTrigger, policyID string, plan backupPlanInternal) (*VolumeBackup, error) {+ entry, lease, err := s.prepareBackupInternal(ctx, volumeName, trigger, policyID, plan)+ if err != nil {+ return nil, err}- lease, admitted, leaseErr := s.engine.TryAcquireRun(ctx, backup.VolumeAdmissionScope, volumeName)- if leaseErr != nil {- return nil, leaseErr+ defer lease.Release()+ err = s.executeBackupInternal(ctx, entry, user, plan)+ err = s.completeBackupInternal(ctx, entry, err)+ return entry, err+}++func (s *VolumeService) prepareBackupInternal(ctx context.Context, volumeName string, trigger VolumeBackupTrigger, policyID string, plan backupPlanInternal) (*VolumeBackup, *actors.Lease[actors.AdmissionKey], error) {+ lease, admitted, err := s.engine.TryAcquireRun(ctx, backup.VolumeAdmissionScope, volumeName)+ if err != nil {+ return nil, nil, err}if !admitted {- return nil, ErrVolumeBackupAlreadyRunning+ return nil, nil, ErrVolumeBackupAlreadyRunning}- defer lease.Release()entry := &VolumeBackup{VolumeName: volumeName, CreatedAt: time.Now(), Status: VolumeBackupStatusRunning,Trigger: trigger, Destination: plan.destination, Format: VolumeBackupFormatRustic,@@ -642,21 +649,41 @@ func (s *VolumeService) createBackupInternal(ctx context.Context, volumeName str}entry.ID = fmt.Sprintf("%s-%d-%s", volumeName, time.Now().UnixNano(), uuid.New().String()[:8])if err := s.db.WithContext(ctx).Create(entry).Error; err != nil {- return nil, err+ lease.Release()+ return nil, nil, err+ }+ return entry, lease, nil+}++func (s *VolumeService) completeBackupInternal(ctx context.Context, entry *VolumeBackup, err error) error {+ if err == nil {+ err = ctx.Err()+ }+ if err != nil {+ entry.Status, entry.Error = VolumeBackupStatusFailed, err.Error()+ } else {+ entry.Status, entry.Error = VolumeBackupStatusSucceeded, ""+ }+ if saveErr := s.db.WithContext(context.WithoutCancel(ctx)).Save(entry).Error; saveErr != nil {+ return errors.Combine(err, fmt.Errorf("failed to save volume backup result: %w", saveErr))+ }+ return err+}++//nolint:gocognit // backup execution keeps container recovery with snapshot work+func (s *VolumeService) executeBackupInternal(ctx context.Context, entry *VolumeBackup, user common.User, plan backupPlanInternal) (err error) {+ defer utils.RecoverToError(&err, "volume backup")+ volumeName, trigger := entry.VolumeName, entry.Trigger+ workspaceLock, _ := ctx.Value(volumeWorkspaceLockContextKeyInternal{}).(volumeWorkspaceLockContextInternal)+ if workspaceLock.service != s || workspaceLock.volumeName != volumeName {+ defer s.workspaceLocks.Lock(volumeName)()+ }+ if err := ctx.Err(); err != nil {+ return err}- defer func() {- if err != nil {- entry.Status, entry.Error = VolumeBackupStatusFailed, err.Error()- } else {- entry.Status, entry.Error = VolumeBackupStatusSucceeded, ""- }- if saveErr := s.db.WithContext(context.WithoutCancel(ctx)).Save(entry).Error; saveErr != nil {- err = errors.Combine(err, fmt.Errorf("failed to save volume backup result: %w", saveErr))- }- }()dockerClient, err := s.dockerService.GetClient(ctx)if err != nil {- return entry, err+ return err}var stopped []container.SummarycontainersStopped := false@@ -670,7 +697,7 @@ func (s *VolumeService) createBackupInternal(ctx context.Context, volumeName str}}()if err != nil {- return entry, err+ return err}}// The live volume is read once. A local+S3 backup replicates the local@@ -679,11 +706,11 @@ func (s *VolumeService) createBackupInternal(ctx context.Context, volumeName strif plan.localEnabled {repository, repoErr := s.localRusticRepositoryInternal(ctx, dockerClient, false)if repoErr != nil {- return entry, repoErr+ return repoErr}localSnapshot, err = s.engine.CreateSnapshot(ctx, dockerClient, repository, s.rusticPasswordInternal(), volumeName, volumeSourceMountInternal(volumeName))if err != nil {- return entry, fmt.Errorf("failed to create local Rustic snapshot: %w", err)+ return fmt.Errorf("failed to create local Rustic snapshot: %w", err)}entry.LocalSnapshotID = localSnapshot.IDentry.Size = localSnapshot.Size@@ -691,20 +718,20 @@ func (s *VolumeService) createBackupInternal(ctx context.Context, volumeName strif plan.s3Enabled {remoteRepository, repoErr := s.remoteRusticRepositoryInternal(ctx, plan.s3DestinationID)if repoErr != nil {- return entry, repoErr+ return repoErr}var remoteSnapshot backup.Snapshotif plan.localEnabled {localRepository, localErr := s.localRusticRepositoryInternal(ctx, dockerClient, true)if localErr != nil {- return entry, localErr+ return localErr}remoteSnapshot, err = s.engine.Replicate(ctx, dockerClient, localRepository, localSnapshot.ID, remoteRepository, s.rusticPasswordInternal(), volumeName)} else {remoteSnapshot, err = s.engine.CreateSnapshot(ctx, dockerClient, remoteRepository, s.rusticPasswordInternal(), volumeName, volumeSourceMountInternal(volumeName))}if err != nil {- return entry, fmt.Errorf("failed to create S3 Rustic snapshot: %w", err)+ return fmt.Errorf("failed to create S3 Rustic snapshot: %w", err)}entry.RemoteSnapshotID = remoteSnapshot.IDif entry.Size == 0 {@@ -715,7 +742,7 @@ func (s *VolumeService) createBackupInternal(ctx context.Context, volumeName strstopped, err = s.startContainersAfterBackupInternal(context.WithoutCancel(ctx), dockerClient, stopped, user)containersStopped = len(stopped) > 0if err != nil {- return entry, err+ return err}}// Retention failures must not fail the run: the backup itself succeeded,@@ -730,7 +757,7 @@ func (s *VolumeService) createBackupInternal(ctx context.Context, volumeName strif logErr := s.eventService.LogVolumeEvent(ctx, event.EventTypeVolumeBackupCreate, volumeName, volumeName, user.ID, user.Username, "0", metadata); logErr != nil {slog.WarnContext(ctx, "could not log volume backup create event", "volume", volumeName, "error", logErr)}- return entry, nil+ return ctx.Err()}// HasEnabledBackupPolicy reports whether a volume-level schedule takes precedence over centralized backups.diff --git a/backend/internal/volume/backup_run.go b/backend/internal/volume/backup_run.gonew file mode 100644index 000000000..934289298--- /dev/null+++ b/backend/internal/volume/backup_run.go@@ -0,0 +1,51 @@+package volume++import (+ "context"++ "emperror.dev/errors"+ "github.com/getarcaneapp/arcane/backend/v2/internal/common"+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"+ activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"+ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"+ volumetypes "github.com/getarcaneapp/arcane/types/v2/volume"+)++// StartBackup persists a running backup and submits application-owned work.+func (s *VolumeService) StartBackup(ctx context.Context, environmentID, volumeName string, user common.User, request volumetypes.CreateBackupRequest) (volumetypes.BackupEntry, error) {+ plan, err := s.resolveBackupPlanInternal(ctx, volumeName, VolumeBackupTriggerManual, request, nil)+ if err != nil {+ return volumetypes.BackupEntry{}, err+ }+ entry, lease, err := s.prepareBackupInternal(ctx, volumeName, VolumeBackupTriggerManual, request.PolicyID, plan)+ if err != nil {+ return volumetypes.BackupEntry{}, err+ }+ activityID, workCtx := activitylib.StartHandlerActivity(ctx, s.activityService, environmentID, activitytypes.TypeResourceAction,+ "volume", volumeName, volumeName, &user, "Creating backup", "Creating volume backup",+ database.JSON{"action": "create_volume_backup", "backupId": entry.ID, "destination": entry.Destination, "policyId": request.PolicyID, "s3DestinationId": entry.S3DestinationID}, false)+ if activityID == "" {+ defer lease.Release()+ return volumetypes.BackupEntry{}, s.completeBackupInternal(ctx, entry, errors.New("failed to start backup activity"))+ }+ entry.ActivityID = &activityID+ accepted := entry.ToDTO()+ complete := func(runErr error) {+ defer lease.Release()+ runErr = s.completeBackupInternal(workCtx, entry, runErr)+ activitylib.CompleteHandlerActivity(workCtx, s.activityService, activityID, "Volume backup created successfully", runErr)+ }+ if err := s.engine.SubmitRun(workCtx, entry.ID, func(workerCtx context.Context) error {+ return s.executeBackupInternal(workerCtx, entry, user, plan)+ }, complete); err != nil {+ complete(err)+ return volumetypes.BackupEntry{}, err+ }+ return accepted, nil+}++// ReconcileInterruptedBackups runs before this process can accept backup work.+func (s *VolumeService) ReconcileInterruptedBackups(ctx context.Context) error {+ return s.db.WithContext(ctx).Model(&VolumeBackup{}).Where("status = ?", VolumeBackupStatusRunning).+ Updates(map[string]any{"status": VolumeBackupStatusFailed, "error": "Backup interrupted by Arcane restart"}).Error+}diff --git a/backend/internal/volume/handler.go b/backend/internal/volume/handler.goindex 3792f2d71..327c611df 100644--- a/backend/internal/volume/handler.go+++ b/backend/internal/volume/handler.go@@ -365,12 +365,13 @@ func RegisterVolumes(api huma.API, dockerService *docker.DockerClientService, vo}, authz.PermVolumesRead, h.ListBackups)middleware.RegisterWithPermission(api, huma.Operation{- OperationID: "create-volume-backup",- Method: http.MethodPost,- Path: "/environments/{id}/volumes/{volumeName}/backups",- Summary: "Create volume backup",- Tags: []string{"Volume Backup"},- Security: handlerutil.DefaultOperationSecurity(),+ OperationID: "create-volume-backup",+ DefaultStatus: http.StatusAccepted,+ Method: http.MethodPost,+ Path: "/environments/{id}/volumes/{volumeName}/backups",+ Summary: "Create volume backup",+ Tags: []string{"Volume Backup"},+ Security: handlerutil.DefaultOperationSecurity(),}, authz.PermVolumesBackup, h.CreateBackup)middleware.RegisterWithPermission(api, huma.Operation{@@ -904,52 +905,24 @@ func (h *VolumeHandler) ListBackups(ctx context.Context, input *ListBackupsInput}, nil}-func (h *VolumeHandler) CreateBackup(ctx context.Context, input *CreateBackupInput) (*handlerutil.Out[*VolumeBackup], error) {+func (h *VolumeHandler) CreateBackup(ctx context.Context, input *CreateBackupInput) (*handlerutil.Out[volumetypes.BackupEntry], error) {user, err := handlerutil.RequireUser(ctx)if err != nil {return nil, err}-- var backup *VolumeBackup- destination := volumetypes.BackupDestination("")- policyID := ""- s3DestinationID := ""+ request := volumetypes.CreateBackupRequest{}if input.Body != nil {- destination = input.Body.Destination- policyID = input.Body.PolicyID- s3DestinationID = input.Body.S3DestinationID+ request = *input.Body}- runtimeCtx := utils.ActivityRuntimeContext(ctx, h.appCtx)- activityID, err := activitylib.RunHandlerActivity(runtimeCtx, h.activityService, activitylib.HandlerOptions{- EnvironmentID: input.EnvironmentID,- Type: activitytypes.TypeResourceAction,- ResourceType: "volume",- ResourceID: input.VolumeName,- ResourceName: input.VolumeName,- User: user,- Step: "Creating backup",- Message: "Creating volume backup",- SuccessMessage: "Volume backup created successfully",- Metadata: database.JSON{"action": "create_volume_backup", "destination": destination, "policyId": policyID, "s3DestinationId": s3DestinationID},- }, func(runtimeCtx context.Context) error {- var backupErr error- backup, backupErr = h.volumeService.CreateBackup(runtimeCtx, input.VolumeName, *user, VolumeBackupTriggerManual, volumetypes.CreateBackupRequest{- Destination: destination, PolicyID: policyID, S3DestinationID: s3DestinationID,- })- return backupErr- })+ entry, err := h.volumeService.StartBackup(utils.ActivityRuntimeContext(ctx, h.appCtx), input.EnvironmentID, input.VolumeName, *user, request)if errors.Is(err, ErrVolumeBackupAlreadyRunning) {return nil, huma.Error409Conflict(err.Error())}if err != nil {return nil, huma.Error500InternalServerError(err.Error())}- backup.ActivityID = mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()- return &handlerutil.Out[*VolumeBackup]{- Body: base.ApiResponse[*VolumeBackup]{- Success: true,- Data: backup,- },+ return &handlerutil.Out[volumetypes.BackupEntry]{+ Body: base.ApiResponse[volumetypes.BackupEntry]{Success: true, Data: entry},}, nil}diff --git a/backend/internal/volume/model.go b/backend/internal/volume/model.goindex 6a1638cb6..205d8cf57 100644--- a/backend/internal/volume/model.go+++ b/backend/internal/volume/model.go@@ -57,6 +57,7 @@ func (*VolumeBackup) TableName() string {func (b *VolumeBackup) ToDTO() volume.BackupEntry {return volume.BackupEntry{+ ActivityID: b.ActivityID,ID: b.ID,VolumeName: b.VolumeName,Size: b.Size,diff --git a/backend/internal/volume/service_test.go b/backend/internal/volume/service_test.goindex 32a8cc717..5f4d92224 100644--- a/backend/internal/volume/service_test.go+++ b/backend/internal/volume/service_test.go@@ -740,10 +740,10 @@ func TestVolumeBackup_ValidationErrors(t *testing.T) {_, err = service.UpdateBackupPolicies(ctx, "app-data", []volumetypes.UpdateBackupPolicy{{Schedule: "0 0 2 * * *", RetentionCount: 7}})require.ErrorContains(t, err, "select at least one volume backup destination")- _, err = service.CreateBackup(ctx, "app-data", common.User{}, VolumeBackupTriggerManual, volumetypes.CreateBackupRequest{Destination: volumetypes.BackupDestination("invalid")})+ _, err = service.StartBackup(ctx, "0", "app-data", common.User{}, volumetypes.CreateBackupRequest{Destination: volumetypes.BackupDestination("invalid")})require.EqualError(t, err, "invalid volume backup destination")- _, err = service.CreateBackup(ctx, "app-data", common.User{}, VolumeBackupTriggerManual, volumetypes.CreateBackupRequest{Destination: volumetypes.BackupDestinationS3})+ _, err = service.StartBackup(ctx, "0", "app-data", common.User{}, volumetypes.CreateBackupRequest{Destination: volumetypes.BackupDestinationS3})require.EqualError(t, err, "select an S3 destination for the volume backup")}diff --git a/cli/pkg/volumes/backup.go b/cli/pkg/volumes/backup.goindex 4e2222997..715431e4c 100644--- a/cli/pkg/volumes/backup.go+++ b/cli/pkg/volumes/backup.go@@ -380,7 +380,7 @@ var backupsCreateCmd = &cobra.Command{return cmdutil.PrintJSON(result.Data)}- output.Success("Backup created for volume %s", resolved.Name)+ output.Success("Backup started for volume %s", resolved.Name)output.KeyValue("ID", result.Data.ID)output.KeyValue("Status", result.Data.Status)output.KeyValue("Destination", string(result.Data.Destination))diff --git a/frontend/messages/en.json b/frontend/messages/en.jsonindex 048c56079..52148b50e 100644--- a/frontend/messages/en.json+++ b/frontend/messages/en.json@@ -3347,7 +3347,9 @@"system_backups_recovery_key_saved": "Recovery key configured.","system_backups_recovery_key_save_failed": "Failed to configure the recovery key.","system_backups_enabled_description": "Run an encrypted Arcane recovery backup on the configured cron schedule.",- "system_backups_created": "Arcane system backup created successfully.",+ "backups_started": "Backup started",+ "backups_running": "Backup in progress",+ "system_backups_created": "Arcane system backup created successfully.","system_backups_create_failed": "Failed to create Arcane system backup.","system_backups_policy_saved": "System backup schedule saved.","system_backups_policy_failed": "Failed to save the system backup schedule.",diff --git a/frontend/src/lib/hooks/use-backup-activity.svelte.ts b/frontend/src/lib/hooks/use-backup-activity.svelte.tsnew file mode 100644index 000000000..d291ec97b--- /dev/null+++ b/frontend/src/lib/hooks/use-backup-activity.svelte.ts@@ -0,0 +1,86 @@+import { untrack } from 'svelte';+import { createQuery, useQueryClient } from '@tanstack/svelte-query';+import { queryKeys } from '#lib/query/query-keys';+import { activityService } from '#lib/services/activity-service';+import { activityStore } from '#lib/stores/activity.store.svelte';+import { environmentStore } from '#lib/stores/environment.store.svelte';+import type { Activity } from '#lib/types/activity.type';++export function useBackupActivity(+ getEnvironmentId: () => string,+ matches: (activity: Activity) => boolean,+ onChange: () => Promise<unknown>,+ getResourceKey: () => string = () => '',+ getDiscovery: () => ((environmentId: string) => Promise<string[]>) | undefined = () => undefined+) {+ const queryClient = useQueryClient();+ const queryKey = $derived(queryKeys.backupActivities.active(getEnvironmentId(), getResourceKey(), !!getDiscovery()));+ const activeQuery = createQuery(() => {+ const environmentId = getEnvironmentId();+ const discover = getDiscovery();+ return {+ queryKey,+ queryFn: async ({ signal }): Promise<string[]> => {+ await environmentStore.ready;+ signal.throwIfAborted();+ if (discover) return discover(environmentId);+ const activities: Activity[] = [];+ for (const status of ['queued', 'running']) {+ let page = 1;+ let totalPages = 1;+ do {+ const result = await activityService.getActivities(+ { pagination: { page, limit: 100 }, filters: { status } },+ environmentId+ );+ signal.throwIfAborted();+ activities.push(...result.data);+ totalPages = result.pagination.totalPages;+ page++;+ } while (page <= totalPages);+ }+ return activities.filter(matches).map((activity) => activity.id);+ },+ refetchInterval: (query) => (discover || query.state.data?.length ? 5000 : false),+ refetchIntervalInBackground: true,+ staleTime: 0+ };+ });+ const activeIds = $derived(activeQuery.data ?? []);++ $effect(() => {+ if (!activeQuery.dataUpdatedAt) return;+ untrack(() => {+ void onChange().catch((error) => console.warn('Failed to refresh backup history', error));+ });+ });++ $effect(() => {+ const environmentId = getEnvironmentId();+ const key = queryKey;+ activityStore.connected;+ const events = activityStore.activities.filter(+ (activity) => (activity.sourceEnvironmentId || activity.environmentId) === environmentId && matches(activity)+ );+ // Include progress changes as well as terminal transitions.+ JSON.stringify(events);+ untrack(() => {+ void queryClient.cancelQueries({ queryKey: key, exact: true });+ void queryClient.invalidateQueries({ queryKey: key, exact: true });+ });+ });++ return {+ get activeIds() {+ return activeIds;+ },+ accepted(activityId?: string) {+ // Cancel older discovery before adding the accepted operation to the cache.+ void queryClient.cancelQueries({ queryKey, exact: true });+ if (activityId) {+ queryClient.setQueryData<string[]>(queryKey, (ids = []) => (ids.includes(activityId) ? ids : [...ids, activityId]));+ }+ void queryClient.invalidateQueries({ queryKey, exact: true });+ }+ };+}diff --git a/frontend/src/lib/query/query-keys.ts b/frontend/src/lib/query/query-keys.tsindex ef34dc1ab..19fbfd82f 100644--- a/frontend/src/lib/query/query-keys.ts+++ b/frontend/src/lib/query/query-keys.ts@@ -13,6 +13,10 @@ function stableSerialize(value: unknown): string {}export const queryKeys = {+ backupActivities: {+ active: (environmentId: string, resourceKey: string, discoverFromHistory: boolean) =>+ ['backup-activities', environmentId, resourceKey, discoverFromHistory] as const+ },auth: {all: ['auth'] as const,logout: () => ['auth', 'logout'] as const,diff --git a/frontend/src/lib/services/volume-backup-service.ts b/frontend/src/lib/services/volume-backup-service.tsindex ec7afc84d..d1977b819 100644--- a/frontend/src/lib/services/volume-backup-service.ts+++ b/frontend/src/lib/services/volume-backup-service.ts@@ -30,8 +30,12 @@ class VolumeBackupService extends BaseAPIService {return res.data.data;}- async listBackups(volumeName: string, options?: SearchPaginationSortRequest): Promise<VolumeBackupListResponse> {- const envId = await environmentStore.getCurrentEnvironmentId();+ async listBackups(+ volumeName: string,+ options?: SearchPaginationSortRequest,+ environmentId?: string+ ): Promise<VolumeBackupListResponse> {+ const envId = environmentId ?? (await environmentStore.getCurrentEnvironmentId());const params = transformPaginationParams(options);const res = await this.api.get(`/environments/${envId}/volumes/${volumeName}/backups`, { params });return res.data;diff --git a/frontend/src/lib/types/backup.ts b/frontend/src/lib/types/backup.tsindex a5970db2c..06be6ff3a 100644--- a/frontend/src/lib/types/backup.ts+++ b/frontend/src/lib/types/backup.ts@@ -6,6 +6,7 @@ export type BackupTrigger = 'manual' | 'scheduled' | 'safety';export type BackupManagementType = 'system' | 'volume';export type BackupRun = {+ activityId?: string;id: string;size: number;createdAt: string;diff --git a/frontend/src/lib/types/system-backup.ts b/frontend/src/lib/types/system-backup.tsindex 0e6f190ec..155fee8eb 100644--- a/frontend/src/lib/types/system-backup.ts+++ b/frontend/src/lib/types/system-backup.ts@@ -35,11 +35,8 @@ export type SystemVolumeBackupOption = {};export type SystemVolumeBackupRunResult = {- matched: number;- succeeded: number;- failed: number;- skipped: number;- failures: { volumeName: string; error: string }[];+ activityId: string;+ status: 'running';};export type SystemVolumeBackupCustomRun = {diff --git a/frontend/src/lib/utils/activity-toast.ts b/frontend/src/lib/utils/activity-toast.tsindex 1ecb32808..8cbc01acc 100644--- a/frontend/src/lib/utils/activity-toast.ts+++ b/frontend/src/lib/utils/activity-toast.ts@@ -2,14 +2,14 @@ import { activityStore } from '#lib/stores/activity.store.svelte';import { markActivityToastShown } from '#lib/components/activity/activity-completion-toasts';import { m } from '#lib/paraglide/messages';-export function activityToastOptions(activityId?: string) {+export function activityToastOptions(activityId?: string, suppressCompletion = true) {if (!activityId) {return undefined;}// The caller is about to show a response toast for this activity; keep the// stream-driven completion toast from duplicating it.- markActivityToastShown(activityId);+ if (suppressCompletion) markActivityToastShown(activityId);return {action: {diff --git a/frontend/src/routes/(app)/settings/backups/+page.svelte b/frontend/src/routes/(app)/settings/backups/+page.svelteindex cd497becc..8f137adb4 100644--- a/frontend/src/routes/(app)/settings/backups/+page.svelte+++ b/frontend/src/routes/(app)/settings/backups/+page.svelte@@ -1,5 +1,7 @@<script lang="ts">import { goto } from '$app/navigation';+ import { useBackupActivity } from '#lib/hooks/use-backup-activity.svelte';+ import { activityStore } from '#lib/stores/activity.store.svelte';import { toast } from 'svelte-sonner';import settingsStore from '#lib/stores/config-store';import { SettingsPageLayout, type SettingsActionButton } from '#lib/layouts';@@ -300,18 +302,28 @@})();});+ const backupActivity = useBackupActivity(+ () => '0',+ (activity) =>+ activity.metadata?.['action'] === 'create_system_backup' ||+ activity.metadata?.['action'] === 'scheduled_system_backup' ||+ activity.metadata?.['action'] === 'run_system_volume_backups',+ refresh+ );+async function submitAction() {- if (loading || invalid) return;+ if (loading || invalid || (action === 'create' && backupActivity.activeIds.length)) return;loading = true;try {if (action === 'create') {if (backupType === 'system') {- await systemBackupService.create(+ const result = await systemBackupService.create(backupConfiguration === 'custom'? { destination, s3DestinationId: destination === 'local' ? '' : s3DestinationId, recoveryKey }: { policyId: backupConfiguration, recoveryKey });- toast.success(m.system_backups_created());+ backupActivity.accepted(extractActivityId(result));+ toast.success(m.backups_started(), activityToastOptions(extractActivityId(result), false));} else {const result = await systemBackupService.runSystemVolumeBackups(backupConfiguration === 'custom'@@ -327,16 +339,12 @@}: { policyId: backupConfiguration });- toast.success(- m.system_volume_backups_run_result({- matched: result.matched,- succeeded: result.succeeded,- failed: result.failed,- skipped: result.skipped- })- );+ backupActivity.accepted(result.activityId);+ toast.success(m.backups_started(), activityToastOptions(result.activityId, false));}- await refresh();+ actionOpen = false;+ loading = false;+ void refresh().catch((error) => console.warn('Failed to refresh backup history', error));} else if (action === 'restore' && selected) {await systemBackupService.restore(selected.id, recoveryKey);toast.success(m.system_backups_restore_started());@@ -412,7 +420,11 @@disabled: isReadOnly,options: [{ label: m.jobs_schedule(), onclick: () => openSchedule() },- { label: m.volumes_workspace_backup(), onclick: () => openAction('create') }+ {+ label: m.volumes_workspace_backup(),+ onclick: () => openAction('create'),+ disabled: backupActivity.activeIds.length > 0+ }]}]);@@ -573,7 +585,7 @@customLabel={dialogTitle()}onclick={submitAction}{loading}- disabled={loading || invalid}+ disabled={loading || invalid || (action === 'create' && backupActivity.activeIds.length > 0)}/>{/snippet}</ResponsiveDialog>@@ -690,6 +702,20 @@{/if}</div>+ {#each backupActivity.activeIds as activityId (activityId)}+ <Alert.Root+ ><Alert.Description>+ {m.backups_running()}+ <ArcaneButton+ action="inspect"+ size="sm"+ customLabel={m.activity_view_activity()}+ onclick={() => activityStore.openCenter(activityId)}+ />+ </Alert.Description></Alert.Root+ >+ {/each}+<SystemBackupTablebind:backupsbind:requestOptionsdiff --git a/frontend/src/routes/(app)/volumes/components/volume-backup-table.svelte b/frontend/src/routes/(app)/volumes/components/volume-backup-table.svelteindex b6e3421ed..a388538e1 100644--- a/frontend/src/routes/(app)/volumes/components/volume-backup-table.svelte+++ b/frontend/src/routes/(app)/volumes/components/volume-backup-table.svelte@@ -6,6 +6,8 @@import type { BackupEntry, CreateVolumeBackupRequest, VolumeBackupPolicy } from '#lib/types/shared';import type { S3Destination } from '#lib/types/s3-destination';import { onMount } from 'svelte';+ import { useBackupActivity } from '#lib/hooks/use-backup-activity.svelte';+ import { activityStore } from '#lib/stores/activity.store.svelte';import {TrashIcon,AddIcon,@@ -73,6 +75,7 @@} = $props();const currentEnvId = $derived(environmentStore.selected?.id || '0');+ const canReadActivities = $derived(hasPermission('activities:read', currentEnvId));const canBackupVolume = $derived(hasPermission('volumes:backup', currentEnvId));const canDeleteBackup = $derived(hasPermission('volumes:backup', currentEnvId));@@ -123,12 +126,55 @@}}+ const backupActivity = useBackupActivity(+ () => currentEnvId,+ (activity) => {+ const action = activity.metadata?.['action'];+ return (+ ((action === 'create_volume_backup' || action === 'scheduled_volume_backup') && activity.resourceName === volumeName) ||+ (action === 'run_system_volume_backups' &&+ Array.isArray(activity.metadata?.['volumeNames']) &&+ activity.metadata['volumeNames'].includes(volumeName))+ );+ },+ () => loadData(requestOptions),+ () => volumeName,+ () => {+ const name = volumeName;+ if (canReadActivities) return undefined;+ return async (environmentId) => {+ const ids: string[] = [];+ let page = 1;+ let totalPages = 1;+ do {+ const result = await volumeBackupService.listBackups(+ name,+ {+ search: 'running',+ pagination: { page, limit: 100 }+ },+ environmentId+ );+ ids.push(...result.data.filter((backup) => backup.status === 'running').map((backup) => backup.id));+ totalPages = result.pagination.totalPages;+ page++;+ } while (page <= totalPages);+ return ids;+ };+ }+ );+async function handleCreate(request?: CreateVolumeBackupRequest) {+ if (creating || backupActivity.activeIds.length) return false;creating = true;try {const result = await volumeBackupService.createBackup(volumeName, request);- toast.success(m.common_success(), activityToastOptions(extractActivityId(result)));- await loadData(requestOptions);+ showS3DestinationDialog = false;+ onDemandS3DestinationId = '';+ creating = false;+ backupActivity.accepted(canReadActivities ? extractActivityId(result) : result.id);+ toast.success(m.backups_started(), canReadActivities ? activityToastOptions(extractActivityId(result), false) : undefined);+ void loadData(requestOptions);return true;} catch (error) {toast.error(error instanceof Error ? error.message : m.common_failed());@@ -434,7 +480,7 @@action="create"customLabel={m.volumes_backup_create()}loading={creating}- disabled={creating}+ disabled={creating || backupActivity.activeIds.length > 0}onclick={() => handleCreate()}size="sm"icon={AddIcon}@@ -443,7 +489,7 @@<DropdownMenu.Triggerclass={cn(arcaneButtonVariants({ tone: 'outline-primary', size: 'icon' }), 'size-8 rounded-md')}aria-label={m.common_open_menu()}- disabled={creating}+ disabled={creating || backupActivity.activeIds.length > 0}><ArrowDownIcon class="size-4" /></DropdownMenu.Trigger>@@ -522,6 +568,21 @@{/snippet}<div class="space-y-4">+ {#each backupActivity.activeIds as activityId (activityId)}+ <Alert.Root+ ><Alert.Description>+ {m.backups_running()}+ {#if canReadActivities}+ <ArcaneButton+ action="inspect"+ size="sm"+ customLabel={m.activity_view_activity()}+ onclick={() => activityStore.openCenter(activityId)}+ />+ {/if}+ </Alert.Description></Alert.Root+ >+ {/each}<div class="flex items-center justify-between"><h2 class="text-lg font-semibold">{m.volumes_backups_title()}</h2></div>@@ -665,7 +726,7 @@customLabel={m.volumes_backup_create()}onclick={createS3Backup}loading={creating}- disabled={creating || !onDemandS3DestinationId}+ disabled={creating || backupActivity.activeIds.length > 0 || !onDemandS3DestinationId}/>{/snippet}</ResponsiveDialog>diff --git a/tests/spec/system-volume-backups.spec.ts b/tests/spec/system-volume-backups.spec.tsindex 5f12d7da0..28cf54e9f 100644--- a/tests/spec/system-volume-backups.spec.ts+++ b/tests/spec/system-volume-backups.spec.ts@@ -1,4 +1,5 @@import { expect, test, type Page } from '../fixtures/test.fixture';+import type { Activity } from '../../frontend/src/lib/types/activity.type';type ManagementType = 'system' | 'volume';@@ -93,6 +94,13 @@ async function mockSystemBackupPage() {let savedCollection = structuredClone(collection);const runRequests: unknown[] = [];+ const activities: Activity[] = [];+ await page.route('**/api/environments/0/activities?**', (route) => {+ const status = new URL(route.request().url()).searchParams.get('status');+ return route.fulfill({+ json: paginated(activities.filter((activity) => !status || activity.status === status))+ });+ });await page.route('**/api/backups/volumes/config', async (route) => {if (route.request().method() === 'PUT') {@@ -128,9 +136,18 @@ async function mockSystemBackupPage(await page.route('**/api/backups/volumes/options', (route) => route.fulfill({ json: options }));await page.route('**/api/backups/volumes/run', async (route) => {runRequests.push(await route.request().postDataJSON());- return route.fulfill({- json: { matched: 2, succeeded: 1, failed: 0, skipped: 1, failures: [] }- });+ const now = new Date().toISOString();+ const activity: Activity = {+ id: `system-volume-run-${runRequests.length}`,+ environmentId: '0',+ type: 'resource_action',+ status: 'running',+ metadata: { action: 'run_system_volume_backups' },+ startedAt: now,+ createdAt: now+ };+ activities.push(activity);+ return route.fulfill({ status: 202, json: { activityId: activity.id, status: 'running' } });});await page.route('**/api/backups/history**', (route) => {const type = new URL(route.request().url()).searchParams.get('type');@@ -139,7 +156,29 @@ async function mockSystemBackupPage(return route.fulfill({ json: paginated(rows) });});- return { savedCollection: () => savedCollection, runRequests: () => runRequests };+ return {+ savedCollection: () => savedCollection,+ runRequests: () => runRequests,+ completeRun: () => {+ const activity = activities.at(-1);+ if (!activity) throw new Error('No backup run was accepted');+ activity.status = 'success';+ activity.endedAt = new Date().toISOString();+ activity.metadata = {+ ...activity.metadata,+ matched: 2,+ succeeded: 1,+ failed: 0,+ skipped: 1,+ failures: []+ };+ history.push({+ ...historyEntry('completed-batch-volume', 'system'),+ id: activity.id,+ trigger: 'manual'+ });+ }+ };}test.describe('System-managed volume backups', () => {@@ -200,12 +239,25 @@ test.describe('System-managed volume backups', () => {await page.getByRole('button', { name: 'Create', exact: true }).click();await page.getByRole('menuitem', { name: 'Backup' }).click();const createBackup = page.getByRole('dialog', { name: 'Create Backup' });+ const runningBackup = page.getByRole('alert').filter({ hasText: 'Backup in progress' });await createBackup.getByLabel('Backup type').click();await page.getByRole('option', { name: 'Volume' }).click();await createBackup.getByLabel('Backup configuration').click();await page.getByRole('option', { name: '0 0 2 * * *' }).click();await createBackup.getByRole('button', { name: 'Create Backup' }).click();- await expect(page.getByText('Matched 2; 1 succeeded, 0 failed, and 1 skipped.')).toBeVisible();+ await expect(createBackup).toBeHidden();+ await expect(page.getByText('Backup started', { exact: true })).toBeVisible();+ await expect(runningBackup).toBeVisible();+ await page.getByRole('button', { name: 'Create', exact: true }).click();+ await expect(page.getByRole('menuitem', { name: 'Backup', exact: true })).toBeDisabled();+ await page.keyboard.press('Escape');+ await page.reload();+ await expect(runningBackup).toBeVisible();+ mock.completeRun();+ await expect(runningBackup).toBeHidden({+ timeout: 15000+ });+ await expect(page.getByRole('table').getByText('completed-batch-volume')).toBeVisible();expect(mock.runRequests()).toContainEqual({ policyId: 'volume-nightly' });await page.getByRole('button', { name: 'Create', exact: true }).click();@@ -217,6 +269,8 @@ test.describe('System-managed volume backups', () => {await page.getByRole('option', { name: 'Allowlist' }).click();await customBackup.locator('label').filter({ hasText: liveName }).getByRole('checkbox').click();await customBackup.getByRole('button', { name: 'Create Backup' }).click();+ await expect(customBackup).toBeHidden();+ await expect(runningBackup).toBeVisible();expect(mock.runRequests()).toContainEqual({custom: {destination: 'local',diff --git a/types/backup/system_backup.go b/types/backup/system_backup.goindex c9a53a7b4..58bf2926d 100644--- a/types/backup/system_backup.go+++ b/types/backup/system_backup.go@@ -11,6 +11,7 @@ const ()type SystemBackupRun struct {+ ActivityID string `json:"activityId,omitempty"`ID string `json:"id"`Size int64 `json:"size"`CreatedAt time.Time `json:"createdAt"`diff --git a/types/backup/system_volume_backup.go b/types/backup/system_volume_backup.goindex acac23276..78ee13ad7 100644--- a/types/backup/system_volume_backup.go+++ b/types/backup/system_volume_backup.go@@ -113,3 +113,9 @@ type HistoryEntry struct {ResourceType string `json:"resourceType"`ResourceName string `json:"resourceName" sortable:"true"`}++// BackupRunAccepted identifies a backup operation accepted for background execution.+type BackupRunAccepted struct {+ ActivityID string `json:"activityId"`+ Status string `json:"status"`+}diff --git a/types/volume/backup.go b/types/volume/backup.goindex c923d6f7e..7b6feb799 100644--- a/types/volume/backup.go+++ b/types/volume/backup.go@@ -28,6 +28,7 @@ const ()type BackupEntry struct {+ ActivityID *string `json:"activityId,omitempty"`ID string `json:"id" doc:"Unique identifier of the backup"`VolumeName string `json:"volumeName" doc:"Name of the volume"`Size int64 `json:"size" doc:"Total size of the backup contents"`